Bug Summary

File:llvm/lib/Target/X86/X86ISelLowering.cpp
Warning:line 14996, column 21
The result of the '/' expression is undefined

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name X86ISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/lib/Target/X86 -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86 -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/include -I /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/build-llvm/lib/Target/X86 -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-01-11-115256-23437-1 -x c++ /build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp

/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp

→
1//===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "X86ISelLowering.h"
15#include "Utils/X86ShuffleDecode.h"
16#include "X86CallingConv.h"
17#include "X86FrameLowering.h"
18#include "X86InstrBuilder.h"
19#include "X86IntrinsicsInfo.h"
20#include "X86MachineFunctionInfo.h"
21#include "X86TargetMachine.h"
22#include "X86TargetObjectFile.h"
23#include "llvm/ADT/SmallBitVector.h"
24#include "llvm/ADT/SmallSet.h"
25#include "llvm/ADT/Statistic.h"
26#include "llvm/ADT/StringExtras.h"
27#include "llvm/ADT/StringSwitch.h"
28#include "llvm/Analysis/BlockFrequencyInfo.h"
29#include "llvm/Analysis/EHPersonalities.h"
30#include "llvm/Analysis/ProfileSummaryInfo.h"
31#include "llvm/CodeGen/IntrinsicLowering.h"
32#include "llvm/CodeGen/MachineFrameInfo.h"
33#include "llvm/CodeGen/MachineFunction.h"
34#include "llvm/CodeGen/MachineInstrBuilder.h"
35#include "llvm/CodeGen/MachineJumpTableInfo.h"
36#include "llvm/CodeGen/MachineModuleInfo.h"
37#include "llvm/CodeGen/MachineRegisterInfo.h"
38#include "llvm/CodeGen/TargetLowering.h"
39#include "llvm/CodeGen/WinEHFuncInfo.h"
40#include "llvm/IR/CallSite.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/Constants.h"
43#include "llvm/IR/DerivedTypes.h"
44#include "llvm/IR/DiagnosticInfo.h"
45#include "llvm/IR/Function.h"
46#include "llvm/IR/GlobalAlias.h"
47#include "llvm/IR/GlobalVariable.h"
48#include "llvm/IR/Instructions.h"
49#include "llvm/IR/Intrinsics.h"
50#include "llvm/MC/MCAsmInfo.h"
51#include "llvm/MC/MCContext.h"
52#include "llvm/MC/MCExpr.h"
53#include "llvm/MC/MCSymbol.h"
54#include "llvm/Support/CommandLine.h"
55#include "llvm/Support/Debug.h"
56#include "llvm/Support/ErrorHandling.h"
57#include "llvm/Support/KnownBits.h"
58#include "llvm/Support/MathExtras.h"
59#include "llvm/Target/TargetOptions.h"
60#include <algorithm>
61#include <bitset>
62#include <cctype>
63#include <numeric>
64using namespace llvm;
65
66#define DEBUG_TYPE"x86-isel" "x86-isel"
67
68STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"x86-isel", "NumTailCalls"
, "Number of tail calls"};
69
70static cl::opt<int> ExperimentalPrefLoopAlignment(
71 "x86-experimental-pref-loop-alignment", cl::init(4),
72 cl::desc(
73 "Sets the preferable loop alignment for experiments (as log2 bytes)"
74 "(the last x86-experimental-pref-loop-alignment bits"
75 " of the loop header PC will be 0)."),
76 cl::Hidden);
77
78// Added in 10.0.
79static cl::opt<bool> EnableOldKNLABI(
80 "x86-enable-old-knl-abi", cl::init(false),
81 cl::desc("Enables passing v32i16 and v64i8 in 2 YMM registers instead of "
82 "one ZMM register on AVX512F, but not AVX512BW targets."),
83 cl::Hidden);
84
85static cl::opt<bool> MulConstantOptimization(
86 "mul-constant-optimization", cl::init(true),
87 cl::desc("Replace 'mul x, Const' with more effective instructions like "
88 "SHIFT, LEA, etc."),
89 cl::Hidden);
90
91static cl::opt<bool> ExperimentalUnorderedISEL(
92 "x86-experimental-unordered-atomic-isel", cl::init(false),
93 cl::desc("Use LoadSDNode and StoreSDNode instead of "
94 "AtomicSDNode for unordered atomic loads and "
95 "stores respectively."),
96 cl::Hidden);
97
98/// Call this when the user attempts to do something unsupported, like
99/// returning a double without SSE2 enabled on x86_64. This is not fatal, unlike
100/// report_fatal_error, so calling code should attempt to recover without
101/// crashing.
102static void errorUnsupported(SelectionDAG &DAG, const SDLoc &dl,
103 const char *Msg) {
104 MachineFunction &MF = DAG.getMachineFunction();
105 DAG.getContext()->diagnose(
106 DiagnosticInfoUnsupported(MF.getFunction(), Msg, dl.getDebugLoc()));
107}
108
109X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
110 const X86Subtarget &STI)
111 : TargetLowering(TM), Subtarget(STI) {
112 bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
113 X86ScalarSSEf64 = Subtarget.hasSSE2();
114 X86ScalarSSEf32 = Subtarget.hasSSE1();
115 MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
116
117 // Set up the TargetLowering object.
118
119 // X86 is weird. It always uses i8 for shift amounts and setcc results.
120 setBooleanContents(ZeroOrOneBooleanContent);
121 // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
122 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
123
124 // For 64-bit, since we have so many registers, use the ILP scheduler.
125 // For 32-bit, use the register pressure specific scheduling.
126 // For Atom, always use ILP scheduling.
127 if (Subtarget.isAtom())
128 setSchedulingPreference(Sched::ILP);
129 else if (Subtarget.is64Bit())
130 setSchedulingPreference(Sched::ILP);
131 else
132 setSchedulingPreference(Sched::RegPressure);
133 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
134 setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
135
136 // Bypass expensive divides and use cheaper ones.
137 if (TM.getOptLevel() >= CodeGenOpt::Default) {
138 if (Subtarget.hasSlowDivide32())
139 addBypassSlowDiv(32, 8);
140 if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
141 addBypassSlowDiv(64, 32);
142 }
143
144 if (Subtarget.isTargetWindowsMSVC() ||
145 Subtarget.isTargetWindowsItanium()) {
146 // Setup Windows compiler runtime calls.
147 setLibcallName(RTLIB::SDIV_I64, "_alldiv");
148 setLibcallName(RTLIB::UDIV_I64, "_aulldiv");
149 setLibcallName(RTLIB::SREM_I64, "_allrem");
150 setLibcallName(RTLIB::UREM_I64, "_aullrem");
151 setLibcallName(RTLIB::MUL_I64, "_allmul");
152 setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall);
153 setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall);
154 setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall);
155 setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall);
156 setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall);
157 }
158
159 if (Subtarget.getTargetTriple().isOSMSVCRT()) {
160 // MSVCRT doesn't have powi; fall back to pow
161 setLibcallName(RTLIB::POWI_F32, nullptr);
162 setLibcallName(RTLIB::POWI_F64, nullptr);
163 }
164
165 // If we don't have cmpxchg8b(meaing this is a 386/486), limit atomic size to
166 // 32 bits so the AtomicExpandPass will expand it so we don't need cmpxchg8b.
167 // FIXME: Should we be limitting the atomic size on other configs? Default is
168 // 1024.
169 if (!Subtarget.hasCmpxchg8b())
170 setMaxAtomicSizeInBitsSupported(32);
171
172 // Set up the register classes.
173 addRegisterClass(MVT::i8, &X86::GR8RegClass);
174 addRegisterClass(MVT::i16, &X86::GR16RegClass);
175 addRegisterClass(MVT::i32, &X86::GR32RegClass);
176 if (Subtarget.is64Bit())
177 addRegisterClass(MVT::i64, &X86::GR64RegClass);
178
179 for (MVT VT : MVT::integer_valuetypes())
180 setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
181
182 // We don't accept any truncstore of integer registers.
183 setTruncStoreAction(MVT::i64, MVT::i32, Expand);
184 setTruncStoreAction(MVT::i64, MVT::i16, Expand);
185 setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
186 setTruncStoreAction(MVT::i32, MVT::i16, Expand);
187 setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
188 setTruncStoreAction(MVT::i16, MVT::i8, Expand);
189
190 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
191
192 // SETOEQ and SETUNE require checking two conditions.
193 setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand);
194 setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand);
195 setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand);
196 setCondCodeAction(ISD::SETUNE, MVT::f32, Expand);
197 setCondCodeAction(ISD::SETUNE, MVT::f64, Expand);
198 setCondCodeAction(ISD::SETUNE, MVT::f80, Expand);
199
200 // Integer absolute.
201 if (Subtarget.hasCMov()) {
202 setOperationAction(ISD::ABS , MVT::i16 , Custom);
203 setOperationAction(ISD::ABS , MVT::i32 , Custom);
204 }
205 setOperationAction(ISD::ABS , MVT::i64 , Custom);
206
207 // Funnel shifts.
208 for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
209 setOperationAction(ShiftOp , MVT::i16 , Custom);
210 setOperationAction(ShiftOp , MVT::i32 , Custom);
211 if (Subtarget.is64Bit())
212 setOperationAction(ShiftOp , MVT::i64 , Custom);
213 }
214
215 if (!Subtarget.useSoftFloat()) {
216 // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
217 // operation.
218 setOperationAction(ISD::UINT_TO_FP, MVT::i8, Promote);
219 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i8, Promote);
220 setOperationAction(ISD::UINT_TO_FP, MVT::i16, Promote);
221 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i16, Promote);
222 // We have an algorithm for SSE2, and we turn this into a 64-bit
223 // FILD or VCVTUSI2SS/SD for other targets.
224 setOperationAction(ISD::UINT_TO_FP, MVT::i32, Custom);
225 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i32, Custom);
226 // We have an algorithm for SSE2->double, and we turn this into a
227 // 64-bit FILD followed by conditional FADD for other targets.
228 setOperationAction(ISD::UINT_TO_FP, MVT::i64, Custom);
229 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::i64, Custom);
230
231 // Promote i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
232 // this operation.
233 setOperationAction(ISD::SINT_TO_FP, MVT::i8, Promote);
234 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i8, Promote);
235 // SSE has no i16 to fp conversion, only i32. We promote in the handler
236 // to allow f80 to use i16 and f64 to use i16 with sse1 only
237 setOperationAction(ISD::SINT_TO_FP, MVT::i16, Custom);
238 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i16, Custom);
239 // f32 and f64 cases are Legal with SSE1/SSE2, f80 case is not
240 setOperationAction(ISD::SINT_TO_FP, MVT::i32, Custom);
241 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i32, Custom);
242 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
243 // are Legal, f80 is custom lowered.
244 setOperationAction(ISD::SINT_TO_FP, MVT::i64, Custom);
245 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::i64, Custom);
246
247 // Promote i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
248 // this operation.
249 setOperationAction(ISD::FP_TO_SINT, MVT::i8, Promote);
250 // FIXME: This doesn't generate invalid exception when it should. PR44019.
251 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i8, Promote);
252 setOperationAction(ISD::FP_TO_SINT, MVT::i16, Custom);
253 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i16, Custom);
254 setOperationAction(ISD::FP_TO_SINT, MVT::i32, Custom);
255 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i32, Custom);
256 // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
257 // are Legal, f80 is custom lowered.
258 setOperationAction(ISD::FP_TO_SINT, MVT::i64, Custom);
259 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::i64, Custom);
260
261 // Handle FP_TO_UINT by promoting the destination to a larger signed
262 // conversion.
263 setOperationAction(ISD::FP_TO_UINT, MVT::i8, Promote);
264 // FIXME: This doesn't generate invalid exception when it should. PR44019.
265 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i8, Promote);
266 setOperationAction(ISD::FP_TO_UINT, MVT::i16, Promote);
267 // FIXME: This doesn't generate invalid exception when it should. PR44019.
268 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i16, Promote);
269 setOperationAction(ISD::FP_TO_UINT, MVT::i32, Custom);
270 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i32, Custom);
271 setOperationAction(ISD::FP_TO_UINT, MVT::i64, Custom);
272 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::i64, Custom);
273 }
274
275 // Handle address space casts between mixed sized pointers.
276 setOperationAction(ISD::ADDRSPACECAST, MVT::i32, Custom);
277 setOperationAction(ISD::ADDRSPACECAST, MVT::i64, Custom);
278
279 // TODO: when we have SSE, these could be more efficient, by using movd/movq.
280 if (!X86ScalarSSEf64) {
281 setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
282 setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
283 if (Subtarget.is64Bit()) {
284 setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
285 // Without SSE, i64->f64 goes through memory.
286 setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
287 }
288 } else if (!Subtarget.is64Bit())
289 setOperationAction(ISD::BITCAST , MVT::i64 , Custom);
290
291 // Scalar integer divide and remainder are lowered to use operations that
292 // produce two results, to match the available instructions. This exposes
293 // the two-result form to trivial CSE, which is able to combine x/y and x%y
294 // into a single instruction.
295 //
296 // Scalar integer multiply-high is also lowered to use two-result
297 // operations, to match the available instructions. However, plain multiply
298 // (low) operations are left as Legal, as there are single-result
299 // instructions for this in x86. Using the two-result multiply instructions
300 // when both high and low results are needed must be arranged by dagcombine.
301 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
302 setOperationAction(ISD::MULHS, VT, Expand);
303 setOperationAction(ISD::MULHU, VT, Expand);
304 setOperationAction(ISD::SDIV, VT, Expand);
305 setOperationAction(ISD::UDIV, VT, Expand);
306 setOperationAction(ISD::SREM, VT, Expand);
307 setOperationAction(ISD::UREM, VT, Expand);
308 }
309
310 setOperationAction(ISD::BR_JT , MVT::Other, Expand);
311 setOperationAction(ISD::BRCOND , MVT::Other, Custom);
312 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
313 MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
314 setOperationAction(ISD::BR_CC, VT, Expand);
315 setOperationAction(ISD::SELECT_CC, VT, Expand);
316 }
317 if (Subtarget.is64Bit())
318 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
319 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
320 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
321 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
322
323 setOperationAction(ISD::FREM , MVT::f32 , Expand);
324 setOperationAction(ISD::FREM , MVT::f64 , Expand);
325 setOperationAction(ISD::FREM , MVT::f80 , Expand);
326 setOperationAction(ISD::FREM , MVT::f128 , Expand);
327 setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom);
328
329 // Promote the i8 variants and force them on up to i32 which has a shorter
330 // encoding.
331 setOperationPromotedToType(ISD::CTTZ , MVT::i8 , MVT::i32);
332 setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
333 if (!Subtarget.hasBMI()) {
334 setOperationAction(ISD::CTTZ , MVT::i16 , Custom);
335 setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
336 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Legal);
337 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Legal);
338 if (Subtarget.is64Bit()) {
339 setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
340 setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);
341 }
342 }
343
344 if (Subtarget.hasLZCNT()) {
345 // When promoting the i8 variants, force them to i32 for a shorter
346 // encoding.
347 setOperationPromotedToType(ISD::CTLZ , MVT::i8 , MVT::i32);
348 setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
349 } else {
350 setOperationAction(ISD::CTLZ , MVT::i8 , Custom);
351 setOperationAction(ISD::CTLZ , MVT::i16 , Custom);
352 setOperationAction(ISD::CTLZ , MVT::i32 , Custom);
353 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom);
354 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom);
355 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom);
356 if (Subtarget.is64Bit()) {
357 setOperationAction(ISD::CTLZ , MVT::i64 , Custom);
358 setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom);
359 }
360 }
361
362 // Special handling for half-precision floating point conversions.
363 // If we don't have F16C support, then lower half float conversions
364 // into library calls.
365 if (Subtarget.useSoftFloat() || !Subtarget.hasF16C()) {
366 setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
367 setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
368 }
369
370 // There's never any support for operations beyond MVT::f32.
371 setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
372 setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand);
373 setOperationAction(ISD::FP16_TO_FP, MVT::f128, Expand);
374 setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
375 setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand);
376 setOperationAction(ISD::FP_TO_FP16, MVT::f128, Expand);
377
378 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
379 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
380 setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand);
381 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f16, Expand);
382 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
383 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
384 setTruncStoreAction(MVT::f80, MVT::f16, Expand);
385 setTruncStoreAction(MVT::f128, MVT::f16, Expand);
386
387 if (Subtarget.hasPOPCNT()) {
388 setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
389 } else {
390 setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
391 setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
392 setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
393 if (Subtarget.is64Bit())
394 setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
395 else
396 setOperationAction(ISD::CTPOP , MVT::i64 , Custom);
397 }
398
399 setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
400
401 if (!Subtarget.hasMOVBE())
402 setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
403
404 // X86 wants to expand cmov itself.
405 for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
406 setOperationAction(ISD::SELECT, VT, Custom);
407 setOperationAction(ISD::SETCC, VT, Custom);
408 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
409 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
410 }
411 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
412 if (VT == MVT::i64 && !Subtarget.is64Bit())
413 continue;
414 setOperationAction(ISD::SELECT, VT, Custom);
415 setOperationAction(ISD::SETCC, VT, Custom);
416 }
417
418 // Custom action for SELECT MMX and expand action for SELECT_CC MMX
419 setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
420 setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
421
422 setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
423 // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
424 // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
425 setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
426 setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
427 setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
428 if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
429 setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
430
431 // Darwin ABI issue.
432 for (auto VT : { MVT::i32, MVT::i64 }) {
433 if (VT == MVT::i64 && !Subtarget.is64Bit())
434 continue;
435 setOperationAction(ISD::ConstantPool , VT, Custom);
436 setOperationAction(ISD::JumpTable , VT, Custom);
437 setOperationAction(ISD::GlobalAddress , VT, Custom);
438 setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
439 setOperationAction(ISD::ExternalSymbol , VT, Custom);
440 setOperationAction(ISD::BlockAddress , VT, Custom);
441 }
442
443 // 64-bit shl, sra, srl (iff 32-bit x86)
444 for (auto VT : { MVT::i32, MVT::i64 }) {
445 if (VT == MVT::i64 && !Subtarget.is64Bit())
446 continue;
447 setOperationAction(ISD::SHL_PARTS, VT, Custom);
448 setOperationAction(ISD::SRA_PARTS, VT, Custom);
449 setOperationAction(ISD::SRL_PARTS, VT, Custom);
450 }
451
452 if (Subtarget.hasSSEPrefetch() || Subtarget.has3DNow())
453 setOperationAction(ISD::PREFETCH , MVT::Other, Legal);
454
455 setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
456
457 // Expand certain atomics
458 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
459 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
460 setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
461 setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
462 setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
463 setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
464 setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
465 setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
466 }
467
468 if (!Subtarget.is64Bit())
469 setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
470
471 if (Subtarget.hasCmpxchg16b()) {
472 setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
473 }
474
475 // FIXME - use subtarget debug flags
476 if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
477 !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
478 TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
479 setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
480 }
481
482 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
483 setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
484
485 setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
486 setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
487
488 setOperationAction(ISD::TRAP, MVT::Other, Legal);
489 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
490
491 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
492 setOperationAction(ISD::VASTART , MVT::Other, Custom);
493 setOperationAction(ISD::VAEND , MVT::Other, Expand);
494 bool Is64Bit = Subtarget.is64Bit();
495 setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
496 setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
497
498 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
499 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
500
501 setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
502
503 // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
504 setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
505 setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
506
507 if (!Subtarget.useSoftFloat() && X86ScalarSSEf64) {
508 // f32 and f64 use SSE.
509 // Set up the FP register classes.
510 addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
511 : &X86::FR32RegClass);
512 addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
513 : &X86::FR64RegClass);
514
515 // Disable f32->f64 extload as we can only generate this in one instruction
516 // under optsize. So its easier to pattern match (fpext (load)) for that
517 // case instead of needing to emit 2 instructions for extload in the
518 // non-optsize case.
519 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
520
521 for (auto VT : { MVT::f32, MVT::f64 }) {
522 // Use ANDPD to simulate FABS.
523 setOperationAction(ISD::FABS, VT, Custom);
524
525 // Use XORP to simulate FNEG.
526 setOperationAction(ISD::FNEG, VT, Custom);
527
528 // Use ANDPD and ORPD to simulate FCOPYSIGN.
529 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
530
531 // These might be better off as horizontal vector ops.
532 setOperationAction(ISD::FADD, VT, Custom);
533 setOperationAction(ISD::FSUB, VT, Custom);
534
535 // We don't support sin/cos/fmod
536 setOperationAction(ISD::FSIN , VT, Expand);
537 setOperationAction(ISD::FCOS , VT, Expand);
538 setOperationAction(ISD::FSINCOS, VT, Expand);
539 }
540
541 // Lower this to MOVMSK plus an AND.
542 setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
543 setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
544
545 } else if (!useSoftFloat() && X86ScalarSSEf32 && (UseX87 || Is64Bit)) {
546 // Use SSE for f32, x87 for f64.
547 // Set up the FP register classes.
548 addRegisterClass(MVT::f32, &X86::FR32RegClass);
549 if (UseX87)
550 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
551
552 // Use ANDPS to simulate FABS.
553 setOperationAction(ISD::FABS , MVT::f32, Custom);
554
555 // Use XORP to simulate FNEG.
556 setOperationAction(ISD::FNEG , MVT::f32, Custom);
557
558 if (UseX87)
559 setOperationAction(ISD::UNDEF, MVT::f64, Expand);
560
561 // Use ANDPS and ORPS to simulate FCOPYSIGN.
562 if (UseX87)
563 setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
564 setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
565
566 // We don't support sin/cos/fmod
567 setOperationAction(ISD::FSIN , MVT::f32, Expand);
568 setOperationAction(ISD::FCOS , MVT::f32, Expand);
569 setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
570
571 if (UseX87) {
572 // Always expand sin/cos functions even though x87 has an instruction.
573 setOperationAction(ISD::FSIN, MVT::f64, Expand);
574 setOperationAction(ISD::FCOS, MVT::f64, Expand);
575 setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
576 }
577 } else if (UseX87) {
578 // f32 and f64 in x87.
579 // Set up the FP register classes.
580 addRegisterClass(MVT::f64, &X86::RFP64RegClass);
581 addRegisterClass(MVT::f32, &X86::RFP32RegClass);
582
583 for (auto VT : { MVT::f32, MVT::f64 }) {
584 setOperationAction(ISD::UNDEF, VT, Expand);
585 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
586
587 // Always expand sin/cos functions even though x87 has an instruction.
588 setOperationAction(ISD::FSIN , VT, Expand);
589 setOperationAction(ISD::FCOS , VT, Expand);
590 setOperationAction(ISD::FSINCOS, VT, Expand);
591 }
592 }
593
594 // Expand FP32 immediates into loads from the stack, save special cases.
595 if (isTypeLegal(MVT::f32)) {
596 if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
597 addLegalFPImmediate(APFloat(+0.0f)); // FLD0
598 addLegalFPImmediate(APFloat(+1.0f)); // FLD1
599 addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
600 addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
601 } else // SSE immediates.
602 addLegalFPImmediate(APFloat(+0.0f)); // xorps
603 }
604 // Expand FP64 immediates into loads from the stack, save special cases.
605 if (isTypeLegal(MVT::f64)) {
606 if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
607 addLegalFPImmediate(APFloat(+0.0)); // FLD0
608 addLegalFPImmediate(APFloat(+1.0)); // FLD1
609 addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
610 addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
611 } else // SSE immediates.
612 addLegalFPImmediate(APFloat(+0.0)); // xorpd
613 }
614 // Handle constrained floating-point operations of scalar.
615 setOperationAction(ISD::STRICT_FADD, MVT::f32, Legal);
616 setOperationAction(ISD::STRICT_FADD, MVT::f64, Legal);
617 setOperationAction(ISD::STRICT_FSUB, MVT::f32, Legal);
618 setOperationAction(ISD::STRICT_FSUB, MVT::f64, Legal);
619 setOperationAction(ISD::STRICT_FMUL, MVT::f32, Legal);
620 setOperationAction(ISD::STRICT_FMUL, MVT::f64, Legal);
621 setOperationAction(ISD::STRICT_FDIV, MVT::f32, Legal);
622 setOperationAction(ISD::STRICT_FDIV, MVT::f64, Legal);
623 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f64, Legal);
624 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Legal);
625 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Legal);
626 setOperationAction(ISD::STRICT_FSQRT, MVT::f32, Legal);
627 setOperationAction(ISD::STRICT_FSQRT, MVT::f64, Legal);
628
629 // We don't support FMA.
630 setOperationAction(ISD::FMA, MVT::f64, Expand);
631 setOperationAction(ISD::FMA, MVT::f32, Expand);
632
633 // f80 always uses X87.
634 if (UseX87) {
635 addRegisterClass(MVT::f80, &X86::RFP80RegClass);
636 setOperationAction(ISD::UNDEF, MVT::f80, Expand);
637 setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
638 {
639 APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
640 addLegalFPImmediate(TmpFlt); // FLD0
641 TmpFlt.changeSign();
642 addLegalFPImmediate(TmpFlt); // FLD0/FCHS
643
644 bool ignored;
645 APFloat TmpFlt2(+1.0);
646 TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
647 &ignored);
648 addLegalFPImmediate(TmpFlt2); // FLD1
649 TmpFlt2.changeSign();
650 addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
651 }
652
653 // Always expand sin/cos functions even though x87 has an instruction.
654 setOperationAction(ISD::FSIN , MVT::f80, Expand);
655 setOperationAction(ISD::FCOS , MVT::f80, Expand);
656 setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
657
658 setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
659 setOperationAction(ISD::FCEIL, MVT::f80, Expand);
660 setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
661 setOperationAction(ISD::FRINT, MVT::f80, Expand);
662 setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
663 setOperationAction(ISD::FMA, MVT::f80, Expand);
664 setOperationAction(ISD::LROUND, MVT::f80, Expand);
665 setOperationAction(ISD::LLROUND, MVT::f80, Expand);
666 setOperationAction(ISD::LRINT, MVT::f80, Expand);
667 setOperationAction(ISD::LLRINT, MVT::f80, Expand);
668
669 // Handle constrained floating-point operations of scalar.
670 setOperationAction(ISD::STRICT_FADD , MVT::f80, Legal);
671 setOperationAction(ISD::STRICT_FSUB , MVT::f80, Legal);
672 setOperationAction(ISD::STRICT_FMUL , MVT::f80, Legal);
673 setOperationAction(ISD::STRICT_FDIV , MVT::f80, Legal);
674 setOperationAction(ISD::STRICT_FSQRT , MVT::f80, Legal);
675 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f80, Legal);
676 // FIXME: When the target is 64-bit, STRICT_FP_ROUND will be overwritten
677 // as Custom.
678 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Legal);
679 }
680
681 // f128 uses xmm registers, but most operations require libcalls.
682 if (!Subtarget.useSoftFloat() && Subtarget.is64Bit() && Subtarget.hasSSE1()) {
683 addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
684 : &X86::VR128RegClass);
685
686 addLegalFPImmediate(APFloat::getZero(APFloat::IEEEquad())); // xorps
687
688 setOperationAction(ISD::FADD, MVT::f128, LibCall);
689 setOperationAction(ISD::STRICT_FADD, MVT::f128, LibCall);
690 setOperationAction(ISD::FSUB, MVT::f128, LibCall);
691 setOperationAction(ISD::STRICT_FSUB, MVT::f128, LibCall);
692 setOperationAction(ISD::FDIV, MVT::f128, LibCall);
693 setOperationAction(ISD::STRICT_FDIV, MVT::f128, LibCall);
694 setOperationAction(ISD::FMUL, MVT::f128, LibCall);
695 setOperationAction(ISD::STRICT_FMUL, MVT::f128, LibCall);
696 setOperationAction(ISD::FMA, MVT::f128, LibCall);
697 setOperationAction(ISD::STRICT_FMA, MVT::f128, LibCall);
698
699 setOperationAction(ISD::FABS, MVT::f128, Custom);
700 setOperationAction(ISD::FNEG, MVT::f128, Custom);
701 setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
702
703 setOperationAction(ISD::FSIN, MVT::f128, LibCall);
704 setOperationAction(ISD::STRICT_FSIN, MVT::f128, LibCall);
705 setOperationAction(ISD::FCOS, MVT::f128, LibCall);
706 setOperationAction(ISD::STRICT_FCOS, MVT::f128, LibCall);
707 setOperationAction(ISD::FSINCOS, MVT::f128, LibCall);
708 // No STRICT_FSINCOS
709 setOperationAction(ISD::FSQRT, MVT::f128, LibCall);
710 setOperationAction(ISD::STRICT_FSQRT, MVT::f128, LibCall);
711
712 setOperationAction(ISD::FP_EXTEND, MVT::f128, Custom);
713 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::f128, Custom);
714 // We need to custom handle any FP_ROUND with an f128 input, but
715 // LegalizeDAG uses the result type to know when to run a custom handler.
716 // So we have to list all legal floating point result types here.
717 if (isTypeLegal(MVT::f32)) {
718 setOperationAction(ISD::FP_ROUND, MVT::f32, Custom);
719 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f32, Custom);
720 }
721 if (isTypeLegal(MVT::f64)) {
722 setOperationAction(ISD::FP_ROUND, MVT::f64, Custom);
723 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f64, Custom);
724 }
725 if (isTypeLegal(MVT::f80)) {
726 setOperationAction(ISD::FP_ROUND, MVT::f80, Custom);
727 setOperationAction(ISD::STRICT_FP_ROUND, MVT::f80, Custom);
728 }
729
730 setOperationAction(ISD::SETCC, MVT::f128, Custom);
731
732 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f32, Expand);
733 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f64, Expand);
734 setLoadExtAction(ISD::EXTLOAD, MVT::f128, MVT::f80, Expand);
735 setTruncStoreAction(MVT::f128, MVT::f32, Expand);
736 setTruncStoreAction(MVT::f128, MVT::f64, Expand);
737 setTruncStoreAction(MVT::f128, MVT::f80, Expand);
738 }
739
740 // Always use a library call for pow.
741 setOperationAction(ISD::FPOW , MVT::f32 , Expand);
742 setOperationAction(ISD::FPOW , MVT::f64 , Expand);
743 setOperationAction(ISD::FPOW , MVT::f80 , Expand);
744 setOperationAction(ISD::FPOW , MVT::f128 , Expand);
745
746 setOperationAction(ISD::FLOG, MVT::f80, Expand);
747 setOperationAction(ISD::FLOG2, MVT::f80, Expand);
748 setOperationAction(ISD::FLOG10, MVT::f80, Expand);
749 setOperationAction(ISD::FEXP, MVT::f80, Expand);
750 setOperationAction(ISD::FEXP2, MVT::f80, Expand);
751 setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
752 setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
753
754 // Some FP actions are always expanded for vector types.
755 for (auto VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32,
756 MVT::v2f64, MVT::v4f64, MVT::v8f64 }) {
757 setOperationAction(ISD::FSIN, VT, Expand);
758 setOperationAction(ISD::FSINCOS, VT, Expand);
759 setOperationAction(ISD::FCOS, VT, Expand);
760 setOperationAction(ISD::FREM, VT, Expand);
761 setOperationAction(ISD::FCOPYSIGN, VT, Expand);
762 setOperationAction(ISD::FPOW, VT, Expand);
763 setOperationAction(ISD::FLOG, VT, Expand);
764 setOperationAction(ISD::FLOG2, VT, Expand);
765 setOperationAction(ISD::FLOG10, VT, Expand);
766 setOperationAction(ISD::FEXP, VT, Expand);
767 setOperationAction(ISD::FEXP2, VT, Expand);
768 }
769
770 // First set operation action for all vector types to either promote
771 // (for widening) or expand (for scalarization). Then we will selectively
772 // turn on ones that can be effectively codegen'd.
773 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
774 setOperationAction(ISD::SDIV, VT, Expand);
775 setOperationAction(ISD::UDIV, VT, Expand);
776 setOperationAction(ISD::SREM, VT, Expand);
777 setOperationAction(ISD::UREM, VT, Expand);
778 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
779 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
780 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
781 setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
782 setOperationAction(ISD::FMA, VT, Expand);
783 setOperationAction(ISD::FFLOOR, VT, Expand);
784 setOperationAction(ISD::FCEIL, VT, Expand);
785 setOperationAction(ISD::FTRUNC, VT, Expand);
786 setOperationAction(ISD::FRINT, VT, Expand);
787 setOperationAction(ISD::FNEARBYINT, VT, Expand);
788 setOperationAction(ISD::SMUL_LOHI, VT, Expand);
789 setOperationAction(ISD::MULHS, VT, Expand);
790 setOperationAction(ISD::UMUL_LOHI, VT, Expand);
791 setOperationAction(ISD::MULHU, VT, Expand);
792 setOperationAction(ISD::SDIVREM, VT, Expand);
793 setOperationAction(ISD::UDIVREM, VT, Expand);
794 setOperationAction(ISD::CTPOP, VT, Expand);
795 setOperationAction(ISD::CTTZ, VT, Expand);
796 setOperationAction(ISD::CTLZ, VT, Expand);
797 setOperationAction(ISD::ROTL, VT, Expand);
798 setOperationAction(ISD::ROTR, VT, Expand);
799 setOperationAction(ISD::BSWAP, VT, Expand);
800 setOperationAction(ISD::SETCC, VT, Expand);
801 setOperationAction(ISD::FP_TO_UINT, VT, Expand);
802 setOperationAction(ISD::FP_TO_SINT, VT, Expand);
803 setOperationAction(ISD::UINT_TO_FP, VT, Expand);
804 setOperationAction(ISD::SINT_TO_FP, VT, Expand);
805 setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
806 setOperationAction(ISD::TRUNCATE, VT, Expand);
807 setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
808 setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
809 setOperationAction(ISD::ANY_EXTEND, VT, Expand);
810 setOperationAction(ISD::SELECT_CC, VT, Expand);
811 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
812 setTruncStoreAction(InnerVT, VT, Expand);
813
814 setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
815 setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
816
817 // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
818 // types, we have to deal with them whether we ask for Expansion or not.
819 // Setting Expand causes its own optimisation problems though, so leave
820 // them legal.
821 if (VT.getVectorElementType() == MVT::i1)
822 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
823
824 // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
825 // split/scalarized right now.
826 if (VT.getVectorElementType() == MVT::f16)
827 setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
828 }
829 }
830
831 // FIXME: In order to prevent SSE instructions being expanded to MMX ones
832 // with -msoft-float, disable use of MMX as well.
833 if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
834 addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
835 // No operations on x86mmx supported, everything uses intrinsics.
836 }
837
838 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
839 addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
840 : &X86::VR128RegClass);
841
842 setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
843 setOperationAction(ISD::FABS, MVT::v4f32, Custom);
844 setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
845 setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
846 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
847 setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
848 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
849 setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
850
851 setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
852 setOperationAction(ISD::STORE, MVT::v2f32, Custom);
853
854 setOperationAction(ISD::STRICT_FADD, MVT::v4f32, Legal);
855 setOperationAction(ISD::STRICT_FSUB, MVT::v4f32, Legal);
856 setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
857 setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
858 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
859 }
860
861 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
862 addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
863 : &X86::VR128RegClass);
864
865 // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
866 // registers cannot be used even for integer operations.
867 addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
868 : &X86::VR128RegClass);
869 addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
870 : &X86::VR128RegClass);
871 addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
872 : &X86::VR128RegClass);
873 addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
874 : &X86::VR128RegClass);
875
876 for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
877 MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
878 setOperationAction(ISD::SDIV, VT, Custom);
879 setOperationAction(ISD::SREM, VT, Custom);
880 setOperationAction(ISD::UDIV, VT, Custom);
881 setOperationAction(ISD::UREM, VT, Custom);
882 }
883
884 setOperationAction(ISD::MUL, MVT::v2i8, Custom);
885 setOperationAction(ISD::MUL, MVT::v4i8, Custom);
886 setOperationAction(ISD::MUL, MVT::v8i8, Custom);
887
888 setOperationAction(ISD::MUL, MVT::v16i8, Custom);
889 setOperationAction(ISD::MUL, MVT::v4i32, Custom);
890 setOperationAction(ISD::MUL, MVT::v2i64, Custom);
891 setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
892 setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
893 setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
894 setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
895 setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
896 setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
897 setOperationAction(ISD::MUL, MVT::v8i16, Legal);
898 setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
899 setOperationAction(ISD::FABS, MVT::v2f64, Custom);
900 setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
901
902 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
903 setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
904 setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
905 setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
906 setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
907 }
908
909 setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
910 setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
911 setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
912 setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
913 setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
914 setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
915 setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
916 setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
917 setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
918 setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
919 setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
920 setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
921
922 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
923 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
924 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
925
926 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
927 setOperationAction(ISD::SETCC, VT, Custom);
928 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
929 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
930 setOperationAction(ISD::CTPOP, VT, Custom);
931 setOperationAction(ISD::ABS, VT, Custom);
932
933 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
934 // setcc all the way to isel and prefer SETGT in some isel patterns.
935 setCondCodeAction(ISD::SETLT, VT, Custom);
936 setCondCodeAction(ISD::SETLE, VT, Custom);
937 }
938
939 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
940 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
941 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
942 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
943 setOperationAction(ISD::VSELECT, VT, Custom);
944 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
945 }
946
947 for (auto VT : { MVT::v2f64, MVT::v2i64 }) {
948 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
949 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
950 setOperationAction(ISD::VSELECT, VT, Custom);
951
952 if (VT == MVT::v2i64 && !Subtarget.is64Bit())
953 continue;
954
955 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
956 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
957 }
958
959 // Custom lower v2i64 and v2f64 selects.
960 setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
961 setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
962 setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
963 setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
964 setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
965
966 setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
967 setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
968 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v4i32, Legal);
969 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i32, Custom);
970
971 // Custom legalize these to avoid over promotion or custom promotion.
972 for (auto VT : {MVT::v2i8, MVT::v4i8, MVT::v8i8, MVT::v2i16, MVT::v4i16}) {
973 setOperationAction(ISD::FP_TO_SINT, VT, Custom);
974 setOperationAction(ISD::FP_TO_UINT, VT, Custom);
975 setOperationAction(ISD::STRICT_FP_TO_SINT, VT, Custom);
976 setOperationAction(ISD::STRICT_FP_TO_UINT, VT, Custom);
977 }
978
979 setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
980 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i32, Legal);
981 setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
982 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2i32, Custom);
983
984 setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
985 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2i32, Custom);
986
987 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
988 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32, Custom);
989
990 // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
991 setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
992 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v2f32, Custom);
993 setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
994 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v2f32, Custom);
995
996 setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
997 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v2f32, Custom);
998 setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
999 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v2f32, Custom);
1000
1001 // We want to legalize this to an f64 load rather than an i64 load on
1002 // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
1003 // store.
1004 setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
1005 setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
1006 setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
1007 setOperationAction(ISD::STORE, MVT::v2i32, Custom);
1008 setOperationAction(ISD::STORE, MVT::v4i16, Custom);
1009 setOperationAction(ISD::STORE, MVT::v8i8, Custom);
1010
1011 setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
1012 setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
1013 setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
1014 if (!Subtarget.hasAVX512())
1015 setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
1016
1017 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
1018 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
1019 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
1020
1021 setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
1022
1023 setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
1024 setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
1025 setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
1026 setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
1027 setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
1028 setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
1029
1030 // In the customized shift lowering, the legal v4i32/v2i64 cases
1031 // in AVX2 will be recognized.
1032 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1033 setOperationAction(ISD::SRL, VT, Custom);
1034 setOperationAction(ISD::SHL, VT, Custom);
1035 setOperationAction(ISD::SRA, VT, Custom);
1036 }
1037
1038 setOperationAction(ISD::ROTL, MVT::v4i32, Custom);
1039 setOperationAction(ISD::ROTL, MVT::v8i16, Custom);
1040
1041 // With AVX512, expanding (and promoting the shifts) is better.
1042 if (!Subtarget.hasAVX512())
1043 setOperationAction(ISD::ROTL, MVT::v16i8, Custom);
1044
1045 setOperationAction(ISD::STRICT_FSQRT, MVT::v2f64, Legal);
1046 setOperationAction(ISD::STRICT_FADD, MVT::v2f64, Legal);
1047 setOperationAction(ISD::STRICT_FSUB, MVT::v2f64, Legal);
1048 setOperationAction(ISD::STRICT_FMUL, MVT::v2f64, Legal);
1049 setOperationAction(ISD::STRICT_FDIV, MVT::v2f64, Legal);
1050 }
1051
1052 if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
1053 setOperationAction(ISD::ABS, MVT::v16i8, Legal);
1054 setOperationAction(ISD::ABS, MVT::v8i16, Legal);
1055 setOperationAction(ISD::ABS, MVT::v4i32, Legal);
1056 setOperationAction(ISD::BITREVERSE, MVT::v16i8, Custom);
1057 setOperationAction(ISD::CTLZ, MVT::v16i8, Custom);
1058 setOperationAction(ISD::CTLZ, MVT::v8i16, Custom);
1059 setOperationAction(ISD::CTLZ, MVT::v4i32, Custom);
1060 setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
1061
1062 // These might be better off as horizontal vector ops.
1063 setOperationAction(ISD::ADD, MVT::i16, Custom);
1064 setOperationAction(ISD::ADD, MVT::i32, Custom);
1065 setOperationAction(ISD::SUB, MVT::i16, Custom);
1066 setOperationAction(ISD::SUB, MVT::i32, Custom);
1067 }
1068
1069 if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
1070 for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
1071 setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
1072 setOperationAction(ISD::STRICT_FFLOOR, RoundedTy, Legal);
1073 setOperationAction(ISD::FCEIL, RoundedTy, Legal);
1074 setOperationAction(ISD::STRICT_FCEIL, RoundedTy, Legal);
1075 setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
1076 setOperationAction(ISD::STRICT_FTRUNC, RoundedTy, Legal);
1077 setOperationAction(ISD::FRINT, RoundedTy, Legal);
1078 setOperationAction(ISD::STRICT_FRINT, RoundedTy, Legal);
1079 setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
1080 setOperationAction(ISD::STRICT_FNEARBYINT, RoundedTy, Legal);
1081 }
1082
1083 setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
1084 setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
1085 setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
1086 setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
1087 setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
1088 setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
1089 setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
1090 setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
1091
1092 // FIXME: Do we need to handle scalar-to-vector here?
1093 setOperationAction(ISD::MUL, MVT::v4i32, Legal);
1094
1095 // We directly match byte blends in the backend as they match the VSELECT
1096 // condition form.
1097 setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
1098
1099 // SSE41 brings specific instructions for doing vector sign extend even in
1100 // cases where we don't have SRA.
1101 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1102 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
1103 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
1104 }
1105
1106 // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1107 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1108 setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
1109 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
1110 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
1111 setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
1112 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
1113 setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
1114 }
1115
1116 // i8 vectors are custom because the source register and source
1117 // source memory operand types are not the same width.
1118 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1119
1120 if (Subtarget.is64Bit() && !Subtarget.hasAVX512()) {
1121 // We need to scalarize v4i64->v432 uint_to_fp using cvtsi2ss, but we can
1122 // do the pre and post work in the vector domain.
1123 setOperationAction(ISD::UINT_TO_FP, MVT::v4i64, Custom);
1124 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i64, Custom);
1125 // We need to mark SINT_TO_FP as Custom even though we want to expand it
1126 // so that DAG combine doesn't try to turn it into uint_to_fp.
1127 setOperationAction(ISD::SINT_TO_FP, MVT::v4i64, Custom);
1128 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v4i64, Custom);
1129 }
1130 }
1131
1132 if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1133 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1134 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1135 setOperationAction(ISD::ROTL, VT, Custom);
1136
1137 // XOP can efficiently perform BITREVERSE with VPPERM.
1138 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
1139 setOperationAction(ISD::BITREVERSE, VT, Custom);
1140
1141 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1142 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1143 setOperationAction(ISD::BITREVERSE, VT, Custom);
1144 }
1145
1146 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
1147 bool HasInt256 = Subtarget.hasInt256();
1148
1149 addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
1150 : &X86::VR256RegClass);
1151 addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1152 : &X86::VR256RegClass);
1153 addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1154 : &X86::VR256RegClass);
1155 addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1156 : &X86::VR256RegClass);
1157 addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1158 : &X86::VR256RegClass);
1159 addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1160 : &X86::VR256RegClass);
1161
1162 for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
1163 setOperationAction(ISD::FFLOOR, VT, Legal);
1164 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1165 setOperationAction(ISD::FCEIL, VT, Legal);
1166 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1167 setOperationAction(ISD::FTRUNC, VT, Legal);
1168 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1169 setOperationAction(ISD::FRINT, VT, Legal);
1170 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1171 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1172 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1173 setOperationAction(ISD::FNEG, VT, Custom);
1174 setOperationAction(ISD::FABS, VT, Custom);
1175 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1176 }
1177
1178 // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1179 // even though v8i16 is a legal type.
1180 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1181 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1182 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1183 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1184 setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal);
1185 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i32, Legal);
1186
1187 setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal);
1188 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i32, Legal);
1189
1190 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v4f32, Legal);
1191 setOperationAction(ISD::STRICT_FADD, MVT::v8f32, Legal);
1192 setOperationAction(ISD::STRICT_FADD, MVT::v4f64, Legal);
1193 setOperationAction(ISD::STRICT_FSUB, MVT::v8f32, Legal);
1194 setOperationAction(ISD::STRICT_FSUB, MVT::v4f64, Legal);
1195 setOperationAction(ISD::STRICT_FMUL, MVT::v8f32, Legal);
1196 setOperationAction(ISD::STRICT_FMUL, MVT::v4f64, Legal);
1197 setOperationAction(ISD::STRICT_FDIV, MVT::v8f32, Legal);
1198 setOperationAction(ISD::STRICT_FDIV, MVT::v4f64, Legal);
1199 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v4f64, Legal);
1200 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
1201 setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
1202
1203 if (!Subtarget.hasAVX512())
1204 setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
1205
1206 // In the customized shift lowering, the legal v8i32/v4i64 cases
1207 // in AVX2 will be recognized.
1208 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1209 setOperationAction(ISD::SRL, VT, Custom);
1210 setOperationAction(ISD::SHL, VT, Custom);
1211 setOperationAction(ISD::SRA, VT, Custom);
1212 }
1213
1214 // These types need custom splitting if their input is a 128-bit vector.
1215 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1216 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1217 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1218 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1219
1220 setOperationAction(ISD::ROTL, MVT::v8i32, Custom);
1221 setOperationAction(ISD::ROTL, MVT::v16i16, Custom);
1222
1223 // With BWI, expanding (and promoting the shifts) is the better.
1224 if (!Subtarget.hasBWI())
1225 setOperationAction(ISD::ROTL, MVT::v32i8, Custom);
1226
1227 setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
1228 setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
1229 setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
1230 setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
1231 setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
1232 setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
1233
1234 for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1235 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1236 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1237 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1238 }
1239
1240 setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1241 setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
1242 setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
1243 setOperationAction(ISD::BITREVERSE, MVT::v32i8, Custom);
1244
1245 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1246 setOperationAction(ISD::SETCC, VT, Custom);
1247 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
1248 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
1249 setOperationAction(ISD::CTPOP, VT, Custom);
1250 setOperationAction(ISD::CTLZ, VT, Custom);
1251
1252 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1253 // setcc all the way to isel and prefer SETGT in some isel patterns.
1254 setCondCodeAction(ISD::SETLT, VT, Custom);
1255 setCondCodeAction(ISD::SETLE, VT, Custom);
1256 }
1257
1258 if (Subtarget.hasAnyFMA()) {
1259 for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
1260 MVT::v2f64, MVT::v4f64 }) {
1261 setOperationAction(ISD::FMA, VT, Legal);
1262 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1263 }
1264 }
1265
1266 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1267 setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
1268 setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
1269 }
1270
1271 setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1272 setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
1273 setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
1274 setOperationAction(ISD::MUL, MVT::v32i8, Custom);
1275
1276 setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
1277 setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
1278 setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
1279 setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
1280 setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
1281 setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
1282
1283 setOperationAction(ISD::ABS, MVT::v4i64, Custom);
1284 setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
1285 setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
1286 setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
1287 setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
1288
1289 setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1290 setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1291 setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1292 setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1293 setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1294 setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1295 setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1296 setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1297
1298 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
1299 setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
1300 setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
1301 setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
1302 setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
1303 setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
1304 }
1305
1306 for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
1307 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1308 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1309 }
1310
1311 if (HasInt256) {
1312 // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1313 // when we have a 256bit-wide blend with immediate.
1314 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1315 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32, Custom);
1316
1317 // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1318 for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1319 setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
1320 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
1321 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
1322 setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
1323 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
1324 setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
1325 }
1326 }
1327
1328 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1329 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
1330 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
1331 setOperationAction(ISD::MSTORE, VT, Legal);
1332 }
1333
1334 // Extract subvector is special because the value type
1335 // (result) is 128-bit but the source is 256-bit wide.
1336 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1337 MVT::v4f32, MVT::v2f64 }) {
1338 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1339 }
1340
1341 // Custom lower several nodes for 256-bit types.
1342 for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1343 MVT::v8f32, MVT::v4f64 }) {
1344 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1345 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1346 setOperationAction(ISD::VSELECT, VT, Custom);
1347 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1348 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1349 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1350 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1351 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1352 setOperationAction(ISD::STORE, VT, Custom);
1353 }
1354
1355 if (HasInt256) {
1356 setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1357
1358 // Custom legalize 2x32 to get a little better code.
1359 setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
1360 setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
1361
1362 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1363 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1364 setOperationAction(ISD::MGATHER, VT, Custom);
1365 }
1366 }
1367
1368 // This block controls legalization of the mask vector sizes that are
1369 // available with AVX512. 512-bit vectors are in a separate block controlled
1370 // by useAVX512Regs.
1371 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1372 addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
1373 addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1374 addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1375 addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1376 addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1377
1378 setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
1379 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
1380 setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
1381
1382 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1383 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1384 setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1385 setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1386 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1387 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1388 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1389 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1390 setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
1391 setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
1392 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2i1, Custom);
1393 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i1, Custom);
1394
1395 // There is no byte sized k-register load or store without AVX512DQ.
1396 if (!Subtarget.hasDQI()) {
1397 setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
1398 setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
1399 setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
1400 setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
1401
1402 setOperationAction(ISD::STORE, MVT::v1i1, Custom);
1403 setOperationAction(ISD::STORE, MVT::v2i1, Custom);
1404 setOperationAction(ISD::STORE, MVT::v4i1, Custom);
1405 setOperationAction(ISD::STORE, MVT::v8i1, Custom);
1406 }
1407
1408 // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
1409 for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1410 setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1411 setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1412 setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1413 }
1414
1415 for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
1416 setOperationAction(ISD::ADD, VT, Custom);
1417 setOperationAction(ISD::SUB, VT, Custom);
1418 setOperationAction(ISD::MUL, VT, Custom);
1419 setOperationAction(ISD::SETCC, VT, Custom);
1420 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
1421 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
1422 setOperationAction(ISD::SELECT, VT, Custom);
1423 setOperationAction(ISD::TRUNCATE, VT, Custom);
1424 setOperationAction(ISD::UADDSAT, VT, Custom);
1425 setOperationAction(ISD::SADDSAT, VT, Custom);
1426 setOperationAction(ISD::USUBSAT, VT, Custom);
1427 setOperationAction(ISD::SSUBSAT, VT, Custom);
1428
1429 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1430 setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1431 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1432 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Custom);
1433 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1434 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1435 setOperationAction(ISD::VSELECT, VT, Expand);
1436 }
1437
1438 for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
1439 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1440 }
1441
1442 // This block controls legalization for 512-bit operations with 32/64 bit
1443 // elements. 512-bits can be disabled based on prefer-vector-width and
1444 // required-vector-width function attributes.
1445 if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
1446 addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1447 addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1448 addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1449 addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1450
1451 for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1452 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
1453 setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
1454 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
1455 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
1456 setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
1457 }
1458
1459 for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
1460 setOperationAction(ISD::FNEG, VT, Custom);
1461 setOperationAction(ISD::FABS, VT, Custom);
1462 setOperationAction(ISD::FMA, VT, Legal);
1463 setOperationAction(ISD::STRICT_FMA, VT, Legal);
1464 setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1465 }
1466
1467 for (MVT VT : { MVT::v16i1, MVT::v16i8, MVT::v16i16 }) {
1468 setOperationPromotedToType(ISD::FP_TO_SINT , VT, MVT::v16i32);
1469 setOperationPromotedToType(ISD::FP_TO_UINT , VT, MVT::v16i32);
1470 setOperationPromotedToType(ISD::STRICT_FP_TO_SINT, VT, MVT::v16i32);
1471 setOperationPromotedToType(ISD::STRICT_FP_TO_UINT, VT, MVT::v16i32);
1472 }
1473 setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal);
1474 setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal);
1475 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v16i32, Legal);
1476 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v16i32, Legal);
1477 setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal);
1478 setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal);
1479 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v16i32, Legal);
1480 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v16i32, Legal);
1481
1482 setOperationAction(ISD::STRICT_FADD, MVT::v16f32, Legal);
1483 setOperationAction(ISD::STRICT_FADD, MVT::v8f64, Legal);
1484 setOperationAction(ISD::STRICT_FSUB, MVT::v16f32, Legal);
1485 setOperationAction(ISD::STRICT_FSUB, MVT::v8f64, Legal);
1486 setOperationAction(ISD::STRICT_FMUL, MVT::v16f32, Legal);
1487 setOperationAction(ISD::STRICT_FMUL, MVT::v8f64, Legal);
1488 setOperationAction(ISD::STRICT_FDIV, MVT::v16f32, Legal);
1489 setOperationAction(ISD::STRICT_FDIV, MVT::v8f64, Legal);
1490 setOperationAction(ISD::STRICT_FSQRT, MVT::v16f32, Legal);
1491 setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
1492 setOperationAction(ISD::STRICT_FP_EXTEND, MVT::v8f64, Legal);
1493 setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
1494
1495 setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
1496 setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
1497 setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
1498 setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
1499 setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
1500
1501 // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
1502 // to 512-bit rather than use the AVX2 instructions so that we can use
1503 // k-masks.
1504 if (!Subtarget.hasVLX()) {
1505 for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1506 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1507 setOperationAction(ISD::MLOAD, VT, Custom);
1508 setOperationAction(ISD::MSTORE, VT, Custom);
1509 }
1510 }
1511
1512 setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
1513 setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
1514 setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1515 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1516 setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
1517 setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
1518 setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1519 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1520
1521 // Need to custom widen this if we don't have AVX512BW.
1522 setOperationAction(ISD::ANY_EXTEND, MVT::v8i8, Custom);
1523 setOperationAction(ISD::ZERO_EXTEND, MVT::v8i8, Custom);
1524 setOperationAction(ISD::SIGN_EXTEND, MVT::v8i8, Custom);
1525
1526 for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
1527 setOperationAction(ISD::FFLOOR, VT, Legal);
1528 setOperationAction(ISD::STRICT_FFLOOR, VT, Legal);
1529 setOperationAction(ISD::FCEIL, VT, Legal);
1530 setOperationAction(ISD::STRICT_FCEIL, VT, Legal);
1531 setOperationAction(ISD::FTRUNC, VT, Legal);
1532 setOperationAction(ISD::STRICT_FTRUNC, VT, Legal);
1533 setOperationAction(ISD::FRINT, VT, Legal);
1534 setOperationAction(ISD::STRICT_FRINT, VT, Legal);
1535 setOperationAction(ISD::FNEARBYINT, VT, Legal);
1536 setOperationAction(ISD::STRICT_FNEARBYINT, VT, Legal);
1537
1538 setOperationAction(ISD::SELECT, VT, Custom);
1539 }
1540
1541 // Without BWI we need to use custom lowering to handle MVT::v64i8 input.
1542 for (auto VT : {MVT::v16i32, MVT::v8i64, MVT::v64i8}) {
1543 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1544 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1545 }
1546
1547 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom);
1548 setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom);
1549 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom);
1550 setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom);
1551
1552 setOperationAction(ISD::MUL, MVT::v8i64, Custom);
1553 setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1554
1555 setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
1556 setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
1557
1558 for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1559 setOperationAction(ISD::SMAX, VT, Legal);
1560 setOperationAction(ISD::UMAX, VT, Legal);
1561 setOperationAction(ISD::SMIN, VT, Legal);
1562 setOperationAction(ISD::UMIN, VT, Legal);
1563 setOperationAction(ISD::ABS, VT, Legal);
1564 setOperationAction(ISD::SRL, VT, Custom);
1565 setOperationAction(ISD::SHL, VT, Custom);
1566 setOperationAction(ISD::SRA, VT, Custom);
1567 setOperationAction(ISD::CTPOP, VT, Custom);
1568 setOperationAction(ISD::ROTL, VT, Custom);
1569 setOperationAction(ISD::ROTR, VT, Custom);
1570 setOperationAction(ISD::SETCC, VT, Custom);
1571 setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
1572 setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
1573 setOperationAction(ISD::SELECT, VT, Custom);
1574
1575 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1576 // setcc all the way to isel and prefer SETGT in some isel patterns.
1577 setCondCodeAction(ISD::SETLT, VT, Custom);
1578 setCondCodeAction(ISD::SETLE, VT, Custom);
1579 }
1580
1581 if (Subtarget.hasDQI()) {
1582 setOperationAction(ISD::SINT_TO_FP, MVT::v8i64, Legal);
1583 setOperationAction(ISD::UINT_TO_FP, MVT::v8i64, Legal);
1584 setOperationAction(ISD::STRICT_SINT_TO_FP, MVT::v8i64, Legal);
1585 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i64, Legal);
1586 setOperationAction(ISD::FP_TO_SINT, MVT::v8i64, Legal);
1587 setOperationAction(ISD::FP_TO_UINT, MVT::v8i64, Legal);
1588 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v8i64, Legal);
1589 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i64, Legal);
1590
1591 setOperationAction(ISD::MUL, MVT::v8i64, Legal);
1592 }
1593
1594 if (Subtarget.hasCDI()) {
1595 // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
1596 for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
1597 setOperationAction(ISD::CTLZ, VT, Legal);
1598 }
1599 } // Subtarget.hasCDI()
1600
1601 if (Subtarget.hasVPOPCNTDQ()) {
1602 for (auto VT : { MVT::v16i32, MVT::v8i64 })
1603 setOperationAction(ISD::CTPOP, VT, Legal);
1604 }
1605
1606 // Extract subvector is special because the value type
1607 // (result) is 256-bit but the source is 512-bit wide.
1608 // 128-bit was made Legal under AVX1.
1609 for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1610 MVT::v8f32, MVT::v4f64 })
1611 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1612
1613 for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
1614 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1615 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1616 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1617 setOperationAction(ISD::VSELECT, VT, Custom);
1618 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1619 setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1620 setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1621 setOperationAction(ISD::MLOAD, VT, Legal);
1622 setOperationAction(ISD::MSTORE, VT, Legal);
1623 setOperationAction(ISD::MGATHER, VT, Custom);
1624 setOperationAction(ISD::MSCATTER, VT, Custom);
1625 }
1626 if (!Subtarget.hasBWI()) {
1627 // Need to custom split v32i16/v64i8 bitcasts.
1628 setOperationAction(ISD::BITCAST, MVT::v32i16, Custom);
1629 setOperationAction(ISD::BITCAST, MVT::v64i8, Custom);
1630
1631 // Better to split these into two 256-bit ops.
1632 setOperationAction(ISD::BITREVERSE, MVT::v8i64, Custom);
1633 setOperationAction(ISD::BITREVERSE, MVT::v16i32, Custom);
1634 }
1635
1636 if (Subtarget.hasVBMI2()) {
1637 for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1638 setOperationAction(ISD::FSHL, VT, Custom);
1639 setOperationAction(ISD::FSHR, VT, Custom);
1640 }
1641 }
1642 }// has AVX-512
1643
1644 // This block controls legalization for operations that don't have
1645 // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
1646 // narrower widths.
1647 if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1648 // These operations are handled on non-VLX by artificially widening in
1649 // isel patterns.
1650
1651 setOperationAction(ISD::FP_TO_UINT, MVT::v8i32,
1652 Subtarget.hasVLX() ? Legal : Custom);
1653 setOperationAction(ISD::FP_TO_UINT, MVT::v4i32,
1654 Subtarget.hasVLX() ? Legal : Custom);
1655 setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
1656 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v8i32,
1657 Subtarget.hasVLX() ? Legal : Custom);
1658 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v4i32,
1659 Subtarget.hasVLX() ? Legal : Custom);
1660 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2i32, Custom);
1661 setOperationAction(ISD::UINT_TO_FP, MVT::v8i32,
1662 Subtarget.hasVLX() ? Legal : Custom);
1663 setOperationAction(ISD::UINT_TO_FP, MVT::v4i32,
1664 Subtarget.hasVLX() ? Legal : Custom);
1665 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v8i32,
1666 Subtarget.hasVLX() ? Legal : Custom);
1667 setOperationAction(ISD::STRICT_UINT_TO_FP, MVT::v4i32,
1668 Subtarget.hasVLX() ? Legal : Custom);
1669
1670 for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
1671 setOperationAction(ISD::SMAX, VT, Legal);
1672 setOperationAction(ISD::UMAX, VT, Legal);
1673 setOperationAction(ISD::SMIN, VT, Legal);
1674 setOperationAction(ISD::UMIN, VT, Legal);
1675 setOperationAction(ISD::ABS, VT, Legal);
1676 }
1677
1678 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
1679 setOperationAction(ISD::ROTL, VT, Custom);
1680 setOperationAction(ISD::ROTR, VT, Custom);
1681 }
1682
1683 // Custom legalize 2x32 to get a little better code.
1684 setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
1685 setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
1686
1687 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1688 MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1689 setOperationAction(ISD::MSCATTER, VT, Custom);
1690
1691 if (Subtarget.hasDQI()) {
1692 for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
1693 setOperationAction(ISD::SINT_TO_FP, VT,
1694 Subtarget.hasVLX() ? Legal : Custom);
1695 setOperationAction(ISD::UINT_TO_FP, VT,
1696 Subtarget.hasVLX() ? Legal : Custom);
1697 setOperationAction(ISD::STRICT_SINT_TO_FP, VT,
1698 Subtarget.hasVLX() ? Legal : Custom);
1699 setOperationAction(ISD::STRICT_UINT_TO_FP, VT,
1700 Subtarget.hasVLX() ? Legal : Custom);
1701 setOperationAction(ISD::FP_TO_SINT, VT,
1702 Subtarget.hasVLX() ? Legal : Custom);
1703 setOperationAction(ISD::FP_TO_UINT, VT,
1704 Subtarget.hasVLX() ? Legal : Custom);
1705 setOperationAction(ISD::STRICT_FP_TO_SINT, VT,
1706 Subtarget.hasVLX() ? Legal : Custom);
1707 setOperationAction(ISD::STRICT_FP_TO_UINT, VT,
1708 Subtarget.hasVLX() ? Legal : Custom);
1709 setOperationAction(ISD::MUL, VT, Legal);
1710 }
1711 }
1712
1713 if (Subtarget.hasCDI()) {
1714 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
1715 setOperationAction(ISD::CTLZ, VT, Legal);
1716 }
1717 } // Subtarget.hasCDI()
1718
1719 if (Subtarget.hasVPOPCNTDQ()) {
1720 for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })
1721 setOperationAction(ISD::CTPOP, VT, Legal);
1722 }
1723 }
1724
1725 // This block control legalization of v32i1/v64i1 which are available with
1726 // AVX512BW. 512-bit v32i16 and v64i8 vector legalization is controlled with
1727 // useBWIRegs.
1728 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
1729 addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
1730 addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
1731
1732 for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
1733 setOperationAction(ISD::ADD, VT, Custom);
1734 setOperationAction(ISD::SUB, VT, Custom);
1735 setOperationAction(ISD::MUL, VT, Custom);
1736 setOperationAction(ISD::VSELECT, VT, Expand);
1737 setOperationAction(ISD::UADDSAT, VT, Custom);
1738 setOperationAction(ISD::SADDSAT, VT, Custom);
1739 setOperationAction(ISD::USUBSAT, VT, Custom);
1740 setOperationAction(ISD::SSUBSAT, VT, Custom);
1741
1742 setOperationAction(ISD::TRUNCATE, VT, Custom);
1743 setOperationAction(ISD::SETCC, VT, Custom);
1744 setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1745 setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1746 setOperationAction(ISD::SELECT, VT, Custom);
1747 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1748 setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1749 }
1750
1751 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i1, Custom);
1752 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i1, Custom);
1753 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i1, Custom);
1754 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i1, Custom);
1755 for (auto VT : { MVT::v16i1, MVT::v32i1 })
1756 setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1757
1758 // Extends from v32i1 masks to 256-bit vectors.
1759 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
1760 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
1761 setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
1762 }
1763
1764 // This block controls legalization for v32i16 and v64i8. 512-bits can be
1765 // disabled based on prefer-vector-width and required-vector-width function
1766 // attributes.
1767 if (!Subtarget.useSoftFloat() && Subtarget.useBWIRegs()) {
1768 addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1769 addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
1770
1771 // Extends from v64i1 masks to 512-bit vectors.
1772 setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
1773 setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
1774 setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
1775
1776 setOperationAction(ISD::MUL, MVT::v32i16, Legal);
1777 setOperationAction(ISD::MUL, MVT::v64i8, Custom);
1778 setOperationAction(ISD::MULHS, MVT::v32i16, Legal);
1779 setOperationAction(ISD::MULHU, MVT::v32i16, Legal);
1780 setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
1781 setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
1782 setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i16, Custom);
1783 setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i8, Custom);
1784 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i16, Legal);
1785 setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i8, Legal);
1786 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v32i16, Custom);
1787 setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v64i8, Custom);
1788 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32i16, Custom);
1789 setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v64i8, Custom);
1790 setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
1791 setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
1792 setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
1793 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32i16, Custom);
1794 setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v64i8, Custom);
1795 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32i16, Custom);
1796 setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v64i8, Custom);
1797 setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
1798 setOperationAction(ISD::BITREVERSE, MVT::v64i8, Custom);
1799
1800 setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v32i16, Custom);
1801 setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, MVT::v32i16, Custom);
1802
1803 setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
1804
1805 for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1806 setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1807 setOperationAction(ISD::VSELECT, VT, Custom);
1808 setOperationAction(ISD::ABS, VT, Legal);
1809 setOperationAction(ISD::SRL, VT, Custom);
1810 setOperationAction(ISD::SHL, VT, Custom);
1811 setOperationAction(ISD::SRA, VT, Custom);
1812 setOperationAction(ISD::MLOAD, VT, Legal);
1813 setOperationAction(ISD::MSTORE, VT, Legal);
1814 setOperationAction(ISD::CTPOP, VT, Custom);
1815 setOperationAction(ISD::CTLZ, VT, Custom);
1816 setOperationAction(ISD::SMAX, VT, Legal);
1817 setOperationAction(ISD::UMAX, VT, Legal);
1818 setOperationAction(ISD::SMIN, VT, Legal);
1819 setOperationAction(ISD::UMIN, VT, Legal);
1820 setOperationAction(ISD::SETCC, VT, Custom);
1821 setOperationAction(ISD::UADDSAT, VT, Legal);
1822 setOperationAction(ISD::SADDSAT, VT, Legal);
1823 setOperationAction(ISD::USUBSAT, VT, Legal);
1824 setOperationAction(ISD::SSUBSAT, VT, Legal);
1825 setOperationAction(ISD::SELECT, VT, Custom);
1826
1827 // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1828 // setcc all the way to isel and prefer SETGT in some isel patterns.
1829 setCondCodeAction(ISD::SETLT, VT, Custom);
1830 setCondCodeAction(ISD::SETLE, VT, Custom);
1831 }
1832
1833 for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1834 setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
1835 }
1836
1837 if (Subtarget.hasBITALG()) {
1838 for (auto VT : { MVT::v64i8, MVT::v32i16 })
1839 setOperationAction(ISD::CTPOP, VT, Legal);
1840 }
1841
1842 if (Subtarget.hasVBMI2()) {
1843 setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
1844 setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
1845 }
1846 }
1847
1848 if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
1849 for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
1850 setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
1851 setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
1852 }
1853
1854 // These operations are handled on non-VLX by artificially widening in
1855 // isel patterns.
1856 // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
1857
1858 if (Subtarget.hasBITALG()) {
1859 for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
1860 setOperationAction(ISD::CTPOP, VT, Legal);
1861 }
1862 }
1863
1864 if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
1865 setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
1866 setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
1867 setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
1868 setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
1869 setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
1870
1871 setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
1872 setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
1873 setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
1874 setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
1875 setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
1876
1877 if (Subtarget.hasDQI()) {
1878 // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
1879 // v2f32 UINT_TO_FP is already custom under SSE2.
1880 assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&((isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) && isOperationCustom
(ISD::STRICT_UINT_TO_FP, MVT::v2f32) && "Unexpected operation action!"
) ? static_cast<void> (0) : __assert_fail ("isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) && isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) && \"Unexpected operation action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 1882, __PRETTY_FUNCTION__))
1881 isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) &&((isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) && isOperationCustom
(ISD::STRICT_UINT_TO_FP, MVT::v2f32) && "Unexpected operation action!"
) ? static_cast<void> (0) : __assert_fail ("isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) && isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) && \"Unexpected operation action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 1882, __PRETTY_FUNCTION__))
1882 "Unexpected operation action!")((isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) && isOperationCustom
(ISD::STRICT_UINT_TO_FP, MVT::v2f32) && "Unexpected operation action!"
) ? static_cast<void> (0) : __assert_fail ("isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) && isOperationCustom(ISD::STRICT_UINT_TO_FP, MVT::v2f32) && \"Unexpected operation action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 1882, __PRETTY_FUNCTION__));
1883 // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
1884 setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
1885 setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
1886 setOperationAction(ISD::STRICT_FP_TO_SINT, MVT::v2f32, Custom);
1887 setOperationAction(ISD::STRICT_FP_TO_UINT, MVT::v2f32, Custom);
1888 }
1889
1890 if (Subtarget.hasBWI()) {
1891 setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
1892 setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
1893 }
1894
1895 if (Subtarget.hasVBMI2()) {
1896 // TODO: Make these legal even without VLX?
1897 for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64,
1898 MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1899 setOperationAction(ISD::FSHL, VT, Custom);
1900 setOperationAction(ISD::FSHR, VT, Custom);
1901 }
1902 }
1903
1904 setOperationAction(ISD::TRUNCATE, MVT::v16i32, Custom);
1905 setOperationAction(ISD::TRUNCATE, MVT::v8i64, Custom);
1906 setOperationAction(ISD::TRUNCATE, MVT::v16i64, Custom);
1907 }
1908
1909 // We want to custom lower some of our intrinsics.
1910 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
1911 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
1912 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1913 if (!Subtarget.is64Bit()) {
1914 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
1915 }
1916
1917 // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
1918 // handle type legalization for these operations here.
1919 //
1920 // FIXME: We really should do custom legalization for addition and
1921 // subtraction on x86-32 once PR3203 is fixed. We really can't do much better
1922 // than generic legalization for 64-bit multiplication-with-overflow, though.
1923 for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
1924 if (VT == MVT::i64 && !Subtarget.is64Bit())
1925 continue;
1926 // Add/Sub/Mul with overflow operations are custom lowered.
1927 setOperationAction(ISD::SADDO, VT, Custom);
1928 setOperationAction(ISD::UADDO, VT, Custom);
1929 setOperationAction(ISD::SSUBO, VT, Custom);
1930 setOperationAction(ISD::USUBO, VT, Custom);
1931 setOperationAction(ISD::SMULO, VT, Custom);
1932 setOperationAction(ISD::UMULO, VT, Custom);
1933
1934 // Support carry in as value rather than glue.
1935 setOperationAction(ISD::ADDCARRY, VT, Custom);
1936 setOperationAction(ISD::SUBCARRY, VT, Custom);
1937 setOperationAction(ISD::SETCCCARRY, VT, Custom);
1938 }
1939
1940 if (!Subtarget.is64Bit()) {
1941 // These libcalls are not available in 32-bit.
1942 setLibcallName(RTLIB::SHL_I128, nullptr);
1943 setLibcallName(RTLIB::SRL_I128, nullptr);
1944 setLibcallName(RTLIB::SRA_I128, nullptr);
1945 setLibcallName(RTLIB::MUL_I128, nullptr);
1946 }
1947
1948 // Combine sin / cos into _sincos_stret if it is available.
1949 if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
1950 getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
1951 setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1952 setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1953 }
1954
1955 if (Subtarget.isTargetWin64()) {
1956 setOperationAction(ISD::SDIV, MVT::i128, Custom);
1957 setOperationAction(ISD::UDIV, MVT::i128, Custom);
1958 setOperationAction(ISD::SREM, MVT::i128, Custom);
1959 setOperationAction(ISD::UREM, MVT::i128, Custom);
1960 setOperationAction(ISD::SDIVREM, MVT::i128, Custom);
1961 setOperationAction(ISD::UDIVREM, MVT::i128, Custom);
1962 }
1963
1964 // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
1965 // is. We should promote the value to 64-bits to solve this.
1966 // This is what the CRT headers do - `fmodf` is an inline header
1967 // function casting to f64 and calling `fmod`.
1968 if (Subtarget.is32Bit() &&
1969 (Subtarget.isTargetWindowsMSVC() || Subtarget.isTargetWindowsItanium()))
1970 for (ISD::NodeType Op :
1971 {ISD::FCEIL, ISD::STRICT_FCEIL,
1972 ISD::FCOS, ISD::STRICT_FCOS,
1973 ISD::FEXP, ISD::STRICT_FEXP,
1974 ISD::FFLOOR, ISD::STRICT_FFLOOR,
1975 ISD::FREM, ISD::STRICT_FREM,
1976 ISD::FLOG, ISD::STRICT_FLOG,
1977 ISD::FLOG10, ISD::STRICT_FLOG10,
1978 ISD::FPOW, ISD::STRICT_FPOW,
1979 ISD::FSIN, ISD::STRICT_FSIN})
1980 if (isOperationExpand(Op, MVT::f32))
1981 setOperationAction(Op, MVT::f32, Promote);
1982
1983 // We have target-specific dag combine patterns for the following nodes:
1984 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
1985 setTargetDAGCombine(ISD::SCALAR_TO_VECTOR);
1986 setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
1987 setTargetDAGCombine(ISD::CONCAT_VECTORS);
1988 setTargetDAGCombine(ISD::INSERT_SUBVECTOR);
1989 setTargetDAGCombine(ISD::EXTRACT_SUBVECTOR);
1990 setTargetDAGCombine(ISD::BITCAST);
1991 setTargetDAGCombine(ISD::VSELECT);
1992 setTargetDAGCombine(ISD::SELECT);
1993 setTargetDAGCombine(ISD::SHL);
1994 setTargetDAGCombine(ISD::SRA);
1995 setTargetDAGCombine(ISD::SRL);
1996 setTargetDAGCombine(ISD::OR);
1997 setTargetDAGCombine(ISD::AND);
1998 setTargetDAGCombine(ISD::ADD);
1999 setTargetDAGCombine(ISD::FADD);
2000 setTargetDAGCombine(ISD::FSUB);
2001 setTargetDAGCombine(ISD::FNEG);
2002 setTargetDAGCombine(ISD::FMA);
2003 setTargetDAGCombine(ISD::FMINNUM);
2004 setTargetDAGCombine(ISD::FMAXNUM);
2005 setTargetDAGCombine(ISD::SUB);
2006 setTargetDAGCombine(ISD::LOAD);
2007 setTargetDAGCombine(ISD::MLOAD);
2008 setTargetDAGCombine(ISD::STORE);
2009 setTargetDAGCombine(ISD::MSTORE);
2010 setTargetDAGCombine(ISD::TRUNCATE);
2011 setTargetDAGCombine(ISD::ZERO_EXTEND);
2012 setTargetDAGCombine(ISD::ANY_EXTEND);
2013 setTargetDAGCombine(ISD::SIGN_EXTEND);
2014 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
2015 setTargetDAGCombine(ISD::ANY_EXTEND_VECTOR_INREG);
2016 setTargetDAGCombine(ISD::SIGN_EXTEND_VECTOR_INREG);
2017 setTargetDAGCombine(ISD::ZERO_EXTEND_VECTOR_INREG);
2018 setTargetDAGCombine(ISD::SINT_TO_FP);
2019 setTargetDAGCombine(ISD::UINT_TO_FP);
2020 setTargetDAGCombine(ISD::STRICT_SINT_TO_FP);
2021 setTargetDAGCombine(ISD::STRICT_UINT_TO_FP);
2022 setTargetDAGCombine(ISD::SETCC);
2023 setTargetDAGCombine(ISD::MUL);
2024 setTargetDAGCombine(ISD::XOR);
2025 setTargetDAGCombine(ISD::MSCATTER);
2026 setTargetDAGCombine(ISD::MGATHER);
2027
2028 computeRegisterProperties(Subtarget.getRegisterInfo());
2029
2030 MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
2031 MaxStoresPerMemsetOptSize = 8;
2032 MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
2033 MaxStoresPerMemcpyOptSize = 4;
2034 MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
2035 MaxStoresPerMemmoveOptSize = 4;
2036
2037 // TODO: These control memcmp expansion in CGP and could be raised higher, but
2038 // that needs to benchmarked and balanced with the potential use of vector
2039 // load/store types (PR33329, PR33914).
2040 MaxLoadsPerMemcmp = 2;
2041 MaxLoadsPerMemcmpOptSize = 2;
2042
2043 // Set loop alignment to 2^ExperimentalPrefLoopAlignment bytes (default: 2^4).
2044 setPrefLoopAlignment(Align(1ULL << ExperimentalPrefLoopAlignment));
2045
2046 // An out-of-order CPU can speculatively execute past a predictable branch,
2047 // but a conditional move could be stalled by an expensive earlier operation.
2048 PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
2049 EnableExtLdPromotion = true;
2050 setPrefFunctionAlignment(Align(16));
2051
2052 verifyIntrinsicTables();
2053
2054 // Default to having -disable-strictnode-mutation on
2055 IsStrictFPEnabled = true;
2056}
2057
2058// This has so far only been implemented for 64-bit MachO.
2059bool X86TargetLowering::useLoadStackGuardNode() const {
2060 return Subtarget.isTargetMachO() && Subtarget.is64Bit();
2061}
2062
2063bool X86TargetLowering::useStackGuardXorFP() const {
2064 // Currently only MSVC CRTs XOR the frame pointer into the stack guard value.
2065 return Subtarget.getTargetTriple().isOSMSVCRT() && !Subtarget.isTargetMachO();
2066}
2067
2068SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
2069 const SDLoc &DL) const {
2070 EVT PtrTy = getPointerTy(DAG.getDataLayout());
2071 unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
2072 MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
2073 return SDValue(Node, 0);
2074}
2075
2076TargetLoweringBase::LegalizeTypeAction
2077X86TargetLowering::getPreferredVectorAction(MVT VT) const {
2078 if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
2079 return TypeSplitVector;
2080
2081 if (VT.getVectorNumElements() != 1 &&
2082 VT.getVectorElementType() != MVT::i1)
2083 return TypeWidenVector;
2084
2085 return TargetLoweringBase::getPreferredVectorAction(VT);
2086}
2087
2088MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
2089 CallingConv::ID CC,
2090 EVT VT) const {
2091 // v32i1 vectors should be promoted to v32i8 to match avx2.
2092 if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
2093 return MVT::v32i8;
2094 // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
2095 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
2096 Subtarget.hasAVX512() &&
2097 (!isPowerOf2_32(VT.getVectorNumElements()) ||
2098 (VT.getVectorNumElements() > 16 && !Subtarget.hasBWI()) ||
2099 (VT.getVectorNumElements() > 64 && Subtarget.hasBWI())))
2100 return MVT::i8;
2101 // Split v64i1 vectors if we don't have v64i8 available.
2102 if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
2103 CC != CallingConv::X86_RegCall)
2104 return MVT::v32i1;
2105 // FIXME: Should we just make these types legal and custom split operations?
2106 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !EnableOldKNLABI &&
2107 Subtarget.useAVX512Regs() && !Subtarget.hasBWI())
2108 return MVT::v16i32;
2109 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2110}
2111
2112unsigned X86TargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
2113 CallingConv::ID CC,
2114 EVT VT) const {
2115 // v32i1 vectors should be promoted to v32i8 to match avx2.
2116 if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
2117 return 1;
2118 // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
2119 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
2120 Subtarget.hasAVX512() &&
2121 (!isPowerOf2_32(VT.getVectorNumElements()) ||
2122 (VT.getVectorNumElements() > 16 && !Subtarget.hasBWI()) ||
2123 (VT.getVectorNumElements() > 64 && Subtarget.hasBWI())))
2124 return VT.getVectorNumElements();
2125 // Split v64i1 vectors if we don't have v64i8 available.
2126 if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
2127 CC != CallingConv::X86_RegCall)
2128 return 2;
2129 // FIXME: Should we just make these types legal and custom split operations?
2130 if ((VT == MVT::v32i16 || VT == MVT::v64i8) && !EnableOldKNLABI &&
2131 Subtarget.useAVX512Regs() && !Subtarget.hasBWI())
2132 return 1;
2133 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2134}
2135
2136unsigned X86TargetLowering::getVectorTypeBreakdownForCallingConv(
2137 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
2138 unsigned &NumIntermediates, MVT &RegisterVT) const {
2139 // Break wide or odd vXi1 vectors into scalars to match avx2 behavior.
2140 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
2141 Subtarget.hasAVX512() &&
2142 (!isPowerOf2_32(VT.getVectorNumElements()) ||
2143 (VT.getVectorNumElements() > 16 && !Subtarget.hasBWI()) ||
2144 (VT.getVectorNumElements() > 64 && Subtarget.hasBWI()))) {
2145 RegisterVT = MVT::i8;
2146 IntermediateVT = MVT::i1;
2147 NumIntermediates = VT.getVectorNumElements();
2148 return NumIntermediates;
2149 }
2150
2151 // Split v64i1 vectors if we don't have v64i8 available.
2152 if (VT == MVT::v64i1 && Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
2153 CC != CallingConv::X86_RegCall) {
2154 RegisterVT = MVT::v32i1;
2155 IntermediateVT = MVT::v32i1;
2156 NumIntermediates = 2;
2157 return 2;
2158 }
2159
2160 return TargetLowering::getVectorTypeBreakdownForCallingConv(Context, CC, VT, IntermediateVT,
2161 NumIntermediates, RegisterVT);
2162}
2163
2164EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,
2165 LLVMContext& Context,
2166 EVT VT) const {
2167 if (!VT.isVector())
2168 return MVT::i8;
2169
2170 if (Subtarget.hasAVX512()) {
2171 const unsigned NumElts = VT.getVectorNumElements();
2172
2173 // Figure out what this type will be legalized to.
2174 EVT LegalVT = VT;
2175 while (getTypeAction(Context, LegalVT) != TypeLegal)
2176 LegalVT = getTypeToTransformTo(Context, LegalVT);
2177
2178 // If we got a 512-bit vector then we'll definitely have a vXi1 compare.
2179 if (LegalVT.getSimpleVT().is512BitVector())
2180 return EVT::getVectorVT(Context, MVT::i1, NumElts);
2181
2182 if (LegalVT.getSimpleVT().isVector() && Subtarget.hasVLX()) {
2183 // If we legalized to less than a 512-bit vector, then we will use a vXi1
2184 // compare for vXi32/vXi64 for sure. If we have BWI we will also support
2185 // vXi16/vXi8.
2186 MVT EltVT = LegalVT.getSimpleVT().getVectorElementType();
2187 if (Subtarget.hasBWI() || EltVT.getSizeInBits() >= 32)
2188 return EVT::getVectorVT(Context, MVT::i1, NumElts);
2189 }
2190 }
2191
2192 return VT.changeVectorElementTypeToInteger();
2193}
2194
2195/// Helper for getByValTypeAlignment to determine
2196/// the desired ByVal argument alignment.
2197static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
2198 if (MaxAlign == 16)
2199 return;
2200 if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
2201 if (VTy->getBitWidth() == 128)
2202 MaxAlign = 16;
2203 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
2204 unsigned EltAlign = 0;
2205 getMaxByValAlign(ATy->getElementType(), EltAlign);
2206 if (EltAlign > MaxAlign)
2207 MaxAlign = EltAlign;
2208 } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
2209 for (auto *EltTy : STy->elements()) {
2210 unsigned EltAlign = 0;
2211 getMaxByValAlign(EltTy, EltAlign);
2212 if (EltAlign > MaxAlign)
2213 MaxAlign = EltAlign;
2214 if (MaxAlign == 16)
2215 break;
2216 }
2217 }
2218}
2219
2220/// Return the desired alignment for ByVal aggregate
2221/// function arguments in the caller parameter area. For X86, aggregates
2222/// that contain SSE vectors are placed at 16-byte boundaries while the rest
2223/// are at 4-byte boundaries.
2224unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty,
2225 const DataLayout &DL) const {
2226 if (Subtarget.is64Bit()) {
2227 // Max of 8 and alignment of type.
2228 unsigned TyAlign = DL.getABITypeAlignment(Ty);
2229 if (TyAlign > 8)
2230 return TyAlign;
2231 return 8;
2232 }
2233
2234 unsigned Align = 4;
2235 if (Subtarget.hasSSE1())
2236 getMaxByValAlign(Ty, Align);
2237 return Align;
2238}
2239
2240/// Returns the target specific optimal type for load
2241/// and store operations as a result of memset, memcpy, and memmove
2242/// lowering. If DstAlign is zero that means it's safe to destination
2243/// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
2244/// means there isn't a need to check it against alignment requirement,
2245/// probably because the source does not need to be loaded. If 'IsMemset' is
2246/// true, that means it's expanding a memset. If 'ZeroMemset' is true, that
2247/// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy
2248/// source is constant so it does not need to be loaded.
2249/// It returns EVT::Other if the type should be determined using generic
2250/// target-independent logic.
2251/// For vector ops we check that the overall size isn't larger than our
2252/// preferred vector width.
2253EVT X86TargetLowering::getOptimalMemOpType(
2254 uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset,
2255 bool ZeroMemset, bool MemcpyStrSrc,
2256 const AttributeList &FuncAttributes) const {
2257 if (!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
2258 if (Size >= 16 && (!Subtarget.isUnalignedMem16Slow() ||
2259 ((DstAlign == 0 || DstAlign >= 16) &&
2260 (SrcAlign == 0 || SrcAlign >= 16)))) {
2261 // FIXME: Check if unaligned 64-byte accesses are slow.
2262 if (Size >= 64 && Subtarget.hasAVX512() &&
2263 (Subtarget.getPreferVectorWidth() >= 512)) {
2264 return Subtarget.hasBWI() ? MVT::v64i8 : MVT::v16i32;
2265 }
2266 // FIXME: Check if unaligned 32-byte accesses are slow.
2267 if (Size >= 32 && Subtarget.hasAVX() &&
2268 (Subtarget.getPreferVectorWidth() >= 256)) {
2269 // Although this isn't a well-supported type for AVX1, we'll let
2270 // legalization and shuffle lowering produce the optimal codegen. If we
2271 // choose an optimal type with a vector element larger than a byte,
2272 // getMemsetStores() may create an intermediate splat (using an integer
2273 // multiply) before we splat as a vector.
2274 return MVT::v32i8;
2275 }
2276 if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128))
2277 return MVT::v16i8;
2278 // TODO: Can SSE1 handle a byte vector?
2279 // If we have SSE1 registers we should be able to use them.
2280 if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
2281 (Subtarget.getPreferVectorWidth() >= 128))
2282 return MVT::v4f32;
2283 } else if ((!IsMemset || ZeroMemset) && !MemcpyStrSrc && Size >= 8 &&
2284 !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
2285 // Do not use f64 to lower memcpy if source is string constant. It's
2286 // better to use i32 to avoid the loads.
2287 // Also, do not use f64 to lower memset unless this is a memset of zeros.
2288 // The gymnastics of splatting a byte value into an XMM register and then
2289 // only using 8-byte stores (because this is a CPU with slow unaligned
2290 // 16-byte accesses) makes that a loser.
2291 return MVT::f64;
2292 }
2293 }
2294 // This is a compromise. If we reach here, unaligned accesses may be slow on
2295 // this target. However, creating smaller, aligned accesses could be even
2296 // slower and would certainly be a lot more code.
2297 if (Subtarget.is64Bit() && Size >= 8)
2298 return MVT::i64;
2299 return MVT::i32;
2300}
2301
2302bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
2303 if (VT == MVT::f32)
2304 return X86ScalarSSEf32;
2305 else if (VT == MVT::f64)
2306 return X86ScalarSSEf64;
2307 return true;
2308}
2309
2310bool X86TargetLowering::allowsMisalignedMemoryAccesses(
2311 EVT VT, unsigned, unsigned Align, MachineMemOperand::Flags Flags,
2312 bool *Fast) const {
2313 if (Fast) {
2314 switch (VT.getSizeInBits()) {
2315 default:
2316 // 8-byte and under are always assumed to be fast.
2317 *Fast = true;
2318 break;
2319 case 128:
2320 *Fast = !Subtarget.isUnalignedMem16Slow();
2321 break;
2322 case 256:
2323 *Fast = !Subtarget.isUnalignedMem32Slow();
2324 break;
2325 // TODO: What about AVX-512 (512-bit) accesses?
2326 }
2327 }
2328 // NonTemporal vector memory ops must be aligned.
2329 if (!!(Flags & MachineMemOperand::MONonTemporal) && VT.isVector()) {
2330 // NT loads can only be vector aligned, so if its less aligned than the
2331 // minimum vector size (which we can split the vector down to), we might as
2332 // well use a regular unaligned vector load.
2333 // We don't have any NT loads pre-SSE41.
2334 if (!!(Flags & MachineMemOperand::MOLoad))
2335 return (Align < 16 || !Subtarget.hasSSE41());
2336 return false;
2337 }
2338 // Misaligned accesses of any size are always allowed.
2339 return true;
2340}
2341
2342/// Return the entry encoding for a jump table in the
2343/// current function. The returned value is a member of the
2344/// MachineJumpTableInfo::JTEntryKind enum.
2345unsigned X86TargetLowering::getJumpTableEncoding() const {
2346 // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
2347 // symbol.
2348 if (isPositionIndependent() && Subtarget.isPICStyleGOT())
2349 return MachineJumpTableInfo::EK_Custom32;
2350
2351 // Otherwise, use the normal jump table encoding heuristics.
2352 return TargetLowering::getJumpTableEncoding();
2353}
2354
2355bool X86TargetLowering::useSoftFloat() const {
2356 return Subtarget.useSoftFloat();
2357}
2358
2359void X86TargetLowering::markLibCallAttributes(MachineFunction *MF, unsigned CC,
2360 ArgListTy &Args) const {
2361
2362 // Only relabel X86-32 for C / Stdcall CCs.
2363 if (Subtarget.is64Bit())
2364 return;
2365 if (CC != CallingConv::C && CC != CallingConv::X86_StdCall)
2366 return;
2367 unsigned ParamRegs = 0;
2368 if (auto *M = MF->getFunction().getParent())
2369 ParamRegs = M->getNumberRegisterParameters();
2370
2371 // Mark the first N int arguments as having reg
2372 for (unsigned Idx = 0; Idx < Args.size(); Idx++) {
2373 Type *T = Args[Idx].Ty;
2374 if (T->isIntOrPtrTy())
2375 if (MF->getDataLayout().getTypeAllocSize(T) <= 8) {
2376 unsigned numRegs = 1;
2377 if (MF->getDataLayout().getTypeAllocSize(T) > 4)
2378 numRegs = 2;
2379 if (ParamRegs < numRegs)
2380 return;
2381 ParamRegs -= numRegs;
2382 Args[Idx].IsInReg = true;
2383 }
2384 }
2385}
2386
2387const MCExpr *
2388X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
2389 const MachineBasicBlock *MBB,
2390 unsigned uid,MCContext &Ctx) const{
2391 assert(isPositionIndependent() && Subtarget.isPICStyleGOT())((isPositionIndependent() && Subtarget.isPICStyleGOT(
)) ? static_cast<void> (0) : __assert_fail ("isPositionIndependent() && Subtarget.isPICStyleGOT()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2391, __PRETTY_FUNCTION__));
2392 // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
2393 // entries.
2394 return MCSymbolRefExpr::create(MBB->getSymbol(),
2395 MCSymbolRefExpr::VK_GOTOFF, Ctx);
2396}
2397
2398/// Returns relocation base for the given PIC jumptable.
2399SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
2400 SelectionDAG &DAG) const {
2401 if (!Subtarget.is64Bit())
2402 // This doesn't have SDLoc associated with it, but is not really the
2403 // same as a Register.
2404 return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
2405 getPointerTy(DAG.getDataLayout()));
2406 return Table;
2407}
2408
2409/// This returns the relocation base for the given PIC jumptable,
2410/// the same as getPICJumpTableRelocBase, but as an MCExpr.
2411const MCExpr *X86TargetLowering::
2412getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
2413 MCContext &Ctx) const {
2414 // X86-64 uses RIP relative addressing based on the jump table label.
2415 if (Subtarget.isPICStyleRIPRel())
2416 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
2417
2418 // Otherwise, the reference is relative to the PIC base.
2419 return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);
2420}
2421
2422std::pair<const TargetRegisterClass *, uint8_t>
2423X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
2424 MVT VT) const {
2425 const TargetRegisterClass *RRC = nullptr;
2426 uint8_t Cost = 1;
2427 switch (VT.SimpleTy) {
2428 default:
2429 return TargetLowering::findRepresentativeClass(TRI, VT);
2430 case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:
2431 RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;
2432 break;
2433 case MVT::x86mmx:
2434 RRC = &X86::VR64RegClass;
2435 break;
2436 case MVT::f32: case MVT::f64:
2437 case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
2438 case MVT::v4f32: case MVT::v2f64:
2439 case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64:
2440 case MVT::v8f32: case MVT::v4f64:
2441 case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64:
2442 case MVT::v16f32: case MVT::v8f64:
2443 RRC = &X86::VR128XRegClass;
2444 break;
2445 }
2446 return std::make_pair(RRC, Cost);
2447}
2448
2449unsigned X86TargetLowering::getAddressSpace() const {
2450 if (Subtarget.is64Bit())
2451 return (getTargetMachine().getCodeModel() == CodeModel::Kernel) ? 256 : 257;
2452 return 256;
2453}
2454
2455static bool hasStackGuardSlotTLS(const Triple &TargetTriple) {
2456 return TargetTriple.isOSGlibc() || TargetTriple.isOSFuchsia() ||
2457 (TargetTriple.isAndroid() && !TargetTriple.isAndroidVersionLT(17));
2458}
2459
2460static Constant* SegmentOffset(IRBuilder<> &IRB,
2461 unsigned Offset, unsigned AddressSpace) {
2462 return ConstantExpr::getIntToPtr(
2463 ConstantInt::get(Type::getInt32Ty(IRB.getContext()), Offset),
2464 Type::getInt8PtrTy(IRB.getContext())->getPointerTo(AddressSpace));
2465}
2466
2467Value *X86TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const {
2468 // glibc, bionic, and Fuchsia have a special slot for the stack guard in
2469 // tcbhead_t; use it instead of the usual global variable (see
2470 // sysdeps/{i386,x86_64}/nptl/tls.h)
2471 if (hasStackGuardSlotTLS(Subtarget.getTargetTriple())) {
2472 if (Subtarget.isTargetFuchsia()) {
2473 // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
2474 return SegmentOffset(IRB, 0x10, getAddressSpace());
2475 } else {
2476 // %fs:0x28, unless we're using a Kernel code model, in which case
2477 // it's %gs:0x28. gs:0x14 on i386.
2478 unsigned Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14;
2479 return SegmentOffset(IRB, Offset, getAddressSpace());
2480 }
2481 }
2482
2483 return TargetLowering::getIRStackGuard(IRB);
2484}
2485
2486void X86TargetLowering::insertSSPDeclarations(Module &M) const {
2487 // MSVC CRT provides functionalities for stack protection.
2488 if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
2489 Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
2490 // MSVC CRT has a global variable holding security cookie.
2491 M.getOrInsertGlobal("__security_cookie",
2492 Type::getInt8PtrTy(M.getContext()));
2493
2494 // MSVC CRT has a function to validate security cookie.
2495 FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
2496 "__security_check_cookie", Type::getVoidTy(M.getContext()),
2497 Type::getInt8PtrTy(M.getContext()));
2498 if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {
2499 F->setCallingConv(CallingConv::X86_FastCall);
2500 F->addAttribute(1, Attribute::AttrKind::InReg);
2501 }
2502 return;
2503 }
2504 // glibc, bionic, and Fuchsia have a special slot for the stack guard.
2505 if (hasStackGuardSlotTLS(Subtarget.getTargetTriple()))
2506 return;
2507 TargetLowering::insertSSPDeclarations(M);
2508}
2509
2510Value *X86TargetLowering::getSDagStackGuard(const Module &M) const {
2511 // MSVC CRT has a global variable holding security cookie.
2512 if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
2513 Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
2514 return M.getGlobalVariable("__security_cookie");
2515 }
2516 return TargetLowering::getSDagStackGuard(M);
2517}
2518
2519Function *X86TargetLowering::getSSPStackGuardCheck(const Module &M) const {
2520 // MSVC CRT has a function to validate security cookie.
2521 if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
2522 Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
2523 return M.getFunction("__security_check_cookie");
2524 }
2525 return TargetLowering::getSSPStackGuardCheck(M);
2526}
2527
2528Value *X86TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
2529 if (Subtarget.getTargetTriple().isOSContiki())
2530 return getDefaultSafeStackPointerLocation(IRB, false);
2531
2532 // Android provides a fixed TLS slot for the SafeStack pointer. See the
2533 // definition of TLS_SLOT_SAFESTACK in
2534 // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
2535 if (Subtarget.isTargetAndroid()) {
2536 // %fs:0x48, unless we're using a Kernel code model, in which case it's %gs:
2537 // %gs:0x24 on i386
2538 unsigned Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24;
2539 return SegmentOffset(IRB, Offset, getAddressSpace());
2540 }
2541
2542 // Fuchsia is similar.
2543 if (Subtarget.isTargetFuchsia()) {
2544 // <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.
2545 return SegmentOffset(IRB, 0x18, getAddressSpace());
2546 }
2547
2548 return TargetLowering::getSafeStackPointerLocation(IRB);
2549}
2550
2551bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
2552 unsigned DestAS) const {
2553 assert(SrcAS != DestAS && "Expected different address spaces!")((SrcAS != DestAS && "Expected different address spaces!"
) ? static_cast<void> (0) : __assert_fail ("SrcAS != DestAS && \"Expected different address spaces!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2553, __PRETTY_FUNCTION__));
2554
2555 const TargetMachine &TM = getTargetMachine();
2556 if (TM.getPointerSize(SrcAS) != TM.getPointerSize(DestAS))
2557 return false;
2558
2559 return SrcAS < 256 && DestAS < 256;
2560}
2561
2562//===----------------------------------------------------------------------===//
2563// Return Value Calling Convention Implementation
2564//===----------------------------------------------------------------------===//
2565
2566bool X86TargetLowering::CanLowerReturn(
2567 CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
2568 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2569 SmallVector<CCValAssign, 16> RVLocs;
2570 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2571 return CCInfo.CheckReturn(Outs, RetCC_X86);
2572}
2573
2574const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {
2575 static const MCPhysReg ScratchRegs[] = { X86::R11, 0 };
2576 return ScratchRegs;
2577}
2578
2579/// Lowers masks values (v*i1) to the local register values
2580/// \returns DAG node after lowering to register type
2581static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
2582 const SDLoc &Dl, SelectionDAG &DAG) {
2583 EVT ValVT = ValArg.getValueType();
2584
2585 if (ValVT == MVT::v1i1)
2586 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Dl, ValLoc, ValArg,
2587 DAG.getIntPtrConstant(0, Dl));
2588
2589 if ((ValVT == MVT::v8i1 && (ValLoc == MVT::i8 || ValLoc == MVT::i32)) ||
2590 (ValVT == MVT::v16i1 && (ValLoc == MVT::i16 || ValLoc == MVT::i32))) {
2591 // Two stage lowering might be required
2592 // bitcast: v8i1 -> i8 / v16i1 -> i16
2593 // anyextend: i8 -> i32 / i16 -> i32
2594 EVT TempValLoc = ValVT == MVT::v8i1 ? MVT::i8 : MVT::i16;
2595 SDValue ValToCopy = DAG.getBitcast(TempValLoc, ValArg);
2596 if (ValLoc == MVT::i32)
2597 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, Dl, ValLoc, ValToCopy);
2598 return ValToCopy;
2599 }
2600
2601 if ((ValVT == MVT::v32i1 && ValLoc == MVT::i32) ||
2602 (ValVT == MVT::v64i1 && ValLoc == MVT::i64)) {
2603 // One stage lowering is required
2604 // bitcast: v32i1 -> i32 / v64i1 -> i64
2605 return DAG.getBitcast(ValLoc, ValArg);
2606 }
2607
2608 return DAG.getNode(ISD::ANY_EXTEND, Dl, ValLoc, ValArg);
2609}
2610
2611/// Breaks v64i1 value into two registers and adds the new node to the DAG
2612static void Passv64i1ArgInRegs(
2613 const SDLoc &Dl, SelectionDAG &DAG, SDValue &Arg,
2614 SmallVectorImpl<std::pair<unsigned, SDValue>> &RegsToPass, CCValAssign &VA,
2615 CCValAssign &NextVA, const X86Subtarget &Subtarget) {
2616 assert(Subtarget.hasBWI() && "Expected AVX512BW target!")((Subtarget.hasBWI() && "Expected AVX512BW target!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI() && \"Expected AVX512BW target!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2616, __PRETTY_FUNCTION__));
2617 assert(Subtarget.is32Bit() && "Expecting 32 bit target")((Subtarget.is32Bit() && "Expecting 32 bit target") ?
static_cast<void> (0) : __assert_fail ("Subtarget.is32Bit() && \"Expecting 32 bit target\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2617, __PRETTY_FUNCTION__));
2618 assert(Arg.getValueType() == MVT::i64 && "Expecting 64 bit value")((Arg.getValueType() == MVT::i64 && "Expecting 64 bit value"
) ? static_cast<void> (0) : __assert_fail ("Arg.getValueType() == MVT::i64 && \"Expecting 64 bit value\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2618, __PRETTY_FUNCTION__));
2619 assert(VA.isRegLoc() && NextVA.isRegLoc() &&((VA.isRegLoc() && NextVA.isRegLoc() && "The value should reside in two registers"
) ? static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && NextVA.isRegLoc() && \"The value should reside in two registers\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2620, __PRETTY_FUNCTION__))
2620 "The value should reside in two registers")((VA.isRegLoc() && NextVA.isRegLoc() && "The value should reside in two registers"
) ? static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && NextVA.isRegLoc() && \"The value should reside in two registers\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2620, __PRETTY_FUNCTION__));
2621
2622 // Before splitting the value we cast it to i64
2623 Arg = DAG.getBitcast(MVT::i64, Arg);
2624
2625 // Splitting the value into two i32 types
2626 SDValue Lo, Hi;
2627 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg,
2628 DAG.getConstant(0, Dl, MVT::i32));
2629 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg,
2630 DAG.getConstant(1, Dl, MVT::i32));
2631
2632 // Attach the two i32 types into corresponding registers
2633 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Lo));
2634 RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), Hi));
2635}
2636
2637SDValue
2638X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2639 bool isVarArg,
2640 const SmallVectorImpl<ISD::OutputArg> &Outs,
2641 const SmallVectorImpl<SDValue> &OutVals,
2642 const SDLoc &dl, SelectionDAG &DAG) const {
2643 MachineFunction &MF = DAG.getMachineFunction();
2644 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2645
2646 // In some cases we need to disable registers from the default CSR list.
2647 // For example, when they are used for argument passing.
2648 bool ShouldDisableCalleeSavedRegister =
2649 CallConv == CallingConv::X86_RegCall ||
2650 MF.getFunction().hasFnAttribute("no_caller_saved_registers");
2651
2652 if (CallConv == CallingConv::X86_INTR && !Outs.empty())
2653 report_fatal_error("X86 interrupts may not return any value");
2654
2655 SmallVector<CCValAssign, 16> RVLocs;
2656 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
2657 CCInfo.AnalyzeReturn(Outs, RetCC_X86);
2658
2659 SDValue Flag;
2660 SmallVector<SDValue, 6> RetOps;
2661 RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2662 // Operand #1 = Bytes To Pop
2663 RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), dl,
2664 MVT::i32));
2665
2666 // Copy the result values into the output registers.
2667 for (unsigned I = 0, OutsIndex = 0, E = RVLocs.size(); I != E;
2668 ++I, ++OutsIndex) {
2669 CCValAssign &VA = RVLocs[I];
2670 assert(VA.isRegLoc() && "Can only return in registers!")((VA.isRegLoc() && "Can only return in registers!") ?
static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2670, __PRETTY_FUNCTION__));
2671
2672 // Add the register to the CalleeSaveDisableRegs list.
2673 if (ShouldDisableCalleeSavedRegister)
2674 MF.getRegInfo().disableCalleeSavedRegister(VA.getLocReg());
2675
2676 SDValue ValToCopy = OutVals[OutsIndex];
2677 EVT ValVT = ValToCopy.getValueType();
2678
2679 // Promote values to the appropriate types.
2680 if (VA.getLocInfo() == CCValAssign::SExt)
2681 ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);
2682 else if (VA.getLocInfo() == CCValAssign::ZExt)
2683 ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy);
2684 else if (VA.getLocInfo() == CCValAssign::AExt) {
2685 if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1)
2686 ValToCopy = lowerMasksToReg(ValToCopy, VA.getLocVT(), dl, DAG);
2687 else
2688 ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);
2689 }
2690 else if (VA.getLocInfo() == CCValAssign::BCvt)
2691 ValToCopy = DAG.getBitcast(VA.getLocVT(), ValToCopy);
2692
2693 assert(VA.getLocInfo() != CCValAssign::FPExt &&((VA.getLocInfo() != CCValAssign::FPExt && "Unexpected FP-extend for return value."
) ? static_cast<void> (0) : __assert_fail ("VA.getLocInfo() != CCValAssign::FPExt && \"Unexpected FP-extend for return value.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2694, __PRETTY_FUNCTION__))
2694 "Unexpected FP-extend for return value.")((VA.getLocInfo() != CCValAssign::FPExt && "Unexpected FP-extend for return value."
) ? static_cast<void> (0) : __assert_fail ("VA.getLocInfo() != CCValAssign::FPExt && \"Unexpected FP-extend for return value.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2694, __PRETTY_FUNCTION__));
2695
2696 // If this is x86-64, and we disabled SSE, we can't return FP values,
2697 // or SSE or MMX vectors.
2698 if ((ValVT == MVT::f32 || ValVT == MVT::f64 ||
2699 VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) &&
2700 (Subtarget.is64Bit() && !Subtarget.hasSSE1())) {
2701 errorUnsupported(DAG, dl, "SSE register return with SSE disabled");
2702 VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
2703 } else if (ValVT == MVT::f64 &&
2704 (Subtarget.is64Bit() && !Subtarget.hasSSE2())) {
2705 // Likewise we can't return F64 values with SSE1 only. gcc does so, but
2706 // llvm-gcc has never done it right and no one has noticed, so this
2707 // should be OK for now.
2708 errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");
2709 VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
2710 }
2711
2712 // Returns in ST0/ST1 are handled specially: these are pushed as operands to
2713 // the RET instruction and handled by the FP Stackifier.
2714 if (VA.getLocReg() == X86::FP0 ||
2715 VA.getLocReg() == X86::FP1) {
2716 // If this is a copy from an xmm register to ST(0), use an FPExtend to
2717 // change the value to the FP stack register class.
2718 if (isScalarFPTypeInSSEReg(VA.getValVT()))
2719 ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);
2720 RetOps.push_back(ValToCopy);
2721 // Don't emit a copytoreg.
2722 continue;
2723 }
2724
2725 // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
2726 // which is returned in RAX / RDX.
2727 if (Subtarget.is64Bit()) {
2728 if (ValVT == MVT::x86mmx) {
2729 if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
2730 ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy);
2731 ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
2732 ValToCopy);
2733 // If we don't have SSE2 available, convert to v4f32 so the generated
2734 // register is legal.
2735 if (!Subtarget.hasSSE2())
2736 ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy);
2737 }
2738 }
2739 }
2740
2741 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2742
2743 if (VA.needsCustom()) {
2744 assert(VA.getValVT() == MVT::v64i1 &&((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2745, __PRETTY_FUNCTION__))
2745 "Currently the only custom case is when we split v64i1 to 2 regs")((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2745, __PRETTY_FUNCTION__));
2746
2747 Passv64i1ArgInRegs(dl, DAG, ValToCopy, RegsToPass, VA, RVLocs[++I],
2748 Subtarget);
2749
2750 assert(2 == RegsToPass.size() &&((2 == RegsToPass.size() && "Expecting two registers after Pass64BitArgInRegs"
) ? static_cast<void> (0) : __assert_fail ("2 == RegsToPass.size() && \"Expecting two registers after Pass64BitArgInRegs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2751, __PRETTY_FUNCTION__))
2751 "Expecting two registers after Pass64BitArgInRegs")((2 == RegsToPass.size() && "Expecting two registers after Pass64BitArgInRegs"
) ? static_cast<void> (0) : __assert_fail ("2 == RegsToPass.size() && \"Expecting two registers after Pass64BitArgInRegs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2751, __PRETTY_FUNCTION__));
2752
2753 // Add the second register to the CalleeSaveDisableRegs list.
2754 if (ShouldDisableCalleeSavedRegister)
2755 MF.getRegInfo().disableCalleeSavedRegister(RVLocs[I].getLocReg());
2756 } else {
2757 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ValToCopy));
2758 }
2759
2760 // Add nodes to the DAG and add the values into the RetOps list
2761 for (auto &Reg : RegsToPass) {
2762 Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, Flag);
2763 Flag = Chain.getValue(1);
2764 RetOps.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
2765 }
2766 }
2767
2768 // Swift calling convention does not require we copy the sret argument
2769 // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
2770
2771 // All x86 ABIs require that for returning structs by value we copy
2772 // the sret argument into %rax/%eax (depending on ABI) for the return.
2773 // We saved the argument into a virtual register in the entry block,
2774 // so now we copy the value out and into %rax/%eax.
2775 //
2776 // Checking Function.hasStructRetAttr() here is insufficient because the IR
2777 // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is
2778 // false, then an sret argument may be implicitly inserted in the SelDAG. In
2779 // either case FuncInfo->setSRetReturnReg() will have been called.
2780 if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
2781 // When we have both sret and another return value, we should use the
2782 // original Chain stored in RetOps[0], instead of the current Chain updated
2783 // in the above loop. If we only have sret, RetOps[0] equals to Chain.
2784
2785 // For the case of sret and another return value, we have
2786 // Chain_0 at the function entry
2787 // Chain_1 = getCopyToReg(Chain_0) in the above loop
2788 // If we use Chain_1 in getCopyFromReg, we will have
2789 // Val = getCopyFromReg(Chain_1)
2790 // Chain_2 = getCopyToReg(Chain_1, Val) from below
2791
2792 // getCopyToReg(Chain_0) will be glued together with
2793 // getCopyToReg(Chain_1, Val) into Unit A, getCopyFromReg(Chain_1) will be
2794 // in Unit B, and we will have cyclic dependency between Unit A and Unit B:
2795 // Data dependency from Unit B to Unit A due to usage of Val in
2796 // getCopyToReg(Chain_1, Val)
2797 // Chain dependency from Unit A to Unit B
2798
2799 // So here, we use RetOps[0] (i.e Chain_0) for getCopyFromReg.
2800 SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg,
2801 getPointerTy(MF.getDataLayout()));
2802
2803 unsigned RetValReg
2804 = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ?
2805 X86::RAX : X86::EAX;
2806 Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag);
2807 Flag = Chain.getValue(1);
2808
2809 // RAX/EAX now acts like a return value.
2810 RetOps.push_back(
2811 DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
2812
2813 // Add the returned register to the CalleeSaveDisableRegs list.
2814 if (ShouldDisableCalleeSavedRegister)
2815 MF.getRegInfo().disableCalleeSavedRegister(RetValReg);
2816 }
2817
2818 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
2819 const MCPhysReg *I =
2820 TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
2821 if (I) {
2822 for (; *I; ++I) {
2823 if (X86::GR64RegClass.contains(*I))
2824 RetOps.push_back(DAG.getRegister(*I, MVT::i64));
2825 else
2826 llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2826);
2827 }
2828 }
2829
2830 RetOps[0] = Chain; // Update chain.
2831
2832 // Add the flag if we have it.
2833 if (Flag.getNode())
2834 RetOps.push_back(Flag);
2835
2836 X86ISD::NodeType opcode = X86ISD::RET_FLAG;
2837 if (CallConv == CallingConv::X86_INTR)
2838 opcode = X86ISD::IRET;
2839 return DAG.getNode(opcode, dl, MVT::Other, RetOps);
2840}
2841
2842bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2843 if (N->getNumValues() != 1 || !N->hasNUsesOfValue(1, 0))
2844 return false;
2845
2846 SDValue TCChain = Chain;
2847 SDNode *Copy = *N->use_begin();
2848 if (Copy->getOpcode() == ISD::CopyToReg) {
2849 // If the copy has a glue operand, we conservatively assume it isn't safe to
2850 // perform a tail call.
2851 if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2852 return false;
2853 TCChain = Copy->getOperand(0);
2854 } else if (Copy->getOpcode() != ISD::FP_EXTEND)
2855 return false;
2856
2857 bool HasRet = false;
2858 for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end();
2859 UI != UE; ++UI) {
2860 if (UI->getOpcode() != X86ISD::RET_FLAG)
2861 return false;
2862 // If we are returning more than one value, we can definitely
2863 // not make a tail call see PR19530
2864 if (UI->getNumOperands() > 4)
2865 return false;
2866 if (UI->getNumOperands() == 4 &&
2867 UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue)
2868 return false;
2869 HasRet = true;
2870 }
2871
2872 if (!HasRet)
2873 return false;
2874
2875 Chain = TCChain;
2876 return true;
2877}
2878
2879EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
2880 ISD::NodeType ExtendKind) const {
2881 MVT ReturnMVT = MVT::i32;
2882
2883 bool Darwin = Subtarget.getTargetTriple().isOSDarwin();
2884 if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) {
2885 // The ABI does not require i1, i8 or i16 to be extended.
2886 //
2887 // On Darwin, there is code in the wild relying on Clang's old behaviour of
2888 // always extending i8/i16 return values, so keep doing that for now.
2889 // (PR26665).
2890 ReturnMVT = MVT::i8;
2891 }
2892
2893 EVT MinVT = getRegisterType(Context, ReturnMVT);
2894 return VT.bitsLT(MinVT) ? MinVT : VT;
2895}
2896
2897/// Reads two 32 bit registers and creates a 64 bit mask value.
2898/// \param VA The current 32 bit value that need to be assigned.
2899/// \param NextVA The next 32 bit value that need to be assigned.
2900/// \param Root The parent DAG node.
2901/// \param [in,out] InFlag Represents SDvalue in the parent DAG node for
2902/// glue purposes. In the case the DAG is already using
2903/// physical register instead of virtual, we should glue
2904/// our new SDValue to InFlag SDvalue.
2905/// \return a new SDvalue of size 64bit.
2906static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,
2907 SDValue &Root, SelectionDAG &DAG,
2908 const SDLoc &Dl, const X86Subtarget &Subtarget,
2909 SDValue *InFlag = nullptr) {
2910 assert((Subtarget.hasBWI()) && "Expected AVX512BW target!")(((Subtarget.hasBWI()) && "Expected AVX512BW target!"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasBWI()) && \"Expected AVX512BW target!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2910, __PRETTY_FUNCTION__));
2911 assert(Subtarget.is32Bit() && "Expecting 32 bit target")((Subtarget.is32Bit() && "Expecting 32 bit target") ?
static_cast<void> (0) : __assert_fail ("Subtarget.is32Bit() && \"Expecting 32 bit target\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2911, __PRETTY_FUNCTION__));
2912 assert(VA.getValVT() == MVT::v64i1 &&((VA.getValVT() == MVT::v64i1 && "Expecting first location of 64 bit width type"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Expecting first location of 64 bit width type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2913, __PRETTY_FUNCTION__))
2913 "Expecting first location of 64 bit width type")((VA.getValVT() == MVT::v64i1 && "Expecting first location of 64 bit width type"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Expecting first location of 64 bit width type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2913, __PRETTY_FUNCTION__));
2914 assert(NextVA.getValVT() == VA.getValVT() &&((NextVA.getValVT() == VA.getValVT() && "The locations should have the same type"
) ? static_cast<void> (0) : __assert_fail ("NextVA.getValVT() == VA.getValVT() && \"The locations should have the same type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2915, __PRETTY_FUNCTION__))
2915 "The locations should have the same type")((NextVA.getValVT() == VA.getValVT() && "The locations should have the same type"
) ? static_cast<void> (0) : __assert_fail ("NextVA.getValVT() == VA.getValVT() && \"The locations should have the same type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2915, __PRETTY_FUNCTION__));
2916 assert(VA.isRegLoc() && NextVA.isRegLoc() &&((VA.isRegLoc() && NextVA.isRegLoc() && "The values should reside in two registers"
) ? static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && NextVA.isRegLoc() && \"The values should reside in two registers\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2917, __PRETTY_FUNCTION__))
2917 "The values should reside in two registers")((VA.isRegLoc() && NextVA.isRegLoc() && "The values should reside in two registers"
) ? static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && NextVA.isRegLoc() && \"The values should reside in two registers\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2917, __PRETTY_FUNCTION__));
2918
2919 SDValue Lo, Hi;
2920 SDValue ArgValueLo, ArgValueHi;
2921
2922 MachineFunction &MF = DAG.getMachineFunction();
2923 const TargetRegisterClass *RC = &X86::GR32RegClass;
2924
2925 // Read a 32 bit value from the registers.
2926 if (nullptr == InFlag) {
2927 // When no physical register is present,
2928 // create an intermediate virtual register.
2929 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
2930 ArgValueLo = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);
2931 Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
2932 ArgValueHi = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);
2933 } else {
2934 // When a physical register is available read the value from it and glue
2935 // the reads together.
2936 ArgValueLo =
2937 DAG.getCopyFromReg(Root, Dl, VA.getLocReg(), MVT::i32, *InFlag);
2938 *InFlag = ArgValueLo.getValue(2);
2939 ArgValueHi =
2940 DAG.getCopyFromReg(Root, Dl, NextVA.getLocReg(), MVT::i32, *InFlag);
2941 *InFlag = ArgValueHi.getValue(2);
2942 }
2943
2944 // Convert the i32 type into v32i1 type.
2945 Lo = DAG.getBitcast(MVT::v32i1, ArgValueLo);
2946
2947 // Convert the i32 type into v32i1 type.
2948 Hi = DAG.getBitcast(MVT::v32i1, ArgValueHi);
2949
2950 // Concatenate the two values together.
2951 return DAG.getNode(ISD::CONCAT_VECTORS, Dl, MVT::v64i1, Lo, Hi);
2952}
2953
2954/// The function will lower a register of various sizes (8/16/32/64)
2955/// to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)
2956/// \returns a DAG node contains the operand after lowering to mask type.
2957static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,
2958 const EVT &ValLoc, const SDLoc &Dl,
2959 SelectionDAG &DAG) {
2960 SDValue ValReturned = ValArg;
2961
2962 if (ValVT == MVT::v1i1)
2963 return DAG.getNode(ISD::SCALAR_TO_VECTOR, Dl, MVT::v1i1, ValReturned);
2964
2965 if (ValVT == MVT::v64i1) {
2966 // In 32 bit machine, this case is handled by getv64i1Argument
2967 assert(ValLoc == MVT::i64 && "Expecting only i64 locations")((ValLoc == MVT::i64 && "Expecting only i64 locations"
) ? static_cast<void> (0) : __assert_fail ("ValLoc == MVT::i64 && \"Expecting only i64 locations\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2967, __PRETTY_FUNCTION__));
2968 // In 64 bit machine, There is no need to truncate the value only bitcast
2969 } else {
2970 MVT maskLen;
2971 switch (ValVT.getSimpleVT().SimpleTy) {
2972 case MVT::v8i1:
2973 maskLen = MVT::i8;
2974 break;
2975 case MVT::v16i1:
2976 maskLen = MVT::i16;
2977 break;
2978 case MVT::v32i1:
2979 maskLen = MVT::i32;
2980 break;
2981 default:
2982 llvm_unreachable("Expecting a vector of i1 types")::llvm::llvm_unreachable_internal("Expecting a vector of i1 types"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 2982);
2983 }
2984
2985 ValReturned = DAG.getNode(ISD::TRUNCATE, Dl, maskLen, ValReturned);
2986 }
2987 return DAG.getBitcast(ValVT, ValReturned);
2988}
2989
2990/// Lower the result values of a call into the
2991/// appropriate copies out of appropriate physical registers.
2992///
2993SDValue X86TargetLowering::LowerCallResult(
2994 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2995 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
2996 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
2997 uint32_t *RegMask) const {
2998
2999 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
3000 // Assign locations to each value returned by this call.
3001 SmallVector<CCValAssign, 16> RVLocs;
3002 bool Is64Bit = Subtarget.is64Bit();
3003 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
3004 *DAG.getContext());
3005 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
3006
3007 // Copy all of the result registers out of their specified physreg.
3008 for (unsigned I = 0, InsIndex = 0, E = RVLocs.size(); I != E;
3009 ++I, ++InsIndex) {
3010 CCValAssign &VA = RVLocs[I];
3011 EVT CopyVT = VA.getLocVT();
3012
3013 // In some calling conventions we need to remove the used registers
3014 // from the register mask.
3015 if (RegMask) {
3016 for (MCSubRegIterator SubRegs(VA.getLocReg(), TRI, /*IncludeSelf=*/true);
3017 SubRegs.isValid(); ++SubRegs)
3018 RegMask[*SubRegs / 32] &= ~(1u << (*SubRegs % 32));
3019 }
3020
3021 // If this is x86-64, and we disabled SSE, we can't return FP values
3022 if ((CopyVT == MVT::f32 || CopyVT == MVT::f64 || CopyVT == MVT::f128) &&
3023 ((Is64Bit || Ins[InsIndex].Flags.isInReg()) && !Subtarget.hasSSE1())) {
3024 errorUnsupported(DAG, dl, "SSE register return with SSE disabled");
3025 if (VA.getLocReg() == X86::XMM1)
3026 VA.convertToReg(X86::FP1); // Set reg to FP1, avoid hitting asserts.
3027 else
3028 VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
3029 } else if (CopyVT == MVT::f64 &&
3030 (Is64Bit && !Subtarget.hasSSE2())) {
3031 errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");
3032 if (VA.getLocReg() == X86::XMM1)
3033 VA.convertToReg(X86::FP1); // Set reg to FP1, avoid hitting asserts.
3034 else
3035 VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
3036 }
3037
3038 // If we prefer to use the value in xmm registers, copy it out as f80 and
3039 // use a truncate to move it from fp stack reg to xmm reg.
3040 bool RoundAfterCopy = false;
3041 if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
3042 isScalarFPTypeInSSEReg(VA.getValVT())) {
3043 if (!Subtarget.hasX87())
3044 report_fatal_error("X87 register return with X87 disabled");
3045 CopyVT = MVT::f80;
3046 RoundAfterCopy = (CopyVT != VA.getLocVT());
3047 }
3048
3049 SDValue Val;
3050 if (VA.needsCustom()) {
3051 assert(VA.getValVT() == MVT::v64i1 &&((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3052, __PRETTY_FUNCTION__))
3052 "Currently the only custom case is when we split v64i1 to 2 regs")((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3052, __PRETTY_FUNCTION__));
3053 Val =
3054 getv64i1Argument(VA, RVLocs[++I], Chain, DAG, dl, Subtarget, &InFlag);
3055 } else {
3056 Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), CopyVT, InFlag)
3057 .getValue(1);
3058 Val = Chain.getValue(0);
3059 InFlag = Chain.getValue(2);
3060 }
3061
3062 if (RoundAfterCopy)
3063 Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
3064 // This truncation won't change the value.
3065 DAG.getIntPtrConstant(1, dl));
3066
3067 if (VA.isExtInLoc() && (VA.getValVT().getScalarType() == MVT::i1)) {
3068 if (VA.getValVT().isVector() &&
3069 ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
3070 (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
3071 // promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
3072 Val = lowerRegToMasks(Val, VA.getValVT(), VA.getLocVT(), dl, DAG);
3073 } else
3074 Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
3075 }
3076
3077 InVals.push_back(Val);
3078 }
3079
3080 return Chain;
3081}
3082
3083//===----------------------------------------------------------------------===//
3084// C & StdCall & Fast Calling Convention implementation
3085//===----------------------------------------------------------------------===//
3086// StdCall calling convention seems to be standard for many Windows' API
3087// routines and around. It differs from C calling convention just a little:
3088// callee should clean up the stack, not caller. Symbols should be also
3089// decorated in some fancy way :) It doesn't support any vector arguments.
3090// For info on fast calling convention see Fast Calling Convention (tail call)
3091// implementation LowerX86_32FastCCCallTo.
3092
3093/// CallIsStructReturn - Determines whether a call uses struct return
3094/// semantics.
3095enum StructReturnType {
3096 NotStructReturn,
3097 RegStructReturn,
3098 StackStructReturn
3099};
3100static StructReturnType
3101callIsStructReturn(ArrayRef<ISD::OutputArg> Outs, bool IsMCU) {
3102 if (Outs.empty())
3103 return NotStructReturn;
3104
3105 const ISD::ArgFlagsTy &Flags = Outs[0].Flags;
3106 if (!Flags.isSRet())
3107 return NotStructReturn;
3108 if (Flags.isInReg() || IsMCU)
3109 return RegStructReturn;
3110 return StackStructReturn;
3111}
3112
3113/// Determines whether a function uses struct return semantics.
3114static StructReturnType
3115argsAreStructReturn(ArrayRef<ISD::InputArg> Ins, bool IsMCU) {
3116 if (Ins.empty())
3117 return NotStructReturn;
3118
3119 const ISD::ArgFlagsTy &Flags = Ins[0].Flags;
3120 if (!Flags.isSRet())
3121 return NotStructReturn;
3122 if (Flags.isInReg() || IsMCU)
3123 return RegStructReturn;
3124 return StackStructReturn;
3125}
3126
3127/// Make a copy of an aggregate at address specified by "Src" to address
3128/// "Dst" with size and alignment information specified by the specific
3129/// parameter attribute. The copy will be passed as a byval function parameter.
3130static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
3131 SDValue Chain, ISD::ArgFlagsTy Flags,
3132 SelectionDAG &DAG, const SDLoc &dl) {
3133 SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
3134
3135 return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
3136 /*isVolatile*/false, /*AlwaysInline=*/true,
3137 /*isTailCall*/false,
3138 MachinePointerInfo(), MachinePointerInfo());
3139}
3140
3141/// Return true if the calling convention is one that we can guarantee TCO for.
3142static bool canGuaranteeTCO(CallingConv::ID CC) {
3143 return (CC == CallingConv::Fast || CC == CallingConv::GHC ||
3144 CC == CallingConv::X86_RegCall || CC == CallingConv::HiPE ||
3145 CC == CallingConv::HHVM || CC == CallingConv::Tail);
3146}
3147
3148/// Return true if we might ever do TCO for calls with this calling convention.
3149static bool mayTailCallThisCC(CallingConv::ID CC) {
3150 switch (CC) {
3151 // C calling conventions:
3152 case CallingConv::C:
3153 case CallingConv::Win64:
3154 case CallingConv::X86_64_SysV:
3155 // Callee pop conventions:
3156 case CallingConv::X86_ThisCall:
3157 case CallingConv::X86_StdCall:
3158 case CallingConv::X86_VectorCall:
3159 case CallingConv::X86_FastCall:
3160 // Swift:
3161 case CallingConv::Swift:
3162 return true;
3163 default:
3164 return canGuaranteeTCO(CC);
3165 }
3166}
3167
3168/// Return true if the function is being made into a tailcall target by
3169/// changing its ABI.
3170static bool shouldGuaranteeTCO(CallingConv::ID CC, bool GuaranteedTailCallOpt) {
3171 return (GuaranteedTailCallOpt && canGuaranteeTCO(CC)) || CC == CallingConv::Tail;
3172}
3173
3174bool X86TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
3175 if (!CI->isTailCall())
3176 return false;
3177
3178 ImmutableCallSite CS(CI);
3179 CallingConv::ID CalleeCC = CS.getCallingConv();
3180 if (!mayTailCallThisCC(CalleeCC))
3181 return false;
3182
3183 return true;
3184}
3185
3186SDValue
3187X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
3188 const SmallVectorImpl<ISD::InputArg> &Ins,
3189 const SDLoc &dl, SelectionDAG &DAG,
3190 const CCValAssign &VA,
3191 MachineFrameInfo &MFI, unsigned i) const {
3192 // Create the nodes corresponding to a load from this parameter slot.
3193 ISD::ArgFlagsTy Flags = Ins[i].Flags;
3194 bool AlwaysUseMutable = shouldGuaranteeTCO(
3195 CallConv, DAG.getTarget().Options.GuaranteedTailCallOpt);
3196 bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
3197 EVT ValVT;
3198 MVT PtrVT = getPointerTy(DAG.getDataLayout());
3199
3200 // If value is passed by pointer we have address passed instead of the value
3201 // itself. No need to extend if the mask value and location share the same
3202 // absolute size.
3203 bool ExtendedInMem =
3204 VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1 &&
3205 VA.getValVT().getSizeInBits() != VA.getLocVT().getSizeInBits();
3206
3207 if (VA.getLocInfo() == CCValAssign::Indirect || ExtendedInMem)
3208 ValVT = VA.getLocVT();
3209 else
3210 ValVT = VA.getValVT();
3211
3212 // FIXME: For now, all byval parameter objects are marked mutable. This can be
3213 // changed with more analysis.
3214 // In case of tail call optimization mark all arguments mutable. Since they
3215 // could be overwritten by lowering of arguments in case of a tail call.
3216 if (Flags.isByVal()) {
3217 unsigned Bytes = Flags.getByValSize();
3218 if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
3219
3220 // FIXME: For now, all byval parameter objects are marked as aliasing. This
3221 // can be improved with deeper analysis.
3222 int FI = MFI.CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable,
3223 /*isAliased=*/true);
3224 return DAG.getFrameIndex(FI, PtrVT);
3225 }
3226
3227 // This is an argument in memory. We might be able to perform copy elision.
3228 // If the argument is passed directly in memory without any extension, then we
3229 // can perform copy elision. Large vector types, for example, may be passed
3230 // indirectly by pointer.
3231 if (Flags.isCopyElisionCandidate() &&
3232 VA.getLocInfo() != CCValAssign::Indirect && !ExtendedInMem) {
3233 EVT ArgVT = Ins[i].ArgVT;
3234 SDValue PartAddr;
3235 if (Ins[i].PartOffset == 0) {
3236 // If this is a one-part value or the first part of a multi-part value,
3237 // create a stack object for the entire argument value type and return a
3238 // load from our portion of it. This assumes that if the first part of an
3239 // argument is in memory, the rest will also be in memory.
3240 int FI = MFI.CreateFixedObject(ArgVT.getStoreSize(), VA.getLocMemOffset(),
3241 /*IsImmutable=*/false);
3242 PartAddr = DAG.getFrameIndex(FI, PtrVT);
3243 return DAG.getLoad(
3244 ValVT, dl, Chain, PartAddr,
3245 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
3246 } else {
3247 // This is not the first piece of an argument in memory. See if there is
3248 // already a fixed stack object including this offset. If so, assume it
3249 // was created by the PartOffset == 0 branch above and create a load from
3250 // the appropriate offset into it.
3251 int64_t PartBegin = VA.getLocMemOffset();
3252 int64_t PartEnd = PartBegin + ValVT.getSizeInBits() / 8;
3253 int FI = MFI.getObjectIndexBegin();
3254 for (; MFI.isFixedObjectIndex(FI); ++FI) {
3255 int64_t ObjBegin = MFI.getObjectOffset(FI);
3256 int64_t ObjEnd = ObjBegin + MFI.getObjectSize(FI);
3257 if (ObjBegin <= PartBegin && PartEnd <= ObjEnd)
3258 break;
3259 }
3260 if (MFI.isFixedObjectIndex(FI)) {
3261 SDValue Addr =
3262 DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getFrameIndex(FI, PtrVT),
3263 DAG.getIntPtrConstant(Ins[i].PartOffset, dl));
3264 return DAG.getLoad(
3265 ValVT, dl, Chain, Addr,
3266 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI,
3267 Ins[i].PartOffset));
3268 }
3269 }
3270 }
3271
3272 int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
3273 VA.getLocMemOffset(), isImmutable);
3274
3275 // Set SExt or ZExt flag.
3276 if (VA.getLocInfo() == CCValAssign::ZExt) {
3277 MFI.setObjectZExt(FI, true);
3278 } else if (VA.getLocInfo() == CCValAssign::SExt) {
3279 MFI.setObjectSExt(FI, true);
3280 }
3281
3282 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3283 SDValue Val = DAG.getLoad(
3284 ValVT, dl, Chain, FIN,
3285 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
3286 return ExtendedInMem
3287 ? (VA.getValVT().isVector()
3288 ? DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VA.getValVT(), Val)
3289 : DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val))
3290 : Val;
3291}
3292
3293// FIXME: Get this from tablegen.
3294static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
3295 const X86Subtarget &Subtarget) {
3296 assert(Subtarget.is64Bit())((Subtarget.is64Bit()) ? static_cast<void> (0) : __assert_fail
("Subtarget.is64Bit()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3296, __PRETTY_FUNCTION__));
3297
3298 if (Subtarget.isCallingConvWin64(CallConv)) {
3299 static const MCPhysReg GPR64ArgRegsWin64[] = {
3300 X86::RCX, X86::RDX, X86::R8, X86::R9
3301 };
3302 return makeArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64));
3303 }
3304
3305 static const MCPhysReg GPR64ArgRegs64Bit[] = {
3306 X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
3307 };
3308 return makeArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit));
3309}
3310
3311// FIXME: Get this from tablegen.
3312static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,
3313 CallingConv::ID CallConv,
3314 const X86Subtarget &Subtarget) {
3315 assert(Subtarget.is64Bit())((Subtarget.is64Bit()) ? static_cast<void> (0) : __assert_fail
("Subtarget.is64Bit()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3315, __PRETTY_FUNCTION__));
3316 if (Subtarget.isCallingConvWin64(CallConv)) {
3317 // The XMM registers which might contain var arg parameters are shadowed
3318 // in their paired GPR. So we only need to save the GPR to their home
3319 // slots.
3320 // TODO: __vectorcall will change this.
3321 return None;
3322 }
3323
3324 const Function &F = MF.getFunction();
3325 bool NoImplicitFloatOps = F.hasFnAttribute(Attribute::NoImplicitFloat);
3326 bool isSoftFloat = Subtarget.useSoftFloat();
3327 assert(!(isSoftFloat && NoImplicitFloatOps) &&((!(isSoftFloat && NoImplicitFloatOps) && "SSE register cannot be used when SSE is disabled!"
) ? static_cast<void> (0) : __assert_fail ("!(isSoftFloat && NoImplicitFloatOps) && \"SSE register cannot be used when SSE is disabled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3328, __PRETTY_FUNCTION__))
3328 "SSE register cannot be used when SSE is disabled!")((!(isSoftFloat && NoImplicitFloatOps) && "SSE register cannot be used when SSE is disabled!"
) ? static_cast<void> (0) : __assert_fail ("!(isSoftFloat && NoImplicitFloatOps) && \"SSE register cannot be used when SSE is disabled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3328, __PRETTY_FUNCTION__));
3329 if (isSoftFloat || NoImplicitFloatOps || !Subtarget.hasSSE1())
3330 // Kernel mode asks for SSE to be disabled, so there are no XMM argument
3331 // registers.
3332 return None;
3333
3334 static const MCPhysReg XMMArgRegs64Bit[] = {
3335 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3336 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3337 };
3338 return makeArrayRef(std::begin(XMMArgRegs64Bit), std::end(XMMArgRegs64Bit));
3339}
3340
3341#ifndef NDEBUG
3342static bool isSortedByValueNo(ArrayRef<CCValAssign> ArgLocs) {
3343 return std::is_sorted(ArgLocs.begin(), ArgLocs.end(),
3344 [](const CCValAssign &A, const CCValAssign &B) -> bool {
3345 return A.getValNo() < B.getValNo();
3346 });
3347}
3348#endif
3349
3350SDValue X86TargetLowering::LowerFormalArguments(
3351 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3352 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
3353 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3354 MachineFunction &MF = DAG.getMachineFunction();
3355 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
3356 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
3357
3358 const Function &F = MF.getFunction();
3359 if (F.hasExternalLinkage() && Subtarget.isTargetCygMing() &&
3360 F.getName() == "main")
3361 FuncInfo->setForceFramePointer(true);
3362
3363 MachineFrameInfo &MFI = MF.getFrameInfo();
3364 bool Is64Bit = Subtarget.is64Bit();
3365 bool IsWin64 = Subtarget.isCallingConvWin64(CallConv);
3366
3367 assert(((!(isVarArg && canGuaranteeTCO(CallConv)) &&
"Var args not supported with calling conv' regcall, fastcc, ghc or hipe"
) ? static_cast<void> (0) : __assert_fail ("!(isVarArg && canGuaranteeTCO(CallConv)) && \"Var args not supported with calling conv' regcall, fastcc, ghc or hipe\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3369, __PRETTY_FUNCTION__))
3368 !(isVarArg && canGuaranteeTCO(CallConv)) &&((!(isVarArg && canGuaranteeTCO(CallConv)) &&
"Var args not supported with calling conv' regcall, fastcc, ghc or hipe"
) ? static_cast<void> (0) : __assert_fail ("!(isVarArg && canGuaranteeTCO(CallConv)) && \"Var args not supported with calling conv' regcall, fastcc, ghc or hipe\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3369, __PRETTY_FUNCTION__))
3369 "Var args not supported with calling conv' regcall, fastcc, ghc or hipe")((!(isVarArg && canGuaranteeTCO(CallConv)) &&
"Var args not supported with calling conv' regcall, fastcc, ghc or hipe"
) ? static_cast<void> (0) : __assert_fail ("!(isVarArg && canGuaranteeTCO(CallConv)) && \"Var args not supported with calling conv' regcall, fastcc, ghc or hipe\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3369, __PRETTY_FUNCTION__));
3370
3371 // Assign locations to all of the incoming arguments.
3372 SmallVector<CCValAssign, 16> ArgLocs;
3373 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
3374
3375 // Allocate shadow area for Win64.
3376 if (IsWin64)
3377 CCInfo.AllocateStack(32, 8);
3378
3379 CCInfo.AnalyzeArguments(Ins, CC_X86);
3380
3381 // In vectorcall calling convention a second pass is required for the HVA
3382 // types.
3383 if (CallingConv::X86_VectorCall == CallConv) {
3384 CCInfo.AnalyzeArgumentsSecondPass(Ins, CC_X86);
3385 }
3386
3387 // The next loop assumes that the locations are in the same order of the
3388 // input arguments.
3389 assert(isSortedByValueNo(ArgLocs) &&((isSortedByValueNo(ArgLocs) && "Argument Location list must be sorted before lowering"
) ? static_cast<void> (0) : __assert_fail ("isSortedByValueNo(ArgLocs) && \"Argument Location list must be sorted before lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3390, __PRETTY_FUNCTION__))
3390 "Argument Location list must be sorted before lowering")((isSortedByValueNo(ArgLocs) && "Argument Location list must be sorted before lowering"
) ? static_cast<void> (0) : __assert_fail ("isSortedByValueNo(ArgLocs) && \"Argument Location list must be sorted before lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3390, __PRETTY_FUNCTION__));
3391
3392 SDValue ArgValue;
3393 for (unsigned I = 0, InsIndex = 0, E = ArgLocs.size(); I != E;
3394 ++I, ++InsIndex) {
3395 assert(InsIndex < Ins.size() && "Invalid Ins index")((InsIndex < Ins.size() && "Invalid Ins index") ? static_cast
<void> (0) : __assert_fail ("InsIndex < Ins.size() && \"Invalid Ins index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3395, __PRETTY_FUNCTION__));
3396 CCValAssign &VA = ArgLocs[I];
3397
3398 if (VA.isRegLoc()) {
3399 EVT RegVT = VA.getLocVT();
3400 if (VA.needsCustom()) {
3401 assert(((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3403, __PRETTY_FUNCTION__))
3402 VA.getValVT() == MVT::v64i1 &&((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3403, __PRETTY_FUNCTION__))
3403 "Currently the only custom case is when we split v64i1 to 2 regs")((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3403, __PRETTY_FUNCTION__));
3404
3405 // v64i1 values, in regcall calling convention, that are
3406 // compiled to 32 bit arch, are split up into two registers.
3407 ArgValue =
3408 getv64i1Argument(VA, ArgLocs[++I], Chain, DAG, dl, Subtarget);
3409 } else {
3410 const TargetRegisterClass *RC;
3411 if (RegVT == MVT::i8)
3412 RC = &X86::GR8RegClass;
3413 else if (RegVT == MVT::i16)
3414 RC = &X86::GR16RegClass;
3415 else if (RegVT == MVT::i32)
3416 RC = &X86::GR32RegClass;
3417 else if (Is64Bit && RegVT == MVT::i64)
3418 RC = &X86::GR64RegClass;
3419 else if (RegVT == MVT::f32)
3420 RC = Subtarget.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
3421 else if (RegVT == MVT::f64)
3422 RC = Subtarget.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
3423 else if (RegVT == MVT::f80)
3424 RC = &X86::RFP80RegClass;
3425 else if (RegVT == MVT::f128)
3426 RC = &X86::VR128RegClass;
3427 else if (RegVT.is512BitVector())
3428 RC = &X86::VR512RegClass;
3429 else if (RegVT.is256BitVector())
3430 RC = Subtarget.hasVLX() ? &X86::VR256XRegClass : &X86::VR256RegClass;
3431 else if (RegVT.is128BitVector())
3432 RC = Subtarget.hasVLX() ? &X86::VR128XRegClass : &X86::VR128RegClass;
3433 else if (RegVT == MVT::x86mmx)
3434 RC = &X86::VR64RegClass;
3435 else if (RegVT == MVT::v1i1)
3436 RC = &X86::VK1RegClass;
3437 else if (RegVT == MVT::v8i1)
3438 RC = &X86::VK8RegClass;
3439 else if (RegVT == MVT::v16i1)
3440 RC = &X86::VK16RegClass;
3441 else if (RegVT == MVT::v32i1)
3442 RC = &X86::VK32RegClass;
3443 else if (RegVT == MVT::v64i1)
3444 RC = &X86::VK64RegClass;
3445 else
3446 llvm_unreachable("Unknown argument type!")::llvm::llvm_unreachable_internal("Unknown argument type!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3446);
3447
3448 unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
3449 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
3450 }
3451
3452 // If this is an 8 or 16-bit value, it is really passed promoted to 32
3453 // bits. Insert an assert[sz]ext to capture this, then truncate to the
3454 // right size.
3455 if (VA.getLocInfo() == CCValAssign::SExt)
3456 ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
3457 DAG.getValueType(VA.getValVT()));
3458 else if (VA.getLocInfo() == CCValAssign::ZExt)
3459 ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
3460 DAG.getValueType(VA.getValVT()));
3461 else if (VA.getLocInfo() == CCValAssign::BCvt)
3462 ArgValue = DAG.getBitcast(VA.getValVT(), ArgValue);
3463
3464 if (VA.isExtInLoc()) {
3465 // Handle MMX values passed in XMM regs.
3466 if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1)
3467 ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
3468 else if (VA.getValVT().isVector() &&
3469 VA.getValVT().getScalarType() == MVT::i1 &&
3470 ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
3471 (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
3472 // Promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
3473 ArgValue = lowerRegToMasks(ArgValue, VA.getValVT(), RegVT, dl, DAG);
3474 } else
3475 ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
3476 }
3477 } else {
3478 assert(VA.isMemLoc())((VA.isMemLoc()) ? static_cast<void> (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3478, __PRETTY_FUNCTION__));
3479 ArgValue =
3480 LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, InsIndex);
3481 }
3482
3483 // If value is passed via pointer - do a load.
3484 if (VA.getLocInfo() == CCValAssign::Indirect && !Ins[I].Flags.isByVal())
3485 ArgValue =
3486 DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, MachinePointerInfo());
3487
3488 InVals.push_back(ArgValue);
3489 }
3490
3491 for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
3492 // Swift calling convention does not require we copy the sret argument
3493 // into %rax/%eax for the return. We don't set SRetReturnReg for Swift.
3494 if (CallConv == CallingConv::Swift)
3495 continue;
3496
3497 // All x86 ABIs require that for returning structs by value we copy the
3498 // sret argument into %rax/%eax (depending on ABI) for the return. Save
3499 // the argument into a virtual register so that we can access it from the
3500 // return points.
3501 if (Ins[I].Flags.isSRet()) {
3502 unsigned Reg = FuncInfo->getSRetReturnReg();
3503 if (!Reg) {
3504 MVT PtrTy = getPointerTy(DAG.getDataLayout());
3505 Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
3506 FuncInfo->setSRetReturnReg(Reg);
3507 }
3508 SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[I]);
3509 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
3510 break;
3511 }
3512 }
3513
3514 unsigned StackSize = CCInfo.getNextStackOffset();
3515 // Align stack specially for tail calls.
3516 if (shouldGuaranteeTCO(CallConv,
3517 MF.getTarget().Options.GuaranteedTailCallOpt))
3518 StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
3519
3520 // If the function takes variable number of arguments, make a frame index for
3521 // the start of the first vararg value... for expansion of llvm.va_start. We
3522 // can skip this if there are no va_start calls.
3523 if (MFI.hasVAStart() &&
3524 (Is64Bit || (CallConv != CallingConv::X86_FastCall &&
3525 CallConv != CallingConv::X86_ThisCall))) {
3526 FuncInfo->setVarArgsFrameIndex(MFI.CreateFixedObject(1, StackSize, true));
3527 }
3528
3529 // Figure out if XMM registers are in use.
3530 assert(!(Subtarget.useSoftFloat() &&((!(Subtarget.useSoftFloat() && F.hasFnAttribute(Attribute
::NoImplicitFloat)) && "SSE register cannot be used when SSE is disabled!"
) ? static_cast<void> (0) : __assert_fail ("!(Subtarget.useSoftFloat() && F.hasFnAttribute(Attribute::NoImplicitFloat)) && \"SSE register cannot be used when SSE is disabled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3532, __PRETTY_FUNCTION__))
3531 F.hasFnAttribute(Attribute::NoImplicitFloat)) &&((!(Subtarget.useSoftFloat() && F.hasFnAttribute(Attribute
::NoImplicitFloat)) && "SSE register cannot be used when SSE is disabled!"
) ? static_cast<void> (0) : __assert_fail ("!(Subtarget.useSoftFloat() && F.hasFnAttribute(Attribute::NoImplicitFloat)) && \"SSE register cannot be used when SSE is disabled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3532, __PRETTY_FUNCTION__))
3532 "SSE register cannot be used when SSE is disabled!")((!(Subtarget.useSoftFloat() && F.hasFnAttribute(Attribute
::NoImplicitFloat)) && "SSE register cannot be used when SSE is disabled!"
) ? static_cast<void> (0) : __assert_fail ("!(Subtarget.useSoftFloat() && F.hasFnAttribute(Attribute::NoImplicitFloat)) && \"SSE register cannot be used when SSE is disabled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3532, __PRETTY_FUNCTION__));
3533
3534 // 64-bit calling conventions support varargs and register parameters, so we
3535 // have to do extra work to spill them in the prologue.
3536 if (Is64Bit && isVarArg && MFI.hasVAStart()) {
3537 // Find the first unallocated argument registers.
3538 ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget);
3539 ArrayRef<MCPhysReg> ArgXMMs = get64BitArgumentXMMs(MF, CallConv, Subtarget);
3540 unsigned NumIntRegs = CCInfo.getFirstUnallocated(ArgGPRs);
3541 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(ArgXMMs);
3542 assert(!(NumXMMRegs && !Subtarget.hasSSE1()) &&((!(NumXMMRegs && !Subtarget.hasSSE1()) && "SSE register cannot be used when SSE is disabled!"
) ? static_cast<void> (0) : __assert_fail ("!(NumXMMRegs && !Subtarget.hasSSE1()) && \"SSE register cannot be used when SSE is disabled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3543, __PRETTY_FUNCTION__))
3543 "SSE register cannot be used when SSE is disabled!")((!(NumXMMRegs && !Subtarget.hasSSE1()) && "SSE register cannot be used when SSE is disabled!"
) ? static_cast<void> (0) : __assert_fail ("!(NumXMMRegs && !Subtarget.hasSSE1()) && \"SSE register cannot be used when SSE is disabled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3543, __PRETTY_FUNCTION__));
3544
3545 // Gather all the live in physical registers.
3546 SmallVector<SDValue, 6> LiveGPRs;
3547 SmallVector<SDValue, 8> LiveXMMRegs;
3548 SDValue ALVal;
3549 for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) {
3550 unsigned GPR = MF.addLiveIn(Reg, &X86::GR64RegClass);
3551 LiveGPRs.push_back(
3552 DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64));
3553 }
3554 if (!ArgXMMs.empty()) {
3555 unsigned AL = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
3556 ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8);
3557 for (MCPhysReg Reg : ArgXMMs.slice(NumXMMRegs)) {
3558 unsigned XMMReg = MF.addLiveIn(Reg, &X86::VR128RegClass);
3559 LiveXMMRegs.push_back(
3560 DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32));
3561 }
3562 }
3563
3564 if (IsWin64) {
3565 // Get to the caller-allocated home save location. Add 8 to account
3566 // for the return address.
3567 int HomeOffset = TFI.getOffsetOfLocalArea() + 8;
3568 FuncInfo->setRegSaveFrameIndex(
3569 MFI.CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false));
3570 // Fixup to set vararg frame on shadow area (4 x i64).
3571 if (NumIntRegs < 4)
3572 FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex());
3573 } else {
3574 // For X86-64, if there are vararg parameters that are passed via
3575 // registers, then we must store them to their spots on the stack so
3576 // they may be loaded by dereferencing the result of va_next.
3577 FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
3578 FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);
3579 FuncInfo->setRegSaveFrameIndex(MFI.CreateStackObject(
3580 ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false));
3581 }
3582
3583 // Store the integer parameter registers.
3584 SmallVector<SDValue, 8> MemOps;
3585 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
3586 getPointerTy(DAG.getDataLayout()));
3587 unsigned Offset = FuncInfo->getVarArgsGPOffset();
3588 for (SDValue Val : LiveGPRs) {
3589 SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
3590 RSFIN, DAG.getIntPtrConstant(Offset, dl));
3591 SDValue Store =
3592 DAG.getStore(Val.getValue(1), dl, Val, FIN,
3593 MachinePointerInfo::getFixedStack(
3594 DAG.getMachineFunction(),
3595 FuncInfo->getRegSaveFrameIndex(), Offset));
3596 MemOps.push_back(Store);
3597 Offset += 8;
3598 }
3599
3600 if (!ArgXMMs.empty() && NumXMMRegs != ArgXMMs.size()) {
3601 // Now store the XMM (fp + vector) parameter registers.
3602 SmallVector<SDValue, 12> SaveXMMOps;
3603 SaveXMMOps.push_back(Chain);
3604 SaveXMMOps.push_back(ALVal);
3605 SaveXMMOps.push_back(DAG.getIntPtrConstant(
3606 FuncInfo->getRegSaveFrameIndex(), dl));
3607 SaveXMMOps.push_back(DAG.getIntPtrConstant(
3608 FuncInfo->getVarArgsFPOffset(), dl));
3609 SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(),
3610 LiveXMMRegs.end());
3611 MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
3612 MVT::Other, SaveXMMOps));
3613 }
3614
3615 if (!MemOps.empty())
3616 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
3617 }
3618
3619 if (isVarArg && MFI.hasMustTailInVarArgFunc()) {
3620 // Find the largest legal vector type.
3621 MVT VecVT = MVT::Other;
3622 // FIXME: Only some x86_32 calling conventions support AVX512.
3623 if (Subtarget.useAVX512Regs() &&
3624 (Is64Bit || (CallConv == CallingConv::X86_VectorCall ||
3625 CallConv == CallingConv::Intel_OCL_BI)))
3626 VecVT = MVT::v16f32;
3627 else if (Subtarget.hasAVX())
3628 VecVT = MVT::v8f32;
3629 else if (Subtarget.hasSSE2())
3630 VecVT = MVT::v4f32;
3631
3632 // We forward some GPRs and some vector types.
3633 SmallVector<MVT, 2> RegParmTypes;
3634 MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32;
3635 RegParmTypes.push_back(IntVT);
3636 if (VecVT != MVT::Other)
3637 RegParmTypes.push_back(VecVT);
3638
3639 // Compute the set of forwarded registers. The rest are scratch.
3640 SmallVectorImpl<ForwardedRegister> &Forwards =
3641 FuncInfo->getForwardedMustTailRegParms();
3642 CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86);
3643
3644 // Forward AL for SysV x86_64 targets, since it is used for varargs.
3645 if (Is64Bit && !IsWin64 && !CCInfo.isAllocated(X86::AL)) {
3646 unsigned ALVReg = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
3647 Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));
3648 }
3649
3650 // Copy all forwards from physical to virtual registers.
3651 for (ForwardedRegister &FR : Forwards) {
3652 // FIXME: Can we use a less constrained schedule?
3653 SDValue RegVal = DAG.getCopyFromReg(Chain, dl, FR.VReg, FR.VT);
3654 FR.VReg = MF.getRegInfo().createVirtualRegister(getRegClassFor(FR.VT));
3655 Chain = DAG.getCopyToReg(Chain, dl, FR.VReg, RegVal);
3656 }
3657 }
3658
3659 // Some CCs need callee pop.
3660 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
3661 MF.getTarget().Options.GuaranteedTailCallOpt)) {
3662 FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
3663 } else if (CallConv == CallingConv::X86_INTR && Ins.size() == 2) {
3664 // X86 interrupts must pop the error code (and the alignment padding) if
3665 // present.
3666 FuncInfo->setBytesToPopOnReturn(Is64Bit ? 16 : 4);
3667 } else {
3668 FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
3669 // If this is an sret function, the return should pop the hidden pointer.
3670 if (!Is64Bit && !canGuaranteeTCO(CallConv) &&
3671 !Subtarget.getTargetTriple().isOSMSVCRT() &&
3672 argsAreStructReturn(Ins, Subtarget.isTargetMCU()) == StackStructReturn)
3673 FuncInfo->setBytesToPopOnReturn(4);
3674 }
3675
3676 if (!Is64Bit) {
3677 // RegSaveFrameIndex is X86-64 only.
3678 FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
3679 if (CallConv == CallingConv::X86_FastCall ||
3680 CallConv == CallingConv::X86_ThisCall)
3681 // fastcc functions can't have varargs.
3682 FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
3683 }
3684
3685 FuncInfo->setArgumentStackSize(StackSize);
3686
3687 if (WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo()) {
3688 EHPersonality Personality = classifyEHPersonality(F.getPersonalityFn());
3689 if (Personality == EHPersonality::CoreCLR) {
3690 assert(Is64Bit)((Is64Bit) ? static_cast<void> (0) : __assert_fail ("Is64Bit"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3690, __PRETTY_FUNCTION__));
3691 // TODO: Add a mechanism to frame lowering that will allow us to indicate
3692 // that we'd prefer this slot be allocated towards the bottom of the frame
3693 // (i.e. near the stack pointer after allocating the frame). Every
3694 // funclet needs a copy of this slot in its (mostly empty) frame, and the
3695 // offset from the bottom of this and each funclet's frame must be the
3696 // same, so the size of funclets' (mostly empty) frames is dictated by
3697 // how far this slot is from the bottom (since they allocate just enough
3698 // space to accommodate holding this slot at the correct offset).
3699 int PSPSymFI = MFI.CreateStackObject(8, 8, /*isSS=*/false);
3700 EHInfo->PSPSymFrameIdx = PSPSymFI;
3701 }
3702 }
3703
3704 if (CallConv == CallingConv::X86_RegCall ||
3705 F.hasFnAttribute("no_caller_saved_registers")) {
3706 MachineRegisterInfo &MRI = MF.getRegInfo();
3707 for (std::pair<unsigned, unsigned> Pair : MRI.liveins())
3708 MRI.disableCalleeSavedRegister(Pair.first);
3709 }
3710
3711 return Chain;
3712}
3713
3714SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
3715 SDValue Arg, const SDLoc &dl,
3716 SelectionDAG &DAG,
3717 const CCValAssign &VA,
3718 ISD::ArgFlagsTy Flags) const {
3719 unsigned LocMemOffset = VA.getLocMemOffset();
3720 SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
3721 PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
3722 StackPtr, PtrOff);
3723 if (Flags.isByVal())
3724 return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
3725
3726 return DAG.getStore(
3727 Chain, dl, Arg, PtrOff,
3728 MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset));
3729}
3730
3731/// Emit a load of return address if tail call
3732/// optimization is performed and it is required.
3733SDValue X86TargetLowering::EmitTailCallLoadRetAddr(
3734 SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,
3735 bool Is64Bit, int FPDiff, const SDLoc &dl) const {
3736 // Adjust the Return address stack slot.
3737 EVT VT = getPointerTy(DAG.getDataLayout());
3738 OutRetAddr = getReturnAddressFrameIndex(DAG);
3739
3740 // Load the "old" Return address.
3741 OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo());
3742 return SDValue(OutRetAddr.getNode(), 1);
3743}
3744
3745/// Emit a store of the return address if tail call
3746/// optimization is performed and it is required (FPDiff!=0).
3747static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
3748 SDValue Chain, SDValue RetAddrFrIdx,
3749 EVT PtrVT, unsigned SlotSize,
3750 int FPDiff, const SDLoc &dl) {
3751 // Store the return address to the appropriate stack slot.
3752 if (!FPDiff) return Chain;
3753 // Calculate the new stack slot for the return address.
3754 int NewReturnAddrFI =
3755 MF.getFrameInfo().CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,
3756 false);
3757 SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
3758 Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
3759 MachinePointerInfo::getFixedStack(
3760 DAG.getMachineFunction(), NewReturnAddrFI));
3761 return Chain;
3762}
3763
3764/// Returns a vector_shuffle mask for an movs{s|d}, movd
3765/// operation of specified width.
3766static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
3767 SDValue V2) {
3768 unsigned NumElems = VT.getVectorNumElements();
3769 SmallVector<int, 8> Mask;
3770 Mask.push_back(NumElems);
3771 for (unsigned i = 1; i != NumElems; ++i)
3772 Mask.push_back(i);
3773 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
3774}
3775
3776SDValue
3777X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
3778 SmallVectorImpl<SDValue> &InVals) const {
3779 SelectionDAG &DAG = CLI.DAG;
3780 SDLoc &dl = CLI.DL;
3781 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
3782 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
3783 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
3784 SDValue Chain = CLI.Chain;
3785 SDValue Callee = CLI.Callee;
3786 CallingConv::ID CallConv = CLI.CallConv;
3787 bool &isTailCall = CLI.IsTailCall;
3788 bool isVarArg = CLI.IsVarArg;
3789
3790 MachineFunction &MF = DAG.getMachineFunction();
3791 bool Is64Bit = Subtarget.is64Bit();
3792 bool IsWin64 = Subtarget.isCallingConvWin64(CallConv);
3793 StructReturnType SR = callIsStructReturn(Outs, Subtarget.isTargetMCU());
3794 bool IsSibcall = false;
3795 bool IsGuaranteeTCO = MF.getTarget().Options.GuaranteedTailCallOpt ||
3796 CallConv == CallingConv::Tail;
3797 X86MachineFunctionInfo *X86Info = MF.getInfo<X86MachineFunctionInfo>();
3798 const auto *CI = dyn_cast_or_null<CallInst>(CLI.CS.getInstruction());
3799 const Function *Fn = CI ? CI->getCalledFunction() : nullptr;
3800 bool HasNCSR = (CI && CI->hasFnAttr("no_caller_saved_registers")) ||
3801 (Fn && Fn->hasFnAttribute("no_caller_saved_registers"));
3802 const auto *II = dyn_cast_or_null<InvokeInst>(CLI.CS.getInstruction());
3803 bool HasNoCfCheck =
3804 (CI && CI->doesNoCfCheck()) || (II && II->doesNoCfCheck());
3805 const Module *M = MF.getMMI().getModule();
3806 Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
3807
3808 MachineFunction::CallSiteInfo CSInfo;
3809
3810 if (CallConv == CallingConv::X86_INTR)
3811 report_fatal_error("X86 interrupts may not be called directly");
3812
3813 if (Subtarget.isPICStyleGOT() && !IsGuaranteeTCO) {
3814 // If we are using a GOT, disable tail calls to external symbols with
3815 // default visibility. Tail calling such a symbol requires using a GOT
3816 // relocation, which forces early binding of the symbol. This breaks code
3817 // that require lazy function symbol resolution. Using musttail or
3818 // GuaranteedTailCallOpt will override this.
3819 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
3820 if (!G || (!G->getGlobal()->hasLocalLinkage() &&
3821 G->getGlobal()->hasDefaultVisibility()))
3822 isTailCall = false;
3823 }
3824
3825 bool IsMustTail = CLI.CS && CLI.CS.isMustTailCall();
3826 if (IsMustTail) {
3827 // Force this to be a tail call. The verifier rules are enough to ensure
3828 // that we can lower this successfully without moving the return address
3829 // around.
3830 isTailCall = true;
3831 } else if (isTailCall) {
3832 // Check if it's really possible to do a tail call.
3833 isTailCall = IsEligibleForTailCallOptimization(Callee, CallConv,
3834 isVarArg, SR != NotStructReturn,
3835 MF.getFunction().hasStructRetAttr(), CLI.RetTy,
3836 Outs, OutVals, Ins, DAG);
3837
3838 // Sibcalls are automatically detected tailcalls which do not require
3839 // ABI changes.
3840 if (!IsGuaranteeTCO && isTailCall)
3841 IsSibcall = true;
3842
3843 if (isTailCall)
3844 ++NumTailCalls;
3845 }
3846
3847 assert(!(isVarArg && canGuaranteeTCO(CallConv)) &&((!(isVarArg && canGuaranteeTCO(CallConv)) &&
"Var args not supported with calling convention fastcc, ghc or hipe"
) ? static_cast<void> (0) : __assert_fail ("!(isVarArg && canGuaranteeTCO(CallConv)) && \"Var args not supported with calling convention fastcc, ghc or hipe\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3848, __PRETTY_FUNCTION__))
3848 "Var args not supported with calling convention fastcc, ghc or hipe")((!(isVarArg && canGuaranteeTCO(CallConv)) &&
"Var args not supported with calling convention fastcc, ghc or hipe"
) ? static_cast<void> (0) : __assert_fail ("!(isVarArg && canGuaranteeTCO(CallConv)) && \"Var args not supported with calling convention fastcc, ghc or hipe\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3848, __PRETTY_FUNCTION__));
3849
3850 // Analyze operands of the call, assigning locations to each operand.
3851 SmallVector<CCValAssign, 16> ArgLocs;
3852 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
3853
3854 // Allocate shadow area for Win64.
3855 if (IsWin64)
3856 CCInfo.AllocateStack(32, 8);
3857
3858 CCInfo.AnalyzeArguments(Outs, CC_X86);
3859
3860 // In vectorcall calling convention a second pass is required for the HVA
3861 // types.
3862 if (CallingConv::X86_VectorCall == CallConv) {
3863 CCInfo.AnalyzeArgumentsSecondPass(Outs, CC_X86);
3864 }
3865
3866 // Get a count of how many bytes are to be pushed on the stack.
3867 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
3868 if (IsSibcall)
3869 // This is a sibcall. The memory operands are available in caller's
3870 // own caller's stack.
3871 NumBytes = 0;
3872 else if (IsGuaranteeTCO && canGuaranteeTCO(CallConv))
3873 NumBytes = GetAlignedArgumentStackSize(NumBytes, DAG);
3874
3875 int FPDiff = 0;
3876 if (isTailCall && !IsSibcall && !IsMustTail) {
3877 // Lower arguments at fp - stackoffset + fpdiff.
3878 unsigned NumBytesCallerPushed = X86Info->getBytesToPopOnReturn();
3879
3880 FPDiff = NumBytesCallerPushed - NumBytes;
3881
3882 // Set the delta of movement of the returnaddr stackslot.
3883 // But only set if delta is greater than previous delta.
3884 if (FPDiff < X86Info->getTCReturnAddrDelta())
3885 X86Info->setTCReturnAddrDelta(FPDiff);
3886 }
3887
3888 unsigned NumBytesToPush = NumBytes;
3889 unsigned NumBytesToPop = NumBytes;
3890
3891 // If we have an inalloca argument, all stack space has already been allocated
3892 // for us and be right at the top of the stack. We don't support multiple
3893 // arguments passed in memory when using inalloca.
3894 if (!Outs.empty() && Outs.back().Flags.isInAlloca()) {
3895 NumBytesToPush = 0;
3896 if (!ArgLocs.back().isMemLoc())
3897 report_fatal_error("cannot use inalloca attribute on a register "
3898 "parameter");
3899 if (ArgLocs.back().getLocMemOffset() != 0)
3900 report_fatal_error("any parameter with the inalloca attribute must be "
3901 "the only memory argument");
3902 }
3903
3904 if (!IsSibcall && !IsMustTail)
3905 Chain = DAG.getCALLSEQ_START(Chain, NumBytesToPush,
3906 NumBytes - NumBytesToPush, dl);
3907
3908 SDValue RetAddrFrIdx;
3909 // Load return address for tail calls.
3910 if (isTailCall && FPDiff)
3911 Chain = EmitTailCallLoadRetAddr(DAG, RetAddrFrIdx, Chain, isTailCall,
3912 Is64Bit, FPDiff, dl);
3913
3914 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
3915 SmallVector<SDValue, 8> MemOpChains;
3916 SDValue StackPtr;
3917
3918 // The next loop assumes that the locations are in the same order of the
3919 // input arguments.
3920 assert(isSortedByValueNo(ArgLocs) &&((isSortedByValueNo(ArgLocs) && "Argument Location list must be sorted before lowering"
) ? static_cast<void> (0) : __assert_fail ("isSortedByValueNo(ArgLocs) && \"Argument Location list must be sorted before lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3921, __PRETTY_FUNCTION__))
3921 "Argument Location list must be sorted before lowering")((isSortedByValueNo(ArgLocs) && "Argument Location list must be sorted before lowering"
) ? static_cast<void> (0) : __assert_fail ("isSortedByValueNo(ArgLocs) && \"Argument Location list must be sorted before lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3921, __PRETTY_FUNCTION__));
3922
3923 // Walk the register/memloc assignments, inserting copies/loads. In the case
3924 // of tail call optimization arguments are handle later.
3925 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
3926 for (unsigned I = 0, OutIndex = 0, E = ArgLocs.size(); I != E;
3927 ++I, ++OutIndex) {
3928 assert(OutIndex < Outs.size() && "Invalid Out index")((OutIndex < Outs.size() && "Invalid Out index") ?
static_cast<void> (0) : __assert_fail ("OutIndex < Outs.size() && \"Invalid Out index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3928, __PRETTY_FUNCTION__));
3929 // Skip inalloca arguments, they have already been written.
3930 ISD::ArgFlagsTy Flags = Outs[OutIndex].Flags;
3931 if (Flags.isInAlloca())
3932 continue;
3933
3934 CCValAssign &VA = ArgLocs[I];
3935 EVT RegVT = VA.getLocVT();
3936 SDValue Arg = OutVals[OutIndex];
3937 bool isByVal = Flags.isByVal();
3938
3939 // Promote the value if needed.
3940 switch (VA.getLocInfo()) {
3941 default: llvm_unreachable("Unknown loc info!")::llvm::llvm_unreachable_internal("Unknown loc info!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3941);
3942 case CCValAssign::Full: break;
3943 case CCValAssign::SExt:
3944 Arg = DAG.getNode(ISD::SIGN_EXTEND, dl, RegVT, Arg);
3945 break;
3946 case CCValAssign::ZExt:
3947 Arg = DAG.getNode(ISD::ZERO_EXTEND, dl, RegVT, Arg);
3948 break;
3949 case CCValAssign::AExt:
3950 if (Arg.getValueType().isVector() &&
3951 Arg.getValueType().getVectorElementType() == MVT::i1)
3952 Arg = lowerMasksToReg(Arg, RegVT, dl, DAG);
3953 else if (RegVT.is128BitVector()) {
3954 // Special case: passing MMX values in XMM registers.
3955 Arg = DAG.getBitcast(MVT::i64, Arg);
3956 Arg = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Arg);
3957 Arg = getMOVL(DAG, dl, MVT::v2i64, DAG.getUNDEF(MVT::v2i64), Arg);
3958 } else
3959 Arg = DAG.getNode(ISD::ANY_EXTEND, dl, RegVT, Arg);
3960 break;
3961 case CCValAssign::BCvt:
3962 Arg = DAG.getBitcast(RegVT, Arg);
3963 break;
3964 case CCValAssign::Indirect: {
3965 if (isByVal) {
3966 // Memcpy the argument to a temporary stack slot to prevent
3967 // the caller from seeing any modifications the callee may make
3968 // as guaranteed by the `byval` attribute.
3969 int FrameIdx = MF.getFrameInfo().CreateStackObject(
3970 Flags.getByValSize(), std::max(16, (int)Flags.getByValAlign()),
3971 false);
3972 SDValue StackSlot =
3973 DAG.getFrameIndex(FrameIdx, getPointerTy(DAG.getDataLayout()));
3974 Chain =
3975 CreateCopyOfByValArgument(Arg, StackSlot, Chain, Flags, DAG, dl);
3976 // From now on treat this as a regular pointer
3977 Arg = StackSlot;
3978 isByVal = false;
3979 } else {
3980 // Store the argument.
3981 SDValue SpillSlot = DAG.CreateStackTemporary(VA.getValVT());
3982 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
3983 Chain = DAG.getStore(
3984 Chain, dl, Arg, SpillSlot,
3985 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
3986 Arg = SpillSlot;
3987 }
3988 break;
3989 }
3990 }
3991
3992 if (VA.needsCustom()) {
3993 assert(VA.getValVT() == MVT::v64i1 &&((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3994, __PRETTY_FUNCTION__))
3994 "Currently the only custom case is when we split v64i1 to 2 regs")((VA.getValVT() == MVT::v64i1 && "Currently the only custom case is when we split v64i1 to 2 regs"
) ? static_cast<void> (0) : __assert_fail ("VA.getValVT() == MVT::v64i1 && \"Currently the only custom case is when we split v64i1 to 2 regs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 3994, __PRETTY_FUNCTION__));
3995 // Split v64i1 value into two registers
3996 Passv64i1ArgInRegs(dl, DAG, Arg, RegsToPass, VA, ArgLocs[++I], Subtarget);
3997 } else if (VA.isRegLoc()) {
3998 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
3999 const TargetOptions &Options = DAG.getTarget().Options;
4000 if (Options.EnableDebugEntryValues)
4001 CSInfo.emplace_back(VA.getLocReg(), I);
4002 if (isVarArg && IsWin64) {
4003 // Win64 ABI requires argument XMM reg to be copied to the corresponding
4004 // shadow reg if callee is a varargs function.
4005 unsigned ShadowReg = 0;
4006 switch (VA.getLocReg()) {
4007 case X86::XMM0: ShadowReg = X86::RCX; break;
4008 case X86::XMM1: ShadowReg = X86::RDX; break;
4009 case X86::XMM2: ShadowReg = X86::R8; break;
4010 case X86::XMM3: ShadowReg = X86::R9; break;
4011 }
4012 if (ShadowReg)
4013 RegsToPass.push_back(std::make_pair(ShadowReg, Arg));
4014 }
4015 } else if (!IsSibcall && (!isTailCall || isByVal)) {
4016 assert(VA.isMemLoc())((VA.isMemLoc()) ? static_cast<void> (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4016, __PRETTY_FUNCTION__));
4017 if (!StackPtr.getNode())
4018 StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
4019 getPointerTy(DAG.getDataLayout()));
4020 MemOpChains.push_back(LowerMemOpCallTo(Chain, StackPtr, Arg,
4021 dl, DAG, VA, Flags));
4022 }
4023 }
4024
4025 if (!MemOpChains.empty())
4026 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains);
4027
4028 if (Subtarget.isPICStyleGOT()) {
4029 // ELF / PIC requires GOT in the EBX register before function calls via PLT
4030 // GOT pointer.
4031 if (!isTailCall) {
4032 RegsToPass.push_back(std::make_pair(
4033 unsigned(X86::EBX), DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
4034 getPointerTy(DAG.getDataLayout()))));
4035 } else {
4036 // If we are tail calling and generating PIC/GOT style code load the
4037 // address of the callee into ECX. The value in ecx is used as target of
4038 // the tail jump. This is done to circumvent the ebx/callee-saved problem
4039 // for tail calls on PIC/GOT architectures. Normally we would just put the
4040 // address of GOT into ebx and then call target@PLT. But for tail calls
4041 // ebx would be restored (since ebx is callee saved) before jumping to the
4042 // target@PLT.
4043
4044 // Note: The actual moving to ECX is done further down.
4045 GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee);
4046 if (G && !G->getGlobal()->hasLocalLinkage() &&
4047 G->getGlobal()->hasDefaultVisibility())
4048 Callee = LowerGlobalAddress(Callee, DAG);
4049 else if (isa<ExternalSymbolSDNode>(Callee))
4050 Callee = LowerExternalSymbol(Callee, DAG);
4051 }
4052 }
4053
4054 if (Is64Bit && isVarArg && !IsWin64 && !IsMustTail) {
4055 // From AMD64 ABI document:
4056 // For calls that may call functions that use varargs or stdargs
4057 // (prototype-less calls or calls to functions containing ellipsis (...) in
4058 // the declaration) %al is used as hidden argument to specify the number
4059 // of SSE registers used. The contents of %al do not need to match exactly
4060 // the number of registers, but must be an ubound on the number of SSE
4061 // registers used and is in the range 0 - 8 inclusive.
4062
4063 // Count the number of XMM registers allocated.
4064 static const MCPhysReg XMMArgRegs[] = {
4065 X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
4066 X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
4067 };
4068 unsigned NumXMMRegs = CCInfo.getFirstUnallocated(XMMArgRegs);
4069 assert((Subtarget.hasSSE1() || !NumXMMRegs)(((Subtarget.hasSSE1() || !NumXMMRegs) && "SSE registers cannot be used when SSE is disabled"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasSSE1() || !NumXMMRegs) && \"SSE registers cannot be used when SSE is disabled\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4070, __PRETTY_FUNCTION__))
4070 && "SSE registers cannot be used when SSE is disabled")(((Subtarget.hasSSE1() || !NumXMMRegs) && "SSE registers cannot be used when SSE is disabled"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasSSE1() || !NumXMMRegs) && \"SSE registers cannot be used when SSE is disabled\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4070, __PRETTY_FUNCTION__));
4071
4072 RegsToPass.push_back(std::make_pair(unsigned(X86::AL),
4073 DAG.getConstant(NumXMMRegs, dl,
4074 MVT::i8)));
4075 }
4076
4077 if (isVarArg && IsMustTail) {
4078 const auto &Forwards = X86Info->getForwardedMustTailRegParms();
4079 for (const auto &F : Forwards) {
4080 SDValue Val = DAG.getCopyFromReg(Chain, dl, F.VReg, F.VT);
4081 RegsToPass.push_back(std::make_pair(unsigned(F.PReg), Val));
4082 }
4083 }
4084
4085 // For tail calls lower the arguments to the 'real' stack slots. Sibcalls
4086 // don't need this because the eligibility check rejects calls that require
4087 // shuffling arguments passed in memory.
4088 if (!IsSibcall && isTailCall) {
4089 // Force all the incoming stack arguments to be loaded from the stack
4090 // before any new outgoing arguments are stored to the stack, because the
4091 // outgoing stack slots may alias the incoming argument stack slots, and
4092 // the alias isn't otherwise explicit. This is slightly more conservative
4093 // than necessary, because it means that each store effectively depends
4094 // on every argument instead of just those arguments it would clobber.
4095 SDValue ArgChain = DAG.getStackArgumentTokenFactor(Chain);
4096
4097 SmallVector<SDValue, 8> MemOpChains2;
4098 SDValue FIN;
4099 int FI = 0;
4100 for (unsigned I = 0, OutsIndex = 0, E = ArgLocs.size(); I != E;
4101 ++I, ++OutsIndex) {
4102 CCValAssign &VA = ArgLocs[I];
4103
4104 if (VA.isRegLoc()) {
4105 if (VA.needsCustom()) {
4106 assert((CallConv == CallingConv::X86_RegCall) &&(((CallConv == CallingConv::X86_RegCall) && "Expecting custom case only in regcall calling convention"
) ? static_cast<void> (0) : __assert_fail ("(CallConv == CallingConv::X86_RegCall) && \"Expecting custom case only in regcall calling convention\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4107, __PRETTY_FUNCTION__))
4107 "Expecting custom case only in regcall calling convention")(((CallConv == CallingConv::X86_RegCall) && "Expecting custom case only in regcall calling convention"
) ? static_cast<void> (0) : __assert_fail ("(CallConv == CallingConv::X86_RegCall) && \"Expecting custom case only in regcall calling convention\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4107, __PRETTY_FUNCTION__));
4108 // This means that we are in special case where one argument was
4109 // passed through two register locations - Skip the next location
4110 ++I;
4111 }
4112
4113 continue;
4114 }
4115
4116 assert(VA.isMemLoc())((VA.isMemLoc()) ? static_cast<void> (0) : __assert_fail
("VA.isMemLoc()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4116, __PRETTY_FUNCTION__));
4117 SDValue Arg = OutVals[OutsIndex];
4118 ISD::ArgFlagsTy Flags = Outs[OutsIndex].Flags;
4119 // Skip inalloca arguments. They don't require any work.
4120 if (Flags.isInAlloca())
4121 continue;
4122 // Create frame index.
4123 int32_t Offset = VA.getLocMemOffset()+FPDiff;
4124 uint32_t OpSize = (VA.getLocVT().getSizeInBits()+7)/8;
4125 FI = MF.getFrameInfo().CreateFixedObject(OpSize, Offset, true);
4126 FIN = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4127
4128 if (Flags.isByVal()) {
4129 // Copy relative to framepointer.
4130 SDValue Source = DAG.getIntPtrConstant(VA.getLocMemOffset(), dl);
4131 if (!StackPtr.getNode())
4132 StackPtr = DAG.getCopyFromReg(Chain, dl, RegInfo->getStackRegister(),
4133 getPointerTy(DAG.getDataLayout()));
4134 Source = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
4135 StackPtr, Source);
4136
4137 MemOpChains2.push_back(CreateCopyOfByValArgument(Source, FIN,
4138 ArgChain,
4139 Flags, DAG, dl));
4140 } else {
4141 // Store relative to framepointer.
4142 MemOpChains2.push_back(DAG.getStore(
4143 ArgChain, dl, Arg, FIN,
4144 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI)));
4145 }
4146 }
4147
4148 if (!MemOpChains2.empty())
4149 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOpChains2);
4150
4151 // Store the return address to the appropriate stack slot.
4152 Chain = EmitTailCallStoreRetAddr(DAG, MF, Chain, RetAddrFrIdx,
4153 getPointerTy(DAG.getDataLayout()),
4154 RegInfo->getSlotSize(), FPDiff, dl);
4155 }
4156
4157 // Build a sequence of copy-to-reg nodes chained together with token chain
4158 // and flag operands which copy the outgoing args into registers.
4159 SDValue InFlag;
4160 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i) {
4161 Chain = DAG.getCopyToReg(Chain, dl, RegsToPass[i].first,
4162 RegsToPass[i].second, InFlag);
4163 InFlag = Chain.getValue(1);
4164 }
4165
4166 if (DAG.getTarget().getCodeModel() == CodeModel::Large) {
4167 assert(Is64Bit && "Large code model is only legal in 64-bit mode.")((Is64Bit && "Large code model is only legal in 64-bit mode."
) ? static_cast<void> (0) : __assert_fail ("Is64Bit && \"Large code model is only legal in 64-bit mode.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4167, __PRETTY_FUNCTION__));
4168 // In the 64-bit large code model, we have to make all calls
4169 // through a register, since the call instruction's 32-bit
4170 // pc-relative offset may not be large enough to hold the whole
4171 // address.
4172 } else if (Callee->getOpcode() == ISD::GlobalAddress ||
4173 Callee->getOpcode() == ISD::ExternalSymbol) {
4174 // Lower direct calls to global addresses and external symbols. Setting
4175 // ForCall to true here has the effect of removing WrapperRIP when possible
4176 // to allow direct calls to be selected without first materializing the
4177 // address into a register.
4178 Callee = LowerGlobalOrExternal(Callee, DAG, /*ForCall=*/true);
4179 } else if (Subtarget.isTarget64BitILP32() &&
4180 Callee->getValueType(0) == MVT::i32) {
4181 // Zero-extend the 32-bit Callee address into a 64-bit according to x32 ABI
4182 Callee = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Callee);
4183 }
4184
4185 // Returns a chain & a flag for retval copy to use.
4186 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
4187 SmallVector<SDValue, 8> Ops;
4188
4189 if (!IsSibcall && isTailCall && !IsMustTail) {
4190 Chain = DAG.getCALLSEQ_END(Chain,
4191 DAG.getIntPtrConstant(NumBytesToPop, dl, true),
4192 DAG.getIntPtrConstant(0, dl, true), InFlag, dl);
4193 InFlag = Chain.getValue(1);
4194 }
4195
4196 Ops.push_back(Chain);
4197 Ops.push_back(Callee);
4198
4199 if (isTailCall)
4200 Ops.push_back(DAG.getConstant(FPDiff, dl, MVT::i32));
4201
4202 // Add argument registers to the end of the list so that they are known live
4203 // into the call.
4204 for (unsigned i = 0, e = RegsToPass.size(); i != e; ++i)
4205 Ops.push_back(DAG.getRegister(RegsToPass[i].first,
4206 RegsToPass[i].second.getValueType()));
4207
4208 // Add a register mask operand representing the call-preserved registers.
4209 // If HasNCSR is asserted (attribute NoCallerSavedRegisters exists) then we
4210 // set X86_INTR calling convention because it has the same CSR mask
4211 // (same preserved registers).
4212 const uint32_t *Mask = RegInfo->getCallPreservedMask(
4213 MF, HasNCSR ? (CallingConv::ID)CallingConv::X86_INTR : CallConv);
4214 assert(Mask && "Missing call preserved mask for calling convention")((Mask && "Missing call preserved mask for calling convention"
) ? static_cast<void> (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4214, __PRETTY_FUNCTION__));
4215
4216 // If this is an invoke in a 32-bit function using a funclet-based
4217 // personality, assume the function clobbers all registers. If an exception
4218 // is thrown, the runtime will not restore CSRs.
4219 // FIXME: Model this more precisely so that we can register allocate across
4220 // the normal edge and spill and fill across the exceptional edge.
4221 if (!Is64Bit && CLI.CS && CLI.CS.isInvoke()) {
4222 const Function &CallerFn = MF.getFunction();
4223 EHPersonality Pers =
4224 CallerFn.hasPersonalityFn()
4225 ? classifyEHPersonality(CallerFn.getPersonalityFn())
4226 : EHPersonality::Unknown;
4227 if (isFuncletEHPersonality(Pers))
4228 Mask = RegInfo->getNoPreservedMask();
4229 }
4230
4231 // Define a new register mask from the existing mask.
4232 uint32_t *RegMask = nullptr;
4233
4234 // In some calling conventions we need to remove the used physical registers
4235 // from the reg mask.
4236 if (CallConv == CallingConv::X86_RegCall || HasNCSR) {
4237 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
4238
4239 // Allocate a new Reg Mask and copy Mask.
4240 RegMask = MF.allocateRegMask();
4241 unsigned RegMaskSize = MachineOperand::getRegMaskSize(TRI->getNumRegs());
4242 memcpy(RegMask, Mask, sizeof(RegMask[0]) * RegMaskSize);
4243
4244 // Make sure all sub registers of the argument registers are reset
4245 // in the RegMask.
4246 for (auto const &RegPair : RegsToPass)
4247 for (MCSubRegIterator SubRegs(RegPair.first, TRI, /*IncludeSelf=*/true);
4248 SubRegs.isValid(); ++SubRegs)
4249 RegMask[*SubRegs / 32] &= ~(1u << (*SubRegs % 32));
4250
4251 // Create the RegMask Operand according to our updated mask.
4252 Ops.push_back(DAG.getRegisterMask(RegMask));
4253 } else {
4254 // Create the RegMask Operand according to the static mask.
4255 Ops.push_back(DAG.getRegisterMask(Mask));
4256 }
4257
4258 if (InFlag.getNode())
4259 Ops.push_back(InFlag);
4260
4261 if (isTailCall) {
4262 // We used to do:
4263 //// If this is the first return lowered for this function, add the regs
4264 //// to the liveout set for the function.
4265 // This isn't right, although it's probably harmless on x86; liveouts
4266 // should be computed from returns not tail calls. Consider a void
4267 // function making a tail call to a function returning int.
4268 MF.getFrameInfo().setHasTailCall();
4269 SDValue Ret = DAG.getNode(X86ISD::TC_RETURN, dl, NodeTys, Ops);
4270 DAG.addCallSiteInfo(Ret.getNode(), std::move(CSInfo));
4271 return Ret;
4272 }
4273
4274 if (HasNoCfCheck && IsCFProtectionSupported) {
4275 Chain = DAG.getNode(X86ISD::NT_CALL, dl, NodeTys, Ops);
4276 } else {
4277 Chain = DAG.getNode(X86ISD::CALL, dl, NodeTys, Ops);
4278 }
4279 InFlag = Chain.getValue(1);
4280 DAG.addCallSiteInfo(Chain.getNode(), std::move(CSInfo));
4281
4282 // Save heapallocsite metadata.
4283 if (CLI.CS)
4284 if (MDNode *HeapAlloc = CLI.CS->getMetadata("heapallocsite"))
4285 DAG.addHeapAllocSite(Chain.getNode(), HeapAlloc);
4286
4287 // Create the CALLSEQ_END node.
4288 unsigned NumBytesForCalleeToPop;
4289 if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
4290 DAG.getTarget().Options.GuaranteedTailCallOpt))
4291 NumBytesForCalleeToPop = NumBytes; // Callee pops everything
4292 else if (!Is64Bit && !canGuaranteeTCO(CallConv) &&
4293 !Subtarget.getTargetTriple().isOSMSVCRT() &&
4294 SR == StackStructReturn)
4295 // If this is a call to a struct-return function, the callee
4296 // pops the hidden struct pointer, so we have to push it back.
4297 // This is common for Darwin/X86, Linux & Mingw32 targets.
4298 // For MSVC Win32 targets, the caller pops the hidden struct pointer.
4299 NumBytesForCalleeToPop = 4;
4300 else
4301 NumBytesForCalleeToPop = 0; // Callee pops nothing.
4302
4303 if (CLI.DoesNotReturn && !getTargetMachine().Options.TrapUnreachable) {
4304 // No need to reset the stack after the call if the call doesn't return. To
4305 // make the MI verify, we'll pretend the callee does it for us.
4306 NumBytesForCalleeToPop = NumBytes;
4307 }
4308
4309 // Returns a flag for retval copy to use.
4310 if (!IsSibcall) {
4311 Chain = DAG.getCALLSEQ_END(Chain,
4312 DAG.getIntPtrConstant(NumBytesToPop, dl, true),
4313 DAG.getIntPtrConstant(NumBytesForCalleeToPop, dl,
4314 true),
4315 InFlag, dl);
4316 InFlag = Chain.getValue(1);
4317 }
4318
4319 // Handle result values, copying them out of physregs into vregs that we
4320 // return.
4321 return LowerCallResult(Chain, InFlag, CallConv, isVarArg, Ins, dl, DAG,
4322 InVals, RegMask);
4323}
4324
4325//===----------------------------------------------------------------------===//
4326// Fast Calling Convention (tail call) implementation
4327//===----------------------------------------------------------------------===//
4328
4329// Like std call, callee cleans arguments, convention except that ECX is
4330// reserved for storing the tail called function address. Only 2 registers are
4331// free for argument passing (inreg). Tail call optimization is performed
4332// provided:
4333// * tailcallopt is enabled
4334// * caller/callee are fastcc
4335// On X86_64 architecture with GOT-style position independent code only local
4336// (within module) calls are supported at the moment.
4337// To keep the stack aligned according to platform abi the function
4338// GetAlignedArgumentStackSize ensures that argument delta is always multiples
4339// of stack alignment. (Dynamic linkers need this - darwin's dyld for example)
4340// If a tail called function callee has more arguments than the caller the
4341// caller needs to make sure that there is room to move the RETADDR to. This is
4342// achieved by reserving an area the size of the argument delta right after the
4343// original RETADDR, but before the saved framepointer or the spilled registers
4344// e.g. caller(arg1, arg2) calls callee(arg1, arg2,arg3,arg4)
4345// stack layout:
4346// arg1
4347// arg2
4348// RETADDR
4349// [ new RETADDR
4350// move area ]
4351// (possible EBP)
4352// ESI
4353// EDI
4354// local1 ..
4355
4356/// Make the stack size align e.g 16n + 12 aligned for a 16-byte align
4357/// requirement.
4358unsigned
4359X86TargetLowering::GetAlignedArgumentStackSize(const unsigned StackSize,
4360 SelectionDAG &DAG) const {
4361 const Align StackAlignment(Subtarget.getFrameLowering()->getStackAlignment());
4362 const uint64_t SlotSize = Subtarget.getRegisterInfo()->getSlotSize();
4363 assert(StackSize % SlotSize == 0 &&((StackSize % SlotSize == 0 && "StackSize must be a multiple of SlotSize"
) ? static_cast<void> (0) : __assert_fail ("StackSize % SlotSize == 0 && \"StackSize must be a multiple of SlotSize\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4364, __PRETTY_FUNCTION__))
4364 "StackSize must be a multiple of SlotSize")((StackSize % SlotSize == 0 && "StackSize must be a multiple of SlotSize"
) ? static_cast<void> (0) : __assert_fail ("StackSize % SlotSize == 0 && \"StackSize must be a multiple of SlotSize\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4364, __PRETTY_FUNCTION__));
4365 return alignTo(StackSize + SlotSize, StackAlignment) - SlotSize;
4366}
4367
4368/// Return true if the given stack call argument is already available in the
4369/// same position (relatively) of the caller's incoming argument stack.
4370static
4371bool MatchingStackOffset(SDValue Arg, unsigned Offset, ISD::ArgFlagsTy Flags,
4372 MachineFrameInfo &MFI, const MachineRegisterInfo *MRI,
4373 const X86InstrInfo *TII, const CCValAssign &VA) {
4374 unsigned Bytes = Arg.getValueSizeInBits() / 8;
4375
4376 for (;;) {
4377 // Look through nodes that don't alter the bits of the incoming value.
4378 unsigned Op = Arg.getOpcode();
4379 if (Op == ISD::ZERO_EXTEND || Op == ISD::ANY_EXTEND || Op == ISD::BITCAST) {
4380 Arg = Arg.getOperand(0);
4381 continue;
4382 }
4383 if (Op == ISD::TRUNCATE) {
4384 const SDValue &TruncInput = Arg.getOperand(0);
4385 if (TruncInput.getOpcode() == ISD::AssertZext &&
4386 cast<VTSDNode>(TruncInput.getOperand(1))->getVT() ==
4387 Arg.getValueType()) {
4388 Arg = TruncInput.getOperand(0);
4389 continue;
4390 }
4391 }
4392 break;
4393 }
4394
4395 int FI = INT_MAX2147483647;
4396 if (Arg.getOpcode() == ISD::CopyFromReg) {
4397 unsigned VR = cast<RegisterSDNode>(Arg.getOperand(1))->getReg();
4398 if (!Register::isVirtualRegister(VR))
4399 return false;
4400 MachineInstr *Def = MRI->getVRegDef(VR);
4401 if (!Def)
4402 return false;
4403 if (!Flags.isByVal()) {
4404 if (!TII->isLoadFromStackSlot(*Def, FI))
4405 return false;
4406 } else {
4407 unsigned Opcode = Def->getOpcode();
4408 if ((Opcode == X86::LEA32r || Opcode == X86::LEA64r ||
4409 Opcode == X86::LEA64_32r) &&
4410 Def->getOperand(1).isFI()) {
4411 FI = Def->getOperand(1).getIndex();
4412 Bytes = Flags.getByValSize();
4413 } else
4414 return false;
4415 }
4416 } else if (LoadSDNode *Ld = dyn_cast<LoadSDNode>(Arg)) {
4417 if (Flags.isByVal())
4418 // ByVal argument is passed in as a pointer but it's now being
4419 // dereferenced. e.g.
4420 // define @foo(%struct.X* %A) {
4421 // tail call @bar(%struct.X* byval %A)
4422 // }
4423 return false;
4424 SDValue Ptr = Ld->getBasePtr();
4425 FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr);
4426 if (!FINode)
4427 return false;
4428 FI = FINode->getIndex();
4429 } else if (Arg.getOpcode() == ISD::FrameIndex && Flags.isByVal()) {
4430 FrameIndexSDNode *FINode = cast<FrameIndexSDNode>(Arg);
4431 FI = FINode->getIndex();
4432 Bytes = Flags.getByValSize();
4433 } else
4434 return false;
4435
4436 assert(FI != INT_MAX)((FI != 2147483647) ? static_cast<void> (0) : __assert_fail
("FI != INT_MAX", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4436, __PRETTY_FUNCTION__));
4437 if (!MFI.isFixedObjectIndex(FI))
4438 return false;
4439
4440 if (Offset != MFI.getObjectOffset(FI))
4441 return false;
4442
4443 // If this is not byval, check that the argument stack object is immutable.
4444 // inalloca and argument copy elision can create mutable argument stack
4445 // objects. Byval objects can be mutated, but a byval call intends to pass the
4446 // mutated memory.
4447 if (!Flags.isByVal() && !MFI.isImmutableObjectIndex(FI))
4448 return false;
4449
4450 if (VA.getLocVT().getSizeInBits() > Arg.getValueSizeInBits()) {
4451 // If the argument location is wider than the argument type, check that any
4452 // extension flags match.
4453 if (Flags.isZExt() != MFI.isObjectZExt(FI) ||
4454 Flags.isSExt() != MFI.isObjectSExt(FI)) {
4455 return false;
4456 }
4457 }
4458
4459 return Bytes == MFI.getObjectSize(FI);
4460}
4461
4462/// Check whether the call is eligible for tail call optimization. Targets
4463/// that want to do tail call optimization should implement this function.
4464bool X86TargetLowering::IsEligibleForTailCallOptimization(
4465 SDValue Callee, CallingConv::ID CalleeCC, bool isVarArg,
4466 bool isCalleeStructRet, bool isCallerStructRet, Type *RetTy,
4467 const SmallVectorImpl<ISD::OutputArg> &Outs,
4468 const SmallVectorImpl<SDValue> &OutVals,
4469 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const {
4470 if (!mayTailCallThisCC(CalleeCC))
4471 return false;
4472
4473 // If -tailcallopt is specified, make fastcc functions tail-callable.
4474 MachineFunction &MF = DAG.getMachineFunction();
4475 const Function &CallerF = MF.getFunction();
4476
4477 // If the function return type is x86_fp80 and the callee return type is not,
4478 // then the FP_EXTEND of the call result is not a nop. It's not safe to
4479 // perform a tailcall optimization here.
4480 if (CallerF.getReturnType()->isX86_FP80Ty() && !RetTy->isX86_FP80Ty())
4481 return false;
4482
4483 CallingConv::ID CallerCC = CallerF.getCallingConv();
4484 bool CCMatch = CallerCC == CalleeCC;
4485 bool IsCalleeWin64 = Subtarget.isCallingConvWin64(CalleeCC);
4486 bool IsCallerWin64 = Subtarget.isCallingConvWin64(CallerCC);
4487 bool IsGuaranteeTCO = DAG.getTarget().Options.GuaranteedTailCallOpt ||
4488 CalleeCC == CallingConv::Tail;
4489
4490 // Win64 functions have extra shadow space for argument homing. Don't do the
4491 // sibcall if the caller and callee have mismatched expectations for this
4492 // space.
4493 if (IsCalleeWin64 != IsCallerWin64)
4494 return false;
4495
4496 if (IsGuaranteeTCO) {
4497 if (canGuaranteeTCO(CalleeCC) && CCMatch)
4498 return true;
4499 return false;
4500 }
4501
4502 // Look for obvious safe cases to perform tail call optimization that do not
4503 // require ABI changes. This is what gcc calls sibcall.
4504
4505 // Can't do sibcall if stack needs to be dynamically re-aligned. PEI needs to
4506 // emit a special epilogue.
4507 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
4508 if (RegInfo->needsStackRealignment(MF))
4509 return false;
4510
4511 // Also avoid sibcall optimization if either caller or callee uses struct
4512 // return semantics.
4513 if (isCalleeStructRet || isCallerStructRet)
4514 return false;
4515
4516 // Do not sibcall optimize vararg calls unless all arguments are passed via
4517 // registers.
4518 LLVMContext &C = *DAG.getContext();
4519 if (isVarArg && !Outs.empty()) {
4520 // Optimizing for varargs on Win64 is unlikely to be safe without
4521 // additional testing.
4522 if (IsCalleeWin64 || IsCallerWin64)
4523 return false;
4524
4525 SmallVector<CCValAssign, 16> ArgLocs;
4526 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
4527
4528 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
4529 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i)
4530 if (!ArgLocs[i].isRegLoc())
4531 return false;
4532 }
4533
4534 // If the call result is in ST0 / ST1, it needs to be popped off the x87
4535 // stack. Therefore, if it's not used by the call it is not safe to optimize
4536 // this into a sibcall.
4537 bool Unused = false;
4538 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
4539 if (!Ins[i].Used) {
4540 Unused = true;
4541 break;
4542 }
4543 }
4544 if (Unused) {
4545 SmallVector<CCValAssign, 16> RVLocs;
4546 CCState CCInfo(CalleeCC, false, MF, RVLocs, C);
4547 CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
4548 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
4549 CCValAssign &VA = RVLocs[i];
4550 if (VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1)
4551 return false;
4552 }
4553 }
4554
4555 // Check that the call results are passed in the same way.
4556 if (!CCState::resultsCompatible(CalleeCC, CallerCC, MF, C, Ins,
4557 RetCC_X86, RetCC_X86))
4558 return false;
4559 // The callee has to preserve all registers the caller needs to preserve.
4560 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
4561 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
4562 if (!CCMatch) {
4563 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
4564 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
4565 return false;
4566 }
4567
4568 unsigned StackArgsSize = 0;
4569
4570 // If the callee takes no arguments then go on to check the results of the
4571 // call.
4572 if (!Outs.empty()) {
4573 // Check if stack adjustment is needed. For now, do not do this if any
4574 // argument is passed on the stack.
4575 SmallVector<CCValAssign, 16> ArgLocs;
4576 CCState CCInfo(CalleeCC, isVarArg, MF, ArgLocs, C);
4577
4578 // Allocate shadow area for Win64
4579 if (IsCalleeWin64)
4580 CCInfo.AllocateStack(32, 8);
4581
4582 CCInfo.AnalyzeCallOperands(Outs, CC_X86);
4583 StackArgsSize = CCInfo.getNextStackOffset();
4584
4585 if (CCInfo.getNextStackOffset()) {
4586 // Check if the arguments are already laid out in the right way as
4587 // the caller's fixed stack objects.
4588 MachineFrameInfo &MFI = MF.getFrameInfo();
4589 const MachineRegisterInfo *MRI = &MF.getRegInfo();
4590 const X86InstrInfo *TII = Subtarget.getInstrInfo();
4591 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4592 CCValAssign &VA = ArgLocs[i];
4593 SDValue Arg = OutVals[i];
4594 ISD::ArgFlagsTy Flags = Outs[i].Flags;
4595 if (VA.getLocInfo() == CCValAssign::Indirect)
4596 return false;
4597 if (!VA.isRegLoc()) {
4598 if (!MatchingStackOffset(Arg, VA.getLocMemOffset(), Flags,
4599 MFI, MRI, TII, VA))
4600 return false;
4601 }
4602 }
4603 }
4604
4605 bool PositionIndependent = isPositionIndependent();
4606 // If the tailcall address may be in a register, then make sure it's
4607 // possible to register allocate for it. In 32-bit, the call address can
4608 // only target EAX, EDX, or ECX since the tail call must be scheduled after
4609 // callee-saved registers are restored. These happen to be the same
4610 // registers used to pass 'inreg' arguments so watch out for those.
4611 if (!Subtarget.is64Bit() && ((!isa<GlobalAddressSDNode>(Callee) &&
4612 !isa<ExternalSymbolSDNode>(Callee)) ||
4613 PositionIndependent)) {
4614 unsigned NumInRegs = 0;
4615 // In PIC we need an extra register to formulate the address computation
4616 // for the callee.
4617 unsigned MaxInRegs = PositionIndependent ? 2 : 3;
4618
4619 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4620 CCValAssign &VA = ArgLocs[i];
4621 if (!VA.isRegLoc())
4622 continue;
4623 Register Reg = VA.getLocReg();
4624 switch (Reg) {
4625 default: break;
4626 case X86::EAX: case X86::EDX: case X86::ECX:
4627 if (++NumInRegs == MaxInRegs)
4628 return false;
4629 break;
4630 }
4631 }
4632 }
4633
4634 const MachineRegisterInfo &MRI = MF.getRegInfo();
4635 if (!parametersInCSRMatch(MRI, CallerPreserved, ArgLocs, OutVals))
4636 return false;
4637 }
4638
4639 bool CalleeWillPop =
4640 X86::isCalleePop(CalleeCC, Subtarget.is64Bit(), isVarArg,
4641 MF.getTarget().Options.GuaranteedTailCallOpt);
4642
4643 if (unsigned BytesToPop =
4644 MF.getInfo<X86MachineFunctionInfo>()->getBytesToPopOnReturn()) {
4645 // If we have bytes to pop, the callee must pop them.
4646 bool CalleePopMatches = CalleeWillPop && BytesToPop == StackArgsSize;
4647 if (!CalleePopMatches)
4648 return false;
4649 } else if (CalleeWillPop && StackArgsSize > 0) {
4650 // If we don't have bytes to pop, make sure the callee doesn't pop any.
4651 return false;
4652 }
4653
4654 return true;
4655}
4656
4657FastISel *
4658X86TargetLowering::createFastISel(FunctionLoweringInfo &funcInfo,
4659 const TargetLibraryInfo *libInfo) const {
4660 return X86::createFastISel(funcInfo, libInfo);
4661}
4662
4663//===----------------------------------------------------------------------===//
4664// Other Lowering Hooks
4665//===----------------------------------------------------------------------===//
4666
4667static bool MayFoldLoad(SDValue Op) {
4668 return Op.hasOneUse() && ISD::isNormalLoad(Op.getNode());
4669}
4670
4671static bool MayFoldIntoStore(SDValue Op) {
4672 return Op.hasOneUse() && ISD::isNormalStore(*Op.getNode()->use_begin());
4673}
4674
4675static bool MayFoldIntoZeroExtend(SDValue Op) {
4676 if (Op.hasOneUse()) {
4677 unsigned Opcode = Op.getNode()->use_begin()->getOpcode();
4678 return (ISD::ZERO_EXTEND == Opcode);
4679 }
4680 return false;
4681}
4682
4683static bool isTargetShuffle(unsigned Opcode) {
4684 switch(Opcode) {
4685 default: return false;
4686 case X86ISD::BLENDI:
4687 case X86ISD::PSHUFB:
4688 case X86ISD::PSHUFD:
4689 case X86ISD::PSHUFHW:
4690 case X86ISD::PSHUFLW:
4691 case X86ISD::SHUFP:
4692 case X86ISD::INSERTPS:
4693 case X86ISD::EXTRQI:
4694 case X86ISD::INSERTQI:
4695 case X86ISD::PALIGNR:
4696 case X86ISD::VSHLDQ:
4697 case X86ISD::VSRLDQ:
4698 case X86ISD::MOVLHPS:
4699 case X86ISD::MOVHLPS:
4700 case X86ISD::MOVSHDUP:
4701 case X86ISD::MOVSLDUP:
4702 case X86ISD::MOVDDUP:
4703 case X86ISD::MOVSS:
4704 case X86ISD::MOVSD:
4705 case X86ISD::UNPCKL:
4706 case X86ISD::UNPCKH:
4707 case X86ISD::VBROADCAST:
4708 case X86ISD::VPERMILPI:
4709 case X86ISD::VPERMILPV:
4710 case X86ISD::VPERM2X128:
4711 case X86ISD::SHUF128:
4712 case X86ISD::VPERMIL2:
4713 case X86ISD::VPERMI:
4714 case X86ISD::VPPERM:
4715 case X86ISD::VPERMV:
4716 case X86ISD::VPERMV3:
4717 case X86ISD::VZEXT_MOVL:
4718 return true;
4719 }
4720}
4721
4722static bool isTargetShuffleVariableMask(unsigned Opcode) {
4723 switch (Opcode) {
4724 default: return false;
4725 // Target Shuffles.
4726 case X86ISD::PSHUFB:
4727 case X86ISD::VPERMILPV:
4728 case X86ISD::VPERMIL2:
4729 case X86ISD::VPPERM:
4730 case X86ISD::VPERMV:
4731 case X86ISD::VPERMV3:
4732 return true;
4733 // 'Faux' Target Shuffles.
4734 case ISD::OR:
4735 case ISD::AND:
4736 case X86ISD::ANDNP:
4737 return true;
4738 }
4739}
4740
4741SDValue X86TargetLowering::getReturnAddressFrameIndex(SelectionDAG &DAG) const {
4742 MachineFunction &MF = DAG.getMachineFunction();
4743 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
4744 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
4745 int ReturnAddrIndex = FuncInfo->getRAIndex();
4746
4747 if (ReturnAddrIndex == 0) {
4748 // Set up a frame object for the return address.
4749 unsigned SlotSize = RegInfo->getSlotSize();
4750 ReturnAddrIndex = MF.getFrameInfo().CreateFixedObject(SlotSize,
4751 -(int64_t)SlotSize,
4752 false);
4753 FuncInfo->setRAIndex(ReturnAddrIndex);
4754 }
4755
4756 return DAG.getFrameIndex(ReturnAddrIndex, getPointerTy(DAG.getDataLayout()));
4757}
4758
4759bool X86::isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M,
4760 bool hasSymbolicDisplacement) {
4761 // Offset should fit into 32 bit immediate field.
4762 if (!isInt<32>(Offset))
4763 return false;
4764
4765 // If we don't have a symbolic displacement - we don't have any extra
4766 // restrictions.
4767 if (!hasSymbolicDisplacement)
4768 return true;
4769
4770 // FIXME: Some tweaks might be needed for medium code model.
4771 if (M != CodeModel::Small && M != CodeModel::Kernel)
4772 return false;
4773
4774 // For small code model we assume that latest object is 16MB before end of 31
4775 // bits boundary. We may also accept pretty large negative constants knowing
4776 // that all objects are in the positive half of address space.
4777 if (M == CodeModel::Small && Offset < 16*1024*1024)
4778 return true;
4779
4780 // For kernel code model we know that all object resist in the negative half
4781 // of 32bits address space. We may not accept negative offsets, since they may
4782 // be just off and we may accept pretty large positive ones.
4783 if (M == CodeModel::Kernel && Offset >= 0)
4784 return true;
4785
4786 return false;
4787}
4788
4789/// Determines whether the callee is required to pop its own arguments.
4790/// Callee pop is necessary to support tail calls.
4791bool X86::isCalleePop(CallingConv::ID CallingConv,
4792 bool is64Bit, bool IsVarArg, bool GuaranteeTCO) {
4793 // If GuaranteeTCO is true, we force some calls to be callee pop so that we
4794 // can guarantee TCO.
4795 if (!IsVarArg && shouldGuaranteeTCO(CallingConv, GuaranteeTCO))
4796 return true;
4797
4798 switch (CallingConv) {
4799 default:
4800 return false;
4801 case CallingConv::X86_StdCall:
4802 case CallingConv::X86_FastCall:
4803 case CallingConv::X86_ThisCall:
4804 case CallingConv::X86_VectorCall:
4805 return !is64Bit;
4806 }
4807}
4808
4809/// Return true if the condition is an signed comparison operation.
4810static bool isX86CCSigned(unsigned X86CC) {
4811 switch (X86CC) {
4812 default:
4813 llvm_unreachable("Invalid integer condition!")::llvm::llvm_unreachable_internal("Invalid integer condition!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4813);
4814 case X86::COND_E:
4815 case X86::COND_NE:
4816 case X86::COND_B:
4817 case X86::COND_A:
4818 case X86::COND_BE:
4819 case X86::COND_AE:
4820 return false;
4821 case X86::COND_G:
4822 case X86::COND_GE:
4823 case X86::COND_L:
4824 case X86::COND_LE:
4825 return true;
4826 }
4827}
4828
4829static X86::CondCode TranslateIntegerX86CC(ISD::CondCode SetCCOpcode) {
4830 switch (SetCCOpcode) {
4831 default: llvm_unreachable("Invalid integer condition!")::llvm::llvm_unreachable_internal("Invalid integer condition!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4831);
4832 case ISD::SETEQ: return X86::COND_E;
4833 case ISD::SETGT: return X86::COND_G;
4834 case ISD::SETGE: return X86::COND_GE;
4835 case ISD::SETLT: return X86::COND_L;
4836 case ISD::SETLE: return X86::COND_LE;
4837 case ISD::SETNE: return X86::COND_NE;
4838 case ISD::SETULT: return X86::COND_B;
4839 case ISD::SETUGT: return X86::COND_A;
4840 case ISD::SETULE: return X86::COND_BE;
4841 case ISD::SETUGE: return X86::COND_AE;
4842 }
4843}
4844
4845/// Do a one-to-one translation of a ISD::CondCode to the X86-specific
4846/// condition code, returning the condition code and the LHS/RHS of the
4847/// comparison to make.
4848static X86::CondCode TranslateX86CC(ISD::CondCode SetCCOpcode, const SDLoc &DL,
4849 bool isFP, SDValue &LHS, SDValue &RHS,
4850 SelectionDAG &DAG) {
4851 if (!isFP) {
4852 if (ConstantSDNode *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
4853 if (SetCCOpcode == ISD::SETGT && RHSC->isAllOnesValue()) {
4854 // X > -1 -> X == 0, jump !sign.
4855 RHS = DAG.getConstant(0, DL, RHS.getValueType());
4856 return X86::COND_NS;
4857 }
4858 if (SetCCOpcode == ISD::SETLT && RHSC->isNullValue()) {
4859 // X < 0 -> X == 0, jump on sign.
4860 return X86::COND_S;
4861 }
4862 if (SetCCOpcode == ISD::SETGE && RHSC->isNullValue()) {
4863 // X >= 0 -> X == 0, jump on !sign.
4864 return X86::COND_NS;
4865 }
4866 if (SetCCOpcode == ISD::SETLT && RHSC->isOne()) {
4867 // X < 1 -> X <= 0
4868 RHS = DAG.getConstant(0, DL, RHS.getValueType());
4869 return X86::COND_LE;
4870 }
4871 }
4872
4873 return TranslateIntegerX86CC(SetCCOpcode);
4874 }
4875
4876 // First determine if it is required or is profitable to flip the operands.
4877
4878 // If LHS is a foldable load, but RHS is not, flip the condition.
4879 if (ISD::isNON_EXTLoad(LHS.getNode()) &&
4880 !ISD::isNON_EXTLoad(RHS.getNode())) {
4881 SetCCOpcode = getSetCCSwappedOperands(SetCCOpcode);
4882 std::swap(LHS, RHS);
4883 }
4884
4885 switch (SetCCOpcode) {
4886 default: break;
4887 case ISD::SETOLT:
4888 case ISD::SETOLE:
4889 case ISD::SETUGT:
4890 case ISD::SETUGE:
4891 std::swap(LHS, RHS);
4892 break;
4893 }
4894
4895 // On a floating point condition, the flags are set as follows:
4896 // ZF PF CF op
4897 // 0 | 0 | 0 | X > Y
4898 // 0 | 0 | 1 | X < Y
4899 // 1 | 0 | 0 | X == Y
4900 // 1 | 1 | 1 | unordered
4901 switch (SetCCOpcode) {
4902 default: llvm_unreachable("Condcode should be pre-legalized away")::llvm::llvm_unreachable_internal("Condcode should be pre-legalized away"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 4902);
4903 case ISD::SETUEQ:
4904 case ISD::SETEQ: return X86::COND_E;
4905 case ISD::SETOLT: // flipped
4906 case ISD::SETOGT:
4907 case ISD::SETGT: return X86::COND_A;
4908 case ISD::SETOLE: // flipped
4909 case ISD::SETOGE:
4910 case ISD::SETGE: return X86::COND_AE;
4911 case ISD::SETUGT: // flipped
4912 case ISD::SETULT:
4913 case ISD::SETLT: return X86::COND_B;
4914 case ISD::SETUGE: // flipped
4915 case ISD::SETULE:
4916 case ISD::SETLE: return X86::COND_BE;
4917 case ISD::SETONE:
4918 case ISD::SETNE: return X86::COND_NE;
4919 case ISD::SETUO: return X86::COND_P;
4920 case ISD::SETO: return X86::COND_NP;
4921 case ISD::SETOEQ:
4922 case ISD::SETUNE: return X86::COND_INVALID;
4923 }
4924}
4925
4926/// Is there a floating point cmov for the specific X86 condition code?
4927/// Current x86 isa includes the following FP cmov instructions:
4928/// fcmovb, fcomvbe, fcomve, fcmovu, fcmovae, fcmova, fcmovne, fcmovnu.
4929static bool hasFPCMov(unsigned X86CC) {
4930 switch (X86CC) {
4931 default:
4932 return false;
4933 case X86::COND_B:
4934 case X86::COND_BE:
4935 case X86::COND_E:
4936 case X86::COND_P:
4937 case X86::COND_A:
4938 case X86::COND_AE:
4939 case X86::COND_NE:
4940 case X86::COND_NP:
4941 return true;
4942 }
4943}
4944
4945
4946bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
4947 const CallInst &I,
4948 MachineFunction &MF,
4949 unsigned Intrinsic) const {
4950
4951 const IntrinsicData* IntrData = getIntrinsicWithChain(Intrinsic);
4952 if (!IntrData)
4953 return false;
4954
4955 Info.flags = MachineMemOperand::MONone;
4956 Info.offset = 0;
4957
4958 switch (IntrData->Type) {
4959 case TRUNCATE_TO_MEM_VI8:
4960 case TRUNCATE_TO_MEM_VI16:
4961 case TRUNCATE_TO_MEM_VI32: {
4962 Info.opc = ISD::INTRINSIC_VOID;
4963 Info.ptrVal = I.getArgOperand(0);
4964 MVT VT = MVT::getVT(I.getArgOperand(1)->getType());
4965 MVT ScalarVT = MVT::INVALID_SIMPLE_VALUE_TYPE;
4966 if (IntrData->Type == TRUNCATE_TO_MEM_VI8)
4967 ScalarVT = MVT::i8;
4968 else if (IntrData->Type == TRUNCATE_TO_MEM_VI16)
4969 ScalarVT = MVT::i16;
4970 else if (IntrData->Type == TRUNCATE_TO_MEM_VI32)
4971 ScalarVT = MVT::i32;
4972
4973 Info.memVT = MVT::getVectorVT(ScalarVT, VT.getVectorNumElements());
4974 Info.align = Align::None();
4975 Info.flags |= MachineMemOperand::MOStore;
4976 break;
4977 }
4978 case GATHER:
4979 case GATHER_AVX2: {
4980 Info.opc = ISD::INTRINSIC_W_CHAIN;
4981 Info.ptrVal = nullptr;
4982 MVT DataVT = MVT::getVT(I.getType());
4983 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
4984 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
4985 IndexVT.getVectorNumElements());
4986 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
4987 Info.align = Align::None();
4988 Info.flags |= MachineMemOperand::MOLoad;
4989 break;
4990 }
4991 case SCATTER: {
4992 Info.opc = ISD::INTRINSIC_VOID;
4993 Info.ptrVal = nullptr;
4994 MVT DataVT = MVT::getVT(I.getArgOperand(3)->getType());
4995 MVT IndexVT = MVT::getVT(I.getArgOperand(2)->getType());
4996 unsigned NumElts = std::min(DataVT.getVectorNumElements(),
4997 IndexVT.getVectorNumElements());
4998 Info.memVT = MVT::getVectorVT(DataVT.getVectorElementType(), NumElts);
4999 Info.align = Align::None();
5000 Info.flags |= MachineMemOperand::MOStore;
5001 break;
5002 }
5003 default:
5004 return false;
5005 }
5006
5007 return true;
5008}
5009
5010/// Returns true if the target can instruction select the
5011/// specified FP immediate natively. If false, the legalizer will
5012/// materialize the FP immediate as a load from a constant pool.
5013bool X86TargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
5014 bool ForCodeSize) const {
5015 for (unsigned i = 0, e = LegalFPImmediates.size(); i != e; ++i) {
5016 if (Imm.bitwiseIsEqual(LegalFPImmediates[i]))
5017 return true;
5018 }
5019 return false;
5020}
5021
5022bool X86TargetLowering::shouldReduceLoadWidth(SDNode *Load,
5023 ISD::LoadExtType ExtTy,
5024 EVT NewVT) const {
5025 assert(cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow")((cast<LoadSDNode>(Load)->isSimple() && "illegal to narrow"
) ? static_cast<void> (0) : __assert_fail ("cast<LoadSDNode>(Load)->isSimple() && \"illegal to narrow\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5025, __PRETTY_FUNCTION__));
5026
5027 // "ELF Handling for Thread-Local Storage" specifies that R_X86_64_GOTTPOFF
5028 // relocation target a movq or addq instruction: don't let the load shrink.
5029 SDValue BasePtr = cast<LoadSDNode>(Load)->getBasePtr();
5030 if (BasePtr.getOpcode() == X86ISD::WrapperRIP)
5031 if (const auto *GA = dyn_cast<GlobalAddressSDNode>(BasePtr.getOperand(0)))
5032 return GA->getTargetFlags() != X86II::MO_GOTTPOFF;
5033
5034 // If this is an (1) AVX vector load with (2) multiple uses and (3) all of
5035 // those uses are extracted directly into a store, then the extract + store
5036 // can be store-folded. Therefore, it's probably not worth splitting the load.
5037 EVT VT = Load->getValueType(0);
5038 if ((VT.is256BitVector() || VT.is512BitVector()) && !Load->hasOneUse()) {
5039 for (auto UI = Load->use_begin(), UE = Load->use_end(); UI != UE; ++UI) {
5040 // Skip uses of the chain value. Result 0 of the node is the load value.
5041 if (UI.getUse().getResNo() != 0)
5042 continue;
5043
5044 // If this use is not an extract + store, it's probably worth splitting.
5045 if (UI->getOpcode() != ISD::EXTRACT_SUBVECTOR || !UI->hasOneUse() ||
5046 UI->use_begin()->getOpcode() != ISD::STORE)
5047 return true;
5048 }
5049 // All non-chain uses are extract + store.
5050 return false;
5051 }
5052
5053 return true;
5054}
5055
5056/// Returns true if it is beneficial to convert a load of a constant
5057/// to just the constant itself.
5058bool X86TargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
5059 Type *Ty) const {
5060 assert(Ty->isIntegerTy())((Ty->isIntegerTy()) ? static_cast<void> (0) : __assert_fail
("Ty->isIntegerTy()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5060, __PRETTY_FUNCTION__));
5061
5062 unsigned BitSize = Ty->getPrimitiveSizeInBits();
5063 if (BitSize == 0 || BitSize > 64)
5064 return false;
5065 return true;
5066}
5067
5068bool X86TargetLowering::reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
5069 // If we are using XMM registers in the ABI and the condition of the select is
5070 // a floating-point compare and we have blendv or conditional move, then it is
5071 // cheaper to select instead of doing a cross-register move and creating a
5072 // load that depends on the compare result.
5073 bool IsFPSetCC = CmpOpVT.isFloatingPoint() && CmpOpVT != MVT::f128;
5074 return !IsFPSetCC || !Subtarget.isTarget64BitLP64() || !Subtarget.hasAVX();
5075}
5076
5077bool X86TargetLowering::convertSelectOfConstantsToMath(EVT VT) const {
5078 // TODO: It might be a win to ease or lift this restriction, but the generic
5079 // folds in DAGCombiner conflict with vector folds for an AVX512 target.
5080 if (VT.isVector() && Subtarget.hasAVX512())
5081 return false;
5082
5083 return true;
5084}
5085
5086bool X86TargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
5087 SDValue C) const {
5088 // TODO: We handle scalars using custom code, but generic combining could make
5089 // that unnecessary.
5090 APInt MulC;
5091 if (!ISD::isConstantSplatVector(C.getNode(), MulC))
5092 return false;
5093
5094 // Find the type this will be legalized too. Otherwise we might prematurely
5095 // convert this to shl+add/sub and then still have to type legalize those ops.
5096 // Another choice would be to defer the decision for illegal types until
5097 // after type legalization. But constant splat vectors of i64 can't make it
5098 // through type legalization on 32-bit targets so we would need to special
5099 // case vXi64.
5100 while (getTypeAction(Context, VT) != TypeLegal)
5101 VT = getTypeToTransformTo(Context, VT);
5102
5103 // If vector multiply is legal, assume that's faster than shl + add/sub.
5104 // TODO: Multiply is a complex op with higher latency and lower throughput in
5105 // most implementations, so this check could be loosened based on type
5106 // and/or a CPU attribute.
5107 if (isOperationLegal(ISD::MUL, VT))
5108 return false;
5109
5110 // shl+add, shl+sub, shl+add+neg
5111 return (MulC + 1).isPowerOf2() || (MulC - 1).isPowerOf2() ||
5112 (1 - MulC).isPowerOf2() || (-(MulC + 1)).isPowerOf2();
5113}
5114
5115bool X86TargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
5116 unsigned Index) const {
5117 if (!isOperationLegalOrCustom(ISD::EXTRACT_SUBVECTOR, ResVT))
5118 return false;
5119
5120 // Mask vectors support all subregister combinations and operations that
5121 // extract half of vector.
5122 if (ResVT.getVectorElementType() == MVT::i1)
5123 return Index == 0 || ((ResVT.getSizeInBits() == SrcVT.getSizeInBits()*2) &&
5124 (Index == ResVT.getVectorNumElements()));
5125
5126 return (Index % ResVT.getVectorNumElements()) == 0;
5127}
5128
5129bool X86TargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
5130 unsigned Opc = VecOp.getOpcode();
5131
5132 // Assume target opcodes can't be scalarized.
5133 // TODO - do we have any exceptions?
5134 if (Opc >= ISD::BUILTIN_OP_END)
5135 return false;
5136
5137 // If the vector op is not supported, try to convert to scalar.
5138 EVT VecVT = VecOp.getValueType();
5139 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
5140 return true;
5141
5142 // If the vector op is supported, but the scalar op is not, the transform may
5143 // not be worthwhile.
5144 EVT ScalarVT = VecVT.getScalarType();
5145 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
5146}
5147
5148bool X86TargetLowering::shouldFormOverflowOp(unsigned Opcode, EVT VT) const {
5149 // TODO: Allow vectors?
5150 if (VT.isVector())
5151 return false;
5152 return VT.isSimple() || !isOperationExpand(Opcode, VT);
5153}
5154
5155bool X86TargetLowering::isCheapToSpeculateCttz() const {
5156 // Speculate cttz only if we can directly use TZCNT.
5157 return Subtarget.hasBMI();
5158}
5159
5160bool X86TargetLowering::isCheapToSpeculateCtlz() const {
5161 // Speculate ctlz only if we can directly use LZCNT.
5162 return Subtarget.hasLZCNT();
5163}
5164
5165bool X86TargetLowering::isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
5166 const SelectionDAG &DAG,
5167 const MachineMemOperand &MMO) const {
5168 if (!Subtarget.hasAVX512() && !LoadVT.isVector() && BitcastVT.isVector() &&
5169 BitcastVT.getVectorElementType() == MVT::i1)
5170 return false;
5171
5172 if (!Subtarget.hasDQI() && BitcastVT == MVT::v8i1 && LoadVT == MVT::i8)
5173 return false;
5174
5175 // If both types are legal vectors, it's always ok to convert them.
5176 if (LoadVT.isVector() && BitcastVT.isVector() &&
5177 isTypeLegal(LoadVT) && isTypeLegal(BitcastVT))
5178 return true;
5179
5180 return TargetLowering::isLoadBitCastBeneficial(LoadVT, BitcastVT, DAG, MMO);
5181}
5182
5183bool X86TargetLowering::canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
5184 const SelectionDAG &DAG) const {
5185 // Do not merge to float value size (128 bytes) if no implicit
5186 // float attribute is set.
5187 bool NoFloat = DAG.getMachineFunction().getFunction().hasFnAttribute(
5188 Attribute::NoImplicitFloat);
5189
5190 if (NoFloat) {
5191 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
5192 return (MemVT.getSizeInBits() <= MaxIntSize);
5193 }
5194 // Make sure we don't merge greater than our preferred vector
5195 // width.
5196 if (MemVT.getSizeInBits() > Subtarget.getPreferVectorWidth())
5197 return false;
5198 return true;
5199}
5200
5201bool X86TargetLowering::isCtlzFast() const {
5202 return Subtarget.hasFastLZCNT();
5203}
5204
5205bool X86TargetLowering::isMaskAndCmp0FoldingBeneficial(
5206 const Instruction &AndI) const {
5207 return true;
5208}
5209
5210bool X86TargetLowering::hasAndNotCompare(SDValue Y) const {
5211 EVT VT = Y.getValueType();
5212
5213 if (VT.isVector())
5214 return false;
5215
5216 if (!Subtarget.hasBMI())
5217 return false;
5218
5219 // There are only 32-bit and 64-bit forms for 'andn'.
5220 if (VT != MVT::i32 && VT != MVT::i64)
5221 return false;
5222
5223 return !isa<ConstantSDNode>(Y);
5224}
5225
5226bool X86TargetLowering::hasAndNot(SDValue Y) const {
5227 EVT VT = Y.getValueType();
5228
5229 if (!VT.isVector())
5230 return hasAndNotCompare(Y);
5231
5232 // Vector.
5233
5234 if (!Subtarget.hasSSE1() || VT.getSizeInBits() < 128)
5235 return false;
5236
5237 if (VT == MVT::v4i32)
5238 return true;
5239
5240 return Subtarget.hasSSE2();
5241}
5242
5243bool X86TargetLowering::hasBitTest(SDValue X, SDValue Y) const {
5244 return X.getValueType().isScalarInteger(); // 'bt'
5245}
5246
5247bool X86TargetLowering::
5248 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
5249 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
5250 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
5251 SelectionDAG &DAG) const {
5252 // Does baseline recommend not to perform the fold by default?
5253 if (!TargetLowering::shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
5254 X, XC, CC, Y, OldShiftOpcode, NewShiftOpcode, DAG))
5255 return false;
5256 // For scalars this transform is always beneficial.
5257 if (X.getValueType().isScalarInteger())
5258 return true;
5259 // If all the shift amounts are identical, then transform is beneficial even
5260 // with rudimentary SSE2 shifts.
5261 if (DAG.isSplatValue(Y, /*AllowUndefs=*/true))
5262 return true;
5263 // If we have AVX2 with it's powerful shift operations, then it's also good.
5264 if (Subtarget.hasAVX2())
5265 return true;
5266 // Pre-AVX2 vector codegen for this pattern is best for variant with 'shl'.
5267 return NewShiftOpcode == ISD::SHL;
5268}
5269
5270bool X86TargetLowering::shouldFoldConstantShiftPairToMask(
5271 const SDNode *N, CombineLevel Level) const {
5272 assert(((N->getOpcode() == ISD::SHL &&((((N->getOpcode() == ISD::SHL && N->getOperand
(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL
&& N->getOperand(0).getOpcode() == ISD::SHL)) &&
"Expected shift-shift mask") ? static_cast<void> (0) :
__assert_fail ("((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && N->getOperand(0).getOpcode() == ISD::SHL)) && \"Expected shift-shift mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5276, __PRETTY_FUNCTION__))
5273 N->getOperand(0).getOpcode() == ISD::SRL) ||((((N->getOpcode() == ISD::SHL && N->getOperand
(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL
&& N->getOperand(0).getOpcode() == ISD::SHL)) &&
"Expected shift-shift mask") ? static_cast<void> (0) :
__assert_fail ("((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && N->getOperand(0).getOpcode() == ISD::SHL)) && \"Expected shift-shift mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5276, __PRETTY_FUNCTION__))
5274 (N->getOpcode() == ISD::SRL &&((((N->getOpcode() == ISD::SHL && N->getOperand
(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL
&& N->getOperand(0).getOpcode() == ISD::SHL)) &&
"Expected shift-shift mask") ? static_cast<void> (0) :
__assert_fail ("((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && N->getOperand(0).getOpcode() == ISD::SHL)) && \"Expected shift-shift mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5276, __PRETTY_FUNCTION__))
5275 N->getOperand(0).getOpcode() == ISD::SHL)) &&((((N->getOpcode() == ISD::SHL && N->getOperand
(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL
&& N->getOperand(0).getOpcode() == ISD::SHL)) &&
"Expected shift-shift mask") ? static_cast<void> (0) :
__assert_fail ("((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && N->getOperand(0).getOpcode() == ISD::SHL)) && \"Expected shift-shift mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5276, __PRETTY_FUNCTION__))
5276 "Expected shift-shift mask")((((N->getOpcode() == ISD::SHL && N->getOperand
(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL
&& N->getOperand(0).getOpcode() == ISD::SHL)) &&
"Expected shift-shift mask") ? static_cast<void> (0) :
__assert_fail ("((N->getOpcode() == ISD::SHL && N->getOperand(0).getOpcode() == ISD::SRL) || (N->getOpcode() == ISD::SRL && N->getOperand(0).getOpcode() == ISD::SHL)) && \"Expected shift-shift mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5276, __PRETTY_FUNCTION__));
5277 EVT VT = N->getValueType(0);
5278 if ((Subtarget.hasFastVectorShiftMasks() && VT.isVector()) ||
5279 (Subtarget.hasFastScalarShiftMasks() && !VT.isVector())) {
5280 // Only fold if the shift values are equal - so it folds to AND.
5281 // TODO - we should fold if either is a non-uniform vector but we don't do
5282 // the fold for non-splats yet.
5283 return N->getOperand(1) == N->getOperand(0).getOperand(1);
5284 }
5285 return TargetLoweringBase::shouldFoldConstantShiftPairToMask(N, Level);
5286}
5287
5288bool X86TargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
5289 EVT VT = Y.getValueType();
5290
5291 // For vectors, we don't have a preference, but we probably want a mask.
5292 if (VT.isVector())
5293 return false;
5294
5295 // 64-bit shifts on 32-bit targets produce really bad bloated code.
5296 if (VT == MVT::i64 && !Subtarget.is64Bit())
5297 return false;
5298
5299 return true;
5300}
5301
5302bool X86TargetLowering::shouldExpandShift(SelectionDAG &DAG,
5303 SDNode *N) const {
5304 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
5305 !Subtarget.isOSWindows())
5306 return false;
5307 return true;
5308}
5309
5310bool X86TargetLowering::shouldSplatInsEltVarIndex(EVT VT) const {
5311 // Any legal vector type can be splatted more efficiently than
5312 // loading/spilling from memory.
5313 return isTypeLegal(VT);
5314}
5315
5316MVT X86TargetLowering::hasFastEqualityCompare(unsigned NumBits) const {
5317 MVT VT = MVT::getIntegerVT(NumBits);
5318 if (isTypeLegal(VT))
5319 return VT;
5320
5321 // PMOVMSKB can handle this.
5322 if (NumBits == 128 && isTypeLegal(MVT::v16i8))
5323 return MVT::v16i8;
5324
5325 // VPMOVMSKB can handle this.
5326 if (NumBits == 256 && isTypeLegal(MVT::v32i8))
5327 return MVT::v32i8;
5328
5329 // TODO: Allow 64-bit type for 32-bit target.
5330 // TODO: 512-bit types should be allowed, but make sure that those
5331 // cases are handled in combineVectorSizedSetCCEquality().
5332
5333 return MVT::INVALID_SIMPLE_VALUE_TYPE;
5334}
5335
5336/// Val is the undef sentinel value or equal to the specified value.
5337static bool isUndefOrEqual(int Val, int CmpVal) {
5338 return ((Val == SM_SentinelUndef) || (Val == CmpVal));
5339}
5340
5341/// Val is either the undef or zero sentinel value.
5342static bool isUndefOrZero(int Val) {
5343 return ((Val == SM_SentinelUndef) || (Val == SM_SentinelZero));
5344}
5345
5346/// Return true if every element in Mask, beginning from position Pos and ending
5347/// in Pos+Size is the undef sentinel value.
5348static bool isUndefInRange(ArrayRef<int> Mask, unsigned Pos, unsigned Size) {
5349 return llvm::all_of(Mask.slice(Pos, Size),
5350 [](int M) { return M == SM_SentinelUndef; });
5351}
5352
5353/// Return true if the mask creates a vector whose lower half is undefined.
5354static bool isUndefLowerHalf(ArrayRef<int> Mask) {
5355 unsigned NumElts = Mask.size();
5356 return isUndefInRange(Mask, 0, NumElts / 2);
5357}
5358
5359/// Return true if the mask creates a vector whose upper half is undefined.
5360static bool isUndefUpperHalf(ArrayRef<int> Mask) {
5361 unsigned NumElts = Mask.size();
5362 return isUndefInRange(Mask, NumElts / 2, NumElts / 2);
5363}
5364
5365/// Return true if Val falls within the specified range (L, H].
5366static bool isInRange(int Val, int Low, int Hi) {
5367 return (Val >= Low && Val < Hi);
5368}
5369
5370/// Return true if the value of any element in Mask falls within the specified
5371/// range (L, H].
5372static bool isAnyInRange(ArrayRef<int> Mask, int Low, int Hi) {
5373 return llvm::any_of(Mask, [Low, Hi](int M) { return isInRange(M, Low, Hi); });
5374}
5375
5376/// Return true if Val is undef or if its value falls within the
5377/// specified range (L, H].
5378static bool isUndefOrInRange(int Val, int Low, int Hi) {
5379 return (Val == SM_SentinelUndef) || isInRange(Val, Low, Hi);
5380}
5381
5382/// Return true if every element in Mask is undef or if its value
5383/// falls within the specified range (L, H].
5384static bool isUndefOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
5385 return llvm::all_of(
5386 Mask, [Low, Hi](int M) { return isUndefOrInRange(M, Low, Hi); });
5387}
5388
5389/// Return true if Val is undef, zero or if its value falls within the
5390/// specified range (L, H].
5391static bool isUndefOrZeroOrInRange(int Val, int Low, int Hi) {
5392 return isUndefOrZero(Val) || isInRange(Val, Low, Hi);
5393}
5394
5395/// Return true if every element in Mask is undef, zero or if its value
5396/// falls within the specified range (L, H].
5397static bool isUndefOrZeroOrInRange(ArrayRef<int> Mask, int Low, int Hi) {
5398 return llvm::all_of(
5399 Mask, [Low, Hi](int M) { return isUndefOrZeroOrInRange(M, Low, Hi); });
5400}
5401
5402/// Return true if every element in Mask, beginning
5403/// from position Pos and ending in Pos + Size, falls within the specified
5404/// sequence (Low, Low + Step, ..., Low + (Size - 1) * Step) or is undef.
5405static bool isSequentialOrUndefInRange(ArrayRef<int> Mask, unsigned Pos,
5406 unsigned Size, int Low, int Step = 1) {
5407 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
5408 if (!isUndefOrEqual(Mask[i], Low))
5409 return false;
5410 return true;
5411}
5412
5413/// Return true if every element in Mask, beginning
5414/// from position Pos and ending in Pos+Size, falls within the specified
5415/// sequential range (Low, Low+Size], or is undef or is zero.
5416static bool isSequentialOrUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
5417 unsigned Size, int Low,
5418 int Step = 1) {
5419 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
5420 if (!isUndefOrZero(Mask[i]) && Mask[i] != Low)
5421 return false;
5422 return true;
5423}
5424
5425/// Return true if every element in Mask, beginning
5426/// from position Pos and ending in Pos+Size is undef or is zero.
5427static bool isUndefOrZeroInRange(ArrayRef<int> Mask, unsigned Pos,
5428 unsigned Size) {
5429 return llvm::all_of(Mask.slice(Pos, Size),
5430 [](int M) { return isUndefOrZero(M); });
5431}
5432
5433/// Helper function to test whether a shuffle mask could be
5434/// simplified by widening the elements being shuffled.
5435///
5436/// Appends the mask for wider elements in WidenedMask if valid. Otherwise
5437/// leaves it in an unspecified state.
5438///
5439/// NOTE: This must handle normal vector shuffle masks and *target* vector
5440/// shuffle masks. The latter have the special property of a '-2' representing
5441/// a zero-ed lane of a vector.
5442static bool canWidenShuffleElements(ArrayRef<int> Mask,
5443 SmallVectorImpl<int> &WidenedMask) {
5444 WidenedMask.assign(Mask.size() / 2, 0);
5445 for (int i = 0, Size = Mask.size(); i < Size; i += 2) {
5446 int M0 = Mask[i];
5447 int M1 = Mask[i + 1];
5448
5449 // If both elements are undef, its trivial.
5450 if (M0 == SM_SentinelUndef && M1 == SM_SentinelUndef) {
5451 WidenedMask[i / 2] = SM_SentinelUndef;
5452 continue;
5453 }
5454
5455 // Check for an undef mask and a mask value properly aligned to fit with
5456 // a pair of values. If we find such a case, use the non-undef mask's value.
5457 if (M0 == SM_SentinelUndef && M1 >= 0 && (M1 % 2) == 1) {
5458 WidenedMask[i / 2] = M1 / 2;
5459 continue;
5460 }
5461 if (M1 == SM_SentinelUndef && M0 >= 0 && (M0 % 2) == 0) {
5462 WidenedMask[i / 2] = M0 / 2;
5463 continue;
5464 }
5465
5466 // When zeroing, we need to spread the zeroing across both lanes to widen.
5467 if (M0 == SM_SentinelZero || M1 == SM_SentinelZero) {
5468 if ((M0 == SM_SentinelZero || M0 == SM_SentinelUndef) &&
5469 (M1 == SM_SentinelZero || M1 == SM_SentinelUndef)) {
5470 WidenedMask[i / 2] = SM_SentinelZero;
5471 continue;
5472 }
5473 return false;
5474 }
5475
5476 // Finally check if the two mask values are adjacent and aligned with
5477 // a pair.
5478 if (M0 != SM_SentinelUndef && (M0 % 2) == 0 && (M0 + 1) == M1) {
5479 WidenedMask[i / 2] = M0 / 2;
5480 continue;
5481 }
5482
5483 // Otherwise we can't safely widen the elements used in this shuffle.
5484 return false;
5485 }
5486 assert(WidenedMask.size() == Mask.size() / 2 &&((WidenedMask.size() == Mask.size() / 2 && "Incorrect size of mask after widening the elements!"
) ? static_cast<void> (0) : __assert_fail ("WidenedMask.size() == Mask.size() / 2 && \"Incorrect size of mask after widening the elements!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5487, __PRETTY_FUNCTION__))
5487 "Incorrect size of mask after widening the elements!")((WidenedMask.size() == Mask.size() / 2 && "Incorrect size of mask after widening the elements!"
) ? static_cast<void> (0) : __assert_fail ("WidenedMask.size() == Mask.size() / 2 && \"Incorrect size of mask after widening the elements!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5487, __PRETTY_FUNCTION__));
5488
5489 return true;
5490}
5491
5492static bool canWidenShuffleElements(ArrayRef<int> Mask,
5493 const APInt &Zeroable,
5494 bool V2IsZero,
5495 SmallVectorImpl<int> &WidenedMask) {
5496 // Create an alternative mask with info about zeroable elements.
5497 // Here we do not set undef elements as zeroable.
5498 SmallVector<int, 64> ZeroableMask(Mask.begin(), Mask.end());
5499 if (V2IsZero) {
5500 assert(!Zeroable.isNullValue() && "V2's non-undef elements are used?!")((!Zeroable.isNullValue() && "V2's non-undef elements are used?!"
) ? static_cast<void> (0) : __assert_fail ("!Zeroable.isNullValue() && \"V2's non-undef elements are used?!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5500, __PRETTY_FUNCTION__));
5501 for (int i = 0, Size = Mask.size(); i != Size; ++i)
5502 if (Mask[i] != SM_SentinelUndef && Zeroable[i])
5503 ZeroableMask[i] = SM_SentinelZero;
5504 }
5505 return canWidenShuffleElements(ZeroableMask, WidenedMask);
5506}
5507
5508static bool canWidenShuffleElements(ArrayRef<int> Mask) {
5509 SmallVector<int, 32> WidenedMask;
5510 return canWidenShuffleElements(Mask, WidenedMask);
5511}
5512
5513/// Returns true if Elt is a constant zero or a floating point constant +0.0.
5514bool X86::isZeroNode(SDValue Elt) {
5515 return isNullConstant(Elt) || isNullFPConstant(Elt);
5516}
5517
5518// Build a vector of constants.
5519// Use an UNDEF node if MaskElt == -1.
5520// Split 64-bit constants in the 32-bit mode.
5521static SDValue getConstVector(ArrayRef<int> Values, MVT VT, SelectionDAG &DAG,
5522 const SDLoc &dl, bool IsMask = false) {
5523
5524 SmallVector<SDValue, 32> Ops;
5525 bool Split = false;
5526
5527 MVT ConstVecVT = VT;
5528 unsigned NumElts = VT.getVectorNumElements();
5529 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
5530 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
5531 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
5532 Split = true;
5533 }
5534
5535 MVT EltVT = ConstVecVT.getVectorElementType();
5536 for (unsigned i = 0; i < NumElts; ++i) {
5537 bool IsUndef = Values[i] < 0 && IsMask;
5538 SDValue OpNode = IsUndef ? DAG.getUNDEF(EltVT) :
5539 DAG.getConstant(Values[i], dl, EltVT);
5540 Ops.push_back(OpNode);
5541 if (Split)
5542 Ops.push_back(IsUndef ? DAG.getUNDEF(EltVT) :
5543 DAG.getConstant(0, dl, EltVT));
5544 }
5545 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
5546 if (Split)
5547 ConstsNode = DAG.getBitcast(VT, ConstsNode);
5548 return ConstsNode;
5549}
5550
5551static SDValue getConstVector(ArrayRef<APInt> Bits, APInt &Undefs,
5552 MVT VT, SelectionDAG &DAG, const SDLoc &dl) {
5553 assert(Bits.size() == Undefs.getBitWidth() &&((Bits.size() == Undefs.getBitWidth() && "Unequal constant and undef arrays"
) ? static_cast<void> (0) : __assert_fail ("Bits.size() == Undefs.getBitWidth() && \"Unequal constant and undef arrays\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5554, __PRETTY_FUNCTION__))
5554 "Unequal constant and undef arrays")((Bits.size() == Undefs.getBitWidth() && "Unequal constant and undef arrays"
) ? static_cast<void> (0) : __assert_fail ("Bits.size() == Undefs.getBitWidth() && \"Unequal constant and undef arrays\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5554, __PRETTY_FUNCTION__));
5555 SmallVector<SDValue, 32> Ops;
5556 bool Split = false;
5557
5558 MVT ConstVecVT = VT;
5559 unsigned NumElts = VT.getVectorNumElements();
5560 bool In64BitMode = DAG.getTargetLoweringInfo().isTypeLegal(MVT::i64);
5561 if (!In64BitMode && VT.getVectorElementType() == MVT::i64) {
5562 ConstVecVT = MVT::getVectorVT(MVT::i32, NumElts * 2);
5563 Split = true;
5564 }
5565
5566 MVT EltVT = ConstVecVT.getVectorElementType();
5567 for (unsigned i = 0, e = Bits.size(); i != e; ++i) {
5568 if (Undefs[i]) {
5569 Ops.append(Split ? 2 : 1, DAG.getUNDEF(EltVT));
5570 continue;
5571 }
5572 const APInt &V = Bits[i];
5573 assert(V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes")((V.getBitWidth() == VT.getScalarSizeInBits() && "Unexpected sizes"
) ? static_cast<void> (0) : __assert_fail ("V.getBitWidth() == VT.getScalarSizeInBits() && \"Unexpected sizes\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5573, __PRETTY_FUNCTION__));
5574 if (Split) {
5575 Ops.push_back(DAG.getConstant(V.trunc(32), dl, EltVT));
5576 Ops.push_back(DAG.getConstant(V.lshr(32).trunc(32), dl, EltVT));
5577 } else if (EltVT == MVT::f32) {
5578 APFloat FV(APFloat::IEEEsingle(), V);
5579 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
5580 } else if (EltVT == MVT::f64) {
5581 APFloat FV(APFloat::IEEEdouble(), V);
5582 Ops.push_back(DAG.getConstantFP(FV, dl, EltVT));
5583 } else {
5584 Ops.push_back(DAG.getConstant(V, dl, EltVT));
5585 }
5586 }
5587
5588 SDValue ConstsNode = DAG.getBuildVector(ConstVecVT, dl, Ops);
5589 return DAG.getBitcast(VT, ConstsNode);
5590}
5591
5592/// Returns a vector of specified type with all zero elements.
5593static SDValue getZeroVector(MVT VT, const X86Subtarget &Subtarget,
5594 SelectionDAG &DAG, const SDLoc &dl) {
5595 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() ||(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
() || VT.getVectorElementType() == MVT::i1) && "Unexpected vector type"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() || VT.getVectorElementType() == MVT::i1) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5597, __PRETTY_FUNCTION__))
5596 VT.getVectorElementType() == MVT::i1) &&(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
() || VT.getVectorElementType() == MVT::i1) && "Unexpected vector type"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() || VT.getVectorElementType() == MVT::i1) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5597, __PRETTY_FUNCTION__))
5597 "Unexpected vector type")(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
() || VT.getVectorElementType() == MVT::i1) && "Unexpected vector type"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector() || VT.getVectorElementType() == MVT::i1) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5597, __PRETTY_FUNCTION__));
5598
5599 // Try to build SSE/AVX zero vectors as <N x i32> bitcasted to their dest
5600 // type. This ensures they get CSE'd. But if the integer type is not
5601 // available, use a floating-point +0.0 instead.
5602 SDValue Vec;
5603 if (!Subtarget.hasSSE2() && VT.is128BitVector()) {
5604 Vec = DAG.getConstantFP(+0.0, dl, MVT::v4f32);
5605 } else if (VT.isFloatingPoint()) {
5606 Vec = DAG.getConstantFP(+0.0, dl, VT);
5607 } else if (VT.getVectorElementType() == MVT::i1) {
5608 assert((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&(((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&
"Unexpected vector type") ? static_cast<void> (0) : __assert_fail
("(Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5609, __PRETTY_FUNCTION__))
5609 "Unexpected vector type")(((Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) &&
"Unexpected vector type") ? static_cast<void> (0) : __assert_fail
("(Subtarget.hasBWI() || VT.getVectorNumElements() <= 16) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5609, __PRETTY_FUNCTION__));
5610 Vec = DAG.getConstant(0, dl, VT);
5611 } else {
5612 unsigned Num32BitElts = VT.getSizeInBits() / 32;
5613 Vec = DAG.getConstant(0, dl, MVT::getVectorVT(MVT::i32, Num32BitElts));
5614 }
5615 return DAG.getBitcast(VT, Vec);
5616}
5617
5618static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
5619 const SDLoc &dl, unsigned vectorWidth) {
5620 EVT VT = Vec.getValueType();
5621 EVT ElVT = VT.getVectorElementType();
5622 unsigned Factor = VT.getSizeInBits()/vectorWidth;
5623 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
5624 VT.getVectorNumElements()/Factor);
5625
5626 // Extract the relevant vectorWidth bits. Generate an EXTRACT_SUBVECTOR
5627 unsigned ElemsPerChunk = vectorWidth / ElVT.getSizeInBits();
5628 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2")((isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ElemsPerChunk) && \"Elements per chunk not power of 2\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5628, __PRETTY_FUNCTION__));
5629
5630 // This is the index of the first element of the vectorWidth-bit chunk
5631 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
5632 IdxVal &= ~(ElemsPerChunk - 1);
5633
5634 // If the input is a buildvector just emit a smaller one.
5635 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
5636 return DAG.getBuildVector(ResultVT, dl,
5637 Vec->ops().slice(IdxVal, ElemsPerChunk));
5638
5639 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
5640 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResultVT, Vec, VecIdx);
5641}
5642
5643/// Generate a DAG to grab 128-bits from a vector > 128 bits. This
5644/// sets things up to match to an AVX VEXTRACTF128 / VEXTRACTI128
5645/// or AVX-512 VEXTRACTF32x4 / VEXTRACTI32x4
5646/// instructions or a simple subregister reference. Idx is an index in the
5647/// 128 bits we want. It need not be aligned to a 128-bit boundary. That makes
5648/// lowering EXTRACT_VECTOR_ELT operations easier.
5649static SDValue extract128BitVector(SDValue Vec, unsigned IdxVal,
5650 SelectionDAG &DAG, const SDLoc &dl) {
5651 assert((Vec.getValueType().is256BitVector() ||(((Vec.getValueType().is256BitVector() || Vec.getValueType().
is512BitVector()) && "Unexpected vector size!") ? static_cast
<void> (0) : __assert_fail ("(Vec.getValueType().is256BitVector() || Vec.getValueType().is512BitVector()) && \"Unexpected vector size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5652, __PRETTY_FUNCTION__))
5652 Vec.getValueType().is512BitVector()) && "Unexpected vector size!")(((Vec.getValueType().is256BitVector() || Vec.getValueType().
is512BitVector()) && "Unexpected vector size!") ? static_cast
<void> (0) : __assert_fail ("(Vec.getValueType().is256BitVector() || Vec.getValueType().is512BitVector()) && \"Unexpected vector size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5652, __PRETTY_FUNCTION__));
5653 return extractSubVector(Vec, IdxVal, DAG, dl, 128);
5654}
5655
5656/// Generate a DAG to grab 256-bits from a 512-bit vector.
5657static SDValue extract256BitVector(SDValue Vec, unsigned IdxVal,
5658 SelectionDAG &DAG, const SDLoc &dl) {
5659 assert(Vec.getValueType().is512BitVector() && "Unexpected vector size!")((Vec.getValueType().is512BitVector() && "Unexpected vector size!"
) ? static_cast<void> (0) : __assert_fail ("Vec.getValueType().is512BitVector() && \"Unexpected vector size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5659, __PRETTY_FUNCTION__));
5660 return extractSubVector(Vec, IdxVal, DAG, dl, 256);
5661}
5662
5663static SDValue insertSubVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5664 SelectionDAG &DAG, const SDLoc &dl,
5665 unsigned vectorWidth) {
5666 assert((vectorWidth == 128 || vectorWidth == 256) &&(((vectorWidth == 128 || vectorWidth == 256) && "Unsupported vector width"
) ? static_cast<void> (0) : __assert_fail ("(vectorWidth == 128 || vectorWidth == 256) && \"Unsupported vector width\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5667, __PRETTY_FUNCTION__))
5667 "Unsupported vector width")(((vectorWidth == 128 || vectorWidth == 256) && "Unsupported vector width"
) ? static_cast<void> (0) : __assert_fail ("(vectorWidth == 128 || vectorWidth == 256) && \"Unsupported vector width\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5667, __PRETTY_FUNCTION__));
5668 // Inserting UNDEF is Result
5669 if (Vec.isUndef())
5670 return Result;
5671 EVT VT = Vec.getValueType();
5672 EVT ElVT = VT.getVectorElementType();
5673 EVT ResultVT = Result.getValueType();
5674
5675 // Insert the relevant vectorWidth bits.
5676 unsigned ElemsPerChunk = vectorWidth/ElVT.getSizeInBits();
5677 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2")((isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ElemsPerChunk) && \"Elements per chunk not power of 2\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5677, __PRETTY_FUNCTION__));
5678
5679 // This is the index of the first element of the vectorWidth-bit chunk
5680 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
5681 IdxVal &= ~(ElemsPerChunk - 1);
5682
5683 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, dl);
5684 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResultVT, Result, Vec, VecIdx);
5685}
5686
5687/// Generate a DAG to put 128-bits into a vector > 128 bits. This
5688/// sets things up to match to an AVX VINSERTF128/VINSERTI128 or
5689/// AVX-512 VINSERTF32x4/VINSERTI32x4 instructions or a
5690/// simple superregister reference. Idx is an index in the 128 bits
5691/// we want. It need not be aligned to a 128-bit boundary. That makes
5692/// lowering INSERT_VECTOR_ELT operations easier.
5693static SDValue insert128BitVector(SDValue Result, SDValue Vec, unsigned IdxVal,
5694 SelectionDAG &DAG, const SDLoc &dl) {
5695 assert(Vec.getValueType().is128BitVector() && "Unexpected vector size!")((Vec.getValueType().is128BitVector() && "Unexpected vector size!"
) ? static_cast<void> (0) : __assert_fail ("Vec.getValueType().is128BitVector() && \"Unexpected vector size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5695, __PRETTY_FUNCTION__));
5696 return insertSubVector(Result, Vec, IdxVal, DAG, dl, 128);
5697}
5698
5699/// Widen a vector to a larger size with the same scalar type, with the new
5700/// elements either zero or undef.
5701static SDValue widenSubVector(MVT VT, SDValue Vec, bool ZeroNewElements,
5702 const X86Subtarget &Subtarget, SelectionDAG &DAG,
5703 const SDLoc &dl) {
5704 assert(Vec.getValueSizeInBits() < VT.getSizeInBits() &&((Vec.getValueSizeInBits() < VT.getSizeInBits() &&
Vec.getValueType().getScalarType() == VT.getScalarType() &&
"Unsupported vector widening type") ? static_cast<void>
(0) : __assert_fail ("Vec.getValueSizeInBits() < VT.getSizeInBits() && Vec.getValueType().getScalarType() == VT.getScalarType() && \"Unsupported vector widening type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5706, __PRETTY_FUNCTION__))
5705 Vec.getValueType().getScalarType() == VT.getScalarType() &&((Vec.getValueSizeInBits() < VT.getSizeInBits() &&
Vec.getValueType().getScalarType() == VT.getScalarType() &&
"Unsupported vector widening type") ? static_cast<void>
(0) : __assert_fail ("Vec.getValueSizeInBits() < VT.getSizeInBits() && Vec.getValueType().getScalarType() == VT.getScalarType() && \"Unsupported vector widening type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5706, __PRETTY_FUNCTION__))
5706 "Unsupported vector widening type")((Vec.getValueSizeInBits() < VT.getSizeInBits() &&
Vec.getValueType().getScalarType() == VT.getScalarType() &&
"Unsupported vector widening type") ? static_cast<void>
(0) : __assert_fail ("Vec.getValueSizeInBits() < VT.getSizeInBits() && Vec.getValueType().getScalarType() == VT.getScalarType() && \"Unsupported vector widening type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5706, __PRETTY_FUNCTION__));
5707 SDValue Res = ZeroNewElements ? getZeroVector(VT, Subtarget, DAG, dl)
5708 : DAG.getUNDEF(VT);
5709 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT, Res, Vec,
5710 DAG.getIntPtrConstant(0, dl));
5711}
5712
5713/// Widen a vector to a larger size with the same scalar type, with the new
5714/// elements either zero or undef.
5715static SDValue widenSubVector(SDValue Vec, bool ZeroNewElements,
5716 const X86Subtarget &Subtarget, SelectionDAG &DAG,
5717 const SDLoc &dl, unsigned WideSizeInBits) {
5718 assert(Vec.getValueSizeInBits() < WideSizeInBits &&((Vec.getValueSizeInBits() < WideSizeInBits && (WideSizeInBits
% Vec.getScalarValueSizeInBits()) == 0 && "Unsupported vector widening type"
) ? static_cast<void> (0) : __assert_fail ("Vec.getValueSizeInBits() < WideSizeInBits && (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 && \"Unsupported vector widening type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5720, __PRETTY_FUNCTION__))
5719 (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 &&((Vec.getValueSizeInBits() < WideSizeInBits && (WideSizeInBits
% Vec.getScalarValueSizeInBits()) == 0 && "Unsupported vector widening type"
) ? static_cast<void> (0) : __assert_fail ("Vec.getValueSizeInBits() < WideSizeInBits && (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 && \"Unsupported vector widening type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5720, __PRETTY_FUNCTION__))
5720 "Unsupported vector widening type")((Vec.getValueSizeInBits() < WideSizeInBits && (WideSizeInBits
% Vec.getScalarValueSizeInBits()) == 0 && "Unsupported vector widening type"
) ? static_cast<void> (0) : __assert_fail ("Vec.getValueSizeInBits() < WideSizeInBits && (WideSizeInBits % Vec.getScalarValueSizeInBits()) == 0 && \"Unsupported vector widening type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5720, __PRETTY_FUNCTION__));
5721 unsigned WideNumElts = WideSizeInBits / Vec.getScalarValueSizeInBits();
5722 MVT SVT = Vec.getSimpleValueType().getScalarType();
5723 MVT VT = MVT::getVectorVT(SVT, WideNumElts);
5724 return widenSubVector(VT, Vec, ZeroNewElements, Subtarget, DAG, dl);
5725}
5726
5727// Helper function to collect subvector ops that are concated together,
5728// either by ISD::CONCAT_VECTORS or a ISD::INSERT_SUBVECTOR series.
5729// The subvectors in Ops are guaranteed to be the same type.
5730static bool collectConcatOps(SDNode *N, SmallVectorImpl<SDValue> &Ops) {
5731 assert(Ops.empty() && "Expected an empty ops vector")((Ops.empty() && "Expected an empty ops vector") ? static_cast
<void> (0) : __assert_fail ("Ops.empty() && \"Expected an empty ops vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5731, __PRETTY_FUNCTION__));
5732
5733 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
5734 Ops.append(N->op_begin(), N->op_end());
5735 return true;
5736 }
5737
5738 if (N->getOpcode() == ISD::INSERT_SUBVECTOR &&
5739 isa<ConstantSDNode>(N->getOperand(2))) {
5740 SDValue Src = N->getOperand(0);
5741 SDValue Sub = N->getOperand(1);
5742 const APInt &Idx = N->getConstantOperandAPInt(2);
5743 EVT VT = Src.getValueType();
5744 EVT SubVT = Sub.getValueType();
5745
5746 // TODO - Handle more general insert_subvector chains.
5747 if (VT.getSizeInBits() == (SubVT.getSizeInBits() * 2) &&
5748 Idx == (VT.getVectorNumElements() / 2) &&
5749 Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
5750 Src.getOperand(1).getValueType() == SubVT &&
5751 isNullConstant(Src.getOperand(2))) {
5752 Ops.push_back(Src.getOperand(1));
5753 Ops.push_back(Sub);
5754 return true;
5755 }
5756 }
5757
5758 return false;
5759}
5760
5761// Helper for splitting operands of an operation to legal target size and
5762// apply a function on each part.
5763// Useful for operations that are available on SSE2 in 128-bit, on AVX2 in
5764// 256-bit and on AVX512BW in 512-bit. The argument VT is the type used for
5765// deciding if/how to split Ops. Ops elements do *not* have to be of type VT.
5766// The argument Builder is a function that will be applied on each split part:
5767// SDValue Builder(SelectionDAG&G, SDLoc, ArrayRef<SDValue>)
5768template <typename F>
5769SDValue SplitOpsAndApply(SelectionDAG &DAG, const X86Subtarget &Subtarget,
5770 const SDLoc &DL, EVT VT, ArrayRef<SDValue> Ops,
5771 F Builder, bool CheckBWI = true) {
5772 assert(Subtarget.hasSSE2() && "Target assumed to support at least SSE2")((Subtarget.hasSSE2() && "Target assumed to support at least SSE2"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Target assumed to support at least SSE2\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5772, __PRETTY_FUNCTION__));
5773 unsigned NumSubs = 1;
5774 if ((CheckBWI && Subtarget.useBWIRegs()) ||
5775 (!CheckBWI && Subtarget.useAVX512Regs())) {
5776 if (VT.getSizeInBits() > 512) {
5777 NumSubs = VT.getSizeInBits() / 512;
5778 assert((VT.getSizeInBits() % 512) == 0 && "Illegal vector size")(((VT.getSizeInBits() % 512) == 0 && "Illegal vector size"
) ? static_cast<void> (0) : __assert_fail ("(VT.getSizeInBits() % 512) == 0 && \"Illegal vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5778, __PRETTY_FUNCTION__));
5779 }
5780 } else if (Subtarget.hasAVX2()) {
5781 if (VT.getSizeInBits() > 256) {
5782 NumSubs = VT.getSizeInBits() / 256;
5783 assert((VT.getSizeInBits() % 256) == 0 && "Illegal vector size")(((VT.getSizeInBits() % 256) == 0 && "Illegal vector size"
) ? static_cast<void> (0) : __assert_fail ("(VT.getSizeInBits() % 256) == 0 && \"Illegal vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5783, __PRETTY_FUNCTION__));
5784 }
5785 } else {
5786 if (VT.getSizeInBits() > 128) {
5787 NumSubs = VT.getSizeInBits() / 128;
5788 assert((VT.getSizeInBits() % 128) == 0 && "Illegal vector size")(((VT.getSizeInBits() % 128) == 0 && "Illegal vector size"
) ? static_cast<void> (0) : __assert_fail ("(VT.getSizeInBits() % 128) == 0 && \"Illegal vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5788, __PRETTY_FUNCTION__));
5789 }
5790 }
5791
5792 if (NumSubs == 1)
5793 return Builder(DAG, DL, Ops);
5794
5795 SmallVector<SDValue, 4> Subs;
5796 for (unsigned i = 0; i != NumSubs; ++i) {
5797 SmallVector<SDValue, 2> SubOps;
5798 for (SDValue Op : Ops) {
5799 EVT OpVT = Op.getValueType();
5800 unsigned NumSubElts = OpVT.getVectorNumElements() / NumSubs;
5801 unsigned SizeSub = OpVT.getSizeInBits() / NumSubs;
5802 SubOps.push_back(extractSubVector(Op, i * NumSubElts, DAG, DL, SizeSub));
5803 }
5804 Subs.push_back(Builder(DAG, DL, SubOps));
5805 }
5806 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Subs);
5807}
5808
5809/// Insert i1-subvector to i1-vector.
5810static SDValue insert1BitVector(SDValue Op, SelectionDAG &DAG,
5811 const X86Subtarget &Subtarget) {
5812
5813 SDLoc dl(Op);
5814 SDValue Vec = Op.getOperand(0);
5815 SDValue SubVec = Op.getOperand(1);
5816 SDValue Idx = Op.getOperand(2);
5817
5818 if (!isa<ConstantSDNode>(Idx))
5819 return SDValue();
5820
5821 // Inserting undef is a nop. We can just return the original vector.
5822 if (SubVec.isUndef())
5823 return Vec;
5824
5825 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
5826 if (IdxVal == 0 && Vec.isUndef()) // the operation is legal
5827 return Op;
5828
5829 MVT OpVT = Op.getSimpleValueType();
5830 unsigned NumElems = OpVT.getVectorNumElements();
5831
5832 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
5833
5834 // Extend to natively supported kshift.
5835 MVT WideOpVT = OpVT;
5836 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8)
5837 WideOpVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
5838
5839 // Inserting into the lsbs of a zero vector is legal. ISel will insert shifts
5840 // if necessary.
5841 if (IdxVal == 0 && ISD::isBuildVectorAllZeros(Vec.getNode())) {
5842 // May need to promote to a legal type.
5843 Op = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
5844 DAG.getConstant(0, dl, WideOpVT),
5845 SubVec, Idx);
5846 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
5847 }
5848
5849 MVT SubVecVT = SubVec.getSimpleValueType();
5850 unsigned SubVecNumElems = SubVecVT.getVectorNumElements();
5851
5852 assert(IdxVal + SubVecNumElems <= NumElems &&((IdxVal + SubVecNumElems <= NumElems && IdxVal % SubVecVT
.getSizeInBits() == 0 && "Unexpected index value in INSERT_SUBVECTOR"
) ? static_cast<void> (0) : __assert_fail ("IdxVal + SubVecNumElems <= NumElems && IdxVal % SubVecVT.getSizeInBits() == 0 && \"Unexpected index value in INSERT_SUBVECTOR\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5854, __PRETTY_FUNCTION__))
5853 IdxVal % SubVecVT.getSizeInBits() == 0 &&((IdxVal + SubVecNumElems <= NumElems && IdxVal % SubVecVT
.getSizeInBits() == 0 && "Unexpected index value in INSERT_SUBVECTOR"
) ? static_cast<void> (0) : __assert_fail ("IdxVal + SubVecNumElems <= NumElems && IdxVal % SubVecVT.getSizeInBits() == 0 && \"Unexpected index value in INSERT_SUBVECTOR\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5854, __PRETTY_FUNCTION__))
5854 "Unexpected index value in INSERT_SUBVECTOR")((IdxVal + SubVecNumElems <= NumElems && IdxVal % SubVecVT
.getSizeInBits() == 0 && "Unexpected index value in INSERT_SUBVECTOR"
) ? static_cast<void> (0) : __assert_fail ("IdxVal + SubVecNumElems <= NumElems && IdxVal % SubVecVT.getSizeInBits() == 0 && \"Unexpected index value in INSERT_SUBVECTOR\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5854, __PRETTY_FUNCTION__));
5855
5856 SDValue Undef = DAG.getUNDEF(WideOpVT);
5857
5858 if (IdxVal == 0) {
5859 // Zero lower bits of the Vec
5860 SDValue ShiftBits = DAG.getTargetConstant(SubVecNumElems, dl, MVT::i8);
5861 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec,
5862 ZeroIdx);
5863 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
5864 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
5865 // Merge them together, SubVec should be zero extended.
5866 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
5867 DAG.getConstant(0, dl, WideOpVT),
5868 SubVec, ZeroIdx);
5869 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
5870 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
5871 }
5872
5873 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
5874 Undef, SubVec, ZeroIdx);
5875
5876 if (Vec.isUndef()) {
5877 assert(IdxVal != 0 && "Unexpected index")((IdxVal != 0 && "Unexpected index") ? static_cast<
void> (0) : __assert_fail ("IdxVal != 0 && \"Unexpected index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5877, __PRETTY_FUNCTION__));
5878 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5879 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
5880 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
5881 }
5882
5883 if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
5884 assert(IdxVal != 0 && "Unexpected index")((IdxVal != 0 && "Unexpected index") ? static_cast<
void> (0) : __assert_fail ("IdxVal != 0 && \"Unexpected index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5884, __PRETTY_FUNCTION__));
5885 NumElems = WideOpVT.getVectorNumElements();
5886 unsigned ShiftLeft = NumElems - SubVecNumElems;
5887 unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
5888 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5889 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
5890 if (ShiftRight != 0)
5891 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
5892 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
5893 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
5894 }
5895
5896 // Simple case when we put subvector in the upper part
5897 if (IdxVal + SubVecNumElems == NumElems) {
5898 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5899 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
5900 if (SubVecNumElems * 2 == NumElems) {
5901 // Special case, use legal zero extending insert_subvector. This allows
5902 // isel to opimitize when bits are known zero.
5903 Vec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, SubVecVT, Vec, ZeroIdx);
5904 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
5905 DAG.getConstant(0, dl, WideOpVT),
5906 Vec, ZeroIdx);
5907 } else {
5908 // Otherwise use explicit shifts to zero the bits.
5909 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT,
5910 Undef, Vec, ZeroIdx);
5911 NumElems = WideOpVT.getVectorNumElements();
5912 SDValue ShiftBits = DAG.getTargetConstant(NumElems - IdxVal, dl, MVT::i8);
5913 Vec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec, ShiftBits);
5914 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec, ShiftBits);
5915 }
5916 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
5917 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
5918 }
5919
5920 // Inserting into the middle is more complicated.
5921
5922 NumElems = WideOpVT.getVectorNumElements();
5923
5924 // Widen the vector if needed.
5925 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideOpVT, Undef, Vec, ZeroIdx);
5926
5927 unsigned ShiftLeft = NumElems - SubVecNumElems;
5928 unsigned ShiftRight = NumElems - SubVecNumElems - IdxVal;
5929
5930 // Do an optimization for the the most frequently used types.
5931 if (WideOpVT != MVT::v64i1 || Subtarget.is64Bit()) {
5932 APInt Mask0 = APInt::getBitsSet(NumElems, IdxVal, IdxVal + SubVecNumElems);
5933 Mask0.flipAllBits();
5934 SDValue CMask0 = DAG.getConstant(Mask0, dl, MVT::getIntegerVT(NumElems));
5935 SDValue VMask0 = DAG.getNode(ISD::BITCAST, dl, WideOpVT, CMask0);
5936 Vec = DAG.getNode(ISD::AND, dl, WideOpVT, Vec, VMask0);
5937 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5938 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
5939 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
5940 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
5941 Op = DAG.getNode(ISD::OR, dl, WideOpVT, Vec, SubVec);
5942
5943 // Reduce to original width if needed.
5944 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, Op, ZeroIdx);
5945 }
5946
5947 // Clear the upper bits of the subvector and move it to its insert position.
5948 SubVec = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, SubVec,
5949 DAG.getTargetConstant(ShiftLeft, dl, MVT::i8));
5950 SubVec = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, SubVec,
5951 DAG.getTargetConstant(ShiftRight, dl, MVT::i8));
5952
5953 // Isolate the bits below the insertion point.
5954 unsigned LowShift = NumElems - IdxVal;
5955 SDValue Low = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, Vec,
5956 DAG.getTargetConstant(LowShift, dl, MVT::i8));
5957 Low = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Low,
5958 DAG.getTargetConstant(LowShift, dl, MVT::i8));
5959
5960 // Isolate the bits after the last inserted bit.
5961 unsigned HighShift = IdxVal + SubVecNumElems;
5962 SDValue High = DAG.getNode(X86ISD::KSHIFTR, dl, WideOpVT, Vec,
5963 DAG.getTargetConstant(HighShift, dl, MVT::i8));
5964 High = DAG.getNode(X86ISD::KSHIFTL, dl, WideOpVT, High,
5965 DAG.getTargetConstant(HighShift, dl, MVT::i8));
5966
5967 // Now OR all 3 pieces together.
5968 Vec = DAG.getNode(ISD::OR, dl, WideOpVT, Low, High);
5969 SubVec = DAG.getNode(ISD::OR, dl, WideOpVT, SubVec, Vec);
5970
5971 // Reduce to original width if needed.
5972 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OpVT, SubVec, ZeroIdx);
5973}
5974
5975static SDValue concatSubVectors(SDValue V1, SDValue V2, SelectionDAG &DAG,
5976 const SDLoc &dl) {
5977 assert(V1.getValueType() == V2.getValueType() && "subvector type mismatch")((V1.getValueType() == V2.getValueType() && "subvector type mismatch"
) ? static_cast<void> (0) : __assert_fail ("V1.getValueType() == V2.getValueType() && \"subvector type mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5977, __PRETTY_FUNCTION__));
5978 EVT SubVT = V1.getValueType();
5979 EVT SubSVT = SubVT.getScalarType();
5980 unsigned SubNumElts = SubVT.getVectorNumElements();
5981 unsigned SubVectorWidth = SubVT.getSizeInBits();
5982 EVT VT = EVT::getVectorVT(*DAG.getContext(), SubSVT, 2 * SubNumElts);
5983 SDValue V = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, dl, SubVectorWidth);
5984 return insertSubVector(V, V2, SubNumElts, DAG, dl, SubVectorWidth);
5985}
5986
5987/// Returns a vector of specified type with all bits set.
5988/// Always build ones vectors as <4 x i32>, <8 x i32> or <16 x i32>.
5989/// Then bitcast to their original type, ensuring they get CSE'd.
5990static SDValue getOnesVector(EVT VT, SelectionDAG &DAG, const SDLoc &dl) {
5991 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
()) && "Expected a 128/256/512-bit vector type") ? static_cast
<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && \"Expected a 128/256/512-bit vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5992, __PRETTY_FUNCTION__))
5992 "Expected a 128/256/512-bit vector type")(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
()) && "Expected a 128/256/512-bit vector type") ? static_cast
<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && \"Expected a 128/256/512-bit vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 5992, __PRETTY_FUNCTION__));
5993
5994 APInt Ones = APInt::getAllOnesValue(32);
5995 unsigned NumElts = VT.getSizeInBits() / 32;
5996 SDValue Vec = DAG.getConstant(Ones, dl, MVT::getVectorVT(MVT::i32, NumElts));
5997 return DAG.getBitcast(VT, Vec);
5998}
5999
6000// Convert *_EXTEND to *_EXTEND_VECTOR_INREG opcode.
6001static unsigned getOpcode_EXTEND_VECTOR_INREG(unsigned Opcode) {
6002 switch (Opcode) {
6003 case ISD::ANY_EXTEND:
6004 case ISD::ANY_EXTEND_VECTOR_INREG:
6005 return ISD::ANY_EXTEND_VECTOR_INREG;
6006 case ISD::ZERO_EXTEND:
6007 case ISD::ZERO_EXTEND_VECTOR_INREG:
6008 return ISD::ZERO_EXTEND_VECTOR_INREG;
6009 case ISD::SIGN_EXTEND:
6010 case ISD::SIGN_EXTEND_VECTOR_INREG:
6011 return ISD::SIGN_EXTEND_VECTOR_INREG;
6012 }
6013 llvm_unreachable("Unknown opcode")::llvm::llvm_unreachable_internal("Unknown opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6013);
6014}
6015
6016static SDValue getExtendInVec(unsigned Opcode, const SDLoc &DL, EVT VT,
6017 SDValue In, SelectionDAG &DAG) {
6018 EVT InVT = In.getValueType();
6019 assert(VT.isVector() && InVT.isVector() && "Expected vector VTs.")((VT.isVector() && InVT.isVector() && "Expected vector VTs."
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && InVT.isVector() && \"Expected vector VTs.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6019, __PRETTY_FUNCTION__));
6020 assert((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode ||(((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode ||
ISD::ZERO_EXTEND == Opcode) && "Unknown extension opcode"
) ? static_cast<void> (0) : __assert_fail ("(ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode || ISD::ZERO_EXTEND == Opcode) && \"Unknown extension opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6022, __PRETTY_FUNCTION__))
6021 ISD::ZERO_EXTEND == Opcode) &&(((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode ||
ISD::ZERO_EXTEND == Opcode) && "Unknown extension opcode"
) ? static_cast<void> (0) : __assert_fail ("(ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode || ISD::ZERO_EXTEND == Opcode) && \"Unknown extension opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6022, __PRETTY_FUNCTION__))
6022 "Unknown extension opcode")(((ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode ||
ISD::ZERO_EXTEND == Opcode) && "Unknown extension opcode"
) ? static_cast<void> (0) : __assert_fail ("(ISD::ANY_EXTEND == Opcode || ISD::SIGN_EXTEND == Opcode || ISD::ZERO_EXTEND == Opcode) && \"Unknown extension opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6022, __PRETTY_FUNCTION__));
6023
6024 // For 256-bit vectors, we only need the lower (128-bit) input half.
6025 // For 512-bit vectors, we only need the lower input half or quarter.
6026 if (InVT.getSizeInBits() > 128) {
6027 assert(VT.getSizeInBits() == InVT.getSizeInBits() &&((VT.getSizeInBits() == InVT.getSizeInBits() && "Expected VTs to be the same size!"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() == InVT.getSizeInBits() && \"Expected VTs to be the same size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6028, __PRETTY_FUNCTION__))
6028 "Expected VTs to be the same size!")((VT.getSizeInBits() == InVT.getSizeInBits() && "Expected VTs to be the same size!"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() == InVT.getSizeInBits() && \"Expected VTs to be the same size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6028, __PRETTY_FUNCTION__));
6029 unsigned Scale = VT.getScalarSizeInBits() / InVT.getScalarSizeInBits();
6030 In = extractSubVector(In, 0, DAG, DL,
6031 std::max(128U, (unsigned)VT.getSizeInBits() / Scale));
6032 InVT = In.getValueType();
6033 }
6034
6035 if (VT.getVectorNumElements() != InVT.getVectorNumElements())
6036 Opcode = getOpcode_EXTEND_VECTOR_INREG(Opcode);
6037
6038 return DAG.getNode(Opcode, DL, VT, In);
6039}
6040
6041// Match (xor X, -1) -> X.
6042// Match extract_subvector(xor X, -1) -> extract_subvector(X).
6043// Match concat_vectors(xor X, -1, xor Y, -1) -> concat_vectors(X, Y).
6044static SDValue IsNOT(SDValue V, SelectionDAG &DAG) {
6045 V = peekThroughBitcasts(V);
6046 if (V.getOpcode() == ISD::XOR &&
6047 ISD::isBuildVectorAllOnes(V.getOperand(1).getNode()))
6048 return V.getOperand(0);
6049 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6050 (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
6051 if (SDValue Not = IsNOT(V.getOperand(0), DAG)) {
6052 Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
6053 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), V.getValueType(),
6054 Not, V.getOperand(1));
6055 }
6056 }
6057 SmallVector<SDValue, 2> CatOps;
6058 if (collectConcatOps(V.getNode(), CatOps)) {
6059 for (SDValue &CatOp : CatOps) {
6060 SDValue NotCat = IsNOT(CatOp, DAG);
6061 if (!NotCat) return SDValue();
6062 CatOp = DAG.getBitcast(CatOp.getValueType(), NotCat);
6063 }
6064 return DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(V), V.getValueType(), CatOps);
6065 }
6066 return SDValue();
6067}
6068
6069/// Returns a vector_shuffle node for an unpackl operation.
6070static SDValue getUnpackl(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
6071 SDValue V1, SDValue V2) {
6072 SmallVector<int, 8> Mask;
6073 createUnpackShuffleMask(VT, Mask, /* Lo = */ true, /* Unary = */ false);
6074 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
6075}
6076
6077/// Returns a vector_shuffle node for an unpackh operation.
6078static SDValue getUnpackh(SelectionDAG &DAG, const SDLoc &dl, MVT VT,
6079 SDValue V1, SDValue V2) {
6080 SmallVector<int, 8> Mask;
6081 createUnpackShuffleMask(VT, Mask, /* Lo = */ false, /* Unary = */ false);
6082 return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
6083}
6084
6085/// Return a vector_shuffle of the specified vector of zero or undef vector.
6086/// This produces a shuffle where the low element of V2 is swizzled into the
6087/// zero/undef vector, landing at element Idx.
6088/// This produces a shuffle mask like 4,1,2,3 (idx=0) or 0,1,2,4 (idx=3).
6089static SDValue getShuffleVectorZeroOrUndef(SDValue V2, int Idx,
6090 bool IsZero,
6091 const X86Subtarget &Subtarget,
6092 SelectionDAG &DAG) {
6093 MVT VT = V2.getSimpleValueType();
6094 SDValue V1 = IsZero
6095 ? getZeroVector(VT, Subtarget, DAG, SDLoc(V2)) : DAG.getUNDEF(VT);
6096 int NumElems = VT.getVectorNumElements();
6097 SmallVector<int, 16> MaskVec(NumElems);
6098 for (int i = 0; i != NumElems; ++i)
6099 // If this is the insertion idx, put the low elt of V2 here.
6100 MaskVec[i] = (i == Idx) ? NumElems : i;
6101 return DAG.getVectorShuffle(VT, SDLoc(V2), V1, V2, MaskVec);
6102}
6103
6104static const Constant *getTargetConstantFromNode(LoadSDNode *Load) {
6105 if (!Load || !ISD::isNormalLoad(Load))
6106 return nullptr;
6107
6108 SDValue Ptr = Load->getBasePtr();
6109 if (Ptr->getOpcode() == X86ISD::Wrapper ||
6110 Ptr->getOpcode() == X86ISD::WrapperRIP)
6111 Ptr = Ptr->getOperand(0);
6112
6113 auto *CNode = dyn_cast<ConstantPoolSDNode>(Ptr);
6114 if (!CNode || CNode->isMachineConstantPoolEntry() || CNode->getOffset() != 0)
6115 return nullptr;
6116
6117 return CNode->getConstVal();
6118}
6119
6120static const Constant *getTargetConstantFromNode(SDValue Op) {
6121 Op = peekThroughBitcasts(Op);
6122 return getTargetConstantFromNode(dyn_cast<LoadSDNode>(Op));
6123}
6124
6125const Constant *
6126X86TargetLowering::getTargetConstantFromLoad(LoadSDNode *LD) const {
6127 assert(LD && "Unexpected null LoadSDNode")((LD && "Unexpected null LoadSDNode") ? static_cast<
void> (0) : __assert_fail ("LD && \"Unexpected null LoadSDNode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6127, __PRETTY_FUNCTION__));
6128 return getTargetConstantFromNode(LD);
6129}
6130
6131// Extract raw constant bits from constant pools.
6132static bool getTargetConstantBitsFromNode(SDValue Op, unsigned EltSizeInBits,
6133 APInt &UndefElts,
6134 SmallVectorImpl<APInt> &EltBits,
6135 bool AllowWholeUndefs = true,
6136 bool AllowPartialUndefs = true) {
6137 assert(EltBits.empty() && "Expected an empty EltBits vector")((EltBits.empty() && "Expected an empty EltBits vector"
) ? static_cast<void> (0) : __assert_fail ("EltBits.empty() && \"Expected an empty EltBits vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6137, __PRETTY_FUNCTION__));
6138
6139 Op = peekThroughBitcasts(Op);
6140
6141 EVT VT = Op.getValueType();
6142 unsigned SizeInBits = VT.getSizeInBits();
6143 assert((SizeInBits % EltSizeInBits) == 0 && "Can't split constant!")(((SizeInBits % EltSizeInBits) == 0 && "Can't split constant!"
) ? static_cast<void> (0) : __assert_fail ("(SizeInBits % EltSizeInBits) == 0 && \"Can't split constant!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6143, __PRETTY_FUNCTION__));
6144 unsigned NumElts = SizeInBits / EltSizeInBits;
6145
6146 // Bitcast a source array of element bits to the target size.
6147 auto CastBitData = [&](APInt &UndefSrcElts, ArrayRef<APInt> SrcEltBits) {
6148 unsigned NumSrcElts = UndefSrcElts.getBitWidth();
6149 unsigned SrcEltSizeInBits = SrcEltBits[0].getBitWidth();
6150 assert((NumSrcElts * SrcEltSizeInBits) == SizeInBits &&(((NumSrcElts * SrcEltSizeInBits) == SizeInBits && "Constant bit sizes don't match"
) ? static_cast<void> (0) : __assert_fail ("(NumSrcElts * SrcEltSizeInBits) == SizeInBits && \"Constant bit sizes don't match\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6151, __PRETTY_FUNCTION__))
6151 "Constant bit sizes don't match")(((NumSrcElts * SrcEltSizeInBits) == SizeInBits && "Constant bit sizes don't match"
) ? static_cast<void> (0) : __assert_fail ("(NumSrcElts * SrcEltSizeInBits) == SizeInBits && \"Constant bit sizes don't match\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6151, __PRETTY_FUNCTION__));
6152
6153 // Don't split if we don't allow undef bits.
6154 bool AllowUndefs = AllowWholeUndefs || AllowPartialUndefs;
6155 if (UndefSrcElts.getBoolValue() && !AllowUndefs)
6156 return false;
6157
6158 // If we're already the right size, don't bother bitcasting.
6159 if (NumSrcElts == NumElts) {
6160 UndefElts = UndefSrcElts;
6161 EltBits.assign(SrcEltBits.begin(), SrcEltBits.end());
6162 return true;
6163 }
6164
6165 // Extract all the undef/constant element data and pack into single bitsets.
6166 APInt UndefBits(SizeInBits, 0);
6167 APInt MaskBits(SizeInBits, 0);
6168
6169 for (unsigned i = 0; i != NumSrcElts; ++i) {
6170 unsigned BitOffset = i * SrcEltSizeInBits;
6171 if (UndefSrcElts[i])
6172 UndefBits.setBits(BitOffset, BitOffset + SrcEltSizeInBits);
6173 MaskBits.insertBits(SrcEltBits[i], BitOffset);
6174 }
6175
6176 // Split the undef/constant single bitset data into the target elements.
6177 UndefElts = APInt(NumElts, 0);
6178 EltBits.resize(NumElts, APInt(EltSizeInBits, 0));
6179
6180 for (unsigned i = 0; i != NumElts; ++i) {
6181 unsigned BitOffset = i * EltSizeInBits;
6182 APInt UndefEltBits = UndefBits.extractBits(EltSizeInBits, BitOffset);
6183
6184 // Only treat an element as UNDEF if all bits are UNDEF.
6185 if (UndefEltBits.isAllOnesValue()) {
6186 if (!AllowWholeUndefs)
6187 return false;
6188 UndefElts.setBit(i);
6189 continue;
6190 }
6191
6192 // If only some bits are UNDEF then treat them as zero (or bail if not
6193 // supported).
6194 if (UndefEltBits.getBoolValue() && !AllowPartialUndefs)
6195 return false;
6196
6197 EltBits[i] = MaskBits.extractBits(EltSizeInBits, BitOffset);
6198 }
6199 return true;
6200 };
6201
6202 // Collect constant bits and insert into mask/undef bit masks.
6203 auto CollectConstantBits = [](const Constant *Cst, APInt &Mask, APInt &Undefs,
6204 unsigned UndefBitIndex) {
6205 if (!Cst)
6206 return false;
6207 if (isa<UndefValue>(Cst)) {
6208 Undefs.setBit(UndefBitIndex);
6209 return true;
6210 }
6211 if (auto *CInt = dyn_cast<ConstantInt>(Cst)) {
6212 Mask = CInt->getValue();
6213 return true;
6214 }
6215 if (auto *CFP = dyn_cast<ConstantFP>(Cst)) {
6216 Mask = CFP->getValueAPF().bitcastToAPInt();
6217 return true;
6218 }
6219 return false;
6220 };
6221
6222 // Handle UNDEFs.
6223 if (Op.isUndef()) {
6224 APInt UndefSrcElts = APInt::getAllOnesValue(NumElts);
6225 SmallVector<APInt, 64> SrcEltBits(NumElts, APInt(EltSizeInBits, 0));
6226 return CastBitData(UndefSrcElts, SrcEltBits);
6227 }
6228
6229 // Extract scalar constant bits.
6230 if (auto *Cst = dyn_cast<ConstantSDNode>(Op)) {
6231 APInt UndefSrcElts = APInt::getNullValue(1);
6232 SmallVector<APInt, 64> SrcEltBits(1, Cst->getAPIntValue());
6233 return CastBitData(UndefSrcElts, SrcEltBits);
6234 }
6235 if (auto *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
6236 APInt UndefSrcElts = APInt::getNullValue(1);
6237 APInt RawBits = Cst->getValueAPF().bitcastToAPInt();
6238 SmallVector<APInt, 64> SrcEltBits(1, RawBits);
6239 return CastBitData(UndefSrcElts, SrcEltBits);
6240 }
6241
6242 // Extract constant bits from build vector.
6243 if (ISD::isBuildVectorOfConstantSDNodes(Op.getNode())) {
6244 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
6245 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
6246
6247 APInt UndefSrcElts(NumSrcElts, 0);
6248 SmallVector<APInt, 64> SrcEltBits(NumSrcElts, APInt(SrcEltSizeInBits, 0));
6249 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
6250 const SDValue &Src = Op.getOperand(i);
6251 if (Src.isUndef()) {
6252 UndefSrcElts.setBit(i);
6253 continue;
6254 }
6255 auto *Cst = cast<ConstantSDNode>(Src);
6256 SrcEltBits[i] = Cst->getAPIntValue().zextOrTrunc(SrcEltSizeInBits);
6257 }
6258 return CastBitData(UndefSrcElts, SrcEltBits);
6259 }
6260 if (ISD::isBuildVectorOfConstantFPSDNodes(Op.getNode())) {
6261 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
6262 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
6263
6264 APInt UndefSrcElts(NumSrcElts, 0);
6265 SmallVector<APInt, 64> SrcEltBits(NumSrcElts, APInt(SrcEltSizeInBits, 0));
6266 for (unsigned i = 0, e = Op.getNumOperands(); i != e; ++i) {
6267 const SDValue &Src = Op.getOperand(i);
6268 if (Src.isUndef()) {
6269 UndefSrcElts.setBit(i);
6270 continue;
6271 }
6272 auto *Cst = cast<ConstantFPSDNode>(Src);
6273 APInt RawBits = Cst->getValueAPF().bitcastToAPInt();
6274 SrcEltBits[i] = RawBits.zextOrTrunc(SrcEltSizeInBits);
6275 }
6276 return CastBitData(UndefSrcElts, SrcEltBits);
6277 }
6278
6279 // Extract constant bits from constant pool vector.
6280 if (auto *Cst = getTargetConstantFromNode(Op)) {
6281 Type *CstTy = Cst->getType();
6282 unsigned CstSizeInBits = CstTy->getPrimitiveSizeInBits();
6283 if (!CstTy->isVectorTy() || (CstSizeInBits % SizeInBits) != 0)
6284 return false;
6285
6286 unsigned SrcEltSizeInBits = CstTy->getScalarSizeInBits();
6287 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
6288
6289 APInt UndefSrcElts(NumSrcElts, 0);
6290 SmallVector<APInt, 64> SrcEltBits(NumSrcElts, APInt(SrcEltSizeInBits, 0));
6291 for (unsigned i = 0; i != NumSrcElts; ++i)
6292 if (!CollectConstantBits(Cst->getAggregateElement(i), SrcEltBits[i],
6293 UndefSrcElts, i))
6294 return false;
6295
6296 return CastBitData(UndefSrcElts, SrcEltBits);
6297 }
6298
6299 // Extract constant bits from a broadcasted constant pool scalar.
6300 if (Op.getOpcode() == X86ISD::VBROADCAST &&
6301 EltSizeInBits <= VT.getScalarSizeInBits()) {
6302 if (auto *Broadcast = getTargetConstantFromNode(Op.getOperand(0))) {
6303 unsigned SrcEltSizeInBits = Broadcast->getType()->getScalarSizeInBits();
6304 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
6305
6306 APInt UndefSrcElts(NumSrcElts, 0);
6307 SmallVector<APInt, 64> SrcEltBits(1, APInt(SrcEltSizeInBits, 0));
6308 if (CollectConstantBits(Broadcast, SrcEltBits[0], UndefSrcElts, 0)) {
6309 if (UndefSrcElts[0])
6310 UndefSrcElts.setBits(0, NumSrcElts);
6311 SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]);
6312 return CastBitData(UndefSrcElts, SrcEltBits);
6313 }
6314 }
6315 }
6316
6317 if (Op.getOpcode() == X86ISD::VBROADCAST_LOAD &&
6318 EltSizeInBits <= VT.getScalarSizeInBits()) {
6319 auto *MemIntr = cast<MemIntrinsicSDNode>(Op);
6320 if (MemIntr->getMemoryVT().getScalarSizeInBits() != VT.getScalarSizeInBits())
6321 return false;
6322
6323 SDValue Ptr = MemIntr->getBasePtr();
6324 if (Ptr->getOpcode() == X86ISD::Wrapper ||
6325 Ptr->getOpcode() == X86ISD::WrapperRIP)
6326 Ptr = Ptr->getOperand(0);
6327
6328 auto *CNode = dyn_cast<ConstantPoolSDNode>(Ptr);
6329 if (!CNode || CNode->isMachineConstantPoolEntry() ||
6330 CNode->getOffset() != 0)
6331 return false;
6332
6333 if (const Constant *C = CNode->getConstVal()) {
6334 unsigned SrcEltSizeInBits = C->getType()->getScalarSizeInBits();
6335 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
6336
6337 APInt UndefSrcElts(NumSrcElts, 0);
6338 SmallVector<APInt, 64> SrcEltBits(1, APInt(SrcEltSizeInBits, 0));
6339 if (CollectConstantBits(C, SrcEltBits[0], UndefSrcElts, 0)) {
6340 if (UndefSrcElts[0])
6341 UndefSrcElts.setBits(0, NumSrcElts);
6342 SrcEltBits.append(NumSrcElts - 1, SrcEltBits[0]);
6343 return CastBitData(UndefSrcElts, SrcEltBits);
6344 }
6345 }
6346 }
6347
6348 // Extract constant bits from a subvector broadcast.
6349 if (Op.getOpcode() == X86ISD::SUBV_BROADCAST) {
6350 SmallVector<APInt, 16> SubEltBits;
6351 if (getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
6352 UndefElts, SubEltBits, AllowWholeUndefs,
6353 AllowPartialUndefs)) {
6354 UndefElts = APInt::getSplat(NumElts, UndefElts);
6355 while (EltBits.size() < NumElts)
6356 EltBits.append(SubEltBits.begin(), SubEltBits.end());
6357 return true;
6358 }
6359 }
6360
6361 // Extract a rematerialized scalar constant insertion.
6362 if (Op.getOpcode() == X86ISD::VZEXT_MOVL &&
6363 Op.getOperand(0).getOpcode() == ISD::SCALAR_TO_VECTOR &&
6364 isa<ConstantSDNode>(Op.getOperand(0).getOperand(0))) {
6365 unsigned SrcEltSizeInBits = VT.getScalarSizeInBits();
6366 unsigned NumSrcElts = SizeInBits / SrcEltSizeInBits;
6367
6368 APInt UndefSrcElts(NumSrcElts, 0);
6369 SmallVector<APInt, 64> SrcEltBits;
6370 auto *CN = cast<ConstantSDNode>(Op.getOperand(0).getOperand(0));
6371 SrcEltBits.push_back(CN->getAPIntValue().zextOrTrunc(SrcEltSizeInBits));
6372 SrcEltBits.append(NumSrcElts - 1, APInt(SrcEltSizeInBits, 0));
6373 return CastBitData(UndefSrcElts, SrcEltBits);
6374 }
6375
6376 // Insert constant bits from a base and sub vector sources.
6377 if (Op.getOpcode() == ISD::INSERT_SUBVECTOR &&
6378 isa<ConstantSDNode>(Op.getOperand(2))) {
6379 // TODO - support insert_subvector through bitcasts.
6380 if (EltSizeInBits != VT.getScalarSizeInBits())
6381 return false;
6382
6383 APInt UndefSubElts;
6384 SmallVector<APInt, 32> EltSubBits;
6385 if (getTargetConstantBitsFromNode(Op.getOperand(1), EltSizeInBits,
6386 UndefSubElts, EltSubBits,
6387 AllowWholeUndefs, AllowPartialUndefs) &&
6388 getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
6389 UndefElts, EltBits, AllowWholeUndefs,
6390 AllowPartialUndefs)) {
6391 unsigned BaseIdx = Op.getConstantOperandVal(2);
6392 UndefElts.insertBits(UndefSubElts, BaseIdx);
6393 for (unsigned i = 0, e = EltSubBits.size(); i != e; ++i)
6394 EltBits[BaseIdx + i] = EltSubBits[i];
6395 return true;
6396 }
6397 }
6398
6399 // Extract constant bits from a subvector's source.
6400 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6401 isa<ConstantSDNode>(Op.getOperand(1))) {
6402 // TODO - support extract_subvector through bitcasts.
6403 if (EltSizeInBits != VT.getScalarSizeInBits())
6404 return false;
6405
6406 if (getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
6407 UndefElts, EltBits, AllowWholeUndefs,
6408 AllowPartialUndefs)) {
6409 EVT SrcVT = Op.getOperand(0).getValueType();
6410 unsigned NumSrcElts = SrcVT.getVectorNumElements();
6411 unsigned NumSubElts = VT.getVectorNumElements();
6412 unsigned BaseIdx = Op.getConstantOperandVal(1);
6413 UndefElts = UndefElts.extractBits(NumSubElts, BaseIdx);
6414 if ((BaseIdx + NumSubElts) != NumSrcElts)
6415 EltBits.erase(EltBits.begin() + BaseIdx + NumSubElts, EltBits.end());
6416 if (BaseIdx != 0)
6417 EltBits.erase(EltBits.begin(), EltBits.begin() + BaseIdx);
6418 return true;
6419 }
6420 }
6421
6422 // Extract constant bits from shuffle node sources.
6423 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Op)) {
6424 // TODO - support shuffle through bitcasts.
6425 if (EltSizeInBits != VT.getScalarSizeInBits())
6426 return false;
6427
6428 ArrayRef<int> Mask = SVN->getMask();
6429 if ((!AllowWholeUndefs || !AllowPartialUndefs) &&
6430 llvm::any_of(Mask, [](int M) { return M < 0; }))
6431 return false;
6432
6433 APInt UndefElts0, UndefElts1;
6434 SmallVector<APInt, 32> EltBits0, EltBits1;
6435 if (isAnyInRange(Mask, 0, NumElts) &&
6436 !getTargetConstantBitsFromNode(Op.getOperand(0), EltSizeInBits,
6437 UndefElts0, EltBits0, AllowWholeUndefs,
6438 AllowPartialUndefs))
6439 return false;
6440 if (isAnyInRange(Mask, NumElts, 2 * NumElts) &&
6441 !getTargetConstantBitsFromNode(Op.getOperand(1), EltSizeInBits,
6442 UndefElts1, EltBits1, AllowWholeUndefs,
6443 AllowPartialUndefs))
6444 return false;
6445
6446 UndefElts = APInt::getNullValue(NumElts);
6447 for (int i = 0; i != (int)NumElts; ++i) {
6448 int M = Mask[i];
6449 if (M < 0) {
6450 UndefElts.setBit(i);
6451 EltBits.push_back(APInt::getNullValue(EltSizeInBits));
6452 } else if (M < (int)NumElts) {
6453 if (UndefElts0[M])
6454 UndefElts.setBit(i);
6455 EltBits.push_back(EltBits0[M]);
6456 } else {
6457 if (UndefElts1[M - NumElts])
6458 UndefElts.setBit(i);
6459 EltBits.push_back(EltBits1[M - NumElts]);
6460 }
6461 }
6462 return true;
6463 }
6464
6465 return false;
6466}
6467
6468namespace llvm {
6469namespace X86 {
6470bool isConstantSplat(SDValue Op, APInt &SplatVal) {
6471 APInt UndefElts;
6472 SmallVector<APInt, 16> EltBits;
6473 if (getTargetConstantBitsFromNode(Op, Op.getScalarValueSizeInBits(),
6474 UndefElts, EltBits, true, false)) {
6475 int SplatIndex = -1;
6476 for (int i = 0, e = EltBits.size(); i != e; ++i) {
6477 if (UndefElts[i])
6478 continue;
6479 if (0 <= SplatIndex && EltBits[i] != EltBits[SplatIndex]) {
6480 SplatIndex = -1;
6481 break;
6482 }
6483 SplatIndex = i;
6484 }
6485 if (0 <= SplatIndex) {
6486 SplatVal = EltBits[SplatIndex];
6487 return true;
6488 }
6489 }
6490
6491 return false;
6492}
6493} // namespace X86
6494} // namespace llvm
6495
6496static bool getTargetShuffleMaskIndices(SDValue MaskNode,
6497 unsigned MaskEltSizeInBits,
6498 SmallVectorImpl<uint64_t> &RawMask,
6499 APInt &UndefElts) {
6500 // Extract the raw target constant bits.
6501 SmallVector<APInt, 64> EltBits;
6502 if (!getTargetConstantBitsFromNode(MaskNode, MaskEltSizeInBits, UndefElts,
6503 EltBits, /* AllowWholeUndefs */ true,
6504 /* AllowPartialUndefs */ false))
6505 return false;
6506
6507 // Insert the extracted elements into the mask.
6508 for (APInt Elt : EltBits)
6509 RawMask.push_back(Elt.getZExtValue());
6510
6511 return true;
6512}
6513
6514/// Create a shuffle mask that matches the PACKSS/PACKUS truncation.
6515/// Note: This ignores saturation, so inputs must be checked first.
6516static void createPackShuffleMask(MVT VT, SmallVectorImpl<int> &Mask,
6517 bool Unary) {
6518 assert(Mask.empty() && "Expected an empty shuffle mask vector")((Mask.empty() && "Expected an empty shuffle mask vector"
) ? static_cast<void> (0) : __assert_fail ("Mask.empty() && \"Expected an empty shuffle mask vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6518, __PRETTY_FUNCTION__));
6519 unsigned NumElts = VT.getVectorNumElements();
6520 unsigned NumLanes = VT.getSizeInBits() / 128;
6521 unsigned NumEltsPerLane = 128 / VT.getScalarSizeInBits();
6522 unsigned Offset = Unary ? 0 : NumElts;
6523
6524 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
6525 for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += 2)
6526 Mask.push_back(Elt + (Lane * NumEltsPerLane));
6527 for (unsigned Elt = 0; Elt != NumEltsPerLane; Elt += 2)
6528 Mask.push_back(Elt + (Lane * NumEltsPerLane) + Offset);
6529 }
6530}
6531
6532// Split the demanded elts of a PACKSS/PACKUS node between its operands.
6533static void getPackDemandedElts(EVT VT, const APInt &DemandedElts,
6534 APInt &DemandedLHS, APInt &DemandedRHS) {
6535 int NumLanes = VT.getSizeInBits() / 128;
6536 int NumElts = DemandedElts.getBitWidth();
6537 int NumInnerElts = NumElts / 2;
6538 int NumEltsPerLane = NumElts / NumLanes;
6539 int NumInnerEltsPerLane = NumInnerElts / NumLanes;
6540
6541 DemandedLHS = APInt::getNullValue(NumInnerElts);
6542 DemandedRHS = APInt::getNullValue(NumInnerElts);
6543
6544 // Map DemandedElts to the packed operands.
6545 for (int Lane = 0; Lane != NumLanes; ++Lane) {
6546 for (int Elt = 0; Elt != NumInnerEltsPerLane; ++Elt) {
6547 int OuterIdx = (Lane * NumEltsPerLane) + Elt;
6548 int InnerIdx = (Lane * NumInnerEltsPerLane) + Elt;
6549 if (DemandedElts[OuterIdx])
6550 DemandedLHS.setBit(InnerIdx);
6551 if (DemandedElts[OuterIdx + NumInnerEltsPerLane])
6552 DemandedRHS.setBit(InnerIdx);
6553 }
6554 }
6555}
6556
6557// Split the demanded elts of a HADD/HSUB node between its operands.
6558static void getHorizDemandedElts(EVT VT, const APInt &DemandedElts,
6559 APInt &DemandedLHS, APInt &DemandedRHS) {
6560 int NumLanes = VT.getSizeInBits() / 128;
6561 int NumElts = DemandedElts.getBitWidth();
6562 int NumEltsPerLane = NumElts / NumLanes;
6563 int HalfEltsPerLane = NumEltsPerLane / 2;
6564
6565 DemandedLHS = APInt::getNullValue(NumElts);
6566 DemandedRHS = APInt::getNullValue(NumElts);
6567
6568 // Map DemandedElts to the horizontal operands.
6569 for (int Idx = 0; Idx != NumElts; ++Idx) {
6570 if (!DemandedElts[Idx])
6571 continue;
6572 int LaneIdx = (Idx / NumEltsPerLane) * NumEltsPerLane;
6573 int LocalIdx = Idx % NumEltsPerLane;
6574 if (LocalIdx < HalfEltsPerLane) {
6575 DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 0);
6576 DemandedLHS.setBit(LaneIdx + 2 * LocalIdx + 1);
6577 } else {
6578 LocalIdx -= HalfEltsPerLane;
6579 DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 0);
6580 DemandedRHS.setBit(LaneIdx + 2 * LocalIdx + 1);
6581 }
6582 }
6583}
6584
6585/// Calculates the shuffle mask corresponding to the target-specific opcode.
6586/// If the mask could be calculated, returns it in \p Mask, returns the shuffle
6587/// operands in \p Ops, and returns true.
6588/// Sets \p IsUnary to true if only one source is used. Note that this will set
6589/// IsUnary for shuffles which use a single input multiple times, and in those
6590/// cases it will adjust the mask to only have indices within that single input.
6591/// It is an error to call this with non-empty Mask/Ops vectors.
6592static bool getTargetShuffleMask(SDNode *N, MVT VT, bool AllowSentinelZero,
6593 SmallVectorImpl<SDValue> &Ops,
6594 SmallVectorImpl<int> &Mask, bool &IsUnary) {
6595 unsigned NumElems = VT.getVectorNumElements();
6596 unsigned MaskEltSize = VT.getScalarSizeInBits();
6597 SmallVector<uint64_t, 32> RawMask;
6598 APInt RawUndefs;
6599 SDValue ImmN;
6600
6601 assert(Mask.empty() && "getTargetShuffleMask expects an empty Mask vector")((Mask.empty() && "getTargetShuffleMask expects an empty Mask vector"
) ? static_cast<void> (0) : __assert_fail ("Mask.empty() && \"getTargetShuffleMask expects an empty Mask vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6601, __PRETTY_FUNCTION__));
6602 assert(Ops.empty() && "getTargetShuffleMask expects an empty Ops vector")((Ops.empty() && "getTargetShuffleMask expects an empty Ops vector"
) ? static_cast<void> (0) : __assert_fail ("Ops.empty() && \"getTargetShuffleMask expects an empty Ops vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6602, __PRETTY_FUNCTION__));
6603
6604 IsUnary = false;
6605 bool IsFakeUnary = false;
6606 switch (N->getOpcode()) {
6607 case X86ISD::BLENDI:
6608 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6608, __PRETTY_FUNCTION__));
6609 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6609, __PRETTY_FUNCTION__));
6610 ImmN = N->getOperand(N->getNumOperands() - 1);
6611 DecodeBLENDMask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
6612 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6613 break;
6614 case X86ISD::SHUFP:
6615 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6615, __PRETTY_FUNCTION__));
6616 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6616, __PRETTY_FUNCTION__));
6617 ImmN = N->getOperand(N->getNumOperands() - 1);
6618 DecodeSHUFPMask(NumElems, MaskEltSize,
6619 cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
6620 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6621 break;
6622 case X86ISD::INSERTPS:
6623 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6623, __PRETTY_FUNCTION__));
6624 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6624, __PRETTY_FUNCTION__));
6625 ImmN = N->getOperand(N->getNumOperands() - 1);
6626 DecodeINSERTPSMask(cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
6627 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6628 break;
6629 case X86ISD::EXTRQI:
6630 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6630, __PRETTY_FUNCTION__));
6631 if (isa<ConstantSDNode>(N->getOperand(1)) &&
6632 isa<ConstantSDNode>(N->getOperand(2))) {
6633 int BitLen = N->getConstantOperandVal(1);
6634 int BitIdx = N->getConstantOperandVal(2);
6635 DecodeEXTRQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
6636 IsUnary = true;
6637 }
6638 break;
6639 case X86ISD::INSERTQI:
6640 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6640, __PRETTY_FUNCTION__));
6641 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6641, __PRETTY_FUNCTION__));
6642 if (isa<ConstantSDNode>(N->getOperand(2)) &&
6643 isa<ConstantSDNode>(N->getOperand(3))) {
6644 int BitLen = N->getConstantOperandVal(2);
6645 int BitIdx = N->getConstantOperandVal(3);
6646 DecodeINSERTQIMask(NumElems, MaskEltSize, BitLen, BitIdx, Mask);
6647 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6648 }
6649 break;
6650 case X86ISD::UNPCKH:
6651 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6651, __PRETTY_FUNCTION__));
6652 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6652, __PRETTY_FUNCTION__));
6653 DecodeUNPCKHMask(NumElems, MaskEltSize, Mask);
6654 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6655 break;
6656 case X86ISD::UNPCKL:
6657 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6657, __PRETTY_FUNCTION__));
6658 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6658, __PRETTY_FUNCTION__));
6659 DecodeUNPCKLMask(NumElems, MaskEltSize, Mask);
6660 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6661 break;
6662 case X86ISD::MOVHLPS:
6663 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6663, __PRETTY_FUNCTION__));
6664 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6664, __PRETTY_FUNCTION__));
6665 DecodeMOVHLPSMask(NumElems, Mask);
6666 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6667 break;
6668 case X86ISD::MOVLHPS:
6669 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6669, __PRETTY_FUNCTION__));
6670 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6670, __PRETTY_FUNCTION__));
6671 DecodeMOVLHPSMask(NumElems, Mask);
6672 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6673 break;
6674 case X86ISD::PALIGNR:
6675 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected")((VT.getScalarType() == MVT::i8 && "Byte vector expected"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i8 && \"Byte vector expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6675, __PRETTY_FUNCTION__));
6676 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6676, __PRETTY_FUNCTION__));
6677 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6677, __PRETTY_FUNCTION__));
6678 ImmN = N->getOperand(N->getNumOperands() - 1);
6679 DecodePALIGNRMask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(),
6680 Mask);
6681 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6682 Ops.push_back(N->getOperand(1));
6683 Ops.push_back(N->getOperand(0));
6684 break;
6685 case X86ISD::VSHLDQ:
6686 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected")((VT.getScalarType() == MVT::i8 && "Byte vector expected"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i8 && \"Byte vector expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6686, __PRETTY_FUNCTION__));
6687 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6687, __PRETTY_FUNCTION__));
6688 ImmN = N->getOperand(N->getNumOperands() - 1);
6689 DecodePSLLDQMask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(),
6690 Mask);
6691 IsUnary = true;
6692 break;
6693 case X86ISD::VSRLDQ:
6694 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected")((VT.getScalarType() == MVT::i8 && "Byte vector expected"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i8 && \"Byte vector expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6694, __PRETTY_FUNCTION__));
6695 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6695, __PRETTY_FUNCTION__));
6696 ImmN = N->getOperand(N->getNumOperands() - 1);
6697 DecodePSRLDQMask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(),
6698 Mask);
6699 IsUnary = true;
6700 break;
6701 case X86ISD::PSHUFD:
6702 case X86ISD::VPERMILPI:
6703 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6703, __PRETTY_FUNCTION__));
6704 ImmN = N->getOperand(N->getNumOperands() - 1);
6705 DecodePSHUFMask(NumElems, MaskEltSize,
6706 cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
6707 IsUnary = true;
6708 break;
6709 case X86ISD::PSHUFHW:
6710 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6710, __PRETTY_FUNCTION__));
6711 ImmN = N->getOperand(N->getNumOperands() - 1);
6712 DecodePSHUFHWMask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(),
6713 Mask);
6714 IsUnary = true;
6715 break;
6716 case X86ISD::PSHUFLW:
6717 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6717, __PRETTY_FUNCTION__));
6718 ImmN = N->getOperand(N->getNumOperands() - 1);
6719 DecodePSHUFLWMask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(),
6720 Mask);
6721 IsUnary = true;
6722 break;
6723 case X86ISD::VZEXT_MOVL:
6724 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6724, __PRETTY_FUNCTION__));
6725 DecodeZeroMoveLowMask(NumElems, Mask);
6726 IsUnary = true;
6727 break;
6728 case X86ISD::VBROADCAST: {
6729 SDValue N0 = N->getOperand(0);
6730 // See if we're broadcasting from index 0 of an EXTRACT_SUBVECTOR. If so,
6731 // add the pre-extracted value to the Ops vector.
6732 if (N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
6733 N0.getOperand(0).getValueType() == VT &&
6734 N0.getConstantOperandVal(1) == 0)
6735 Ops.push_back(N0.getOperand(0));
6736
6737 // We only decode broadcasts of same-sized vectors, unless the broadcast
6738 // came from an extract from the original width. If we found one, we
6739 // pushed it the Ops vector above.
6740 if (N0.getValueType() == VT || !Ops.empty()) {
6741 DecodeVectorBroadcast(NumElems, Mask);
6742 IsUnary = true;
6743 break;
6744 }
6745 return false;
6746 }
6747 case X86ISD::VPERMILPV: {
6748 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6748, __PRETTY_FUNCTION__));
6749 IsUnary = true;
6750 SDValue MaskNode = N->getOperand(1);
6751 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
6752 RawUndefs)) {
6753 DecodeVPERMILPMask(NumElems, MaskEltSize, RawMask, RawUndefs, Mask);
6754 break;
6755 }
6756 return false;
6757 }
6758 case X86ISD::PSHUFB: {
6759 assert(VT.getScalarType() == MVT::i8 && "Byte vector expected")((VT.getScalarType() == MVT::i8 && "Byte vector expected"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i8 && \"Byte vector expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6759, __PRETTY_FUNCTION__));
6760 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6760, __PRETTY_FUNCTION__));
6761 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6761, __PRETTY_FUNCTION__));
6762 IsUnary = true;
6763 SDValue MaskNode = N->getOperand(1);
6764 if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
6765 DecodePSHUFBMask(RawMask, RawUndefs, Mask);
6766 break;
6767 }
6768 return false;
6769 }
6770 case X86ISD::VPERMI:
6771 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6771, __PRETTY_FUNCTION__));
6772 ImmN = N->getOperand(N->getNumOperands() - 1);
6773 DecodeVPERMMask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
6774 IsUnary = true;
6775 break;
6776 case X86ISD::MOVSS:
6777 case X86ISD::MOVSD:
6778 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6778, __PRETTY_FUNCTION__));
6779 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6779, __PRETTY_FUNCTION__));
6780 DecodeScalarMoveMask(NumElems, /* IsLoad */ false, Mask);
6781 break;
6782 case X86ISD::VPERM2X128:
6783 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6783, __PRETTY_FUNCTION__));
6784 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6784, __PRETTY_FUNCTION__));
6785 ImmN = N->getOperand(N->getNumOperands() - 1);
6786 DecodeVPERM2X128Mask(NumElems, cast<ConstantSDNode>(ImmN)->getZExtValue(),
6787 Mask);
6788 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6789 break;
6790 case X86ISD::SHUF128:
6791 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6791, __PRETTY_FUNCTION__));
6792 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6792, __PRETTY_FUNCTION__));
6793 ImmN = N->getOperand(N->getNumOperands() - 1);
6794 decodeVSHUF64x2FamilyMask(NumElems, MaskEltSize,
6795 cast<ConstantSDNode>(ImmN)->getZExtValue(), Mask);
6796 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6797 break;
6798 case X86ISD::MOVSLDUP:
6799 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6799, __PRETTY_FUNCTION__));
6800 DecodeMOVSLDUPMask(NumElems, Mask);
6801 IsUnary = true;
6802 break;
6803 case X86ISD::MOVSHDUP:
6804 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6804, __PRETTY_FUNCTION__));
6805 DecodeMOVSHDUPMask(NumElems, Mask);
6806 IsUnary = true;
6807 break;
6808 case X86ISD::MOVDDUP:
6809 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6809, __PRETTY_FUNCTION__));
6810 DecodeMOVDDUPMask(NumElems, Mask);
6811 IsUnary = true;
6812 break;
6813 case X86ISD::VPERMIL2: {
6814 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6814, __PRETTY_FUNCTION__));
6815 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6815, __PRETTY_FUNCTION__));
6816 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6817 SDValue MaskNode = N->getOperand(2);
6818 SDValue CtrlNode = N->getOperand(3);
6819 if (ConstantSDNode *CtrlOp = dyn_cast<ConstantSDNode>(CtrlNode)) {
6820 unsigned CtrlImm = CtrlOp->getZExtValue();
6821 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
6822 RawUndefs)) {
6823 DecodeVPERMIL2PMask(NumElems, MaskEltSize, CtrlImm, RawMask, RawUndefs,
6824 Mask);
6825 break;
6826 }
6827 }
6828 return false;
6829 }
6830 case X86ISD::VPPERM: {
6831 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6831, __PRETTY_FUNCTION__));
6832 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6832, __PRETTY_FUNCTION__));
6833 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(1);
6834 SDValue MaskNode = N->getOperand(2);
6835 if (getTargetShuffleMaskIndices(MaskNode, 8, RawMask, RawUndefs)) {
6836 DecodeVPPERMMask(RawMask, RawUndefs, Mask);
6837 break;
6838 }
6839 return false;
6840 }
6841 case X86ISD::VPERMV: {
6842 assert(N->getOperand(1).getValueType() == VT && "Unexpected value type")((N->getOperand(1).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6842, __PRETTY_FUNCTION__));
6843 IsUnary = true;
6844 // Unlike most shuffle nodes, VPERMV's mask operand is operand 0.
6845 Ops.push_back(N->getOperand(1));
6846 SDValue MaskNode = N->getOperand(0);
6847 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
6848 RawUndefs)) {
6849 DecodeVPERMVMask(RawMask, RawUndefs, Mask);
6850 break;
6851 }
6852 return false;
6853 }
6854 case X86ISD::VPERMV3: {
6855 assert(N->getOperand(0).getValueType() == VT && "Unexpected value type")((N->getOperand(0).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(0).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6855, __PRETTY_FUNCTION__));
6856 assert(N->getOperand(2).getValueType() == VT && "Unexpected value type")((N->getOperand(2).getValueType() == VT && "Unexpected value type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(2).getValueType() == VT && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6856, __PRETTY_FUNCTION__));
6857 IsUnary = IsFakeUnary = N->getOperand(0) == N->getOperand(2);
6858 // Unlike most shuffle nodes, VPERMV3's mask operand is the middle one.
6859 Ops.push_back(N->getOperand(0));
6860 Ops.push_back(N->getOperand(2));
6861 SDValue MaskNode = N->getOperand(1);
6862 if (getTargetShuffleMaskIndices(MaskNode, MaskEltSize, RawMask,
6863 RawUndefs)) {
6864 DecodeVPERMV3Mask(RawMask, RawUndefs, Mask);
6865 break;
6866 }
6867 return false;
6868 }
6869 default: llvm_unreachable("unknown target shuffle node")::llvm::llvm_unreachable_internal("unknown target shuffle node"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6869);
6870 }
6871
6872 // Empty mask indicates the decode failed.
6873 if (Mask.empty())
6874 return false;
6875
6876 // Check if we're getting a shuffle mask with zero'd elements.
6877 if (!AllowSentinelZero)
6878 if (any_of(Mask, [](int M) { return M == SM_SentinelZero; }))
6879 return false;
6880
6881 // If we have a fake unary shuffle, the shuffle mask is spread across two
6882 // inputs that are actually the same node. Re-map the mask to always point
6883 // into the first input.
6884 if (IsFakeUnary)
6885 for (int &M : Mask)
6886 if (M >= (int)Mask.size())
6887 M -= Mask.size();
6888
6889 // If we didn't already add operands in the opcode-specific code, default to
6890 // adding 1 or 2 operands starting at 0.
6891 if (Ops.empty()) {
6892 Ops.push_back(N->getOperand(0));
6893 if (!IsUnary || IsFakeUnary)
6894 Ops.push_back(N->getOperand(1));
6895 }
6896
6897 return true;
6898}
6899
6900/// Compute whether each element of a shuffle is zeroable.
6901///
6902/// A "zeroable" vector shuffle element is one which can be lowered to zero.
6903/// Either it is an undef element in the shuffle mask, the element of the input
6904/// referenced is undef, or the element of the input referenced is known to be
6905/// zero. Many x86 shuffles can zero lanes cheaply and we often want to handle
6906/// as many lanes with this technique as possible to simplify the remaining
6907/// shuffle.
6908static void computeZeroableShuffleElements(ArrayRef<int> Mask,
6909 SDValue V1, SDValue V2,
6910 APInt &KnownUndef, APInt &KnownZero) {
6911 int Size = Mask.size();
6912 KnownUndef = KnownZero = APInt::getNullValue(Size);
6913
6914 V1 = peekThroughBitcasts(V1);
6915 V2 = peekThroughBitcasts(V2);
6916
6917 bool V1IsZero = ISD::isBuildVectorAllZeros(V1.getNode());
6918 bool V2IsZero = ISD::isBuildVectorAllZeros(V2.getNode());
6919
6920 int VectorSizeInBits = V1.getValueSizeInBits();
6921 int ScalarSizeInBits = VectorSizeInBits / Size;
6922 assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size")((!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size"
) ? static_cast<void> (0) : __assert_fail ("!(VectorSizeInBits % ScalarSizeInBits) && \"Illegal shuffle mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 6922, __PRETTY_FUNCTION__));
6923
6924 for (int i = 0; i < Size; ++i) {
6925 int M = Mask[i];
6926 // Handle the easy cases.
6927 if (M < 0) {
6928 KnownUndef.setBit(i);
6929 continue;
6930 }
6931 if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
6932 KnownZero.setBit(i);
6933 continue;
6934 }
6935
6936 // Determine shuffle input and normalize the mask.
6937 SDValue V = M < Size ? V1 : V2;
6938 M %= Size;
6939
6940 // Currently we can only search BUILD_VECTOR for UNDEF/ZERO elements.
6941 if (V.getOpcode() != ISD::BUILD_VECTOR)
6942 continue;
6943
6944 // If the BUILD_VECTOR has fewer elements then the bitcasted portion of
6945 // the (larger) source element must be UNDEF/ZERO.
6946 if ((Size % V.getNumOperands()) == 0) {
6947 int Scale = Size / V->getNumOperands();
6948 SDValue Op = V.getOperand(M / Scale);
6949 if (Op.isUndef())
6950 KnownUndef.setBit(i);
6951 if (X86::isZeroNode(Op))
6952 KnownZero.setBit(i);
6953 else if (ConstantSDNode *Cst = dyn_cast<ConstantSDNode>(Op)) {
6954 APInt Val = Cst->getAPIntValue();
6955 Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
6956 if (Val == 0)
6957 KnownZero.setBit(i);
6958 } else if (ConstantFPSDNode *Cst = dyn_cast<ConstantFPSDNode>(Op)) {
6959 APInt Val = Cst->getValueAPF().bitcastToAPInt();
6960 Val = Val.extractBits(ScalarSizeInBits, (M % Scale) * ScalarSizeInBits);
6961 if (Val == 0)
6962 KnownZero.setBit(i);
6963 }
6964 continue;
6965 }
6966
6967 // If the BUILD_VECTOR has more elements then all the (smaller) source
6968 // elements must be UNDEF or ZERO.
6969 if ((V.getNumOperands() % Size) == 0) {
6970 int Scale = V->getNumOperands() / Size;
6971 bool AllUndef = true;
6972 bool AllZero = true;
6973 for (int j = 0; j < Scale; ++j) {
6974 SDValue Op = V.getOperand((M * Scale) + j);
6975 AllUndef &= Op.isUndef();
6976 AllZero &= X86::isZeroNode(Op);
6977 }
6978 if (AllUndef)
6979 KnownUndef.setBit(i);
6980 if (AllZero)
6981 KnownZero.setBit(i);
6982 continue;
6983 }
6984 }
6985}
6986
6987/// Decode a target shuffle mask and inputs and see if any values are
6988/// known to be undef or zero from their inputs.
6989/// Returns true if the target shuffle mask was decoded.
6990/// FIXME: Merge this with computeZeroableShuffleElements?
6991static bool getTargetShuffleAndZeroables(SDValue N, SmallVectorImpl<int> &Mask,
6992 SmallVectorImpl<SDValue> &Ops,
6993 APInt &KnownUndef, APInt &KnownZero) {
6994 bool IsUnary;
6995 if (!isTargetShuffle(N.getOpcode()))
6996 return false;
6997
6998 MVT VT = N.getSimpleValueType();
6999 if (!getTargetShuffleMask(N.getNode(), VT, true, Ops, Mask, IsUnary))
7000 return false;
7001
7002 int Size = Mask.size();
7003 SDValue V1 = Ops[0];
7004 SDValue V2 = IsUnary ? V1 : Ops[1];
7005 KnownUndef = KnownZero = APInt::getNullValue(Size);
7006
7007 V1 = peekThroughBitcasts(V1);
7008 V2 = peekThroughBitcasts(V2);
7009
7010 assert((VT.getSizeInBits() % Size) == 0 &&(((VT.getSizeInBits() % Size) == 0 && "Illegal split of shuffle value type"
) ? static_cast<void> (0) : __assert_fail ("(VT.getSizeInBits() % Size) == 0 && \"Illegal split of shuffle value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7011, __PRETTY_FUNCTION__))
7011 "Illegal split of shuffle value type")(((VT.getSizeInBits() % Size) == 0 && "Illegal split of shuffle value type"
) ? static_cast<void> (0) : __assert_fail ("(VT.getSizeInBits() % Size) == 0 && \"Illegal split of shuffle value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7011, __PRETTY_FUNCTION__));
7012 unsigned EltSizeInBits = VT.getSizeInBits() / Size;
7013
7014 // Extract known constant input data.
7015 APInt UndefSrcElts[2];
7016 SmallVector<APInt, 32> SrcEltBits[2];
7017 bool IsSrcConstant[2] = {
7018 getTargetConstantBitsFromNode(V1, EltSizeInBits, UndefSrcElts[0],
7019 SrcEltBits[0], true, false),
7020 getTargetConstantBitsFromNode(V2, EltSizeInBits, UndefSrcElts[1],
7021 SrcEltBits[1], true, false)};
7022
7023 for (int i = 0; i < Size; ++i) {
7024 int M = Mask[i];
7025
7026 // Already decoded as SM_SentinelZero / SM_SentinelUndef.
7027 if (M < 0) {
7028 assert(isUndefOrZero(M) && "Unknown shuffle sentinel value!")((isUndefOrZero(M) && "Unknown shuffle sentinel value!"
) ? static_cast<void> (0) : __assert_fail ("isUndefOrZero(M) && \"Unknown shuffle sentinel value!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7028, __PRETTY_FUNCTION__));
7029 if (SM_SentinelUndef == M)
7030 KnownUndef.setBit(i);
7031 if (SM_SentinelZero == M)
7032 KnownZero.setBit(i);
7033 continue;
7034 }
7035
7036 // Determine shuffle input and normalize the mask.
7037 unsigned SrcIdx = M / Size;
7038 SDValue V = M < Size ? V1 : V2;
7039 M %= Size;
7040
7041 // We are referencing an UNDEF input.
7042 if (V.isUndef()) {
7043 KnownUndef.setBit(i);
7044 continue;
7045 }
7046
7047 // SCALAR_TO_VECTOR - only the first element is defined, and the rest UNDEF.
7048 // TODO: We currently only set UNDEF for integer types - floats use the same
7049 // registers as vectors and many of the scalar folded loads rely on the
7050 // SCALAR_TO_VECTOR pattern.
7051 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR &&
7052 (Size % V.getValueType().getVectorNumElements()) == 0) {
7053 int Scale = Size / V.getValueType().getVectorNumElements();
7054 int Idx = M / Scale;
7055 if (Idx != 0 && !VT.isFloatingPoint())
7056 KnownUndef.setBit(i);
7057 else if (Idx == 0 && X86::isZeroNode(V.getOperand(0)))
7058 KnownZero.setBit(i);
7059 continue;
7060 }
7061
7062 // Attempt to extract from the source's constant bits.
7063 if (IsSrcConstant[SrcIdx]) {
7064 if (UndefSrcElts[SrcIdx][M])
7065 KnownUndef.setBit(i);
7066 else if (SrcEltBits[SrcIdx][M] == 0)
7067 KnownZero.setBit(i);
7068 }
7069 }
7070
7071 assert(VT.getVectorNumElements() == (unsigned)Size &&((VT.getVectorNumElements() == (unsigned)Size && "Different mask size from vector size!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)Size && \"Different mask size from vector size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7072, __PRETTY_FUNCTION__))
7072 "Different mask size from vector size!")((VT.getVectorNumElements() == (unsigned)Size && "Different mask size from vector size!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == (unsigned)Size && \"Different mask size from vector size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7072, __PRETTY_FUNCTION__));
7073 return true;
7074}
7075
7076// Replace target shuffle mask elements with known undef/zero sentinels.
7077static void resolveTargetShuffleFromZeroables(SmallVectorImpl<int> &Mask,
7078 const APInt &KnownUndef,
7079 const APInt &KnownZero,
7080 bool ResolveKnownZeros= true) {
7081 unsigned NumElts = Mask.size();
7082 assert(KnownUndef.getBitWidth() == NumElts &&((KnownUndef.getBitWidth() == NumElts && KnownZero.getBitWidth
() == NumElts && "Shuffle mask size mismatch") ? static_cast
<void> (0) : __assert_fail ("KnownUndef.getBitWidth() == NumElts && KnownZero.getBitWidth() == NumElts && \"Shuffle mask size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7083, __PRETTY_FUNCTION__))
7083 KnownZero.getBitWidth() == NumElts && "Shuffle mask size mismatch")((KnownUndef.getBitWidth() == NumElts && KnownZero.getBitWidth
() == NumElts && "Shuffle mask size mismatch") ? static_cast
<void> (0) : __assert_fail ("KnownUndef.getBitWidth() == NumElts && KnownZero.getBitWidth() == NumElts && \"Shuffle mask size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7083, __PRETTY_FUNCTION__));
7084
7085 for (unsigned i = 0; i != NumElts; ++i) {
7086 if (KnownUndef[i])
7087 Mask[i] = SM_SentinelUndef;
7088 else if (ResolveKnownZeros && KnownZero[i])
7089 Mask[i] = SM_SentinelZero;
7090 }
7091}
7092
7093// Extract target shuffle mask sentinel elements to known undef/zero bitmasks.
7094static void resolveZeroablesFromTargetShuffle(const SmallVectorImpl<int> &Mask,
7095 APInt &KnownUndef,
7096 APInt &KnownZero) {
7097 unsigned NumElts = Mask.size();
7098 KnownUndef = KnownZero = APInt::getNullValue(NumElts);
7099
7100 for (unsigned i = 0; i != NumElts; ++i) {
7101 int M = Mask[i];
7102 if (SM_SentinelUndef == M)
7103 KnownUndef.setBit(i);
7104 if (SM_SentinelZero == M)
7105 KnownZero.setBit(i);
7106 }
7107}
7108
7109// Forward declaration (for getFauxShuffleMask recursive check).
7110// TODO: Use DemandedElts variant.
7111static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
7112 SmallVectorImpl<int> &Mask,
7113 SelectionDAG &DAG, unsigned Depth,
7114 bool ResolveKnownElts);
7115
7116// Attempt to decode ops that could be represented as a shuffle mask.
7117// The decoded shuffle mask may contain a different number of elements to the
7118// destination value type.
7119static bool getFauxShuffleMask(SDValue N, const APInt &DemandedElts,
7120 SmallVectorImpl<int> &Mask,
7121 SmallVectorImpl<SDValue> &Ops,
7122 SelectionDAG &DAG, unsigned Depth,
7123 bool ResolveKnownElts) {
7124 Mask.clear();
7125 Ops.clear();
7126
7127 MVT VT = N.getSimpleValueType();
7128 unsigned NumElts = VT.getVectorNumElements();
7129 unsigned NumSizeInBits = VT.getSizeInBits();
7130 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
7131 if ((NumBitsPerElt % 8) != 0 || (NumSizeInBits % 8) != 0)
7132 return false;
7133 assert(NumElts == DemandedElts.getBitWidth() && "Unexpected vector size")((NumElts == DemandedElts.getBitWidth() && "Unexpected vector size"
) ? static_cast<void> (0) : __assert_fail ("NumElts == DemandedElts.getBitWidth() && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7133, __PRETTY_FUNCTION__));
7134
7135 unsigned Opcode = N.getOpcode();
7136 switch (Opcode) {
7137 case ISD::VECTOR_SHUFFLE: {
7138 // Don't treat ISD::VECTOR_SHUFFLE as a target shuffle so decode it here.
7139 ArrayRef<int> ShuffleMask = cast<ShuffleVectorSDNode>(N)->getMask();
7140 if (isUndefOrInRange(ShuffleMask, 0, 2 * NumElts)) {
7141 Mask.append(ShuffleMask.begin(), ShuffleMask.end());
7142 Ops.push_back(N.getOperand(0));
7143 Ops.push_back(N.getOperand(1));
7144 return true;
7145 }
7146 return false;
7147 }
7148 case ISD::AND:
7149 case X86ISD::ANDNP: {
7150 // Attempt to decode as a per-byte mask.
7151 APInt UndefElts;
7152 SmallVector<APInt, 32> EltBits;
7153 SDValue N0 = N.getOperand(0);
7154 SDValue N1 = N.getOperand(1);
7155 bool IsAndN = (X86ISD::ANDNP == Opcode);
7156 uint64_t ZeroMask = IsAndN ? 255 : 0;
7157 if (!getTargetConstantBitsFromNode(IsAndN ? N0 : N1, 8, UndefElts, EltBits))
7158 return false;
7159 for (int i = 0, e = (int)EltBits.size(); i != e; ++i) {
7160 if (UndefElts[i]) {
7161 Mask.push_back(SM_SentinelUndef);
7162 continue;
7163 }
7164 const APInt &ByteBits = EltBits[i];
7165 if (ByteBits != 0 && ByteBits != 255)
7166 return false;
7167 Mask.push_back(ByteBits == ZeroMask ? SM_SentinelZero : i);
7168 }
7169 Ops.push_back(IsAndN ? N1 : N0);
7170 return true;
7171 }
7172 case ISD::OR: {
7173 // Inspect each operand at the byte level. We can merge these into a
7174 // blend shuffle mask if for each byte at least one is masked out (zero).
7175 KnownBits Known0 =
7176 DAG.computeKnownBits(N.getOperand(0), DemandedElts, Depth + 1);
7177 KnownBits Known1 =
7178 DAG.computeKnownBits(N.getOperand(1), DemandedElts, Depth + 1);
7179 if (Known0.One.isNullValue() && Known1.One.isNullValue()) {
7180 bool IsByteMask = true;
7181 unsigned NumSizeInBytes = NumSizeInBits / 8;
7182 unsigned NumBytesPerElt = NumBitsPerElt / 8;
7183 APInt ZeroMask = APInt::getNullValue(NumBytesPerElt);
7184 APInt SelectMask = APInt::getNullValue(NumBytesPerElt);
7185 for (unsigned i = 0; i != NumBytesPerElt && IsByteMask; ++i) {
7186 unsigned LHS = Known0.Zero.extractBits(8, i * 8).getZExtValue();
7187 unsigned RHS = Known1.Zero.extractBits(8, i * 8).getZExtValue();
7188 if (LHS == 255 && RHS == 0)
7189 SelectMask.setBit(i);
7190 else if (LHS == 255 && RHS == 255)
7191 ZeroMask.setBit(i);
7192 else if (!(LHS == 0 && RHS == 255))
7193 IsByteMask = false;
7194 }
7195 if (IsByteMask) {
7196 for (unsigned i = 0; i != NumSizeInBytes; i += NumBytesPerElt) {
7197 for (unsigned j = 0; j != NumBytesPerElt; ++j) {
7198 unsigned Ofs = (SelectMask[j] ? NumSizeInBytes : 0);
7199 int Idx = (ZeroMask[j] ? (int)SM_SentinelZero : (i + j + Ofs));
7200 Mask.push_back(Idx);
7201 }
7202 }
7203 Ops.push_back(N.getOperand(0));
7204 Ops.push_back(N.getOperand(1));
7205 return true;
7206 }
7207 }
7208
7209 // Handle OR(SHUFFLE,SHUFFLE) case where one source is zero and the other
7210 // is a valid shuffle index.
7211 SDValue N0 = peekThroughOneUseBitcasts(N.getOperand(0));
7212 SDValue N1 = peekThroughOneUseBitcasts(N.getOperand(1));
7213 if (!N0.getValueType().isVector() || !N1.getValueType().isVector())
7214 return false;
7215 SmallVector<int, 64> SrcMask0, SrcMask1;
7216 SmallVector<SDValue, 2> SrcInputs0, SrcInputs1;
7217 if (!getTargetShuffleInputs(N0, SrcInputs0, SrcMask0, DAG, Depth + 1,
7218 true) ||
7219 !getTargetShuffleInputs(N1, SrcInputs1, SrcMask1, DAG, Depth + 1,
7220 true))
7221 return false;
7222 size_t MaskSize = std::max(SrcMask0.size(), SrcMask1.size());
7223 SmallVector<int, 64> Mask0, Mask1;
7224 scaleShuffleMask<int>(MaskSize / SrcMask0.size(), SrcMask0, Mask0);
7225 scaleShuffleMask<int>(MaskSize / SrcMask1.size(), SrcMask1, Mask1);
7226 for (size_t i = 0; i != MaskSize; ++i) {
7227 if (Mask0[i] == SM_SentinelUndef && Mask1[i] == SM_SentinelUndef)
7228 Mask.push_back(SM_SentinelUndef);
7229 else if (Mask0[i] == SM_SentinelZero && Mask1[i] == SM_SentinelZero)
7230 Mask.push_back(SM_SentinelZero);
7231 else if (Mask1[i] == SM_SentinelZero)
7232 Mask.push_back(Mask0[i]);
7233 else if (Mask0[i] == SM_SentinelZero)
7234 Mask.push_back(Mask1[i] + (int)(MaskSize * SrcInputs0.size()));
7235 else
7236 return false;
7237 }
7238 Ops.append(SrcInputs0.begin(), SrcInputs0.end());
7239 Ops.append(SrcInputs1.begin(), SrcInputs1.end());
7240 return true;
7241 }
7242 case ISD::INSERT_SUBVECTOR: {
7243 SDValue Src = N.getOperand(0);
7244 SDValue Sub = N.getOperand(1);
7245 EVT SubVT = Sub.getValueType();
7246 unsigned NumSubElts = SubVT.getVectorNumElements();
7247 if (!isa<ConstantSDNode>(N.getOperand(2)) ||
7248 !N->isOnlyUserOf(Sub.getNode()))
7249 return false;
7250 uint64_t InsertIdx = N.getConstantOperandVal(2);
7251 // Handle INSERT_SUBVECTOR(SRC0, EXTRACT_SUBVECTOR(SRC1)).
7252 if (Sub.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
7253 Sub.getOperand(0).getValueType() == VT &&
7254 isa<ConstantSDNode>(Sub.getOperand(1))) {
7255 uint64_t ExtractIdx = Sub.getConstantOperandVal(1);
7256 for (int i = 0; i != (int)NumElts; ++i)
7257 Mask.push_back(i);
7258 for (int i = 0; i != (int)NumSubElts; ++i)
7259 Mask[InsertIdx + i] = NumElts + ExtractIdx + i;
7260 Ops.push_back(Src);
7261 Ops.push_back(Sub.getOperand(0));
7262 return true;
7263 }
7264 // Handle INSERT_SUBVECTOR(SRC0, SHUFFLE(SRC1)).
7265 SmallVector<int, 64> SubMask;
7266 SmallVector<SDValue, 2> SubInputs;
7267 if (!getTargetShuffleInputs(peekThroughOneUseBitcasts(Sub), SubInputs,
7268 SubMask, DAG, Depth + 1, ResolveKnownElts))
7269 return false;
7270 if (SubMask.size() != NumSubElts) {
7271 assert(((SubMask.size() % NumSubElts) == 0 ||((((SubMask.size() % NumSubElts) == 0 || (NumSubElts % SubMask
.size()) == 0) && "Illegal submask scale") ? static_cast
<void> (0) : __assert_fail ("((SubMask.size() % NumSubElts) == 0 || (NumSubElts % SubMask.size()) == 0) && \"Illegal submask scale\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7272, __PRETTY_FUNCTION__))
7272 (NumSubElts % SubMask.size()) == 0) && "Illegal submask scale")((((SubMask.size() % NumSubElts) == 0 || (NumSubElts % SubMask
.size()) == 0) && "Illegal submask scale") ? static_cast
<void> (0) : __assert_fail ("((SubMask.size() % NumSubElts) == 0 || (NumSubElts % SubMask.size()) == 0) && \"Illegal submask scale\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7272, __PRETTY_FUNCTION__));
7273 if ((NumSubElts % SubMask.size()) == 0) {
7274 int Scale = NumSubElts / SubMask.size();
7275 SmallVector<int,64> ScaledSubMask;
7276 scaleShuffleMask<int>(Scale, SubMask, ScaledSubMask);
7277 SubMask = ScaledSubMask;
7278 } else {
7279 int Scale = SubMask.size() / NumSubElts;
7280 NumSubElts = SubMask.size();
7281 NumElts *= Scale;
7282 InsertIdx *= Scale;
7283 }
7284 }
7285 Ops.push_back(Src);
7286 for (SDValue &SubInput : SubInputs) {
7287 EVT SubSVT = SubInput.getValueType().getScalarType();
7288 EVT AltVT = EVT::getVectorVT(*DAG.getContext(), SubSVT,
7289 NumSizeInBits / SubSVT.getSizeInBits());
7290 Ops.push_back(DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), AltVT,
7291 DAG.getUNDEF(AltVT), SubInput,
7292 DAG.getIntPtrConstant(0, SDLoc(N))));
7293 }
7294 for (int i = 0; i != (int)NumElts; ++i)
7295 Mask.push_back(i);
7296 for (int i = 0; i != (int)NumSubElts; ++i) {
7297 int M = SubMask[i];
7298 if (0 <= M) {
7299 int InputIdx = M / NumSubElts;
7300 M = (NumElts * (1 + InputIdx)) + (M % NumSubElts);
7301 }
7302 Mask[i + InsertIdx] = M;
7303 }
7304 return true;
7305 }
7306 case ISD::SCALAR_TO_VECTOR: {
7307 // Match against a scalar_to_vector of an extract from a vector,
7308 // for PEXTRW/PEXTRB we must handle the implicit zext of the scalar.
7309 SDValue N0 = N.getOperand(0);
7310 SDValue SrcExtract;
7311
7312 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
7313 N0.getOperand(0).getValueType() == VT) ||
7314 (N0.getOpcode() == X86ISD::PEXTRW &&
7315 N0.getOperand(0).getValueType() == MVT::v8i16) ||
7316 (N0.getOpcode() == X86ISD::PEXTRB &&
7317 N0.getOperand(0).getValueType() == MVT::v16i8)) {
7318 SrcExtract = N0;
7319 }
7320
7321 if (!SrcExtract || !isa<ConstantSDNode>(SrcExtract.getOperand(1)))
7322 return false;
7323
7324 SDValue SrcVec = SrcExtract.getOperand(0);
7325 EVT SrcVT = SrcVec.getValueType();
7326 unsigned NumSrcElts = SrcVT.getVectorNumElements();
7327 unsigned NumZeros = (NumBitsPerElt / SrcVT.getScalarSizeInBits()) - 1;
7328
7329 unsigned SrcIdx = SrcExtract.getConstantOperandVal(1);
7330 if (NumSrcElts <= SrcIdx)
7331 return false;
7332
7333 Ops.push_back(SrcVec);
7334 Mask.push_back(SrcIdx);
7335 Mask.append(NumZeros, SM_SentinelZero);
7336 Mask.append(NumSrcElts - Mask.size(), SM_SentinelUndef);
7337 return true;
7338 }
7339 case X86ISD::PINSRB:
7340 case X86ISD::PINSRW: {
7341 SDValue InVec = N.getOperand(0);
7342 SDValue InScl = N.getOperand(1);
7343 SDValue InIndex = N.getOperand(2);
7344 if (!isa<ConstantSDNode>(InIndex) ||
7345 cast<ConstantSDNode>(InIndex)->getAPIntValue().uge(NumElts))
7346 return false;
7347 uint64_t InIdx = N.getConstantOperandVal(2);
7348
7349 // Attempt to recognise a PINSR*(VEC, 0, Idx) shuffle pattern.
7350 if (X86::isZeroNode(InScl)) {
7351 Ops.push_back(InVec);
7352 for (unsigned i = 0; i != NumElts; ++i)
7353 Mask.push_back(i == InIdx ? SM_SentinelZero : (int)i);
7354 return true;
7355 }
7356
7357 // Attempt to recognise a PINSR*(PEXTR*) shuffle pattern.
7358 // TODO: Expand this to support INSERT_VECTOR_ELT/etc.
7359 unsigned ExOp =
7360 (X86ISD::PINSRB == Opcode ? X86ISD::PEXTRB : X86ISD::PEXTRW);
7361 if (InScl.getOpcode() != ExOp)
7362 return false;
7363
7364 SDValue ExVec = InScl.getOperand(0);
7365 SDValue ExIndex = InScl.getOperand(1);
7366 if (!isa<ConstantSDNode>(ExIndex) ||
7367 cast<ConstantSDNode>(ExIndex)->getAPIntValue().uge(NumElts))
7368 return false;
7369 uint64_t ExIdx = InScl.getConstantOperandVal(1);
7370
7371 Ops.push_back(InVec);
7372 Ops.push_back(ExVec);
7373 for (unsigned i = 0; i != NumElts; ++i)
7374 Mask.push_back(i == InIdx ? NumElts + ExIdx : i);
7375 return true;
7376 }
7377 case X86ISD::PACKSS:
7378 case X86ISD::PACKUS: {
7379 SDValue N0 = N.getOperand(0);
7380 SDValue N1 = N.getOperand(1);
7381 assert(N0.getValueType().getVectorNumElements() == (NumElts / 2) &&((N0.getValueType().getVectorNumElements() == (NumElts / 2) &&
N1.getValueType().getVectorNumElements() == (NumElts / 2) &&
"Unexpected input value type") ? static_cast<void> (0)
: __assert_fail ("N0.getValueType().getVectorNumElements() == (NumElts / 2) && N1.getValueType().getVectorNumElements() == (NumElts / 2) && \"Unexpected input value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7383, __PRETTY_FUNCTION__))
7382 N1.getValueType().getVectorNumElements() == (NumElts / 2) &&((N0.getValueType().getVectorNumElements() == (NumElts / 2) &&
N1.getValueType().getVectorNumElements() == (NumElts / 2) &&
"Unexpected input value type") ? static_cast<void> (0)
: __assert_fail ("N0.getValueType().getVectorNumElements() == (NumElts / 2) && N1.getValueType().getVectorNumElements() == (NumElts / 2) && \"Unexpected input value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7383, __PRETTY_FUNCTION__))
7383 "Unexpected input value type")((N0.getValueType().getVectorNumElements() == (NumElts / 2) &&
N1.getValueType().getVectorNumElements() == (NumElts / 2) &&
"Unexpected input value type") ? static_cast<void> (0)
: __assert_fail ("N0.getValueType().getVectorNumElements() == (NumElts / 2) && N1.getValueType().getVectorNumElements() == (NumElts / 2) && \"Unexpected input value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7383, __PRETTY_FUNCTION__));
7384
7385 APInt EltsLHS, EltsRHS;
7386 getPackDemandedElts(VT, DemandedElts, EltsLHS, EltsRHS);
7387
7388 // If we know input saturation won't happen we can treat this
7389 // as a truncation shuffle.
7390 if (Opcode == X86ISD::PACKSS) {
7391 if ((!N0.isUndef() &&
7392 DAG.ComputeNumSignBits(N0, EltsLHS, Depth + 1) <= NumBitsPerElt) ||
7393 (!N1.isUndef() &&
7394 DAG.ComputeNumSignBits(N1, EltsRHS, Depth + 1) <= NumBitsPerElt))
7395 return false;
7396 } else {
7397 APInt ZeroMask = APInt::getHighBitsSet(2 * NumBitsPerElt, NumBitsPerElt);
7398 if ((!N0.isUndef() &&
7399 !DAG.MaskedValueIsZero(N0, ZeroMask, EltsLHS, Depth + 1)) ||
7400 (!N1.isUndef() &&
7401 !DAG.MaskedValueIsZero(N1, ZeroMask, EltsRHS, Depth + 1)))
7402 return false;
7403 }
7404
7405 bool IsUnary = (N0 == N1);
7406
7407 Ops.push_back(N0);
7408 if (!IsUnary)
7409 Ops.push_back(N1);
7410
7411 createPackShuffleMask(VT, Mask, IsUnary);
7412 return true;
7413 }
7414 case X86ISD::VSHLI:
7415 case X86ISD::VSRLI: {
7416 uint64_t ShiftVal = N.getConstantOperandVal(1);
7417 // Out of range bit shifts are guaranteed to be zero.
7418 if (NumBitsPerElt <= ShiftVal) {
7419 Mask.append(NumElts, SM_SentinelZero);
7420 return true;
7421 }
7422
7423 // We can only decode 'whole byte' bit shifts as shuffles.
7424 if ((ShiftVal % 8) != 0)
7425 break;
7426
7427 uint64_t ByteShift = ShiftVal / 8;
7428 unsigned NumBytes = NumSizeInBits / 8;
7429 unsigned NumBytesPerElt = NumBitsPerElt / 8;
7430 Ops.push_back(N.getOperand(0));
7431
7432 // Clear mask to all zeros and insert the shifted byte indices.
7433 Mask.append(NumBytes, SM_SentinelZero);
7434
7435 if (X86ISD::VSHLI == Opcode) {
7436 for (unsigned i = 0; i != NumBytes; i += NumBytesPerElt)
7437 for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
7438 Mask[i + j] = i + j - ByteShift;
7439 } else {
7440 for (unsigned i = 0; i != NumBytes; i += NumBytesPerElt)
7441 for (unsigned j = ByteShift; j != NumBytesPerElt; ++j)
7442 Mask[i + j - ByteShift] = i + j;
7443 }
7444 return true;
7445 }
7446 case X86ISD::VBROADCAST: {
7447 SDValue Src = N.getOperand(0);
7448 MVT SrcVT = Src.getSimpleValueType();
7449 if (!SrcVT.isVector())
7450 return false;
7451
7452 if (NumSizeInBits != SrcVT.getSizeInBits()) {
7453 assert((NumSizeInBits % SrcVT.getSizeInBits()) == 0 &&(((NumSizeInBits % SrcVT.getSizeInBits()) == 0 && "Illegal broadcast type"
) ? static_cast<void> (0) : __assert_fail ("(NumSizeInBits % SrcVT.getSizeInBits()) == 0 && \"Illegal broadcast type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7454, __PRETTY_FUNCTION__))
7454 "Illegal broadcast type")(((NumSizeInBits % SrcVT.getSizeInBits()) == 0 && "Illegal broadcast type"
) ? static_cast<void> (0) : __assert_fail ("(NumSizeInBits % SrcVT.getSizeInBits()) == 0 && \"Illegal broadcast type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7454, __PRETTY_FUNCTION__));
7455 SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
7456 NumSizeInBits / SrcVT.getScalarSizeInBits());
7457 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), SrcVT,
7458 DAG.getUNDEF(SrcVT), Src,
7459 DAG.getIntPtrConstant(0, SDLoc(N)));
7460 }
7461
7462 Ops.push_back(Src);
7463 Mask.append(NumElts, 0);
7464 return true;
7465 }
7466 case ISD::ZERO_EXTEND:
7467 case ISD::ANY_EXTEND:
7468 case ISD::ZERO_EXTEND_VECTOR_INREG:
7469 case ISD::ANY_EXTEND_VECTOR_INREG: {
7470 SDValue Src = N.getOperand(0);
7471 EVT SrcVT = Src.getValueType();
7472
7473 // Extended source must be a simple vector.
7474 if (!SrcVT.isSimple() || (SrcVT.getSizeInBits() % 128) != 0 ||
7475 (SrcVT.getScalarSizeInBits() % 8) != 0)
7476 return false;
7477
7478 unsigned NumSrcBitsPerElt = SrcVT.getScalarSizeInBits();
7479 bool IsAnyExtend =
7480 (ISD::ANY_EXTEND == Opcode || ISD::ANY_EXTEND_VECTOR_INREG == Opcode);
7481 DecodeZeroExtendMask(NumSrcBitsPerElt, NumBitsPerElt, NumElts, IsAnyExtend,
7482 Mask);
7483
7484 if (NumSizeInBits != SrcVT.getSizeInBits()) {
7485 assert((NumSizeInBits % SrcVT.getSizeInBits()) == 0 &&(((NumSizeInBits % SrcVT.getSizeInBits()) == 0 && "Illegal zero-extension type"
) ? static_cast<void> (0) : __assert_fail ("(NumSizeInBits % SrcVT.getSizeInBits()) == 0 && \"Illegal zero-extension type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7486, __PRETTY_FUNCTION__))
7486 "Illegal zero-extension type")(((NumSizeInBits % SrcVT.getSizeInBits()) == 0 && "Illegal zero-extension type"
) ? static_cast<void> (0) : __assert_fail ("(NumSizeInBits % SrcVT.getSizeInBits()) == 0 && \"Illegal zero-extension type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7486, __PRETTY_FUNCTION__));
7487 SrcVT = MVT::getVectorVT(SrcVT.getSimpleVT().getScalarType(),
7488 NumSizeInBits / NumSrcBitsPerElt);
7489 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, SDLoc(N), SrcVT,
7490 DAG.getUNDEF(SrcVT), Src,
7491 DAG.getIntPtrConstant(0, SDLoc(N)));
7492 }
7493
7494 Ops.push_back(Src);
7495 return true;
7496 }
7497 }
7498
7499 return false;
7500}
7501
7502/// Removes unused/repeated shuffle source inputs and adjusts the shuffle mask.
7503static void resolveTargetShuffleInputsAndMask(SmallVectorImpl<SDValue> &Inputs,
7504 SmallVectorImpl<int> &Mask) {
7505 int MaskWidth = Mask.size();
7506 SmallVector<SDValue, 16> UsedInputs;
7507 for (int i = 0, e = Inputs.size(); i < e; ++i) {
7508 int lo = UsedInputs.size() * MaskWidth;
7509 int hi = lo + MaskWidth;
7510
7511 // Strip UNDEF input usage.
7512 if (Inputs[i].isUndef())
7513 for (int &M : Mask)
7514 if ((lo <= M) && (M < hi))
7515 M = SM_SentinelUndef;
7516
7517 // Check for unused inputs.
7518 if (none_of(Mask, [lo, hi](int i) { return (lo <= i) && (i < hi); })) {
7519 for (int &M : Mask)
7520 if (lo <= M)
7521 M -= MaskWidth;
7522 continue;
7523 }
7524
7525 // Check for repeated inputs.
7526 bool IsRepeat = false;
7527 for (int j = 0, ue = UsedInputs.size(); j != ue; ++j) {
7528 if (UsedInputs[j] != Inputs[i])
7529 continue;
7530 for (int &M : Mask)
7531 if (lo <= M)
7532 M = (M < hi) ? ((M - lo) + (j * MaskWidth)) : (M - MaskWidth);
7533 IsRepeat = true;
7534 break;
7535 }
7536 if (IsRepeat)
7537 continue;
7538
7539 UsedInputs.push_back(Inputs[i]);
7540 }
7541 Inputs = UsedInputs;
7542}
7543
7544/// Calls getTargetShuffleAndZeroables to resolve a target shuffle mask's inputs
7545/// and then sets the SM_SentinelUndef and SM_SentinelZero values.
7546/// Returns true if the target shuffle mask was decoded.
7547static bool getTargetShuffleInputs(SDValue Op, const APInt &DemandedElts,
7548 SmallVectorImpl<SDValue> &Inputs,
7549 SmallVectorImpl<int> &Mask,
7550 APInt &KnownUndef, APInt &KnownZero,
7551 SelectionDAG &DAG, unsigned Depth,
7552 bool ResolveKnownElts) {
7553 EVT VT = Op.getValueType();
7554 if (!VT.isSimple() || !VT.isVector())
7555 return false;
7556
7557 if (getTargetShuffleAndZeroables(Op, Mask, Inputs, KnownUndef, KnownZero)) {
7558 if (ResolveKnownElts)
7559 resolveTargetShuffleFromZeroables(Mask, KnownUndef, KnownZero);
7560 return true;
7561 }
7562 if (getFauxShuffleMask(Op, DemandedElts, Mask, Inputs, DAG, Depth,
7563 ResolveKnownElts)) {
7564 resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
7565 return true;
7566 }
7567 return false;
7568}
7569
7570static bool getTargetShuffleInputs(SDValue Op, SmallVectorImpl<SDValue> &Inputs,
7571 SmallVectorImpl<int> &Mask,
7572 SelectionDAG &DAG, unsigned Depth = 0,
7573 bool ResolveKnownElts = true) {
7574 EVT VT = Op.getValueType();
7575 if (!VT.isSimple() || !VT.isVector())
7576 return false;
7577
7578 APInt KnownUndef, KnownZero;
7579 unsigned NumElts = Op.getValueType().getVectorNumElements();
7580 APInt DemandedElts = APInt::getAllOnesValue(NumElts);
7581 return getTargetShuffleInputs(Op, DemandedElts, Inputs, Mask, KnownUndef,
7582 KnownZero, DAG, Depth, ResolveKnownElts);
7583}
7584
7585/// Returns the scalar element that will make up the ith
7586/// element of the result of the vector shuffle.
7587static SDValue getShuffleScalarElt(SDNode *N, unsigned Index, SelectionDAG &DAG,
7588 unsigned Depth) {
7589 if (Depth == 6)
7590 return SDValue(); // Limit search depth.
7591
7592 SDValue V = SDValue(N, 0);
7593 EVT VT = V.getValueType();
7594 unsigned Opcode = V.getOpcode();
7595
7596 // Recurse into ISD::VECTOR_SHUFFLE node to find scalars.
7597 if (const ShuffleVectorSDNode *SV = dyn_cast<ShuffleVectorSDNode>(N)) {
7598 int Elt = SV->getMaskElt(Index);
7599
7600 if (Elt < 0)
7601 return DAG.getUNDEF(VT.getVectorElementType());
7602
7603 unsigned NumElems = VT.getVectorNumElements();
7604 SDValue NewV = (Elt < (int)NumElems) ? SV->getOperand(0)
7605 : SV->getOperand(1);
7606 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG, Depth+1);
7607 }
7608
7609 // Recurse into target specific vector shuffles to find scalars.
7610 if (isTargetShuffle(Opcode)) {
7611 MVT ShufVT = V.getSimpleValueType();
7612 MVT ShufSVT = ShufVT.getVectorElementType();
7613 int NumElems = (int)ShufVT.getVectorNumElements();
7614 SmallVector<int, 16> ShuffleMask;
7615 SmallVector<SDValue, 16> ShuffleOps;
7616 bool IsUnary;
7617
7618 if (!getTargetShuffleMask(N, ShufVT, true, ShuffleOps, ShuffleMask, IsUnary))
7619 return SDValue();
7620
7621 int Elt = ShuffleMask[Index];
7622 if (Elt == SM_SentinelZero)
7623 return ShufSVT.isInteger() ? DAG.getConstant(0, SDLoc(N), ShufSVT)
7624 : DAG.getConstantFP(+0.0, SDLoc(N), ShufSVT);
7625 if (Elt == SM_SentinelUndef)
7626 return DAG.getUNDEF(ShufSVT);
7627
7628 assert(0 <= Elt && Elt < (2*NumElems) && "Shuffle index out of range")((0 <= Elt && Elt < (2*NumElems) && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= Elt && Elt < (2*NumElems) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7628, __PRETTY_FUNCTION__));
7629 SDValue NewV = (Elt < NumElems) ? ShuffleOps[0] : ShuffleOps[1];
7630 return getShuffleScalarElt(NewV.getNode(), Elt % NumElems, DAG,
7631 Depth+1);
7632 }
7633
7634 // Recurse into insert_subvector base/sub vector to find scalars.
7635 if (Opcode == ISD::INSERT_SUBVECTOR &&
7636 isa<ConstantSDNode>(N->getOperand(2))) {
7637 SDValue Vec = N->getOperand(0);
7638 SDValue Sub = N->getOperand(1);
7639 EVT SubVT = Sub.getValueType();
7640 unsigned NumSubElts = SubVT.getVectorNumElements();
7641 uint64_t SubIdx = N->getConstantOperandVal(2);
7642
7643 if (SubIdx <= Index && Index < (SubIdx + NumSubElts))
7644 return getShuffleScalarElt(Sub.getNode(), Index - SubIdx, DAG, Depth + 1);
7645 return getShuffleScalarElt(Vec.getNode(), Index, DAG, Depth + 1);
7646 }
7647
7648 // Recurse into extract_subvector src vector to find scalars.
7649 if (Opcode == ISD::EXTRACT_SUBVECTOR &&
7650 isa<ConstantSDNode>(N->getOperand(1))) {
7651 SDValue Src = N->getOperand(0);
7652 uint64_t SrcIdx = N->getConstantOperandVal(1);
7653 return getShuffleScalarElt(Src.getNode(), Index + SrcIdx, DAG, Depth + 1);
7654 }
7655
7656 // Actual nodes that may contain scalar elements
7657 if (Opcode == ISD::BITCAST) {
7658 V = V.getOperand(0);
7659 EVT SrcVT = V.getValueType();
7660 unsigned NumElems = VT.getVectorNumElements();
7661
7662 if (!SrcVT.isVector() || SrcVT.getVectorNumElements() != NumElems)
7663 return SDValue();
7664 }
7665
7666 if (V.getOpcode() == ISD::SCALAR_TO_VECTOR)
7667 return (Index == 0) ? V.getOperand(0)
7668 : DAG.getUNDEF(VT.getVectorElementType());
7669
7670 if (V.getOpcode() == ISD::BUILD_VECTOR)
7671 return V.getOperand(Index);
7672
7673 return SDValue();
7674}
7675
7676// Use PINSRB/PINSRW/PINSRD to create a build vector.
7677static SDValue LowerBuildVectorAsInsert(SDValue Op, unsigned NonZeros,
7678 unsigned NumNonZero, unsigned NumZero,
7679 SelectionDAG &DAG,
7680 const X86Subtarget &Subtarget) {
7681 MVT VT = Op.getSimpleValueType();
7682 unsigned NumElts = VT.getVectorNumElements();
7683 assert(((VT == MVT::v8i16 && Subtarget.hasSSE2()) ||((((VT == MVT::v8i16 && Subtarget.hasSSE2()) || ((VT ==
MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41
())) && "Illegal vector insertion") ? static_cast<
void> (0) : __assert_fail ("((VT == MVT::v8i16 && Subtarget.hasSSE2()) || ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) && \"Illegal vector insertion\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7685, __PRETTY_FUNCTION__))
7684 ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) &&((((VT == MVT::v8i16 && Subtarget.hasSSE2()) || ((VT ==
MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41
())) && "Illegal vector insertion") ? static_cast<
void> (0) : __assert_fail ("((VT == MVT::v8i16 && Subtarget.hasSSE2()) || ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) && \"Illegal vector insertion\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7685, __PRETTY_FUNCTION__))
7685 "Illegal vector insertion")((((VT == MVT::v8i16 && Subtarget.hasSSE2()) || ((VT ==
MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41
())) && "Illegal vector insertion") ? static_cast<
void> (0) : __assert_fail ("((VT == MVT::v8i16 && Subtarget.hasSSE2()) || ((VT == MVT::v16i8 || VT == MVT::v4i32) && Subtarget.hasSSE41())) && \"Illegal vector insertion\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7685, __PRETTY_FUNCTION__));
7686
7687 SDLoc dl(Op);
7688 SDValue V;
7689 bool First = true;
7690
7691 for (unsigned i = 0; i < NumElts; ++i) {
7692 bool IsNonZero = (NonZeros & (1 << i)) != 0;
7693 if (!IsNonZero)
7694 continue;
7695
7696 // If the build vector contains zeros or our first insertion is not the
7697 // first index then insert into zero vector to break any register
7698 // dependency else use SCALAR_TO_VECTOR.
7699 if (First) {
7700 First = false;
7701 if (NumZero || 0 != i)
7702 V = getZeroVector(VT, Subtarget, DAG, dl);
7703 else {
7704 assert(0 == i && "Expected insertion into zero-index")((0 == i && "Expected insertion into zero-index") ? static_cast
<void> (0) : __assert_fail ("0 == i && \"Expected insertion into zero-index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7704, __PRETTY_FUNCTION__));
7705 V = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
7706 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, V);
7707 V = DAG.getBitcast(VT, V);
7708 continue;
7709 }
7710 }
7711 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, V, Op.getOperand(i),
7712 DAG.getIntPtrConstant(i, dl));
7713 }
7714
7715 return V;
7716}
7717
7718/// Custom lower build_vector of v16i8.
7719static SDValue LowerBuildVectorv16i8(SDValue Op, unsigned NonZeros,
7720 unsigned NumNonZero, unsigned NumZero,
7721 SelectionDAG &DAG,
7722 const X86Subtarget &Subtarget) {
7723 if (NumNonZero > 8 && !Subtarget.hasSSE41())
7724 return SDValue();
7725
7726 // SSE4.1 - use PINSRB to insert each byte directly.
7727 if (Subtarget.hasSSE41())
7728 return LowerBuildVectorAsInsert(Op, NonZeros, NumNonZero, NumZero, DAG,
7729 Subtarget);
7730
7731 SDLoc dl(Op);
7732 SDValue V;
7733
7734 // Pre-SSE4.1 - merge byte pairs and insert with PINSRW.
7735 for (unsigned i = 0; i < 16; i += 2) {
7736 bool ThisIsNonZero = (NonZeros & (1 << i)) != 0;
7737 bool NextIsNonZero = (NonZeros & (1 << (i + 1))) != 0;
7738 if (!ThisIsNonZero && !NextIsNonZero)
7739 continue;
7740
7741 // FIXME: Investigate combining the first 4 bytes as a i32 instead.
7742 SDValue Elt;
7743 if (ThisIsNonZero) {
7744 if (NumZero || NextIsNonZero)
7745 Elt = DAG.getZExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
7746 else
7747 Elt = DAG.getAnyExtOrTrunc(Op.getOperand(i), dl, MVT::i32);
7748 }
7749
7750 if (NextIsNonZero) {
7751 SDValue NextElt = Op.getOperand(i + 1);
7752 if (i == 0 && NumZero)
7753 NextElt = DAG.getZExtOrTrunc(NextElt, dl, MVT::i32);
7754 else
7755 NextElt = DAG.getAnyExtOrTrunc(NextElt, dl, MVT::i32);
7756 NextElt = DAG.getNode(ISD::SHL, dl, MVT::i32, NextElt,
7757 DAG.getConstant(8, dl, MVT::i8));
7758 if (ThisIsNonZero)
7759 Elt = DAG.getNode(ISD::OR, dl, MVT::i32, NextElt, Elt);
7760 else
7761 Elt = NextElt;
7762 }
7763
7764 // If our first insertion is not the first index then insert into zero
7765 // vector to break any register dependency else use SCALAR_TO_VECTOR.
7766 if (!V) {
7767 if (i != 0)
7768 V = getZeroVector(MVT::v8i16, Subtarget, DAG, dl);
7769 else {
7770 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Elt);
7771 V = DAG.getBitcast(MVT::v8i16, V);
7772 continue;
7773 }
7774 }
7775 Elt = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Elt);
7776 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, MVT::v8i16, V, Elt,
7777 DAG.getIntPtrConstant(i / 2, dl));
7778 }
7779
7780 return DAG.getBitcast(MVT::v16i8, V);
7781}
7782
7783/// Custom lower build_vector of v8i16.
7784static SDValue LowerBuildVectorv8i16(SDValue Op, unsigned NonZeros,
7785 unsigned NumNonZero, unsigned NumZero,
7786 SelectionDAG &DAG,
7787 const X86Subtarget &Subtarget) {
7788 if (NumNonZero > 4 && !Subtarget.hasSSE41())
7789 return SDValue();
7790
7791 // Use PINSRW to insert each byte directly.
7792 return LowerBuildVectorAsInsert(Op, NonZeros, NumNonZero, NumZero, DAG,
7793 Subtarget);
7794}
7795
7796/// Custom lower build_vector of v4i32 or v4f32.
7797static SDValue LowerBuildVectorv4x32(SDValue Op, SelectionDAG &DAG,
7798 const X86Subtarget &Subtarget) {
7799 // If this is a splat of a pair of elements, use MOVDDUP (unless the target
7800 // has XOP; in that case defer lowering to potentially use VPERMIL2PS).
7801 // Because we're creating a less complicated build vector here, we may enable
7802 // further folding of the MOVDDUP via shuffle transforms.
7803 if (Subtarget.hasSSE3() && !Subtarget.hasXOP() &&
7804 Op.getOperand(0) == Op.getOperand(2) &&
7805 Op.getOperand(1) == Op.getOperand(3) &&
7806 Op.getOperand(0) != Op.getOperand(1)) {
7807 SDLoc DL(Op);
7808 MVT VT = Op.getSimpleValueType();
7809 MVT EltVT = VT.getVectorElementType();
7810 // Create a new build vector with the first 2 elements followed by undef
7811 // padding, bitcast to v2f64, duplicate, and bitcast back.
7812 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
7813 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
7814 SDValue NewBV = DAG.getBitcast(MVT::v2f64, DAG.getBuildVector(VT, DL, Ops));
7815 SDValue Dup = DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v2f64, NewBV);
7816 return DAG.getBitcast(VT, Dup);
7817 }
7818
7819 // Find all zeroable elements.
7820 std::bitset<4> Zeroable, Undefs;
7821 for (int i = 0; i < 4; ++i) {
7822 SDValue Elt = Op.getOperand(i);
7823 Undefs[i] = Elt.isUndef();
7824 Zeroable[i] = (Elt.isUndef() || X86::isZeroNode(Elt));
7825 }
7826 assert(Zeroable.size() - Zeroable.count() > 1 &&((Zeroable.size() - Zeroable.count() > 1 && "We expect at least two non-zero elements!"
) ? static_cast<void> (0) : __assert_fail ("Zeroable.size() - Zeroable.count() > 1 && \"We expect at least two non-zero elements!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7827, __PRETTY_FUNCTION__))
7827 "We expect at least two non-zero elements!")((Zeroable.size() - Zeroable.count() > 1 && "We expect at least two non-zero elements!"
) ? static_cast<void> (0) : __assert_fail ("Zeroable.size() - Zeroable.count() > 1 && \"We expect at least two non-zero elements!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7827, __PRETTY_FUNCTION__));
7828
7829 // We only know how to deal with build_vector nodes where elements are either
7830 // zeroable or extract_vector_elt with constant index.
7831 SDValue FirstNonZero;
7832 unsigned FirstNonZeroIdx;
7833 for (unsigned i = 0; i < 4; ++i) {
7834 if (Zeroable[i])
7835 continue;
7836 SDValue Elt = Op.getOperand(i);
7837 if (Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
7838 !isa<ConstantSDNode>(Elt.getOperand(1)))
7839 return SDValue();
7840 // Make sure that this node is extracting from a 128-bit vector.
7841 MVT VT = Elt.getOperand(0).getSimpleValueType();
7842 if (!VT.is128BitVector())
7843 return SDValue();
7844 if (!FirstNonZero.getNode()) {
7845 FirstNonZero = Elt;
7846 FirstNonZeroIdx = i;
7847 }
7848 }
7849
7850 assert(FirstNonZero.getNode() && "Unexpected build vector of all zeros!")((FirstNonZero.getNode() && "Unexpected build vector of all zeros!"
) ? static_cast<void> (0) : __assert_fail ("FirstNonZero.getNode() && \"Unexpected build vector of all zeros!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7850, __PRETTY_FUNCTION__));
7851 SDValue V1 = FirstNonZero.getOperand(0);
7852 MVT VT = V1.getSimpleValueType();
7853
7854 // See if this build_vector can be lowered as a blend with zero.
7855 SDValue Elt;
7856 unsigned EltMaskIdx, EltIdx;
7857 int Mask[4];
7858 for (EltIdx = 0; EltIdx < 4; ++EltIdx) {
7859 if (Zeroable[EltIdx]) {
7860 // The zero vector will be on the right hand side.
7861 Mask[EltIdx] = EltIdx+4;
7862 continue;
7863 }
7864
7865 Elt = Op->getOperand(EltIdx);
7866 // By construction, Elt is a EXTRACT_VECTOR_ELT with constant index.
7867 EltMaskIdx = Elt.getConstantOperandVal(1);
7868 if (Elt.getOperand(0) != V1 || EltMaskIdx != EltIdx)
7869 break;
7870 Mask[EltIdx] = EltIdx;
7871 }
7872
7873 if (EltIdx == 4) {
7874 // Let the shuffle legalizer deal with blend operations.
7875 SDValue VZeroOrUndef = (Zeroable == Undefs)
7876 ? DAG.getUNDEF(VT)
7877 : getZeroVector(VT, Subtarget, DAG, SDLoc(Op));
7878 if (V1.getSimpleValueType() != VT)
7879 V1 = DAG.getBitcast(VT, V1);
7880 return DAG.getVectorShuffle(VT, SDLoc(V1), V1, VZeroOrUndef, Mask);
7881 }
7882
7883 // See if we can lower this build_vector to a INSERTPS.
7884 if (!Subtarget.hasSSE41())
7885 return SDValue();
7886
7887 SDValue V2 = Elt.getOperand(0);
7888 if (Elt == FirstNonZero && EltIdx == FirstNonZeroIdx)
7889 V1 = SDValue();
7890
7891 bool CanFold = true;
7892 for (unsigned i = EltIdx + 1; i < 4 && CanFold; ++i) {
7893 if (Zeroable[i])
7894 continue;
7895
7896 SDValue Current = Op->getOperand(i);
7897 SDValue SrcVector = Current->getOperand(0);
7898 if (!V1.getNode())
7899 V1 = SrcVector;
7900 CanFold = (SrcVector == V1) && (Current.getConstantOperandAPInt(1) == i);
7901 }
7902
7903 if (!CanFold)
7904 return SDValue();
7905
7906 assert(V1.getNode() && "Expected at least two non-zero elements!")((V1.getNode() && "Expected at least two non-zero elements!"
) ? static_cast<void> (0) : __assert_fail ("V1.getNode() && \"Expected at least two non-zero elements!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7906, __PRETTY_FUNCTION__));
7907 if (V1.getSimpleValueType() != MVT::v4f32)
7908 V1 = DAG.getBitcast(MVT::v4f32, V1);
7909 if (V2.getSimpleValueType() != MVT::v4f32)
7910 V2 = DAG.getBitcast(MVT::v4f32, V2);
7911
7912 // Ok, we can emit an INSERTPS instruction.
7913 unsigned ZMask = Zeroable.to_ulong();
7914
7915 unsigned InsertPSMask = EltMaskIdx << 6 | EltIdx << 4 | ZMask;
7916 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!")(((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!"
) ? static_cast<void> (0) : __assert_fail ("(InsertPSMask & ~0xFFu) == 0 && \"Invalid mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7916, __PRETTY_FUNCTION__));
7917 SDLoc DL(Op);
7918 SDValue Result = DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
7919 DAG.getIntPtrConstant(InsertPSMask, DL, true));
7920 return DAG.getBitcast(VT, Result);
7921}
7922
7923/// Return a vector logical shift node.
7924static SDValue getVShift(bool isLeft, EVT VT, SDValue SrcOp, unsigned NumBits,
7925 SelectionDAG &DAG, const TargetLowering &TLI,
7926 const SDLoc &dl) {
7927 assert(VT.is128BitVector() && "Unknown type for VShift")((VT.is128BitVector() && "Unknown type for VShift") ?
static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Unknown type for VShift\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7927, __PRETTY_FUNCTION__));
7928 MVT ShVT = MVT::v16i8;
7929 unsigned Opc = isLeft ? X86ISD::VSHLDQ : X86ISD::VSRLDQ;
7930 SrcOp = DAG.getBitcast(ShVT, SrcOp);
7931 assert(NumBits % 8 == 0 && "Only support byte sized shifts")((NumBits % 8 == 0 && "Only support byte sized shifts"
) ? static_cast<void> (0) : __assert_fail ("NumBits % 8 == 0 && \"Only support byte sized shifts\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 7931, __PRETTY_FUNCTION__));
7932 SDValue ShiftVal = DAG.getTargetConstant(NumBits / 8, dl, MVT::i8);
7933 return DAG.getBitcast(VT, DAG.getNode(Opc, dl, ShVT, SrcOp, ShiftVal));
7934}
7935
7936static SDValue LowerAsSplatVectorLoad(SDValue SrcOp, MVT VT, const SDLoc &dl,
7937 SelectionDAG &DAG) {
7938
7939 // Check if the scalar load can be widened into a vector load. And if
7940 // the address is "base + cst" see if the cst can be "absorbed" into
7941 // the shuffle mask.
7942 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(SrcOp)) {
7943 SDValue Ptr = LD->getBasePtr();
7944 if (!ISD::isNormalLoad(LD) || !LD->isSimple())
7945 return SDValue();
7946 EVT PVT = LD->getValueType(0);
7947 if (PVT != MVT::i32 && PVT != MVT::f32)
7948 return SDValue();
7949
7950 int FI = -1;
7951 int64_t Offset = 0;
7952 if (FrameIndexSDNode *FINode = dyn_cast<FrameIndexSDNode>(Ptr)) {
7953 FI = FINode->getIndex();
7954 Offset = 0;
7955 } else if (DAG.isBaseWithConstantOffset(Ptr) &&
7956 isa<FrameIndexSDNode>(Ptr.getOperand(0))) {
7957 FI = cast<FrameIndexSDNode>(Ptr.getOperand(0))->getIndex();
7958 Offset = Ptr.getConstantOperandVal(1);
7959 Ptr = Ptr.getOperand(0);
7960 } else {
7961 return SDValue();
7962 }
7963
7964 // FIXME: 256-bit vector instructions don't require a strict alignment,
7965 // improve this code to support it better.
7966 unsigned RequiredAlign = VT.getSizeInBits()/8;
7967 SDValue Chain = LD->getChain();
7968 // Make sure the stack object alignment is at least 16 or 32.
7969 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
7970 if (DAG.InferPtrAlignment(Ptr) < RequiredAlign) {
7971 if (MFI.isFixedObjectIndex(FI)) {
7972 // Can't change the alignment. FIXME: It's possible to compute
7973 // the exact stack offset and reference FI + adjust offset instead.
7974 // If someone *really* cares about this. That's the way to implement it.
7975 return SDValue();
7976 } else {
7977 MFI.setObjectAlignment(FI, RequiredAlign);
7978 }
7979 }
7980
7981 // (Offset % 16 or 32) must be multiple of 4. Then address is then
7982 // Ptr + (Offset & ~15).
7983 if (Offset < 0)
7984 return SDValue();
7985 if ((Offset % RequiredAlign) & 3)
7986 return SDValue();
7987 int64_t StartOffset = Offset & ~int64_t(RequiredAlign - 1);
7988 if (StartOffset) {
7989 SDLoc DL(Ptr);
7990 Ptr = DAG.getNode(ISD::ADD, DL, Ptr.getValueType(), Ptr,
7991 DAG.getConstant(StartOffset, DL, Ptr.getValueType()));
7992 }
7993
7994 int EltNo = (Offset - StartOffset) >> 2;
7995 unsigned NumElems = VT.getVectorNumElements();
7996
7997 EVT NVT = EVT::getVectorVT(*DAG.getContext(), PVT, NumElems);
7998 SDValue V1 = DAG.getLoad(NVT, dl, Chain, Ptr,
7999 LD->getPointerInfo().getWithOffset(StartOffset));
8000
8001 SmallVector<int, 8> Mask(NumElems, EltNo);
8002
8003 return DAG.getVectorShuffle(NVT, dl, V1, DAG.getUNDEF(NVT), Mask);
8004 }
8005
8006 return SDValue();
8007}
8008
8009// Recurse to find a LoadSDNode source and the accumulated ByteOffest.
8010static bool findEltLoadSrc(SDValue Elt, LoadSDNode *&Ld, int64_t &ByteOffset) {
8011 if (ISD::isNON_EXTLoad(Elt.getNode())) {
8012 auto *BaseLd = cast<LoadSDNode>(Elt);
8013 if (!BaseLd->isSimple())
8014 return false;
8015 Ld = BaseLd;
8016 ByteOffset = 0;
8017 return true;
8018 }
8019
8020 switch (Elt.getOpcode()) {
8021 case ISD::BITCAST:
8022 case ISD::TRUNCATE:
8023 case ISD::SCALAR_TO_VECTOR:
8024 return findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset);
8025 case ISD::SRL:
8026 if (isa<ConstantSDNode>(Elt.getOperand(1))) {
8027 uint64_t Idx = Elt.getConstantOperandVal(1);
8028 if ((Idx % 8) == 0 && findEltLoadSrc(Elt.getOperand(0), Ld, ByteOffset)) {
8029 ByteOffset += Idx / 8;
8030 return true;
8031 }
8032 }
8033 break;
8034 case ISD::EXTRACT_VECTOR_ELT:
8035 if (isa<ConstantSDNode>(Elt.getOperand(1))) {
8036 SDValue Src = Elt.getOperand(0);
8037 unsigned SrcSizeInBits = Src.getScalarValueSizeInBits();
8038 unsigned DstSizeInBits = Elt.getScalarValueSizeInBits();
8039 if (DstSizeInBits == SrcSizeInBits && (SrcSizeInBits % 8) == 0 &&
8040 findEltLoadSrc(Src, Ld, ByteOffset)) {
8041 uint64_t Idx = Elt.getConstantOperandVal(1);
8042 ByteOffset += Idx * (SrcSizeInBits / 8);
8043 return true;
8044 }
8045 }
8046 break;
8047 }
8048
8049 return false;
8050}
8051
8052/// Given the initializing elements 'Elts' of a vector of type 'VT', see if the
8053/// elements can be replaced by a single large load which has the same value as
8054/// a build_vector or insert_subvector whose loaded operands are 'Elts'.
8055///
8056/// Example: <load i32 *a, load i32 *a+4, zero, undef> -> zextload a
8057static SDValue EltsFromConsecutiveLoads(EVT VT, ArrayRef<SDValue> Elts,
8058 const SDLoc &DL, SelectionDAG &DAG,
8059 const X86Subtarget &Subtarget,
8060 bool isAfterLegalize) {
8061 if ((VT.getScalarSizeInBits() % 8) != 0)
8062 return SDValue();
8063
8064 unsigned NumElems = Elts.size();
8065
8066 int LastLoadedElt = -1;
8067 APInt LoadMask = APInt::getNullValue(NumElems);
8068 APInt ZeroMask = APInt::getNullValue(NumElems);
8069 APInt UndefMask = APInt::getNullValue(NumElems);
8070
8071 SmallVector<LoadSDNode*, 8> Loads(NumElems, nullptr);
8072 SmallVector<int64_t, 8> ByteOffsets(NumElems, 0);
8073
8074 // For each element in the initializer, see if we've found a load, zero or an
8075 // undef.
8076 for (unsigned i = 0; i < NumElems; ++i) {
8077 SDValue Elt = peekThroughBitcasts(Elts[i]);
8078 if (!Elt.getNode())
8079 return SDValue();
8080 if (Elt.isUndef()) {
8081 UndefMask.setBit(i);
8082 continue;
8083 }
8084 if (X86::isZeroNode(Elt) || ISD::isBuildVectorAllZeros(Elt.getNode())) {
8085 ZeroMask.setBit(i);
8086 continue;
8087 }
8088
8089 // Each loaded element must be the correct fractional portion of the
8090 // requested vector load.
8091 unsigned EltSizeInBits = Elt.getValueSizeInBits();
8092 if ((NumElems * EltSizeInBits) != VT.getSizeInBits())
8093 return SDValue();
8094
8095 if (!findEltLoadSrc(Elt, Loads[i], ByteOffsets[i]) || ByteOffsets[i] < 0)
8096 return SDValue();
8097 unsigned LoadSizeInBits = Loads[i]->getValueSizeInBits(0);
8098 if (((ByteOffsets[i] * 8) + EltSizeInBits) > LoadSizeInBits)
8099 return SDValue();
8100
8101 LoadMask.setBit(i);
8102 LastLoadedElt = i;
8103 }
8104 assert((ZeroMask.countPopulation() + UndefMask.countPopulation() +(((ZeroMask.countPopulation() + UndefMask.countPopulation() +
LoadMask.countPopulation()) == NumElems && "Incomplete element masks"
) ? static_cast<void> (0) : __assert_fail ("(ZeroMask.countPopulation() + UndefMask.countPopulation() + LoadMask.countPopulation()) == NumElems && \"Incomplete element masks\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8106, __PRETTY_FUNCTION__))
8105 LoadMask.countPopulation()) == NumElems &&(((ZeroMask.countPopulation() + UndefMask.countPopulation() +
LoadMask.countPopulation()) == NumElems && "Incomplete element masks"
) ? static_cast<void> (0) : __assert_fail ("(ZeroMask.countPopulation() + UndefMask.countPopulation() + LoadMask.countPopulation()) == NumElems && \"Incomplete element masks\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8106, __PRETTY_FUNCTION__))
8106 "Incomplete element masks")(((ZeroMask.countPopulation() + UndefMask.countPopulation() +
LoadMask.countPopulation()) == NumElems && "Incomplete element masks"
) ? static_cast<void> (0) : __assert_fail ("(ZeroMask.countPopulation() + UndefMask.countPopulation() + LoadMask.countPopulation()) == NumElems && \"Incomplete element masks\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8106, __PRETTY_FUNCTION__));
8107
8108 // Handle Special Cases - all undef or undef/zero.
8109 if (UndefMask.countPopulation() == NumElems)
8110 return DAG.getUNDEF(VT);
8111
8112 // FIXME: Should we return this as a BUILD_VECTOR instead?
8113 if ((ZeroMask.countPopulation() + UndefMask.countPopulation()) == NumElems)
8114 return VT.isInteger() ? DAG.getConstant(0, DL, VT)
8115 : DAG.getConstantFP(0.0, DL, VT);
8116
8117 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8118 int FirstLoadedElt = LoadMask.countTrailingZeros();
8119 SDValue EltBase = peekThroughBitcasts(Elts[FirstLoadedElt]);
8120 EVT EltBaseVT = EltBase.getValueType();
8121 assert(EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits() &&((EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits()
&& "Register/Memory size mismatch") ? static_cast<
void> (0) : __assert_fail ("EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits() && \"Register/Memory size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8122, __PRETTY_FUNCTION__))
8122 "Register/Memory size mismatch")((EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits()
&& "Register/Memory size mismatch") ? static_cast<
void> (0) : __assert_fail ("EltBaseVT.getSizeInBits() == EltBaseVT.getStoreSizeInBits() && \"Register/Memory size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8122, __PRETTY_FUNCTION__));
8123 LoadSDNode *LDBase = Loads[FirstLoadedElt];
8124 assert(LDBase && "Did not find base load for merging consecutive loads")((LDBase && "Did not find base load for merging consecutive loads"
) ? static_cast<void> (0) : __assert_fail ("LDBase && \"Did not find base load for merging consecutive loads\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8124, __PRETTY_FUNCTION__));
8125 unsigned BaseSizeInBits = EltBaseVT.getStoreSizeInBits();
8126 unsigned BaseSizeInBytes = BaseSizeInBits / 8;
8127 int LoadSizeInBits = (1 + LastLoadedElt - FirstLoadedElt) * BaseSizeInBits;
8128 assert((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected")(((BaseSizeInBits % 8) == 0 && "Sub-byte element loads detected"
) ? static_cast<void> (0) : __assert_fail ("(BaseSizeInBits % 8) == 0 && \"Sub-byte element loads detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8128, __PRETTY_FUNCTION__));
8129
8130 // TODO: Support offsetting the base load.
8131 if (ByteOffsets[FirstLoadedElt] != 0)
8132 return SDValue();
8133
8134 // Check to see if the element's load is consecutive to the base load
8135 // or offset from a previous (already checked) load.
8136 auto CheckConsecutiveLoad = [&](LoadSDNode *Base, int EltIdx) {
8137 LoadSDNode *Ld = Loads[EltIdx];
8138 int64_t ByteOffset = ByteOffsets[EltIdx];
8139 if (ByteOffset && (ByteOffset % BaseSizeInBytes) == 0) {
8140 int64_t BaseIdx = EltIdx - (ByteOffset / BaseSizeInBytes);
8141 return (0 <= BaseIdx && BaseIdx < (int)NumElems && LoadMask[BaseIdx] &&
8142 Loads[BaseIdx] == Ld && ByteOffsets[BaseIdx] == 0);
8143 }
8144 return DAG.areNonVolatileConsecutiveLoads(Ld, Base, BaseSizeInBytes,
8145 EltIdx - FirstLoadedElt);
8146 };
8147
8148 // Consecutive loads can contain UNDEFS but not ZERO elements.
8149 // Consecutive loads with UNDEFs and ZEROs elements require a
8150 // an additional shuffle stage to clear the ZERO elements.
8151 bool IsConsecutiveLoad = true;
8152 bool IsConsecutiveLoadWithZeros = true;
8153 for (int i = FirstLoadedElt + 1; i <= LastLoadedElt; ++i) {
8154 if (LoadMask[i]) {
8155 if (!CheckConsecutiveLoad(LDBase, i)) {
8156 IsConsecutiveLoad = false;
8157 IsConsecutiveLoadWithZeros = false;
8158 break;
8159 }
8160 } else if (ZeroMask[i]) {
8161 IsConsecutiveLoad = false;
8162 }
8163 }
8164
8165 auto CreateLoad = [&DAG, &DL, &Loads](EVT VT, LoadSDNode *LDBase) {
8166 auto MMOFlags = LDBase->getMemOperand()->getFlags();
8167 assert(LDBase->isSimple() &&((LDBase->isSimple() && "Cannot merge volatile or atomic loads."
) ? static_cast<void> (0) : __assert_fail ("LDBase->isSimple() && \"Cannot merge volatile or atomic loads.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8168, __PRETTY_FUNCTION__))
8168 "Cannot merge volatile or atomic loads.")((LDBase->isSimple() && "Cannot merge volatile or atomic loads."
) ? static_cast<void> (0) : __assert_fail ("LDBase->isSimple() && \"Cannot merge volatile or atomic loads.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8168, __PRETTY_FUNCTION__));
8169 SDValue NewLd =
8170 DAG.getLoad(VT, DL, LDBase->getChain(), LDBase->getBasePtr(),
8171 LDBase->getPointerInfo(), LDBase->getAlignment(), MMOFlags);
8172 for (auto *LD : Loads)
8173 if (LD)
8174 DAG.makeEquivalentMemoryOrdering(LD, NewLd);
8175 return NewLd;
8176 };
8177
8178 // Check if the base load is entirely dereferenceable.
8179 bool IsDereferenceable = LDBase->getPointerInfo().isDereferenceable(
8180 VT.getSizeInBits() / 8, *DAG.getContext(), DAG.getDataLayout());
8181
8182 // LOAD - all consecutive load/undefs (must start/end with a load or be
8183 // entirely dereferenceable). If we have found an entire vector of loads and
8184 // undefs, then return a large load of the entire vector width starting at the
8185 // base pointer. If the vector contains zeros, then attempt to shuffle those
8186 // elements.
8187 if (FirstLoadedElt == 0 &&
8188 (LastLoadedElt == (int)(NumElems - 1) || IsDereferenceable) &&
8189 (IsConsecutiveLoad || IsConsecutiveLoadWithZeros)) {
8190 if (isAfterLegalize && !TLI.isOperationLegal(ISD::LOAD, VT))
8191 return SDValue();
8192
8193 // Don't create 256-bit non-temporal aligned loads without AVX2 as these
8194 // will lower to regular temporal loads and use the cache.
8195 if (LDBase->isNonTemporal() && LDBase->getAlignment() >= 32 &&
8196 VT.is256BitVector() && !Subtarget.hasInt256())
8197 return SDValue();
8198
8199 if (NumElems == 1)
8200 return DAG.getBitcast(VT, Elts[FirstLoadedElt]);
8201
8202 if (!ZeroMask)
8203 return CreateLoad(VT, LDBase);
8204
8205 // IsConsecutiveLoadWithZeros - we need to create a shuffle of the loaded
8206 // vector and a zero vector to clear out the zero elements.
8207 if (!isAfterLegalize && VT.isVector()) {
8208 unsigned NumMaskElts = VT.getVectorNumElements();
8209 if ((NumMaskElts % NumElems) == 0) {
8210 unsigned Scale = NumMaskElts / NumElems;
8211 SmallVector<int, 4> ClearMask(NumMaskElts, -1);
8212 for (unsigned i = 0; i < NumElems; ++i) {
8213 if (UndefMask[i])
8214 continue;
8215 int Offset = ZeroMask[i] ? NumMaskElts : 0;
8216 for (unsigned j = 0; j != Scale; ++j)
8217 ClearMask[(i * Scale) + j] = (i * Scale) + j + Offset;
8218 }
8219 SDValue V = CreateLoad(VT, LDBase);
8220 SDValue Z = VT.isInteger() ? DAG.getConstant(0, DL, VT)
8221 : DAG.getConstantFP(0.0, DL, VT);
8222 return DAG.getVectorShuffle(VT, DL, V, Z, ClearMask);
8223 }
8224 }
8225 }
8226
8227 // If the upper half of a ymm/zmm load is undef then just load the lower half.
8228 if (VT.is256BitVector() || VT.is512BitVector()) {
8229 unsigned HalfNumElems = NumElems / 2;
8230 if (UndefMask.extractBits(HalfNumElems, HalfNumElems).isAllOnesValue()) {
8231 EVT HalfVT =
8232 EVT::getVectorVT(*DAG.getContext(), VT.getScalarType(), HalfNumElems);
8233 SDValue HalfLD =
8234 EltsFromConsecutiveLoads(HalfVT, Elts.drop_back(HalfNumElems), DL,
8235 DAG, Subtarget, isAfterLegalize);
8236 if (HalfLD)
8237 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT),
8238 HalfLD, DAG.getIntPtrConstant(0, DL));
8239 }
8240 }
8241
8242 // VZEXT_LOAD - consecutive 32/64-bit load/undefs followed by zeros/undefs.
8243 if (IsConsecutiveLoad && FirstLoadedElt == 0 &&
8244 (LoadSizeInBits == 32 || LoadSizeInBits == 64) &&
8245 ((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()))) {
8246 MVT VecSVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(LoadSizeInBits)
8247 : MVT::getIntegerVT(LoadSizeInBits);
8248 MVT VecVT = MVT::getVectorVT(VecSVT, VT.getSizeInBits() / LoadSizeInBits);
8249 if (TLI.isTypeLegal(VecVT)) {
8250 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
8251 SDValue Ops[] = { LDBase->getChain(), LDBase->getBasePtr() };
8252 SDValue ResNode =
8253 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, DL, Tys, Ops, VecSVT,
8254 LDBase->getPointerInfo(),
8255 LDBase->getAlignment(),
8256 MachineMemOperand::MOLoad);
8257 for (auto *LD : Loads)
8258 if (LD)
8259 DAG.makeEquivalentMemoryOrdering(LD, ResNode);
8260 return DAG.getBitcast(VT, ResNode);
8261 }
8262 }
8263
8264 // BROADCAST - match the smallest possible repetition pattern, load that
8265 // scalar/subvector element and then broadcast to the entire vector.
8266 if (ZeroMask.isNullValue() && isPowerOf2_32(NumElems) && Subtarget.hasAVX() &&
8267 (VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector())) {
8268 for (unsigned SubElems = 1; SubElems < NumElems; SubElems *= 2) {
8269 unsigned RepeatSize = SubElems * BaseSizeInBits;
8270 unsigned ScalarSize = std::min(RepeatSize, 64u);
8271 if (!Subtarget.hasAVX2() && ScalarSize < 32)
8272 continue;
8273
8274 bool Match = true;
8275 SmallVector<SDValue, 8> RepeatedLoads(SubElems, DAG.getUNDEF(EltBaseVT));
8276 for (unsigned i = 0; i != NumElems && Match; ++i) {
8277 if (!LoadMask[i])
8278 continue;
8279 SDValue Elt = peekThroughBitcasts(Elts[i]);
8280 if (RepeatedLoads[i % SubElems].isUndef())
8281 RepeatedLoads[i % SubElems] = Elt;
8282 else
8283 Match &= (RepeatedLoads[i % SubElems] == Elt);
8284 }
8285
8286 // We must have loads at both ends of the repetition.
8287 Match &= !RepeatedLoads.front().isUndef();
8288 Match &= !RepeatedLoads.back().isUndef();
8289 if (!Match)
8290 continue;
8291
8292 EVT RepeatVT =
8293 VT.isInteger() && (RepeatSize != 64 || TLI.isTypeLegal(MVT::i64))
8294 ? EVT::getIntegerVT(*DAG.getContext(), ScalarSize)
8295 : EVT::getFloatingPointVT(ScalarSize);
8296 if (RepeatSize > ScalarSize)
8297 RepeatVT = EVT::getVectorVT(*DAG.getContext(), RepeatVT,
8298 RepeatSize / ScalarSize);
8299 EVT BroadcastVT =
8300 EVT::getVectorVT(*DAG.getContext(), RepeatVT.getScalarType(),
8301 VT.getSizeInBits() / ScalarSize);
8302 if (TLI.isTypeLegal(BroadcastVT)) {
8303 if (SDValue RepeatLoad = EltsFromConsecutiveLoads(
8304 RepeatVT, RepeatedLoads, DL, DAG, Subtarget, isAfterLegalize)) {
8305 unsigned Opcode = RepeatSize > ScalarSize ? X86ISD::SUBV_BROADCAST
8306 : X86ISD::VBROADCAST;
8307 SDValue Broadcast = DAG.getNode(Opcode, DL, BroadcastVT, RepeatLoad);
8308 return DAG.getBitcast(VT, Broadcast);
8309 }
8310 }
8311 }
8312 }
8313
8314 return SDValue();
8315}
8316
8317// Combine a vector ops (shuffles etc.) that is equal to build_vector load1,
8318// load2, load3, load4, <0, 1, 2, 3> into a vector load if the load addresses
8319// are consecutive, non-overlapping, and in the right order.
8320static SDValue combineToConsecutiveLoads(EVT VT, SDNode *N, const SDLoc &DL,
8321 SelectionDAG &DAG,
8322 const X86Subtarget &Subtarget,
8323 bool isAfterLegalize) {
8324 SmallVector<SDValue, 64> Elts;
8325 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
8326 if (SDValue Elt = getShuffleScalarElt(N, i, DAG, 0)) {
8327 Elts.push_back(Elt);
8328 continue;
8329 }
8330 return SDValue();
8331 }
8332 assert(Elts.size() == VT.getVectorNumElements())((Elts.size() == VT.getVectorNumElements()) ? static_cast<
void> (0) : __assert_fail ("Elts.size() == VT.getVectorNumElements()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8332, __PRETTY_FUNCTION__));
8333 return EltsFromConsecutiveLoads(VT, Elts, DL, DAG, Subtarget,
8334 isAfterLegalize);
8335}
8336
8337static Constant *getConstantVector(MVT VT, const APInt &SplatValue,
8338 unsigned SplatBitSize, LLVMContext &C) {
8339 unsigned ScalarSize = VT.getScalarSizeInBits();
8340 unsigned NumElm = SplatBitSize / ScalarSize;
8341
8342 SmallVector<Constant *, 32> ConstantVec;
8343 for (unsigned i = 0; i < NumElm; i++) {
8344 APInt Val = SplatValue.extractBits(ScalarSize, ScalarSize * i);
8345 Constant *Const;
8346 if (VT.isFloatingPoint()) {
8347 if (ScalarSize == 32) {
8348 Const = ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val));
8349 } else {
8350 assert(ScalarSize == 64 && "Unsupported floating point scalar size")((ScalarSize == 64 && "Unsupported floating point scalar size"
) ? static_cast<void> (0) : __assert_fail ("ScalarSize == 64 && \"Unsupported floating point scalar size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8350, __PRETTY_FUNCTION__));
8351 Const = ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val));
8352 }
8353 } else
8354 Const = Constant::getIntegerValue(Type::getIntNTy(C, ScalarSize), Val);
8355 ConstantVec.push_back(Const);
8356 }
8357 return ConstantVector::get(ArrayRef<Constant *>(ConstantVec));
8358}
8359
8360static bool isFoldableUseOfShuffle(SDNode *N) {
8361 for (auto *U : N->uses()) {
8362 unsigned Opc = U->getOpcode();
8363 // VPERMV/VPERMV3 shuffles can never fold their index operands.
8364 if (Opc == X86ISD::VPERMV && U->getOperand(0).getNode() == N)
8365 return false;
8366 if (Opc == X86ISD::VPERMV3 && U->getOperand(1).getNode() == N)
8367 return false;
8368 if (isTargetShuffle(Opc))
8369 return true;
8370 if (Opc == ISD::BITCAST) // Ignore bitcasts
8371 return isFoldableUseOfShuffle(U);
8372 if (N->hasOneUse())
8373 return true;
8374 }
8375 return false;
8376}
8377
8378// Check if the current node of build vector is a zero extended vector.
8379// // If so, return the value extended.
8380// // For example: (0,0,0,a,0,0,0,a,0,0,0,a,0,0,0,a) returns a.
8381// // NumElt - return the number of zero extended identical values.
8382// // EltType - return the type of the value include the zero extend.
8383static SDValue isSplatZeroExtended(const BuildVectorSDNode *Op,
8384 unsigned &NumElt, MVT &EltType) {
8385 SDValue ExtValue = Op->getOperand(0);
8386 unsigned NumElts = Op->getNumOperands();
8387 unsigned Delta = NumElts;
8388
8389 for (unsigned i = 1; i < NumElts; i++) {
8390 if (Op->getOperand(i) == ExtValue) {
8391 Delta = i;
8392 break;
8393 }
8394 if (!(Op->getOperand(i).isUndef() || isNullConstant(Op->getOperand(i))))
8395 return SDValue();
8396 }
8397 if (!isPowerOf2_32(Delta) || Delta == 1)
8398 return SDValue();
8399
8400 for (unsigned i = Delta; i < NumElts; i++) {
8401 if (i % Delta == 0) {
8402 if (Op->getOperand(i) != ExtValue)
8403 return SDValue();
8404 } else if (!(isNullConstant(Op->getOperand(i)) ||
8405 Op->getOperand(i).isUndef()))
8406 return SDValue();
8407 }
8408 unsigned EltSize = Op->getSimpleValueType(0).getScalarSizeInBits();
8409 unsigned ExtVTSize = EltSize * Delta;
8410 EltType = MVT::getIntegerVT(ExtVTSize);
8411 NumElt = NumElts / Delta;
8412 return ExtValue;
8413}
8414
8415/// Attempt to use the vbroadcast instruction to generate a splat value
8416/// from a splat BUILD_VECTOR which uses:
8417/// a. A single scalar load, or a constant.
8418/// b. Repeated pattern of constants (e.g. <0,1,0,1> or <0,1,2,3,0,1,2,3>).
8419///
8420/// The VBROADCAST node is returned when a pattern is found,
8421/// or SDValue() otherwise.
8422static SDValue lowerBuildVectorAsBroadcast(BuildVectorSDNode *BVOp,
8423 const X86Subtarget &Subtarget,
8424 SelectionDAG &DAG) {
8425 // VBROADCAST requires AVX.
8426 // TODO: Splats could be generated for non-AVX CPUs using SSE
8427 // instructions, but there's less potential gain for only 128-bit vectors.
8428 if (!Subtarget.hasAVX())
8429 return SDValue();
8430
8431 MVT VT = BVOp->getSimpleValueType(0);
8432 SDLoc dl(BVOp);
8433
8434 assert((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) &&(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
()) && "Unsupported vector type for broadcast.") ? static_cast
<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && \"Unsupported vector type for broadcast.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8435, __PRETTY_FUNCTION__))
8435 "Unsupported vector type for broadcast.")(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
()) && "Unsupported vector type for broadcast.") ? static_cast
<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && \"Unsupported vector type for broadcast.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8435, __PRETTY_FUNCTION__));
8436
8437 BitVector UndefElements;
8438 SDValue Ld = BVOp->getSplatValue(&UndefElements);
8439
8440 // Attempt to use VBROADCASTM
8441 // From this paterrn:
8442 // a. t0 = (zext_i64 (bitcast_i8 v2i1 X))
8443 // b. t1 = (build_vector t0 t0)
8444 //
8445 // Create (VBROADCASTM v2i1 X)
8446 if (Subtarget.hasCDI() && (VT.is512BitVector() || Subtarget.hasVLX())) {
8447 MVT EltType = VT.getScalarType();
8448 unsigned NumElts = VT.getVectorNumElements();
8449 SDValue BOperand;
8450 SDValue ZeroExtended = isSplatZeroExtended(BVOp, NumElts, EltType);
8451 if ((ZeroExtended && ZeroExtended.getOpcode() == ISD::BITCAST) ||
8452 (Ld && Ld.getOpcode() == ISD::ZERO_EXTEND &&
8453 Ld.getOperand(0).getOpcode() == ISD::BITCAST)) {
8454 if (ZeroExtended)
8455 BOperand = ZeroExtended.getOperand(0);
8456 else
8457 BOperand = Ld.getOperand(0).getOperand(0);
8458 MVT MaskVT = BOperand.getSimpleValueType();
8459 if ((EltType == MVT::i64 && MaskVT == MVT::v8i1) || // for broadcastmb2q
8460 (EltType == MVT::i32 && MaskVT == MVT::v16i1)) { // for broadcastmw2d
8461 SDValue Brdcst =
8462 DAG.getNode(X86ISD::VBROADCASTM, dl,
8463 MVT::getVectorVT(EltType, NumElts), BOperand);
8464 return DAG.getBitcast(VT, Brdcst);
8465 }
8466 }
8467 }
8468
8469 unsigned NumElts = VT.getVectorNumElements();
8470 unsigned NumUndefElts = UndefElements.count();
8471 if (!Ld || (NumElts - NumUndefElts) <= 1) {
8472 APInt SplatValue, Undef;
8473 unsigned SplatBitSize;
8474 bool HasUndef;
8475 // Check if this is a repeated constant pattern suitable for broadcasting.
8476 if (BVOp->isConstantSplat(SplatValue, Undef, SplatBitSize, HasUndef) &&
8477 SplatBitSize > VT.getScalarSizeInBits() &&
8478 SplatBitSize < VT.getSizeInBits()) {
8479 // Avoid replacing with broadcast when it's a use of a shuffle
8480 // instruction to preserve the present custom lowering of shuffles.
8481 if (isFoldableUseOfShuffle(BVOp))
8482 return SDValue();
8483 // replace BUILD_VECTOR with broadcast of the repeated constants.
8484 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8485 LLVMContext *Ctx = DAG.getContext();
8486 MVT PVT = TLI.getPointerTy(DAG.getDataLayout());
8487 if (Subtarget.hasAVX()) {
8488 if (SplatBitSize <= 64 && Subtarget.hasAVX2() &&
8489 !(SplatBitSize == 64 && Subtarget.is32Bit())) {
8490 // Splatted value can fit in one INTEGER constant in constant pool.
8491 // Load the constant and broadcast it.
8492 MVT CVT = MVT::getIntegerVT(SplatBitSize);
8493 Type *ScalarTy = Type::getIntNTy(*Ctx, SplatBitSize);
8494 Constant *C = Constant::getIntegerValue(ScalarTy, SplatValue);
8495 SDValue CP = DAG.getConstantPool(C, PVT);
8496 unsigned Repeat = VT.getSizeInBits() / SplatBitSize;
8497
8498 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
8499 Ld = DAG.getLoad(
8500 CVT, dl, DAG.getEntryNode(), CP,
8501 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
8502 Alignment);
8503 SDValue Brdcst = DAG.getNode(X86ISD::VBROADCAST, dl,
8504 MVT::getVectorVT(CVT, Repeat), Ld);
8505 return DAG.getBitcast(VT, Brdcst);
8506 } else if (SplatBitSize == 32 || SplatBitSize == 64) {
8507 // Splatted value can fit in one FLOAT constant in constant pool.
8508 // Load the constant and broadcast it.
8509 // AVX have support for 32 and 64 bit broadcast for floats only.
8510 // No 64bit integer in 32bit subtarget.
8511 MVT CVT = MVT::getFloatingPointVT(SplatBitSize);
8512 // Lower the splat via APFloat directly, to avoid any conversion.
8513 Constant *C =
8514 SplatBitSize == 32
8515 ? ConstantFP::get(*Ctx,
8516 APFloat(APFloat::IEEEsingle(), SplatValue))
8517 : ConstantFP::get(*Ctx,
8518 APFloat(APFloat::IEEEdouble(), SplatValue));
8519 SDValue CP = DAG.getConstantPool(C, PVT);
8520 unsigned Repeat = VT.getSizeInBits() / SplatBitSize;
8521
8522 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
8523 Ld = DAG.getLoad(
8524 CVT, dl, DAG.getEntryNode(), CP,
8525 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
8526 Alignment);
8527 SDValue Brdcst = DAG.getNode(X86ISD::VBROADCAST, dl,
8528 MVT::getVectorVT(CVT, Repeat), Ld);
8529 return DAG.getBitcast(VT, Brdcst);
8530 } else if (SplatBitSize > 64) {
8531 // Load the vector of constants and broadcast it.
8532 MVT CVT = VT.getScalarType();
8533 Constant *VecC = getConstantVector(VT, SplatValue, SplatBitSize,
8534 *Ctx);
8535 SDValue VCP = DAG.getConstantPool(VecC, PVT);
8536 unsigned NumElm = SplatBitSize / VT.getScalarSizeInBits();
8537 unsigned Alignment = cast<ConstantPoolSDNode>(VCP)->getAlignment();
8538 Ld = DAG.getLoad(
8539 MVT::getVectorVT(CVT, NumElm), dl, DAG.getEntryNode(), VCP,
8540 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
8541 Alignment);
8542 SDValue Brdcst = DAG.getNode(X86ISD::SUBV_BROADCAST, dl, VT, Ld);
8543 return DAG.getBitcast(VT, Brdcst);
8544 }
8545 }
8546 }
8547
8548 // If we are moving a scalar into a vector (Ld must be set and all elements
8549 // but 1 are undef) and that operation is not obviously supported by
8550 // vmovd/vmovq/vmovss/vmovsd, then keep trying to form a broadcast.
8551 // That's better than general shuffling and may eliminate a load to GPR and
8552 // move from scalar to vector register.
8553 if (!Ld || NumElts - NumUndefElts != 1)
8554 return SDValue();
8555 unsigned ScalarSize = Ld.getValueSizeInBits();
8556 if (!(UndefElements[0] || (ScalarSize != 32 && ScalarSize != 64)))
8557 return SDValue();
8558 }
8559
8560 bool ConstSplatVal =
8561 (Ld.getOpcode() == ISD::Constant || Ld.getOpcode() == ISD::ConstantFP);
8562
8563 // Make sure that all of the users of a non-constant load are from the
8564 // BUILD_VECTOR node.
8565 if (!ConstSplatVal && !BVOp->isOnlyUserOf(Ld.getNode()))
8566 return SDValue();
8567
8568 unsigned ScalarSize = Ld.getValueSizeInBits();
8569 bool IsGE256 = (VT.getSizeInBits() >= 256);
8570
8571 // When optimizing for size, generate up to 5 extra bytes for a broadcast
8572 // instruction to save 8 or more bytes of constant pool data.
8573 // TODO: If multiple splats are generated to load the same constant,
8574 // it may be detrimental to overall size. There needs to be a way to detect
8575 // that condition to know if this is truly a size win.
8576 bool OptForSize = DAG.shouldOptForSize();
8577
8578 // Handle broadcasting a single constant scalar from the constant pool
8579 // into a vector.
8580 // On Sandybridge (no AVX2), it is still better to load a constant vector
8581 // from the constant pool and not to broadcast it from a scalar.
8582 // But override that restriction when optimizing for size.
8583 // TODO: Check if splatting is recommended for other AVX-capable CPUs.
8584 if (ConstSplatVal && (Subtarget.hasAVX2() || OptForSize)) {
8585 EVT CVT = Ld.getValueType();
8586 assert(!CVT.isVector() && "Must not broadcast a vector type")((!CVT.isVector() && "Must not broadcast a vector type"
) ? static_cast<void> (0) : __assert_fail ("!CVT.isVector() && \"Must not broadcast a vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8586, __PRETTY_FUNCTION__));
8587
8588 // Splat f32, i32, v4f64, v4i64 in all cases with AVX2.
8589 // For size optimization, also splat v2f64 and v2i64, and for size opt
8590 // with AVX2, also splat i8 and i16.
8591 // With pattern matching, the VBROADCAST node may become a VMOVDDUP.
8592 if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
8593 (OptForSize && (ScalarSize == 64 || Subtarget.hasAVX2()))) {
8594 const Constant *C = nullptr;
8595 if (ConstantSDNode *CI = dyn_cast<ConstantSDNode>(Ld))
8596 C = CI->getConstantIntValue();
8597 else if (ConstantFPSDNode *CF = dyn_cast<ConstantFPSDNode>(Ld))
8598 C = CF->getConstantFPValue();
8599
8600 assert(C && "Invalid constant type")((C && "Invalid constant type") ? static_cast<void
> (0) : __assert_fail ("C && \"Invalid constant type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8600, __PRETTY_FUNCTION__));
8601
8602 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8603 SDValue CP =
8604 DAG.getConstantPool(C, TLI.getPointerTy(DAG.getDataLayout()));
8605 unsigned Alignment = cast<ConstantPoolSDNode>(CP)->getAlignment();
8606 Ld = DAG.getLoad(
8607 CVT, dl, DAG.getEntryNode(), CP,
8608 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
8609 Alignment);
8610
8611 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
8612 }
8613 }
8614
8615 bool IsLoad = ISD::isNormalLoad(Ld.getNode());
8616
8617 // Handle AVX2 in-register broadcasts.
8618 if (!IsLoad && Subtarget.hasInt256() &&
8619 (ScalarSize == 32 || (IsGE256 && ScalarSize == 64)))
8620 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
8621
8622 // The scalar source must be a normal load.
8623 if (!IsLoad)
8624 return SDValue();
8625
8626 if (ScalarSize == 32 || (IsGE256 && ScalarSize == 64) ||
8627 (Subtarget.hasVLX() && ScalarSize == 64))
8628 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
8629
8630 // The integer check is needed for the 64-bit into 128-bit so it doesn't match
8631 // double since there is no vbroadcastsd xmm
8632 if (Subtarget.hasInt256() && Ld.getValueType().isInteger()) {
8633 if (ScalarSize == 8 || ScalarSize == 16 || ScalarSize == 64)
8634 return DAG.getNode(X86ISD::VBROADCAST, dl, VT, Ld);
8635 }
8636
8637 // Unsupported broadcast.
8638 return SDValue();
8639}
8640
8641/// For an EXTRACT_VECTOR_ELT with a constant index return the real
8642/// underlying vector and index.
8643///
8644/// Modifies \p ExtractedFromVec to the real vector and returns the real
8645/// index.
8646static int getUnderlyingExtractedFromVec(SDValue &ExtractedFromVec,
8647 SDValue ExtIdx) {
8648 int Idx = cast<ConstantSDNode>(ExtIdx)->getZExtValue();
8649 if (!isa<ShuffleVectorSDNode>(ExtractedFromVec))
8650 return Idx;
8651
8652 // For 256-bit vectors, LowerEXTRACT_VECTOR_ELT_SSE4 may have already
8653 // lowered this:
8654 // (extract_vector_elt (v8f32 %1), Constant<6>)
8655 // to:
8656 // (extract_vector_elt (vector_shuffle<2,u,u,u>
8657 // (extract_subvector (v8f32 %0), Constant<4>),
8658 // undef)
8659 // Constant<0>)
8660 // In this case the vector is the extract_subvector expression and the index
8661 // is 2, as specified by the shuffle.
8662 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(ExtractedFromVec);
8663 SDValue ShuffleVec = SVOp->getOperand(0);
8664 MVT ShuffleVecVT = ShuffleVec.getSimpleValueType();
8665 assert(ShuffleVecVT.getVectorElementType() ==((ShuffleVecVT.getVectorElementType() == ExtractedFromVec.getSimpleValueType
().getVectorElementType()) ? static_cast<void> (0) : __assert_fail
("ShuffleVecVT.getVectorElementType() == ExtractedFromVec.getSimpleValueType().getVectorElementType()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8666, __PRETTY_FUNCTION__))
8666 ExtractedFromVec.getSimpleValueType().getVectorElementType())((ShuffleVecVT.getVectorElementType() == ExtractedFromVec.getSimpleValueType
().getVectorElementType()) ? static_cast<void> (0) : __assert_fail
("ShuffleVecVT.getVectorElementType() == ExtractedFromVec.getSimpleValueType().getVectorElementType()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8666, __PRETTY_FUNCTION__));
8667
8668 int ShuffleIdx = SVOp->getMaskElt(Idx);
8669 if (isUndefOrInRange(ShuffleIdx, 0, ShuffleVecVT.getVectorNumElements())) {
8670 ExtractedFromVec = ShuffleVec;
8671 return ShuffleIdx;
8672 }
8673 return Idx;
8674}
8675
8676static SDValue buildFromShuffleMostly(SDValue Op, SelectionDAG &DAG) {
8677 MVT VT = Op.getSimpleValueType();
8678
8679 // Skip if insert_vec_elt is not supported.
8680 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
8681 if (!TLI.isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT))
8682 return SDValue();
8683
8684 SDLoc DL(Op);
8685 unsigned NumElems = Op.getNumOperands();
8686
8687 SDValue VecIn1;
8688 SDValue VecIn2;
8689 SmallVector<unsigned, 4> InsertIndices;
8690 SmallVector<int, 8> Mask(NumElems, -1);
8691
8692 for (unsigned i = 0; i != NumElems; ++i) {
8693 unsigned Opc = Op.getOperand(i).getOpcode();
8694
8695 if (Opc == ISD::UNDEF)
8696 continue;
8697
8698 if (Opc != ISD::EXTRACT_VECTOR_ELT) {
8699 // Quit if more than 1 elements need inserting.
8700 if (InsertIndices.size() > 1)
8701 return SDValue();
8702
8703 InsertIndices.push_back(i);
8704 continue;
8705 }
8706
8707 SDValue ExtractedFromVec = Op.getOperand(i).getOperand(0);
8708 SDValue ExtIdx = Op.getOperand(i).getOperand(1);
8709
8710 // Quit if non-constant index.
8711 if (!isa<ConstantSDNode>(ExtIdx))
8712 return SDValue();
8713 int Idx = getUnderlyingExtractedFromVec(ExtractedFromVec, ExtIdx);
8714
8715 // Quit if extracted from vector of different type.
8716 if (ExtractedFromVec.getValueType() != VT)
8717 return SDValue();
8718
8719 if (!VecIn1.getNode())
8720 VecIn1 = ExtractedFromVec;
8721 else if (VecIn1 != ExtractedFromVec) {
8722 if (!VecIn2.getNode())
8723 VecIn2 = ExtractedFromVec;
8724 else if (VecIn2 != ExtractedFromVec)
8725 // Quit if more than 2 vectors to shuffle
8726 return SDValue();
8727 }
8728
8729 if (ExtractedFromVec == VecIn1)
8730 Mask[i] = Idx;
8731 else if (ExtractedFromVec == VecIn2)
8732 Mask[i] = Idx + NumElems;
8733 }
8734
8735 if (!VecIn1.getNode())
8736 return SDValue();
8737
8738 VecIn2 = VecIn2.getNode() ? VecIn2 : DAG.getUNDEF(VT);
8739 SDValue NV = DAG.getVectorShuffle(VT, DL, VecIn1, VecIn2, Mask);
8740
8741 for (unsigned Idx : InsertIndices)
8742 NV = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, NV, Op.getOperand(Idx),
8743 DAG.getIntPtrConstant(Idx, DL));
8744
8745 return NV;
8746}
8747
8748static SDValue ConvertI1VectorToInteger(SDValue Op, SelectionDAG &DAG) {
8749 assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&((ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
Op.getScalarValueSizeInBits() == 1 && "Can not convert non-constant vector"
) ? static_cast<void> (0) : __assert_fail ("ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && Op.getScalarValueSizeInBits() == 1 && \"Can not convert non-constant vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8751, __PRETTY_FUNCTION__))
8750 Op.getScalarValueSizeInBits() == 1 &&((ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
Op.getScalarValueSizeInBits() == 1 && "Can not convert non-constant vector"
) ? static_cast<void> (0) : __assert_fail ("ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && Op.getScalarValueSizeInBits() == 1 && \"Can not convert non-constant vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8751, __PRETTY_FUNCTION__))
8751 "Can not convert non-constant vector")((ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
Op.getScalarValueSizeInBits() == 1 && "Can not convert non-constant vector"
) ? static_cast<void> (0) : __assert_fail ("ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && Op.getScalarValueSizeInBits() == 1 && \"Can not convert non-constant vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8751, __PRETTY_FUNCTION__));
8752 uint64_t Immediate = 0;
8753 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
8754 SDValue In = Op.getOperand(idx);
8755 if (!In.isUndef())
8756 Immediate |= (cast<ConstantSDNode>(In)->getZExtValue() & 0x1) << idx;
8757 }
8758 SDLoc dl(Op);
8759 MVT VT = MVT::getIntegerVT(std::max((int)Op.getValueSizeInBits(), 8));
8760 return DAG.getConstant(Immediate, dl, VT);
8761}
8762// Lower BUILD_VECTOR operation for v8i1 and v16i1 types.
8763static SDValue LowerBUILD_VECTORvXi1(SDValue Op, SelectionDAG &DAG,
8764 const X86Subtarget &Subtarget) {
8765
8766 MVT VT = Op.getSimpleValueType();
8767 assert((VT.getVectorElementType() == MVT::i1) &&(((VT.getVectorElementType() == MVT::i1) && "Unexpected type in LowerBUILD_VECTORvXi1!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getVectorElementType() == MVT::i1) && \"Unexpected type in LowerBUILD_VECTORvXi1!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8768, __PRETTY_FUNCTION__))
8768 "Unexpected type in LowerBUILD_VECTORvXi1!")(((VT.getVectorElementType() == MVT::i1) && "Unexpected type in LowerBUILD_VECTORvXi1!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getVectorElementType() == MVT::i1) && \"Unexpected type in LowerBUILD_VECTORvXi1!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8768, __PRETTY_FUNCTION__));
8769
8770 SDLoc dl(Op);
8771 if (ISD::isBuildVectorAllZeros(Op.getNode()) ||
8772 ISD::isBuildVectorAllOnes(Op.getNode()))
8773 return Op;
8774
8775 uint64_t Immediate = 0;
8776 SmallVector<unsigned, 16> NonConstIdx;
8777 bool IsSplat = true;
8778 bool HasConstElts = false;
8779 int SplatIdx = -1;
8780 for (unsigned idx = 0, e = Op.getNumOperands(); idx < e; ++idx) {
8781 SDValue In = Op.getOperand(idx);
8782 if (In.isUndef())
8783 continue;
8784 if (!isa<ConstantSDNode>(In))
8785 NonConstIdx.push_back(idx);
8786 else {
8787 Immediate |= (cast<ConstantSDNode>(In)->getZExtValue() & 0x1) << idx;
8788 HasConstElts = true;
8789 }
8790 if (SplatIdx < 0)
8791 SplatIdx = idx;
8792 else if (In != Op.getOperand(SplatIdx))
8793 IsSplat = false;
8794 }
8795
8796 // for splat use " (select i1 splat_elt, all-ones, all-zeroes)"
8797 if (IsSplat) {
8798 // The build_vector allows the scalar element to be larger than the vector
8799 // element type. We need to mask it to use as a condition unless we know
8800 // the upper bits are zero.
8801 // FIXME: Use computeKnownBits instead of checking specific opcode?
8802 SDValue Cond = Op.getOperand(SplatIdx);
8803 assert(Cond.getValueType() == MVT::i8 && "Unexpected VT!")((Cond.getValueType() == MVT::i8 && "Unexpected VT!")
? static_cast<void> (0) : __assert_fail ("Cond.getValueType() == MVT::i8 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8803, __PRETTY_FUNCTION__));
8804 if (Cond.getOpcode() != ISD::SETCC)
8805 Cond = DAG.getNode(ISD::AND, dl, MVT::i8, Cond,
8806 DAG.getConstant(1, dl, MVT::i8));
8807 return DAG.getSelect(dl, VT, Cond,
8808 DAG.getConstant(1, dl, VT),
8809 DAG.getConstant(0, dl, VT));
8810 }
8811
8812 // insert elements one by one
8813 SDValue DstVec;
8814 if (HasConstElts) {
8815 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
8816 SDValue ImmL = DAG.getConstant(Lo_32(Immediate), dl, MVT::i32);
8817 SDValue ImmH = DAG.getConstant(Hi_32(Immediate), dl, MVT::i32);
8818 ImmL = DAG.getBitcast(MVT::v32i1, ImmL);
8819 ImmH = DAG.getBitcast(MVT::v32i1, ImmH);
8820 DstVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, ImmL, ImmH);
8821 } else {
8822 MVT ImmVT = MVT::getIntegerVT(std::max((unsigned)VT.getSizeInBits(), 8U));
8823 SDValue Imm = DAG.getConstant(Immediate, dl, ImmVT);
8824 MVT VecVT = VT.getSizeInBits() >= 8 ? VT : MVT::v8i1;
8825 DstVec = DAG.getBitcast(VecVT, Imm);
8826 DstVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, DstVec,
8827 DAG.getIntPtrConstant(0, dl));
8828 }
8829 } else
8830 DstVec = DAG.getUNDEF(VT);
8831
8832 for (unsigned i = 0, e = NonConstIdx.size(); i != e; ++i) {
8833 unsigned InsertIdx = NonConstIdx[i];
8834 DstVec = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, DstVec,
8835 Op.getOperand(InsertIdx),
8836 DAG.getIntPtrConstant(InsertIdx, dl));
8837 }
8838 return DstVec;
8839}
8840
8841/// This is a helper function of LowerToHorizontalOp().
8842/// This function checks that the build_vector \p N in input implements a
8843/// 128-bit partial horizontal operation on a 256-bit vector, but that operation
8844/// may not match the layout of an x86 256-bit horizontal instruction.
8845/// In other words, if this returns true, then some extraction/insertion will
8846/// be required to produce a valid horizontal instruction.
8847///
8848/// Parameter \p Opcode defines the kind of horizontal operation to match.
8849/// For example, if \p Opcode is equal to ISD::ADD, then this function
8850/// checks if \p N implements a horizontal arithmetic add; if instead \p Opcode
8851/// is equal to ISD::SUB, then this function checks if this is a horizontal
8852/// arithmetic sub.
8853///
8854/// This function only analyzes elements of \p N whose indices are
8855/// in range [BaseIdx, LastIdx).
8856///
8857/// TODO: This function was originally used to match both real and fake partial
8858/// horizontal operations, but the index-matching logic is incorrect for that.
8859/// See the corrected implementation in isHopBuildVector(). Can we reduce this
8860/// code because it is only used for partial h-op matching now?
8861static bool isHorizontalBinOpPart(const BuildVectorSDNode *N, unsigned Opcode,
8862 SelectionDAG &DAG,
8863 unsigned BaseIdx, unsigned LastIdx,
8864 SDValue &V0, SDValue &V1) {
8865 EVT VT = N->getValueType(0);
8866 assert(VT.is256BitVector() && "Only use for matching partial 256-bit h-ops")((VT.is256BitVector() && "Only use for matching partial 256-bit h-ops"
) ? static_cast<void> (0) : __assert_fail ("VT.is256BitVector() && \"Only use for matching partial 256-bit h-ops\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8866, __PRETTY_FUNCTION__));
8867 assert(BaseIdx * 2 <= LastIdx && "Invalid Indices in input!")((BaseIdx * 2 <= LastIdx && "Invalid Indices in input!"
) ? static_cast<void> (0) : __assert_fail ("BaseIdx * 2 <= LastIdx && \"Invalid Indices in input!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8867, __PRETTY_FUNCTION__));
8868 assert(VT.isVector() && VT.getVectorNumElements() >= LastIdx &&((VT.isVector() && VT.getVectorNumElements() >= LastIdx
&& "Invalid Vector in input!") ? static_cast<void
> (0) : __assert_fail ("VT.isVector() && VT.getVectorNumElements() >= LastIdx && \"Invalid Vector in input!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8869, __PRETTY_FUNCTION__))
8869 "Invalid Vector in input!")((VT.isVector() && VT.getVectorNumElements() >= LastIdx
&& "Invalid Vector in input!") ? static_cast<void
> (0) : __assert_fail ("VT.isVector() && VT.getVectorNumElements() >= LastIdx && \"Invalid Vector in input!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8869, __PRETTY_FUNCTION__));
8870
8871 bool IsCommutable = (Opcode == ISD::ADD || Opcode == ISD::FADD);
8872 bool CanFold = true;
8873 unsigned ExpectedVExtractIdx = BaseIdx;
8874 unsigned NumElts = LastIdx - BaseIdx;
8875 V0 = DAG.getUNDEF(VT);
8876 V1 = DAG.getUNDEF(VT);
8877
8878 // Check if N implements a horizontal binop.
8879 for (unsigned i = 0, e = NumElts; i != e && CanFold; ++i) {
8880 SDValue Op = N->getOperand(i + BaseIdx);
8881
8882 // Skip UNDEFs.
8883 if (Op->isUndef()) {
8884 // Update the expected vector extract index.
8885 if (i * 2 == NumElts)
8886 ExpectedVExtractIdx = BaseIdx;
8887 ExpectedVExtractIdx += 2;
8888 continue;
8889 }
8890
8891 CanFold = Op->getOpcode() == Opcode && Op->hasOneUse();
8892
8893 if (!CanFold)
8894 break;
8895
8896 SDValue Op0 = Op.getOperand(0);
8897 SDValue Op1 = Op.getOperand(1);
8898
8899 // Try to match the following pattern:
8900 // (BINOP (extract_vector_elt A, I), (extract_vector_elt A, I+1))
8901 CanFold = (Op0.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8902 Op1.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
8903 Op0.getOperand(0) == Op1.getOperand(0) &&
8904 isa<ConstantSDNode>(Op0.getOperand(1)) &&
8905 isa<ConstantSDNode>(Op1.getOperand(1)));
8906 if (!CanFold)
8907 break;
8908
8909 unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
8910 unsigned I1 = cast<ConstantSDNode>(Op1.getOperand(1))->getZExtValue();
8911
8912 if (i * 2 < NumElts) {
8913 if (V0.isUndef()) {
8914 V0 = Op0.getOperand(0);
8915 if (V0.getValueType() != VT)
8916 return false;
8917 }
8918 } else {
8919 if (V1.isUndef()) {
8920 V1 = Op0.getOperand(0);
8921 if (V1.getValueType() != VT)
8922 return false;
8923 }
8924 if (i * 2 == NumElts)
8925 ExpectedVExtractIdx = BaseIdx;
8926 }
8927
8928 SDValue Expected = (i * 2 < NumElts) ? V0 : V1;
8929 if (I0 == ExpectedVExtractIdx)
8930 CanFold = I1 == I0 + 1 && Op0.getOperand(0) == Expected;
8931 else if (IsCommutable && I1 == ExpectedVExtractIdx) {
8932 // Try to match the following dag sequence:
8933 // (BINOP (extract_vector_elt A, I+1), (extract_vector_elt A, I))
8934 CanFold = I0 == I1 + 1 && Op1.getOperand(0) == Expected;
8935 } else
8936 CanFold = false;
8937
8938 ExpectedVExtractIdx += 2;
8939 }
8940
8941 return CanFold;
8942}
8943
8944/// Emit a sequence of two 128-bit horizontal add/sub followed by
8945/// a concat_vector.
8946///
8947/// This is a helper function of LowerToHorizontalOp().
8948/// This function expects two 256-bit vectors called V0 and V1.
8949/// At first, each vector is split into two separate 128-bit vectors.
8950/// Then, the resulting 128-bit vectors are used to implement two
8951/// horizontal binary operations.
8952///
8953/// The kind of horizontal binary operation is defined by \p X86Opcode.
8954///
8955/// \p Mode specifies how the 128-bit parts of V0 and V1 are passed in input to
8956/// the two new horizontal binop.
8957/// When Mode is set, the first horizontal binop dag node would take as input
8958/// the lower 128-bit of V0 and the upper 128-bit of V0. The second
8959/// horizontal binop dag node would take as input the lower 128-bit of V1
8960/// and the upper 128-bit of V1.
8961/// Example:
8962/// HADD V0_LO, V0_HI
8963/// HADD V1_LO, V1_HI
8964///
8965/// Otherwise, the first horizontal binop dag node takes as input the lower
8966/// 128-bit of V0 and the lower 128-bit of V1, and the second horizontal binop
8967/// dag node takes the upper 128-bit of V0 and the upper 128-bit of V1.
8968/// Example:
8969/// HADD V0_LO, V1_LO
8970/// HADD V0_HI, V1_HI
8971///
8972/// If \p isUndefLO is set, then the algorithm propagates UNDEF to the lower
8973/// 128-bits of the result. If \p isUndefHI is set, then UNDEF is propagated to
8974/// the upper 128-bits of the result.
8975static SDValue ExpandHorizontalBinOp(const SDValue &V0, const SDValue &V1,
8976 const SDLoc &DL, SelectionDAG &DAG,
8977 unsigned X86Opcode, bool Mode,
8978 bool isUndefLO, bool isUndefHI) {
8979 MVT VT = V0.getSimpleValueType();
8980 assert(VT.is256BitVector() && VT == V1.getSimpleValueType() &&((VT.is256BitVector() && VT == V1.getSimpleValueType(
) && "Invalid nodes in input!") ? static_cast<void
> (0) : __assert_fail ("VT.is256BitVector() && VT == V1.getSimpleValueType() && \"Invalid nodes in input!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8981, __PRETTY_FUNCTION__))
8981 "Invalid nodes in input!")((VT.is256BitVector() && VT == V1.getSimpleValueType(
) && "Invalid nodes in input!") ? static_cast<void
> (0) : __assert_fail ("VT.is256BitVector() && VT == V1.getSimpleValueType() && \"Invalid nodes in input!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 8981, __PRETTY_FUNCTION__));
8982
8983 unsigned NumElts = VT.getVectorNumElements();
8984 SDValue V0_LO = extract128BitVector(V0, 0, DAG, DL);
8985 SDValue V0_HI = extract128BitVector(V0, NumElts/2, DAG, DL);
8986 SDValue V1_LO = extract128BitVector(V1, 0, DAG, DL);
8987 SDValue V1_HI = extract128BitVector(V1, NumElts/2, DAG, DL);
8988 MVT NewVT = V0_LO.getSimpleValueType();
8989
8990 SDValue LO = DAG.getUNDEF(NewVT);
8991 SDValue HI = DAG.getUNDEF(NewVT);
8992
8993 if (Mode) {
8994 // Don't emit a horizontal binop if the result is expected to be UNDEF.
8995 if (!isUndefLO && !V0->isUndef())
8996 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V0_HI);
8997 if (!isUndefHI && !V1->isUndef())
8998 HI = DAG.getNode(X86Opcode, DL, NewVT, V1_LO, V1_HI);
8999 } else {
9000 // Don't emit a horizontal binop if the result is expected to be UNDEF.
9001 if (!isUndefLO && (!V0_LO->isUndef() || !V1_LO->isUndef()))
9002 LO = DAG.getNode(X86Opcode, DL, NewVT, V0_LO, V1_LO);
9003
9004 if (!isUndefHI && (!V0_HI->isUndef() || !V1_HI->isUndef()))
9005 HI = DAG.getNode(X86Opcode, DL, NewVT, V0_HI, V1_HI);
9006 }
9007
9008 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, LO, HI);
9009}
9010
9011/// Returns true iff \p BV builds a vector with the result equivalent to
9012/// the result of ADDSUB/SUBADD operation.
9013/// If true is returned then the operands of ADDSUB = Opnd0 +- Opnd1
9014/// (SUBADD = Opnd0 -+ Opnd1) operation are written to the parameters
9015/// \p Opnd0 and \p Opnd1.
9016static bool isAddSubOrSubAdd(const BuildVectorSDNode *BV,
9017 const X86Subtarget &Subtarget, SelectionDAG &DAG,
9018 SDValue &Opnd0, SDValue &Opnd1,
9019 unsigned &NumExtracts,
9020 bool &IsSubAdd) {
9021
9022 MVT VT = BV->getSimpleValueType(0);
9023 if (!Subtarget.hasSSE3() || !VT.isFloatingPoint())
9024 return false;
9025
9026 unsigned NumElts = VT.getVectorNumElements();
9027 SDValue InVec0 = DAG.getUNDEF(VT);
9028 SDValue InVec1 = DAG.getUNDEF(VT);
9029
9030 NumExtracts = 0;
9031
9032 // Odd-numbered elements in the input build vector are obtained from
9033 // adding/subtracting two integer/float elements.
9034 // Even-numbered elements in the input build vector are obtained from
9035 // subtracting/adding two integer/float elements.
9036 unsigned Opc[2] = {0, 0};
9037 for (unsigned i = 0, e = NumElts; i != e; ++i) {
9038 SDValue Op = BV->getOperand(i);
9039
9040 // Skip 'undef' values.
9041 unsigned Opcode = Op.getOpcode();
9042 if (Opcode == ISD::UNDEF)
9043 continue;
9044
9045 // Early exit if we found an unexpected opcode.
9046 if (Opcode != ISD::FADD && Opcode != ISD::FSUB)
9047 return false;
9048
9049 SDValue Op0 = Op.getOperand(0);
9050 SDValue Op1 = Op.getOperand(1);
9051
9052 // Try to match the following pattern:
9053 // (BINOP (extract_vector_elt A, i), (extract_vector_elt B, i))
9054 // Early exit if we cannot match that sequence.
9055 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9056 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9057 !isa<ConstantSDNode>(Op0.getOperand(1)) ||
9058 !isa<ConstantSDNode>(Op1.getOperand(1)) ||
9059 Op0.getOperand(1) != Op1.getOperand(1))
9060 return false;
9061
9062 unsigned I0 = cast<ConstantSDNode>(Op0.getOperand(1))->getZExtValue();
9063 if (I0 != i)
9064 return false;
9065
9066 // We found a valid add/sub node, make sure its the same opcode as previous
9067 // elements for this parity.
9068 if (Opc[i % 2] != 0 && Opc[i % 2] != Opcode)
9069 return false;
9070 Opc[i % 2] = Opcode;
9071
9072 // Update InVec0 and InVec1.
9073 if (InVec0.isUndef()) {
9074 InVec0 = Op0.getOperand(0);
9075 if (InVec0.getSimpleValueType() != VT)
9076 return false;
9077 }
9078 if (InVec1.isUndef()) {
9079 InVec1 = Op1.getOperand(0);
9080 if (InVec1.getSimpleValueType() != VT)
9081 return false;
9082 }
9083
9084 // Make sure that operands in input to each add/sub node always
9085 // come from a same pair of vectors.
9086 if (InVec0 != Op0.getOperand(0)) {
9087 if (Opcode == ISD::FSUB)
9088 return false;
9089
9090 // FADD is commutable. Try to commute the operands
9091 // and then test again.
9092 std::swap(Op0, Op1);
9093 if (InVec0 != Op0.getOperand(0))
9094 return false;
9095 }
9096
9097 if (InVec1 != Op1.getOperand(0))
9098 return false;
9099
9100 // Increment the number of extractions done.
9101 ++NumExtracts;
9102 }
9103
9104 // Ensure we have found an opcode for both parities and that they are
9105 // different. Don't try to fold this build_vector into an ADDSUB/SUBADD if the
9106 // inputs are undef.
9107 if (!Opc[0] || !Opc[1] || Opc[0] == Opc[1] ||
9108 InVec0.isUndef() || InVec1.isUndef())
9109 return false;
9110
9111 IsSubAdd = Opc[0] == ISD::FADD;
9112
9113 Opnd0 = InVec0;
9114 Opnd1 = InVec1;
9115 return true;
9116}
9117
9118/// Returns true if is possible to fold MUL and an idiom that has already been
9119/// recognized as ADDSUB/SUBADD(\p Opnd0, \p Opnd1) into
9120/// FMADDSUB/FMSUBADD(x, y, \p Opnd1). If (and only if) true is returned, the
9121/// operands of FMADDSUB/FMSUBADD are written to parameters \p Opnd0, \p Opnd1, \p Opnd2.
9122///
9123/// Prior to calling this function it should be known that there is some
9124/// SDNode that potentially can be replaced with an X86ISD::ADDSUB operation
9125/// using \p Opnd0 and \p Opnd1 as operands. Also, this method is called
9126/// before replacement of such SDNode with ADDSUB operation. Thus the number
9127/// of \p Opnd0 uses is expected to be equal to 2.
9128/// For example, this function may be called for the following IR:
9129/// %AB = fmul fast <2 x double> %A, %B
9130/// %Sub = fsub fast <2 x double> %AB, %C
9131/// %Add = fadd fast <2 x double> %AB, %C
9132/// %Addsub = shufflevector <2 x double> %Sub, <2 x double> %Add,
9133/// <2 x i32> <i32 0, i32 3>
9134/// There is a def for %Addsub here, which potentially can be replaced by
9135/// X86ISD::ADDSUB operation:
9136/// %Addsub = X86ISD::ADDSUB %AB, %C
9137/// and such ADDSUB can further be replaced with FMADDSUB:
9138/// %Addsub = FMADDSUB %A, %B, %C.
9139///
9140/// The main reason why this method is called before the replacement of the
9141/// recognized ADDSUB idiom with ADDSUB operation is that such replacement
9142/// is illegal sometimes. E.g. 512-bit ADDSUB is not available, while 512-bit
9143/// FMADDSUB is.
9144static bool isFMAddSubOrFMSubAdd(const X86Subtarget &Subtarget,
9145 SelectionDAG &DAG,
9146 SDValue &Opnd0, SDValue &Opnd1, SDValue &Opnd2,
9147 unsigned ExpectedUses) {
9148 if (Opnd0.getOpcode() != ISD::FMUL ||
9149 !Opnd0->hasNUsesOfValue(ExpectedUses, 0) || !Subtarget.hasAnyFMA())
9150 return false;
9151
9152 // FIXME: These checks must match the similar ones in
9153 // DAGCombiner::visitFADDForFMACombine. It would be good to have one
9154 // function that would answer if it is Ok to fuse MUL + ADD to FMADD
9155 // or MUL + ADDSUB to FMADDSUB.
9156 const TargetOptions &Options = DAG.getTarget().Options;
9157 bool AllowFusion =
9158 (Options.AllowFPOpFusion == FPOpFusion::Fast || Options.UnsafeFPMath);
9159 if (!AllowFusion)
9160 return false;
9161
9162 Opnd2 = Opnd1;
9163 Opnd1 = Opnd0.getOperand(1);
9164 Opnd0 = Opnd0.getOperand(0);
9165
9166 return true;
9167}
9168
9169/// Try to fold a build_vector that performs an 'addsub' or 'fmaddsub' or
9170/// 'fsubadd' operation accordingly to X86ISD::ADDSUB or X86ISD::FMADDSUB or
9171/// X86ISD::FMSUBADD node.
9172static SDValue lowerToAddSubOrFMAddSub(const BuildVectorSDNode *BV,
9173 const X86Subtarget &Subtarget,
9174 SelectionDAG &DAG) {
9175 SDValue Opnd0, Opnd1;
9176 unsigned NumExtracts;
9177 bool IsSubAdd;
9178 if (!isAddSubOrSubAdd(BV, Subtarget, DAG, Opnd0, Opnd1, NumExtracts,
9179 IsSubAdd))
9180 return SDValue();
9181
9182 MVT VT = BV->getSimpleValueType(0);
9183 SDLoc DL(BV);
9184
9185 // Try to generate X86ISD::FMADDSUB node here.
9186 SDValue Opnd2;
9187 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, NumExtracts)) {
9188 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
9189 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
9190 }
9191
9192 // We only support ADDSUB.
9193 if (IsSubAdd)
9194 return SDValue();
9195
9196 // Do not generate X86ISD::ADDSUB node for 512-bit types even though
9197 // the ADDSUB idiom has been successfully recognized. There are no known
9198 // X86 targets with 512-bit ADDSUB instructions!
9199 // 512-bit ADDSUB idiom recognition was needed only as part of FMADDSUB idiom
9200 // recognition.
9201 if (VT.is512BitVector())
9202 return SDValue();
9203
9204 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
9205}
9206
9207static bool isHopBuildVector(const BuildVectorSDNode *BV, SelectionDAG &DAG,
9208 unsigned &HOpcode, SDValue &V0, SDValue &V1) {
9209 // Initialize outputs to known values.
9210 MVT VT = BV->getSimpleValueType(0);
9211 HOpcode = ISD::DELETED_NODE;
9212 V0 = DAG.getUNDEF(VT);
9213 V1 = DAG.getUNDEF(VT);
9214
9215 // x86 256-bit horizontal ops are defined in a non-obvious way. Each 128-bit
9216 // half of the result is calculated independently from the 128-bit halves of
9217 // the inputs, so that makes the index-checking logic below more complicated.
9218 unsigned NumElts = VT.getVectorNumElements();
9219 unsigned GenericOpcode = ISD::DELETED_NODE;
9220 unsigned Num128BitChunks = VT.is256BitVector() ? 2 : 1;
9221 unsigned NumEltsIn128Bits = NumElts / Num128BitChunks;
9222 unsigned NumEltsIn64Bits = NumEltsIn128Bits / 2;
9223 for (unsigned i = 0; i != Num128BitChunks; ++i) {
9224 for (unsigned j = 0; j != NumEltsIn128Bits; ++j) {
9225 // Ignore undef elements.
9226 SDValue Op = BV->getOperand(i * NumEltsIn128Bits + j);
9227 if (Op.isUndef())
9228 continue;
9229
9230 // If there's an opcode mismatch, we're done.
9231 if (HOpcode != ISD::DELETED_NODE && Op.getOpcode() != GenericOpcode)
9232 return false;
9233
9234 // Initialize horizontal opcode.
9235 if (HOpcode == ISD::DELETED_NODE) {
9236 GenericOpcode = Op.getOpcode();
9237 switch (GenericOpcode) {
9238 case ISD::ADD: HOpcode = X86ISD::HADD; break;
9239 case ISD::SUB: HOpcode = X86ISD::HSUB; break;
9240 case ISD::FADD: HOpcode = X86ISD::FHADD; break;
9241 case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
9242 default: return false;
9243 }
9244 }
9245
9246 SDValue Op0 = Op.getOperand(0);
9247 SDValue Op1 = Op.getOperand(1);
9248 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9249 Op1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9250 Op0.getOperand(0) != Op1.getOperand(0) ||
9251 !isa<ConstantSDNode>(Op0.getOperand(1)) ||
9252 !isa<ConstantSDNode>(Op1.getOperand(1)) || !Op.hasOneUse())
9253 return false;
9254
9255 // The source vector is chosen based on which 64-bit half of the
9256 // destination vector is being calculated.
9257 if (j < NumEltsIn64Bits) {
9258 if (V0.isUndef())
9259 V0 = Op0.getOperand(0);
9260 } else {
9261 if (V1.isUndef())
9262 V1 = Op0.getOperand(0);
9263 }
9264
9265 SDValue SourceVec = (j < NumEltsIn64Bits) ? V0 : V1;
9266 if (SourceVec != Op0.getOperand(0))
9267 return false;
9268
9269 // op (extract_vector_elt A, I), (extract_vector_elt A, I+1)
9270 unsigned ExtIndex0 = Op0.getConstantOperandVal(1);
9271 unsigned ExtIndex1 = Op1.getConstantOperandVal(1);
9272 unsigned ExpectedIndex = i * NumEltsIn128Bits +
9273 (j % NumEltsIn64Bits) * 2;
9274 if (ExpectedIndex == ExtIndex0 && ExtIndex1 == ExtIndex0 + 1)
9275 continue;
9276
9277 // If this is not a commutative op, this does not match.
9278 if (GenericOpcode != ISD::ADD && GenericOpcode != ISD::FADD)
9279 return false;
9280
9281 // Addition is commutative, so try swapping the extract indexes.
9282 // op (extract_vector_elt A, I+1), (extract_vector_elt A, I)
9283 if (ExpectedIndex == ExtIndex1 && ExtIndex0 == ExtIndex1 + 1)
9284 continue;
9285
9286 // Extract indexes do not match horizontal requirement.
9287 return false;
9288 }
9289 }
9290 // We matched. Opcode and operands are returned by reference as arguments.
9291 return true;
9292}
9293
9294static SDValue getHopForBuildVector(const BuildVectorSDNode *BV,
9295 SelectionDAG &DAG, unsigned HOpcode,
9296 SDValue V0, SDValue V1) {
9297 // If either input vector is not the same size as the build vector,
9298 // extract/insert the low bits to the correct size.
9299 // This is free (examples: zmm --> xmm, xmm --> ymm).
9300 MVT VT = BV->getSimpleValueType(0);
9301 unsigned Width = VT.getSizeInBits();
9302 if (V0.getValueSizeInBits() > Width)
9303 V0 = extractSubVector(V0, 0, DAG, SDLoc(BV), Width);
9304 else if (V0.getValueSizeInBits() < Width)
9305 V0 = insertSubVector(DAG.getUNDEF(VT), V0, 0, DAG, SDLoc(BV), Width);
9306
9307 if (V1.getValueSizeInBits() > Width)
9308 V1 = extractSubVector(V1, 0, DAG, SDLoc(BV), Width);
9309 else if (V1.getValueSizeInBits() < Width)
9310 V1 = insertSubVector(DAG.getUNDEF(VT), V1, 0, DAG, SDLoc(BV), Width);
9311
9312 unsigned NumElts = VT.getVectorNumElements();
9313 APInt DemandedElts = APInt::getAllOnesValue(NumElts);
9314 for (unsigned i = 0; i != NumElts; ++i)
9315 if (BV->getOperand(i).isUndef())
9316 DemandedElts.clearBit(i);
9317
9318 // If we don't need the upper xmm, then perform as a xmm hop.
9319 unsigned HalfNumElts = NumElts / 2;
9320 if (VT.is256BitVector() && DemandedElts.lshr(HalfNumElts) == 0) {
9321 MVT HalfVT = VT.getHalfNumVectorElementsVT();
9322 V0 = extractSubVector(V0, 0, DAG, SDLoc(BV), 128);
9323 V1 = extractSubVector(V1, 0, DAG, SDLoc(BV), 128);
9324 SDValue Half = DAG.getNode(HOpcode, SDLoc(BV), HalfVT, V0, V1);
9325 return insertSubVector(DAG.getUNDEF(VT), Half, 0, DAG, SDLoc(BV), 256);
9326 }
9327
9328 return DAG.getNode(HOpcode, SDLoc(BV), VT, V0, V1);
9329}
9330
9331/// Lower BUILD_VECTOR to a horizontal add/sub operation if possible.
9332static SDValue LowerToHorizontalOp(const BuildVectorSDNode *BV,
9333 const X86Subtarget &Subtarget,
9334 SelectionDAG &DAG) {
9335 // We need at least 2 non-undef elements to make this worthwhile by default.
9336 unsigned NumNonUndefs =
9337 count_if(BV->op_values(), [](SDValue V) { return !V.isUndef(); });
9338 if (NumNonUndefs < 2)
9339 return SDValue();
9340
9341 // There are 4 sets of horizontal math operations distinguished by type:
9342 // int/FP at 128-bit/256-bit. Each type was introduced with a different
9343 // subtarget feature. Try to match those "native" patterns first.
9344 MVT VT = BV->getSimpleValueType(0);
9345 if (((VT == MVT::v4f32 || VT == MVT::v2f64) && Subtarget.hasSSE3()) ||
9346 ((VT == MVT::v8i16 || VT == MVT::v4i32) && Subtarget.hasSSSE3()) ||
9347 ((VT == MVT::v8f32 || VT == MVT::v4f64) && Subtarget.hasAVX()) ||
9348 ((VT == MVT::v16i16 || VT == MVT::v8i32) && Subtarget.hasAVX2())) {
9349 unsigned HOpcode;
9350 SDValue V0, V1;
9351 if (isHopBuildVector(BV, DAG, HOpcode, V0, V1))
9352 return getHopForBuildVector(BV, DAG, HOpcode, V0, V1);
9353 }
9354
9355 // Try harder to match 256-bit ops by using extract/concat.
9356 if (!Subtarget.hasAVX() || !VT.is256BitVector())
9357 return SDValue();
9358
9359 // Count the number of UNDEF operands in the build_vector in input.
9360 unsigned NumElts = VT.getVectorNumElements();
9361 unsigned Half = NumElts / 2;
9362 unsigned NumUndefsLO = 0;
9363 unsigned NumUndefsHI = 0;
9364 for (unsigned i = 0, e = Half; i != e; ++i)
9365 if (BV->getOperand(i)->isUndef())
9366 NumUndefsLO++;
9367
9368 for (unsigned i = Half, e = NumElts; i != e; ++i)
9369 if (BV->getOperand(i)->isUndef())
9370 NumUndefsHI++;
9371
9372 SDLoc DL(BV);
9373 SDValue InVec0, InVec1;
9374 if (VT == MVT::v8i32 || VT == MVT::v16i16) {
9375 SDValue InVec2, InVec3;
9376 unsigned X86Opcode;
9377 bool CanFold = true;
9378
9379 if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, Half, InVec0, InVec1) &&
9380 isHorizontalBinOpPart(BV, ISD::ADD, DAG, Half, NumElts, InVec2,
9381 InVec3) &&
9382 ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
9383 ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
9384 X86Opcode = X86ISD::HADD;
9385 else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, Half, InVec0,
9386 InVec1) &&
9387 isHorizontalBinOpPart(BV, ISD::SUB, DAG, Half, NumElts, InVec2,
9388 InVec3) &&
9389 ((InVec0.isUndef() || InVec2.isUndef()) || InVec0 == InVec2) &&
9390 ((InVec1.isUndef() || InVec3.isUndef()) || InVec1 == InVec3))
9391 X86Opcode = X86ISD::HSUB;
9392 else
9393 CanFold = false;
9394
9395 if (CanFold) {
9396 // Do not try to expand this build_vector into a pair of horizontal
9397 // add/sub if we can emit a pair of scalar add/sub.
9398 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
9399 return SDValue();
9400
9401 // Convert this build_vector into a pair of horizontal binops followed by
9402 // a concat vector. We must adjust the outputs from the partial horizontal
9403 // matching calls above to account for undefined vector halves.
9404 SDValue V0 = InVec0.isUndef() ? InVec2 : InVec0;
9405 SDValue V1 = InVec1.isUndef() ? InVec3 : InVec1;
9406 assert((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?")(((!V0.isUndef() || !V1.isUndef()) && "Horizontal-op of undefs?"
) ? static_cast<void> (0) : __assert_fail ("(!V0.isUndef() || !V1.isUndef()) && \"Horizontal-op of undefs?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9406, __PRETTY_FUNCTION__));
9407 bool isUndefLO = NumUndefsLO == Half;
9408 bool isUndefHI = NumUndefsHI == Half;
9409 return ExpandHorizontalBinOp(V0, V1, DL, DAG, X86Opcode, false, isUndefLO,
9410 isUndefHI);
9411 }
9412 }
9413
9414 if (VT == MVT::v8f32 || VT == MVT::v4f64 || VT == MVT::v8i32 ||
9415 VT == MVT::v16i16) {
9416 unsigned X86Opcode;
9417 if (isHorizontalBinOpPart(BV, ISD::ADD, DAG, 0, NumElts, InVec0, InVec1))
9418 X86Opcode = X86ISD::HADD;
9419 else if (isHorizontalBinOpPart(BV, ISD::SUB, DAG, 0, NumElts, InVec0,
9420 InVec1))
9421 X86Opcode = X86ISD::HSUB;
9422 else if (isHorizontalBinOpPart(BV, ISD::FADD, DAG, 0, NumElts, InVec0,
9423 InVec1))
9424 X86Opcode = X86ISD::FHADD;
9425 else if (isHorizontalBinOpPart(BV, ISD::FSUB, DAG, 0, NumElts, InVec0,
9426 InVec1))
9427 X86Opcode = X86ISD::FHSUB;
9428 else
9429 return SDValue();
9430
9431 // Don't try to expand this build_vector into a pair of horizontal add/sub
9432 // if we can simply emit a pair of scalar add/sub.
9433 if (NumUndefsLO + 1 == Half || NumUndefsHI + 1 == Half)
9434 return SDValue();
9435
9436 // Convert this build_vector into two horizontal add/sub followed by
9437 // a concat vector.
9438 bool isUndefLO = NumUndefsLO == Half;
9439 bool isUndefHI = NumUndefsHI == Half;
9440 return ExpandHorizontalBinOp(InVec0, InVec1, DL, DAG, X86Opcode, true,
9441 isUndefLO, isUndefHI);
9442 }
9443
9444 return SDValue();
9445}
9446
9447/// If a BUILD_VECTOR's source elements all apply the same bit operation and
9448/// one of their operands is constant, lower to a pair of BUILD_VECTOR and
9449/// just apply the bit to the vectors.
9450/// NOTE: Its not in our interest to start make a general purpose vectorizer
9451/// from this, but enough scalar bit operations are created from the later
9452/// legalization + scalarization stages to need basic support.
9453static SDValue lowerBuildVectorToBitOp(BuildVectorSDNode *Op,
9454 SelectionDAG &DAG) {
9455 SDLoc DL(Op);
9456 MVT VT = Op->getSimpleValueType(0);
9457 unsigned NumElems = VT.getVectorNumElements();
9458 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
9459
9460 // Check that all elements have the same opcode.
9461 // TODO: Should we allow UNDEFS and if so how many?
9462 unsigned Opcode = Op->getOperand(0).getOpcode();
9463 for (unsigned i = 1; i < NumElems; ++i)
9464 if (Opcode != Op->getOperand(i).getOpcode())
9465 return SDValue();
9466
9467 // TODO: We may be able to add support for other Ops (ADD/SUB + shifts).
9468 bool IsShift = false;
9469 switch (Opcode) {
9470 default:
9471 return SDValue();
9472 case ISD::SHL:
9473 case ISD::SRL:
9474 case ISD::SRA:
9475 IsShift = true;
9476 break;
9477 case ISD::AND:
9478 case ISD::XOR:
9479 case ISD::OR:
9480 // Don't do this if the buildvector is a splat - we'd replace one
9481 // constant with an entire vector.
9482 if (Op->getSplatValue())
9483 return SDValue();
9484 if (!TLI.isOperationLegalOrPromote(Opcode, VT))
9485 return SDValue();
9486 break;
9487 }
9488
9489 SmallVector<SDValue, 4> LHSElts, RHSElts;
9490 for (SDValue Elt : Op->ops()) {
9491 SDValue LHS = Elt.getOperand(0);
9492 SDValue RHS = Elt.getOperand(1);
9493
9494 // We expect the canonicalized RHS operand to be the constant.
9495 if (!isa<ConstantSDNode>(RHS))
9496 return SDValue();
9497
9498 // Extend shift amounts.
9499 if (RHS.getValueSizeInBits() != VT.getScalarSizeInBits()) {
9500 if (!IsShift)
9501 return SDValue();
9502 RHS = DAG.getZExtOrTrunc(RHS, DL, VT.getScalarType());
9503 }
9504
9505 LHSElts.push_back(LHS);
9506 RHSElts.push_back(RHS);
9507 }
9508
9509 // Limit to shifts by uniform immediates.
9510 // TODO: Only accept vXi8/vXi64 special cases?
9511 // TODO: Permit non-uniform XOP/AVX2/MULLO cases?
9512 if (IsShift && any_of(RHSElts, [&](SDValue V) { return RHSElts[0] != V; }))
9513 return SDValue();
9514
9515 SDValue LHS = DAG.getBuildVector(VT, DL, LHSElts);
9516 SDValue RHS = DAG.getBuildVector(VT, DL, RHSElts);
9517 return DAG.getNode(Opcode, DL, VT, LHS, RHS);
9518}
9519
9520/// Create a vector constant without a load. SSE/AVX provide the bare minimum
9521/// functionality to do this, so it's all zeros, all ones, or some derivation
9522/// that is cheap to calculate.
9523static SDValue materializeVectorConstant(SDValue Op, SelectionDAG &DAG,
9524 const X86Subtarget &Subtarget) {
9525 SDLoc DL(Op);
9526 MVT VT = Op.getSimpleValueType();
9527
9528 // Vectors containing all zeros can be matched by pxor and xorps.
9529 if (ISD::isBuildVectorAllZeros(Op.getNode()))
9530 return Op;
9531
9532 // Vectors containing all ones can be matched by pcmpeqd on 128-bit width
9533 // vectors or broken into v4i32 operations on 256-bit vectors. AVX2 can use
9534 // vpcmpeqd on 256-bit vectors.
9535 if (Subtarget.hasSSE2() && ISD::isBuildVectorAllOnes(Op.getNode())) {
9536 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32)
9537 return Op;
9538
9539 return getOnesVector(VT, DAG, DL);
9540 }
9541
9542 return SDValue();
9543}
9544
9545/// Look for opportunities to create a VPERMV/VPERMILPV/PSHUFB variable permute
9546/// from a vector of source values and a vector of extraction indices.
9547/// The vectors might be manipulated to match the type of the permute op.
9548static SDValue createVariablePermute(MVT VT, SDValue SrcVec, SDValue IndicesVec,
9549 SDLoc &DL, SelectionDAG &DAG,
9550 const X86Subtarget &Subtarget) {
9551 MVT ShuffleVT = VT;
9552 EVT IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
9553 unsigned NumElts = VT.getVectorNumElements();
9554 unsigned SizeInBits = VT.getSizeInBits();
9555
9556 // Adjust IndicesVec to match VT size.
9557 assert(IndicesVec.getValueType().getVectorNumElements() >= NumElts &&((IndicesVec.getValueType().getVectorNumElements() >= NumElts
&& "Illegal variable permute mask size") ? static_cast
<void> (0) : __assert_fail ("IndicesVec.getValueType().getVectorNumElements() >= NumElts && \"Illegal variable permute mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9558, __PRETTY_FUNCTION__))
9558 "Illegal variable permute mask size")((IndicesVec.getValueType().getVectorNumElements() >= NumElts
&& "Illegal variable permute mask size") ? static_cast
<void> (0) : __assert_fail ("IndicesVec.getValueType().getVectorNumElements() >= NumElts && \"Illegal variable permute mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9558, __PRETTY_FUNCTION__));
9559 if (IndicesVec.getValueType().getVectorNumElements() > NumElts)
9560 IndicesVec = extractSubVector(IndicesVec, 0, DAG, SDLoc(IndicesVec),
9561 NumElts * VT.getScalarSizeInBits());
9562 IndicesVec = DAG.getZExtOrTrunc(IndicesVec, SDLoc(IndicesVec), IndicesVT);
9563
9564 // Handle SrcVec that don't match VT type.
9565 if (SrcVec.getValueSizeInBits() != SizeInBits) {
9566 if ((SrcVec.getValueSizeInBits() % SizeInBits) == 0) {
9567 // Handle larger SrcVec by treating it as a larger permute.
9568 unsigned Scale = SrcVec.getValueSizeInBits() / SizeInBits;
9569 VT = MVT::getVectorVT(VT.getScalarType(), Scale * NumElts);
9570 IndicesVT = EVT(VT).changeVectorElementTypeToInteger();
9571 IndicesVec = widenSubVector(IndicesVT.getSimpleVT(), IndicesVec, false,
9572 Subtarget, DAG, SDLoc(IndicesVec));
9573 return extractSubVector(
9574 createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget), 0,
9575 DAG, DL, SizeInBits);
9576 } else if (SrcVec.getValueSizeInBits() < SizeInBits) {
9577 // Widen smaller SrcVec to match VT.
9578 SrcVec = widenSubVector(VT, SrcVec, false, Subtarget, DAG, SDLoc(SrcVec));
9579 } else
9580 return SDValue();
9581 }
9582
9583 auto ScaleIndices = [&DAG](SDValue Idx, uint64_t Scale) {
9584 assert(isPowerOf2_64(Scale) && "Illegal variable permute shuffle scale")((isPowerOf2_64(Scale) && "Illegal variable permute shuffle scale"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_64(Scale) && \"Illegal variable permute shuffle scale\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9584, __PRETTY_FUNCTION__));
9585 EVT SrcVT = Idx.getValueType();
9586 unsigned NumDstBits = SrcVT.getScalarSizeInBits() / Scale;
9587 uint64_t IndexScale = 0;
9588 uint64_t IndexOffset = 0;
9589
9590 // If we're scaling a smaller permute op, then we need to repeat the
9591 // indices, scaling and offsetting them as well.
9592 // e.g. v4i32 -> v16i8 (Scale = 4)
9593 // IndexScale = v4i32 Splat(4 << 24 | 4 << 16 | 4 << 8 | 4)
9594 // IndexOffset = v4i32 Splat(3 << 24 | 2 << 16 | 1 << 8 | 0)
9595 for (uint64_t i = 0; i != Scale; ++i) {
9596 IndexScale |= Scale << (i * NumDstBits);
9597 IndexOffset |= i << (i * NumDstBits);
9598 }
9599
9600 Idx = DAG.getNode(ISD::MUL, SDLoc(Idx), SrcVT, Idx,
9601 DAG.getConstant(IndexScale, SDLoc(Idx), SrcVT));
9602 Idx = DAG.getNode(ISD::ADD, SDLoc(Idx), SrcVT, Idx,
9603 DAG.getConstant(IndexOffset, SDLoc(Idx), SrcVT));
9604 return Idx;
9605 };
9606
9607 unsigned Opcode = 0;
9608 switch (VT.SimpleTy) {
9609 default:
9610 break;
9611 case MVT::v16i8:
9612 if (Subtarget.hasSSSE3())
9613 Opcode = X86ISD::PSHUFB;
9614 break;
9615 case MVT::v8i16:
9616 if (Subtarget.hasVLX() && Subtarget.hasBWI())
9617 Opcode = X86ISD::VPERMV;
9618 else if (Subtarget.hasSSSE3()) {
9619 Opcode = X86ISD::PSHUFB;
9620 ShuffleVT = MVT::v16i8;
9621 }
9622 break;
9623 case MVT::v4f32:
9624 case MVT::v4i32:
9625 if (Subtarget.hasAVX()) {
9626 Opcode = X86ISD::VPERMILPV;
9627 ShuffleVT = MVT::v4f32;
9628 } else if (Subtarget.hasSSSE3()) {
9629 Opcode = X86ISD::PSHUFB;
9630 ShuffleVT = MVT::v16i8;
9631 }
9632 break;
9633 case MVT::v2f64:
9634 case MVT::v2i64:
9635 if (Subtarget.hasAVX()) {
9636 // VPERMILPD selects using bit#1 of the index vector, so scale IndicesVec.
9637 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
9638 Opcode = X86ISD::VPERMILPV;
9639 ShuffleVT = MVT::v2f64;
9640 } else if (Subtarget.hasSSE41()) {
9641 // SSE41 can compare v2i64 - select between indices 0 and 1.
9642 return DAG.getSelectCC(
9643 DL, IndicesVec,
9644 getZeroVector(IndicesVT.getSimpleVT(), Subtarget, DAG, DL),
9645 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {0, 0}),
9646 DAG.getVectorShuffle(VT, DL, SrcVec, SrcVec, {1, 1}),
9647 ISD::CondCode::SETEQ);
9648 }
9649 break;
9650 case MVT::v32i8:
9651 if (Subtarget.hasVLX() && Subtarget.hasVBMI())
9652 Opcode = X86ISD::VPERMV;
9653 else if (Subtarget.hasXOP()) {
9654 SDValue LoSrc = extract128BitVector(SrcVec, 0, DAG, DL);
9655 SDValue HiSrc = extract128BitVector(SrcVec, 16, DAG, DL);
9656 SDValue LoIdx = extract128BitVector(IndicesVec, 0, DAG, DL);
9657 SDValue HiIdx = extract128BitVector(IndicesVec, 16, DAG, DL);
9658 return DAG.getNode(
9659 ISD::CONCAT_VECTORS, DL, VT,
9660 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, LoIdx),
9661 DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, LoSrc, HiSrc, HiIdx));
9662 } else if (Subtarget.hasAVX()) {
9663 SDValue Lo = extract128BitVector(SrcVec, 0, DAG, DL);
9664 SDValue Hi = extract128BitVector(SrcVec, 16, DAG, DL);
9665 SDValue LoLo = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Lo);
9666 SDValue HiHi = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Hi, Hi);
9667 auto PSHUFBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
9668 ArrayRef<SDValue> Ops) {
9669 // Permute Lo and Hi and then select based on index range.
9670 // This works as SHUFB uses bits[3:0] to permute elements and we don't
9671 // care about the bit[7] as its just an index vector.
9672 SDValue Idx = Ops[2];
9673 EVT VT = Idx.getValueType();
9674 return DAG.getSelectCC(DL, Idx, DAG.getConstant(15, DL, VT),
9675 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[1], Idx),
9676 DAG.getNode(X86ISD::PSHUFB, DL, VT, Ops[0], Idx),
9677 ISD::CondCode::SETGT);
9678 };
9679 SDValue Ops[] = {LoLo, HiHi, IndicesVec};
9680 return SplitOpsAndApply(DAG, Subtarget, DL, MVT::v32i8, Ops,
9681 PSHUFBBuilder);
9682 }
9683 break;
9684 case MVT::v16i16:
9685 if (Subtarget.hasVLX() && Subtarget.hasBWI())
9686 Opcode = X86ISD::VPERMV;
9687 else if (Subtarget.hasAVX()) {
9688 // Scale to v32i8 and perform as v32i8.
9689 IndicesVec = ScaleIndices(IndicesVec, 2);
9690 return DAG.getBitcast(
9691 VT, createVariablePermute(
9692 MVT::v32i8, DAG.getBitcast(MVT::v32i8, SrcVec),
9693 DAG.getBitcast(MVT::v32i8, IndicesVec), DL, DAG, Subtarget));
9694 }
9695 break;
9696 case MVT::v8f32:
9697 case MVT::v8i32:
9698 if (Subtarget.hasAVX2())
9699 Opcode = X86ISD::VPERMV;
9700 else if (Subtarget.hasAVX()) {
9701 SrcVec = DAG.getBitcast(MVT::v8f32, SrcVec);
9702 SDValue LoLo = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9703 {0, 1, 2, 3, 0, 1, 2, 3});
9704 SDValue HiHi = DAG.getVectorShuffle(MVT::v8f32, DL, SrcVec, SrcVec,
9705 {4, 5, 6, 7, 4, 5, 6, 7});
9706 if (Subtarget.hasXOP())
9707 return DAG.getBitcast(
9708 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v8f32, LoLo, HiHi,
9709 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
9710 // Permute Lo and Hi and then select based on index range.
9711 // This works as VPERMILPS only uses index bits[0:1] to permute elements.
9712 SDValue Res = DAG.getSelectCC(
9713 DL, IndicesVec, DAG.getConstant(3, DL, MVT::v8i32),
9714 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, HiHi, IndicesVec),
9715 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, LoLo, IndicesVec),
9716 ISD::CondCode::SETGT);
9717 return DAG.getBitcast(VT, Res);
9718 }
9719 break;
9720 case MVT::v4i64:
9721 case MVT::v4f64:
9722 if (Subtarget.hasAVX512()) {
9723 if (!Subtarget.hasVLX()) {
9724 MVT WidenSrcVT = MVT::getVectorVT(VT.getScalarType(), 8);
9725 SrcVec = widenSubVector(WidenSrcVT, SrcVec, false, Subtarget, DAG,
9726 SDLoc(SrcVec));
9727 IndicesVec = widenSubVector(MVT::v8i64, IndicesVec, false, Subtarget,
9728 DAG, SDLoc(IndicesVec));
9729 SDValue Res = createVariablePermute(WidenSrcVT, SrcVec, IndicesVec, DL,
9730 DAG, Subtarget);
9731 return extract256BitVector(Res, 0, DAG, DL);
9732 }
9733 Opcode = X86ISD::VPERMV;
9734 } else if (Subtarget.hasAVX()) {
9735 SrcVec = DAG.getBitcast(MVT::v4f64, SrcVec);
9736 SDValue LoLo =
9737 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {0, 1, 0, 1});
9738 SDValue HiHi =
9739 DAG.getVectorShuffle(MVT::v4f64, DL, SrcVec, SrcVec, {2, 3, 2, 3});
9740 // VPERMIL2PD selects with bit#1 of the index vector, so scale IndicesVec.
9741 IndicesVec = DAG.getNode(ISD::ADD, DL, IndicesVT, IndicesVec, IndicesVec);
9742 if (Subtarget.hasXOP())
9743 return DAG.getBitcast(
9744 VT, DAG.getNode(X86ISD::VPERMIL2, DL, MVT::v4f64, LoLo, HiHi,
9745 IndicesVec, DAG.getTargetConstant(0, DL, MVT::i8)));
9746 // Permute Lo and Hi and then select based on index range.
9747 // This works as VPERMILPD only uses index bit[1] to permute elements.
9748 SDValue Res = DAG.getSelectCC(
9749 DL, IndicesVec, DAG.getConstant(2, DL, MVT::v4i64),
9750 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, HiHi, IndicesVec),
9751 DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v4f64, LoLo, IndicesVec),
9752 ISD::CondCode::SETGT);
9753 return DAG.getBitcast(VT, Res);
9754 }
9755 break;
9756 case MVT::v64i8:
9757 if (Subtarget.hasVBMI())
9758 Opcode = X86ISD::VPERMV;
9759 break;
9760 case MVT::v32i16:
9761 if (Subtarget.hasBWI())
9762 Opcode = X86ISD::VPERMV;
9763 break;
9764 case MVT::v16f32:
9765 case MVT::v16i32:
9766 case MVT::v8f64:
9767 case MVT::v8i64:
9768 if (Subtarget.hasAVX512())
9769 Opcode = X86ISD::VPERMV;
9770 break;
9771 }
9772 if (!Opcode)
9773 return SDValue();
9774
9775 assert((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&(((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&
(VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits())
== 0 && "Illegal variable permute shuffle type") ? static_cast
<void> (0) : __assert_fail ("(VT.getSizeInBits() == ShuffleVT.getSizeInBits()) && (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 && \"Illegal variable permute shuffle type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9777, __PRETTY_FUNCTION__))
9776 (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 &&(((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&
(VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits())
== 0 && "Illegal variable permute shuffle type") ? static_cast
<void> (0) : __assert_fail ("(VT.getSizeInBits() == ShuffleVT.getSizeInBits()) && (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 && \"Illegal variable permute shuffle type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9777, __PRETTY_FUNCTION__))
9777 "Illegal variable permute shuffle type")(((VT.getSizeInBits() == ShuffleVT.getSizeInBits()) &&
(VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits())
== 0 && "Illegal variable permute shuffle type") ? static_cast
<void> (0) : __assert_fail ("(VT.getSizeInBits() == ShuffleVT.getSizeInBits()) && (VT.getScalarSizeInBits() % ShuffleVT.getScalarSizeInBits()) == 0 && \"Illegal variable permute shuffle type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9777, __PRETTY_FUNCTION__));
9778
9779 uint64_t Scale = VT.getScalarSizeInBits() / ShuffleVT.getScalarSizeInBits();
9780 if (Scale > 1)
9781 IndicesVec = ScaleIndices(IndicesVec, Scale);
9782
9783 EVT ShuffleIdxVT = EVT(ShuffleVT).changeVectorElementTypeToInteger();
9784 IndicesVec = DAG.getBitcast(ShuffleIdxVT, IndicesVec);
9785
9786 SrcVec = DAG.getBitcast(ShuffleVT, SrcVec);
9787 SDValue Res = Opcode == X86ISD::VPERMV
9788 ? DAG.getNode(Opcode, DL, ShuffleVT, IndicesVec, SrcVec)
9789 : DAG.getNode(Opcode, DL, ShuffleVT, SrcVec, IndicesVec);
9790 return DAG.getBitcast(VT, Res);
9791}
9792
9793// Tries to lower a BUILD_VECTOR composed of extract-extract chains that can be
9794// reasoned to be a permutation of a vector by indices in a non-constant vector.
9795// (build_vector (extract_elt V, (extract_elt I, 0)),
9796// (extract_elt V, (extract_elt I, 1)),
9797// ...
9798// ->
9799// (vpermv I, V)
9800//
9801// TODO: Handle undefs
9802// TODO: Utilize pshufb and zero mask blending to support more efficient
9803// construction of vectors with constant-0 elements.
9804static SDValue
9805LowerBUILD_VECTORAsVariablePermute(SDValue V, SelectionDAG &DAG,
9806 const X86Subtarget &Subtarget) {
9807 SDValue SrcVec, IndicesVec;
9808 // Check for a match of the permute source vector and permute index elements.
9809 // This is done by checking that the i-th build_vector operand is of the form:
9810 // (extract_elt SrcVec, (extract_elt IndicesVec, i)).
9811 for (unsigned Idx = 0, E = V.getNumOperands(); Idx != E; ++Idx) {
9812 SDValue Op = V.getOperand(Idx);
9813 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
9814 return SDValue();
9815
9816 // If this is the first extract encountered in V, set the source vector,
9817 // otherwise verify the extract is from the previously defined source
9818 // vector.
9819 if (!SrcVec)
9820 SrcVec = Op.getOperand(0);
9821 else if (SrcVec != Op.getOperand(0))
9822 return SDValue();
9823 SDValue ExtractedIndex = Op->getOperand(1);
9824 // Peek through extends.
9825 if (ExtractedIndex.getOpcode() == ISD::ZERO_EXTEND ||
9826 ExtractedIndex.getOpcode() == ISD::SIGN_EXTEND)
9827 ExtractedIndex = ExtractedIndex.getOperand(0);
9828 if (ExtractedIndex.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
9829 return SDValue();
9830
9831 // If this is the first extract from the index vector candidate, set the
9832 // indices vector, otherwise verify the extract is from the previously
9833 // defined indices vector.
9834 if (!IndicesVec)
9835 IndicesVec = ExtractedIndex.getOperand(0);
9836 else if (IndicesVec != ExtractedIndex.getOperand(0))
9837 return SDValue();
9838
9839 auto *PermIdx = dyn_cast<ConstantSDNode>(ExtractedIndex.getOperand(1));
9840 if (!PermIdx || PermIdx->getAPIntValue() != Idx)
9841 return SDValue();
9842 }
9843
9844 SDLoc DL(V);
9845 MVT VT = V.getSimpleValueType();
9846 return createVariablePermute(VT, SrcVec, IndicesVec, DL, DAG, Subtarget);
9847}
9848
9849SDValue
9850X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const {
9851 SDLoc dl(Op);
9852
9853 MVT VT = Op.getSimpleValueType();
9854 MVT EltVT = VT.getVectorElementType();
9855 unsigned NumElems = Op.getNumOperands();
9856
9857 // Generate vectors for predicate vectors.
9858 if (VT.getVectorElementType() == MVT::i1 && Subtarget.hasAVX512())
9859 return LowerBUILD_VECTORvXi1(Op, DAG, Subtarget);
9860
9861 if (SDValue VectorConstant = materializeVectorConstant(Op, DAG, Subtarget))
9862 return VectorConstant;
9863
9864 BuildVectorSDNode *BV = cast<BuildVectorSDNode>(Op.getNode());
9865 if (SDValue AddSub = lowerToAddSubOrFMAddSub(BV, Subtarget, DAG))
9866 return AddSub;
9867 if (SDValue HorizontalOp = LowerToHorizontalOp(BV, Subtarget, DAG))
9868 return HorizontalOp;
9869 if (SDValue Broadcast = lowerBuildVectorAsBroadcast(BV, Subtarget, DAG))
9870 return Broadcast;
9871 if (SDValue BitOp = lowerBuildVectorToBitOp(BV, DAG))
9872 return BitOp;
9873
9874 unsigned EVTBits = EltVT.getSizeInBits();
9875
9876 unsigned NumZero = 0;
9877 unsigned NumNonZero = 0;
9878 uint64_t NonZeros = 0;
9879 bool IsAllConstants = true;
9880 SmallSet<SDValue, 8> Values;
9881 unsigned NumConstants = NumElems;
9882 for (unsigned i = 0; i < NumElems; ++i) {
9883 SDValue Elt = Op.getOperand(i);
9884 if (Elt.isUndef())
9885 continue;
9886 Values.insert(Elt);
9887 if (!isa<ConstantSDNode>(Elt) && !isa<ConstantFPSDNode>(Elt)) {
9888 IsAllConstants = false;
9889 NumConstants--;
9890 }
9891 if (X86::isZeroNode(Elt))
9892 NumZero++;
9893 else {
9894 assert(i < sizeof(NonZeros) * 8)((i < sizeof(NonZeros) * 8) ? static_cast<void> (0) :
__assert_fail ("i < sizeof(NonZeros) * 8", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9894, __PRETTY_FUNCTION__)); // Make sure the shift is within range.
9895 NonZeros |= ((uint64_t)1 << i);
9896 NumNonZero++;
9897 }
9898 }
9899
9900 // All undef vector. Return an UNDEF. All zero vectors were handled above.
9901 if (NumNonZero == 0)
9902 return DAG.getUNDEF(VT);
9903
9904 // If we are inserting one variable into a vector of non-zero constants, try
9905 // to avoid loading each constant element as a scalar. Load the constants as a
9906 // vector and then insert the variable scalar element. If insertion is not
9907 // supported, fall back to a shuffle to get the scalar blended with the
9908 // constants. Insertion into a zero vector is handled as a special-case
9909 // somewhere below here.
9910 if (NumConstants == NumElems - 1 && NumNonZero != 1 &&
9911 (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) ||
9912 isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) {
9913 // Create an all-constant vector. The variable element in the old
9914 // build vector is replaced by undef in the constant vector. Save the
9915 // variable scalar element and its index for use in the insertelement.
9916 LLVMContext &Context = *DAG.getContext();
9917 Type *EltType = Op.getValueType().getScalarType().getTypeForEVT(Context);
9918 SmallVector<Constant *, 16> ConstVecOps(NumElems, UndefValue::get(EltType));
9919 SDValue VarElt;
9920 SDValue InsIndex;
9921 for (unsigned i = 0; i != NumElems; ++i) {
9922 SDValue Elt = Op.getOperand(i);
9923 if (auto *C = dyn_cast<ConstantSDNode>(Elt))
9924 ConstVecOps[i] = ConstantInt::get(Context, C->getAPIntValue());
9925 else if (auto *C = dyn_cast<ConstantFPSDNode>(Elt))
9926 ConstVecOps[i] = ConstantFP::get(Context, C->getValueAPF());
9927 else if (!Elt.isUndef()) {
9928 assert(!VarElt.getNode() && !InsIndex.getNode() &&((!VarElt.getNode() && !InsIndex.getNode() &&
"Expected one variable element in this vector") ? static_cast
<void> (0) : __assert_fail ("!VarElt.getNode() && !InsIndex.getNode() && \"Expected one variable element in this vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9929, __PRETTY_FUNCTION__))
9929 "Expected one variable element in this vector")((!VarElt.getNode() && !InsIndex.getNode() &&
"Expected one variable element in this vector") ? static_cast
<void> (0) : __assert_fail ("!VarElt.getNode() && !InsIndex.getNode() && \"Expected one variable element in this vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9929, __PRETTY_FUNCTION__));
9930 VarElt = Elt;
9931 InsIndex = DAG.getConstant(i, dl, getVectorIdxTy(DAG.getDataLayout()));
9932 }
9933 }
9934 Constant *CV = ConstantVector::get(ConstVecOps);
9935 SDValue DAGConstVec = DAG.getConstantPool(CV, VT);
9936
9937 // The constants we just created may not be legal (eg, floating point). We
9938 // must lower the vector right here because we can not guarantee that we'll
9939 // legalize it before loading it. This is also why we could not just create
9940 // a new build vector here. If the build vector contains illegal constants,
9941 // it could get split back up into a series of insert elements.
9942 // TODO: Improve this by using shorter loads with broadcast/VZEXT_LOAD.
9943 SDValue LegalDAGConstVec = LowerConstantPool(DAGConstVec, DAG);
9944 MachineFunction &MF = DAG.getMachineFunction();
9945 MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF);
9946 SDValue Ld = DAG.getLoad(VT, dl, DAG.getEntryNode(), LegalDAGConstVec, MPI);
9947 unsigned InsertC = cast<ConstantSDNode>(InsIndex)->getZExtValue();
9948 unsigned NumEltsInLow128Bits = 128 / VT.getScalarSizeInBits();
9949 if (InsertC < NumEltsInLow128Bits)
9950 return DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Ld, VarElt, InsIndex);
9951
9952 // There's no good way to insert into the high elements of a >128-bit
9953 // vector, so use shuffles to avoid an extract/insert sequence.
9954 assert(VT.getSizeInBits() > 128 && "Invalid insertion index?")((VT.getSizeInBits() > 128 && "Invalid insertion index?"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() > 128 && \"Invalid insertion index?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9954, __PRETTY_FUNCTION__));
9955 assert(Subtarget.hasAVX() && "Must have AVX with >16-byte vector")((Subtarget.hasAVX() && "Must have AVX with >16-byte vector"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX() && \"Must have AVX with >16-byte vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9955, __PRETTY_FUNCTION__));
9956 SmallVector<int, 8> ShuffleMask;
9957 unsigned NumElts = VT.getVectorNumElements();
9958 for (unsigned i = 0; i != NumElts; ++i)
9959 ShuffleMask.push_back(i == InsertC ? NumElts : i);
9960 SDValue S2V = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, VarElt);
9961 return DAG.getVectorShuffle(VT, dl, Ld, S2V, ShuffleMask);
9962 }
9963
9964 // Special case for single non-zero, non-undef, element.
9965 if (NumNonZero == 1) {
9966 unsigned Idx = countTrailingZeros(NonZeros);
9967 SDValue Item = Op.getOperand(Idx);
9968
9969 // If we have a constant or non-constant insertion into the low element of
9970 // a vector, we can do this with SCALAR_TO_VECTOR + shuffle of zero into
9971 // the rest of the elements. This will be matched as movd/movq/movss/movsd
9972 // depending on what the source datatype is.
9973 if (Idx == 0) {
9974 if (NumZero == 0)
9975 return DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
9976
9977 if (EltVT == MVT::i32 || EltVT == MVT::f32 || EltVT == MVT::f64 ||
9978 (EltVT == MVT::i64 && Subtarget.is64Bit())) {
9979 assert((VT.is128BitVector() || VT.is256BitVector() ||(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
()) && "Expected an SSE value type!") ? static_cast<
void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && \"Expected an SSE value type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9981, __PRETTY_FUNCTION__))
9980 VT.is512BitVector()) &&(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
()) && "Expected an SSE value type!") ? static_cast<
void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && \"Expected an SSE value type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9981, __PRETTY_FUNCTION__))
9981 "Expected an SSE value type!")(((VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector
()) && "Expected an SSE value type!") ? static_cast<
void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector() || VT.is512BitVector()) && \"Expected an SSE value type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 9981, __PRETTY_FUNCTION__));
9982 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
9983 // Turn it into a MOVL (i.e. movss, movsd, or movd) to a zero vector.
9984 return getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
9985 }
9986
9987 // We can't directly insert an i8 or i16 into a vector, so zero extend
9988 // it to i32 first.
9989 if (EltVT == MVT::i16 || EltVT == MVT::i8) {
9990 Item = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, Item);
9991 MVT ShufVT = MVT::getVectorVT(MVT::i32, VT.getSizeInBits()/32);
9992 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, ShufVT, Item);
9993 Item = getShuffleVectorZeroOrUndef(Item, 0, true, Subtarget, DAG);
9994 return DAG.getBitcast(VT, Item);
9995 }
9996 }
9997
9998 // Is it a vector logical left shift?
9999 if (NumElems == 2 && Idx == 1 &&
10000 X86::isZeroNode(Op.getOperand(0)) &&
10001 !X86::isZeroNode(Op.getOperand(1))) {
10002 unsigned NumBits = VT.getSizeInBits();
10003 return getVShift(true, VT,
10004 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl,
10005 VT, Op.getOperand(1)),
10006 NumBits/2, DAG, *this, dl);
10007 }
10008
10009 if (IsAllConstants) // Otherwise, it's better to do a constpool load.
10010 return SDValue();
10011
10012 // Otherwise, if this is a vector with i32 or f32 elements, and the element
10013 // is a non-constant being inserted into an element other than the low one,
10014 // we can't use a constant pool load. Instead, use SCALAR_TO_VECTOR (aka
10015 // movd/movss) to move this into the low element, then shuffle it into
10016 // place.
10017 if (EVTBits == 32) {
10018 Item = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Item);
10019 return getShuffleVectorZeroOrUndef(Item, Idx, NumZero > 0, Subtarget, DAG);
10020 }
10021 }
10022
10023 // Splat is obviously ok. Let legalizer expand it to a shuffle.
10024 if (Values.size() == 1) {
10025 if (EVTBits == 32) {
10026 // Instead of a shuffle like this:
10027 // shuffle (scalar_to_vector (load (ptr + 4))), undef, <0, 0, 0, 0>
10028 // Check if it's possible to issue this instead.
10029 // shuffle (vload ptr)), undef, <1, 1, 1, 1>
10030 unsigned Idx = countTrailingZeros(NonZeros);
10031 SDValue Item = Op.getOperand(Idx);
10032 if (Op.getNode()->isOnlyUserOf(Item.getNode()))
10033 return LowerAsSplatVectorLoad(Item, VT, dl, DAG);
10034 }
10035 return SDValue();
10036 }
10037
10038 // A vector full of immediates; various special cases are already
10039 // handled, so this is best done with a single constant-pool load.
10040 if (IsAllConstants)
10041 return SDValue();
10042
10043 if (SDValue V = LowerBUILD_VECTORAsVariablePermute(Op, DAG, Subtarget))
10044 return V;
10045
10046 // See if we can use a vector load to get all of the elements.
10047 {
10048 SmallVector<SDValue, 64> Ops(Op->op_begin(), Op->op_begin() + NumElems);
10049 if (SDValue LD =
10050 EltsFromConsecutiveLoads(VT, Ops, dl, DAG, Subtarget, false))
10051 return LD;
10052 }
10053
10054 // If this is a splat of pairs of 32-bit elements, we can use a narrower
10055 // build_vector and broadcast it.
10056 // TODO: We could probably generalize this more.
10057 if (Subtarget.hasAVX2() && EVTBits == 32 && Values.size() == 2) {
10058 SDValue Ops[4] = { Op.getOperand(0), Op.getOperand(1),
10059 DAG.getUNDEF(EltVT), DAG.getUNDEF(EltVT) };
10060 auto CanSplat = [](SDValue Op, unsigned NumElems, ArrayRef<SDValue> Ops) {
10061 // Make sure all the even/odd operands match.
10062 for (unsigned i = 2; i != NumElems; ++i)
10063 if (Ops[i % 2] != Op.getOperand(i))
10064 return false;
10065 return true;
10066 };
10067 if (CanSplat(Op, NumElems, Ops)) {
10068 MVT WideEltVT = VT.isFloatingPoint() ? MVT::f64 : MVT::i64;
10069 MVT NarrowVT = MVT::getVectorVT(EltVT, 4);
10070 // Create a new build vector and cast to v2i64/v2f64.
10071 SDValue NewBV = DAG.getBitcast(MVT::getVectorVT(WideEltVT, 2),
10072 DAG.getBuildVector(NarrowVT, dl, Ops));
10073 // Broadcast from v2i64/v2f64 and cast to final VT.
10074 MVT BcastVT = MVT::getVectorVT(WideEltVT, NumElems/2);
10075 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, dl, BcastVT,
10076 NewBV));
10077 }
10078 }
10079
10080 // For AVX-length vectors, build the individual 128-bit pieces and use
10081 // shuffles to put them in place.
10082 if (VT.getSizeInBits() > 128) {
10083 MVT HVT = MVT::getVectorVT(EltVT, NumElems/2);
10084
10085 // Build both the lower and upper subvector.
10086 SDValue Lower =
10087 DAG.getBuildVector(HVT, dl, Op->ops().slice(0, NumElems / 2));
10088 SDValue Upper = DAG.getBuildVector(
10089 HVT, dl, Op->ops().slice(NumElems / 2, NumElems /2));
10090
10091 // Recreate the wider vector with the lower and upper part.
10092 return concatSubVectors(Lower, Upper, DAG, dl);
10093 }
10094
10095 // Let legalizer expand 2-wide build_vectors.
10096 if (EVTBits == 64) {
10097 if (NumNonZero == 1) {
10098 // One half is zero or undef.
10099 unsigned Idx = countTrailingZeros(NonZeros);
10100 SDValue V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT,
10101 Op.getOperand(Idx));
10102 return getShuffleVectorZeroOrUndef(V2, Idx, true, Subtarget, DAG);
10103 }
10104 return SDValue();
10105 }
10106
10107 // If element VT is < 32 bits, convert it to inserts into a zero vector.
10108 if (EVTBits == 8 && NumElems == 16)
10109 if (SDValue V = LowerBuildVectorv16i8(Op, NonZeros, NumNonZero, NumZero,
10110 DAG, Subtarget))
10111 return V;
10112
10113 if (EVTBits == 16 && NumElems == 8)
10114 if (SDValue V = LowerBuildVectorv8i16(Op, NonZeros, NumNonZero, NumZero,
10115 DAG, Subtarget))
10116 return V;
10117
10118 // If element VT is == 32 bits and has 4 elems, try to generate an INSERTPS
10119 if (EVTBits == 32 && NumElems == 4)
10120 if (SDValue V = LowerBuildVectorv4x32(Op, DAG, Subtarget))
10121 return V;
10122
10123 // If element VT is == 32 bits, turn it into a number of shuffles.
10124 if (NumElems == 4 && NumZero > 0) {
10125 SmallVector<SDValue, 8> Ops(NumElems);
10126 for (unsigned i = 0; i < 4; ++i) {
10127 bool isZero = !(NonZeros & (1ULL << i));
10128 if (isZero)
10129 Ops[i] = getZeroVector(VT, Subtarget, DAG, dl);
10130 else
10131 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
10132 }
10133
10134 for (unsigned i = 0; i < 2; ++i) {
10135 switch ((NonZeros >> (i*2)) & 0x3) {
10136 default: llvm_unreachable("Unexpected NonZero count")::llvm::llvm_unreachable_internal("Unexpected NonZero count",
"/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10136);
10137 case 0:
10138 Ops[i] = Ops[i*2]; // Must be a zero vector.
10139 break;
10140 case 1:
10141 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2+1], Ops[i*2]);
10142 break;
10143 case 2:
10144 Ops[i] = getMOVL(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
10145 break;
10146 case 3:
10147 Ops[i] = getUnpackl(DAG, dl, VT, Ops[i*2], Ops[i*2+1]);
10148 break;
10149 }
10150 }
10151
10152 bool Reverse1 = (NonZeros & 0x3) == 2;
10153 bool Reverse2 = ((NonZeros & (0x3 << 2)) >> 2) == 2;
10154 int MaskVec[] = {
10155 Reverse1 ? 1 : 0,
10156 Reverse1 ? 0 : 1,
10157 static_cast<int>(Reverse2 ? NumElems+1 : NumElems),
10158 static_cast<int>(Reverse2 ? NumElems : NumElems+1)
10159 };
10160 return DAG.getVectorShuffle(VT, dl, Ops[0], Ops[1], MaskVec);
10161 }
10162
10163 assert(Values.size() > 1 && "Expected non-undef and non-splat vector")((Values.size() > 1 && "Expected non-undef and non-splat vector"
) ? static_cast<void> (0) : __assert_fail ("Values.size() > 1 && \"Expected non-undef and non-splat vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10163, __PRETTY_FUNCTION__));
10164
10165 // Check for a build vector from mostly shuffle plus few inserting.
10166 if (SDValue Sh = buildFromShuffleMostly(Op, DAG))
10167 return Sh;
10168
10169 // For SSE 4.1, use insertps to put the high elements into the low element.
10170 if (Subtarget.hasSSE41()) {
10171 SDValue Result;
10172 if (!Op.getOperand(0).isUndef())
10173 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(0));
10174 else
10175 Result = DAG.getUNDEF(VT);
10176
10177 for (unsigned i = 1; i < NumElems; ++i) {
10178 if (Op.getOperand(i).isUndef()) continue;
10179 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VT, Result,
10180 Op.getOperand(i), DAG.getIntPtrConstant(i, dl));
10181 }
10182 return Result;
10183 }
10184
10185 // Otherwise, expand into a number of unpckl*, start by extending each of
10186 // our (non-undef) elements to the full vector width with the element in the
10187 // bottom slot of the vector (which generates no code for SSE).
10188 SmallVector<SDValue, 8> Ops(NumElems);
10189 for (unsigned i = 0; i < NumElems; ++i) {
10190 if (!Op.getOperand(i).isUndef())
10191 Ops[i] = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, Op.getOperand(i));
10192 else
10193 Ops[i] = DAG.getUNDEF(VT);
10194 }
10195
10196 // Next, we iteratively mix elements, e.g. for v4f32:
10197 // Step 1: unpcklps 0, 1 ==> X: <?, ?, 1, 0>
10198 // : unpcklps 2, 3 ==> Y: <?, ?, 3, 2>
10199 // Step 2: unpcklpd X, Y ==> <3, 2, 1, 0>
10200 for (unsigned Scale = 1; Scale < NumElems; Scale *= 2) {
10201 // Generate scaled UNPCKL shuffle mask.
10202 SmallVector<int, 16> Mask;
10203 for(unsigned i = 0; i != Scale; ++i)
10204 Mask.push_back(i);
10205 for (unsigned i = 0; i != Scale; ++i)
10206 Mask.push_back(NumElems+i);
10207 Mask.append(NumElems - Mask.size(), SM_SentinelUndef);
10208
10209 for (unsigned i = 0, e = NumElems / (2 * Scale); i != e; ++i)
10210 Ops[i] = DAG.getVectorShuffle(VT, dl, Ops[2*i], Ops[(2*i)+1], Mask);
10211 }
10212 return Ops[0];
10213}
10214
10215// 256-bit AVX can use the vinsertf128 instruction
10216// to create 256-bit vectors from two other 128-bit ones.
10217// TODO: Detect subvector broadcast here instead of DAG combine?
10218static SDValue LowerAVXCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG,
10219 const X86Subtarget &Subtarget) {
10220 SDLoc dl(Op);
10221 MVT ResVT = Op.getSimpleValueType();
10222
10223 assert((ResVT.is256BitVector() ||(((ResVT.is256BitVector() || ResVT.is512BitVector()) &&
"Value type must be 256-/512-bit wide") ? static_cast<void
> (0) : __assert_fail ("(ResVT.is256BitVector() || ResVT.is512BitVector()) && \"Value type must be 256-/512-bit wide\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10224, __PRETTY_FUNCTION__))
10224 ResVT.is512BitVector()) && "Value type must be 256-/512-bit wide")(((ResVT.is256BitVector() || ResVT.is512BitVector()) &&
"Value type must be 256-/512-bit wide") ? static_cast<void
> (0) : __assert_fail ("(ResVT.is256BitVector() || ResVT.is512BitVector()) && \"Value type must be 256-/512-bit wide\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10224, __PRETTY_FUNCTION__));
10225
10226 unsigned NumOperands = Op.getNumOperands();
10227 unsigned NumZero = 0;
10228 unsigned NumNonZero = 0;
10229 unsigned NonZeros = 0;
10230 for (unsigned i = 0; i != NumOperands; ++i) {
10231 SDValue SubVec = Op.getOperand(i);
10232 if (SubVec.isUndef())
10233 continue;
10234 if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
10235 ++NumZero;
10236 else {
10237 assert(i < sizeof(NonZeros) * CHAR_BIT)((i < sizeof(NonZeros) * 8) ? static_cast<void> (0) :
__assert_fail ("i < sizeof(NonZeros) * CHAR_BIT", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10237, __PRETTY_FUNCTION__)); // Ensure the shift is in range.
10238 NonZeros |= 1 << i;
10239 ++NumNonZero;
10240 }
10241 }
10242
10243 // If we have more than 2 non-zeros, build each half separately.
10244 if (NumNonZero > 2) {
10245 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
10246 ArrayRef<SDUse> Ops = Op->ops();
10247 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10248 Ops.slice(0, NumOperands/2));
10249 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10250 Ops.slice(NumOperands/2));
10251 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
10252 }
10253
10254 // Otherwise, build it up through insert_subvectors.
10255 SDValue Vec = NumZero ? getZeroVector(ResVT, Subtarget, DAG, dl)
10256 : DAG.getUNDEF(ResVT);
10257
10258 MVT SubVT = Op.getOperand(0).getSimpleValueType();
10259 unsigned NumSubElems = SubVT.getVectorNumElements();
10260 for (unsigned i = 0; i != NumOperands; ++i) {
10261 if ((NonZeros & (1 << i)) == 0)
10262 continue;
10263
10264 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec,
10265 Op.getOperand(i),
10266 DAG.getIntPtrConstant(i * NumSubElems, dl));
10267 }
10268
10269 return Vec;
10270}
10271
10272// Returns true if the given node is a type promotion (by concatenating i1
10273// zeros) of the result of a node that already zeros all upper bits of
10274// k-register.
10275// TODO: Merge this with LowerAVXCONCAT_VECTORS?
10276static SDValue LowerCONCAT_VECTORSvXi1(SDValue Op,
10277 const X86Subtarget &Subtarget,
10278 SelectionDAG & DAG) {
10279 SDLoc dl(Op);
10280 MVT ResVT = Op.getSimpleValueType();
10281 unsigned NumOperands = Op.getNumOperands();
10282
10283 assert(NumOperands > 1 && isPowerOf2_32(NumOperands) &&((NumOperands > 1 && isPowerOf2_32(NumOperands) &&
"Unexpected number of operands in CONCAT_VECTORS") ? static_cast
<void> (0) : __assert_fail ("NumOperands > 1 && isPowerOf2_32(NumOperands) && \"Unexpected number of operands in CONCAT_VECTORS\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10284, __PRETTY_FUNCTION__))
10284 "Unexpected number of operands in CONCAT_VECTORS")((NumOperands > 1 && isPowerOf2_32(NumOperands) &&
"Unexpected number of operands in CONCAT_VECTORS") ? static_cast
<void> (0) : __assert_fail ("NumOperands > 1 && isPowerOf2_32(NumOperands) && \"Unexpected number of operands in CONCAT_VECTORS\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10284, __PRETTY_FUNCTION__));
10285
10286 uint64_t Zeros = 0;
10287 uint64_t NonZeros = 0;
10288 for (unsigned i = 0; i != NumOperands; ++i) {
10289 SDValue SubVec = Op.getOperand(i);
10290 if (SubVec.isUndef())
10291 continue;
10292 assert(i < sizeof(NonZeros) * CHAR_BIT)((i < sizeof(NonZeros) * 8) ? static_cast<void> (0) :
__assert_fail ("i < sizeof(NonZeros) * CHAR_BIT", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10292, __PRETTY_FUNCTION__)); // Ensure the shift is in range.
10293 if (ISD::isBuildVectorAllZeros(SubVec.getNode()))
10294 Zeros |= (uint64_t)1 << i;
10295 else
10296 NonZeros |= (uint64_t)1 << i;
10297 }
10298
10299 unsigned NumElems = ResVT.getVectorNumElements();
10300
10301 // If we are inserting non-zero vector and there are zeros in LSBs and undef
10302 // in the MSBs we need to emit a KSHIFTL. The generic lowering to
10303 // insert_subvector will give us two kshifts.
10304 if (isPowerOf2_64(NonZeros) && Zeros != 0 && NonZeros > Zeros &&
10305 Log2_64(NonZeros) != NumOperands - 1) {
10306 MVT ShiftVT = ResVT;
10307 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8)
10308 ShiftVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
10309 unsigned Idx = Log2_64(NonZeros);
10310 SDValue SubVec = Op.getOperand(Idx);
10311 unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
10312 SubVec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ShiftVT,
10313 DAG.getUNDEF(ShiftVT), SubVec,
10314 DAG.getIntPtrConstant(0, dl));
10315 Op = DAG.getNode(X86ISD::KSHIFTL, dl, ShiftVT, SubVec,
10316 DAG.getTargetConstant(Idx * SubVecNumElts, dl, MVT::i8));
10317 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, ResVT, Op,
10318 DAG.getIntPtrConstant(0, dl));
10319 }
10320
10321 // If there are zero or one non-zeros we can handle this very simply.
10322 if (NonZeros == 0 || isPowerOf2_64(NonZeros)) {
10323 SDValue Vec = Zeros ? DAG.getConstant(0, dl, ResVT) : DAG.getUNDEF(ResVT);
10324 if (!NonZeros)
10325 return Vec;
10326 unsigned Idx = Log2_64(NonZeros);
10327 SDValue SubVec = Op.getOperand(Idx);
10328 unsigned SubVecNumElts = SubVec.getSimpleValueType().getVectorNumElements();
10329 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, SubVec,
10330 DAG.getIntPtrConstant(Idx * SubVecNumElts, dl));
10331 }
10332
10333 if (NumOperands > 2) {
10334 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
10335 ArrayRef<SDUse> Ops = Op->ops();
10336 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10337 Ops.slice(0, NumOperands/2));
10338 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, dl, HalfVT,
10339 Ops.slice(NumOperands/2));
10340 return DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Lo, Hi);
10341 }
10342
10343 assert(countPopulation(NonZeros) == 2 && "Simple cases not handled?")((countPopulation(NonZeros) == 2 && "Simple cases not handled?"
) ? static_cast<void> (0) : __assert_fail ("countPopulation(NonZeros) == 2 && \"Simple cases not handled?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10343, __PRETTY_FUNCTION__));
10344
10345 if (ResVT.getVectorNumElements() >= 16)
10346 return Op; // The operation is legal with KUNPCK
10347
10348 SDValue Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT,
10349 DAG.getUNDEF(ResVT), Op.getOperand(0),
10350 DAG.getIntPtrConstant(0, dl));
10351 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, ResVT, Vec, Op.getOperand(1),
10352 DAG.getIntPtrConstant(NumElems/2, dl));
10353}
10354
10355static SDValue LowerCONCAT_VECTORS(SDValue Op,
10356 const X86Subtarget &Subtarget,
10357 SelectionDAG &DAG) {
10358 MVT VT = Op.getSimpleValueType();
10359 if (VT.getVectorElementType() == MVT::i1)
10360 return LowerCONCAT_VECTORSvXi1(Op, Subtarget, DAG);
10361
10362 assert((VT.is256BitVector() && Op.getNumOperands() == 2) ||(((VT.is256BitVector() && Op.getNumOperands() == 2) ||
(VT.is512BitVector() && (Op.getNumOperands() == 2 ||
Op.getNumOperands() == 4))) ? static_cast<void> (0) : __assert_fail
("(VT.is256BitVector() && Op.getNumOperands() == 2) || (VT.is512BitVector() && (Op.getNumOperands() == 2 || Op.getNumOperands() == 4))"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10364, __PRETTY_FUNCTION__))
10363 (VT.is512BitVector() && (Op.getNumOperands() == 2 ||(((VT.is256BitVector() && Op.getNumOperands() == 2) ||
(VT.is512BitVector() && (Op.getNumOperands() == 2 ||
Op.getNumOperands() == 4))) ? static_cast<void> (0) : __assert_fail
("(VT.is256BitVector() && Op.getNumOperands() == 2) || (VT.is512BitVector() && (Op.getNumOperands() == 2 || Op.getNumOperands() == 4))"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10364, __PRETTY_FUNCTION__))
10364 Op.getNumOperands() == 4)))(((VT.is256BitVector() && Op.getNumOperands() == 2) ||
(VT.is512BitVector() && (Op.getNumOperands() == 2 ||
Op.getNumOperands() == 4))) ? static_cast<void> (0) : __assert_fail
("(VT.is256BitVector() && Op.getNumOperands() == 2) || (VT.is512BitVector() && (Op.getNumOperands() == 2 || Op.getNumOperands() == 4))"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10364, __PRETTY_FUNCTION__));
10365
10366 // AVX can use the vinsertf128 instruction to create 256-bit vectors
10367 // from two other 128-bit ones.
10368
10369 // 512-bit vector may contain 2 256-bit vectors or 4 128-bit vectors
10370 return LowerAVXCONCAT_VECTORS(Op, DAG, Subtarget);
10371}
10372
10373//===----------------------------------------------------------------------===//
10374// Vector shuffle lowering
10375//
10376// This is an experimental code path for lowering vector shuffles on x86. It is
10377// designed to handle arbitrary vector shuffles and blends, gracefully
10378// degrading performance as necessary. It works hard to recognize idiomatic
10379// shuffles and lower them to optimal instruction patterns without leaving
10380// a framework that allows reasonably efficient handling of all vector shuffle
10381// patterns.
10382//===----------------------------------------------------------------------===//
10383
10384/// Tiny helper function to identify a no-op mask.
10385///
10386/// This is a somewhat boring predicate function. It checks whether the mask
10387/// array input, which is assumed to be a single-input shuffle mask of the kind
10388/// used by the X86 shuffle instructions (not a fully general
10389/// ShuffleVectorSDNode mask) requires any shuffles to occur. Both undef and an
10390/// in-place shuffle are 'no-op's.
10391static bool isNoopShuffleMask(ArrayRef<int> Mask) {
10392 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
10393 assert(Mask[i] >= -1 && "Out of bound mask element!")((Mask[i] >= -1 && "Out of bound mask element!") ?
static_cast<void> (0) : __assert_fail ("Mask[i] >= -1 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10393, __PRETTY_FUNCTION__));
10394 if (Mask[i] >= 0 && Mask[i] != i)
10395 return false;
10396 }
10397 return true;
10398}
10399
10400/// Test whether there are elements crossing LaneSizeInBits lanes in this
10401/// shuffle mask.
10402///
10403/// X86 divides up its shuffles into in-lane and cross-lane shuffle operations
10404/// and we routinely test for these.
10405static bool isLaneCrossingShuffleMask(unsigned LaneSizeInBits,
10406 unsigned ScalarSizeInBits,
10407 ArrayRef<int> Mask) {
10408 assert(LaneSizeInBits && ScalarSizeInBits &&((LaneSizeInBits && ScalarSizeInBits && (LaneSizeInBits
% ScalarSizeInBits) == 0 && "Illegal shuffle lane size"
) ? static_cast<void> (0) : __assert_fail ("LaneSizeInBits && ScalarSizeInBits && (LaneSizeInBits % ScalarSizeInBits) == 0 && \"Illegal shuffle lane size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10410, __PRETTY_FUNCTION__))
10409 (LaneSizeInBits % ScalarSizeInBits) == 0 &&((LaneSizeInBits && ScalarSizeInBits && (LaneSizeInBits
% ScalarSizeInBits) == 0 && "Illegal shuffle lane size"
) ? static_cast<void> (0) : __assert_fail ("LaneSizeInBits && ScalarSizeInBits && (LaneSizeInBits % ScalarSizeInBits) == 0 && \"Illegal shuffle lane size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10410, __PRETTY_FUNCTION__))
10410 "Illegal shuffle lane size")((LaneSizeInBits && ScalarSizeInBits && (LaneSizeInBits
% ScalarSizeInBits) == 0 && "Illegal shuffle lane size"
) ? static_cast<void> (0) : __assert_fail ("LaneSizeInBits && ScalarSizeInBits && (LaneSizeInBits % ScalarSizeInBits) == 0 && \"Illegal shuffle lane size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10410, __PRETTY_FUNCTION__));
10411 int LaneSize = LaneSizeInBits / ScalarSizeInBits;
10412 int Size = Mask.size();
10413 for (int i = 0; i < Size; ++i)
10414 if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
10415 return true;
10416 return false;
10417}
10418
10419/// Test whether there are elements crossing 128-bit lanes in this
10420/// shuffle mask.
10421static bool is128BitLaneCrossingShuffleMask(MVT VT, ArrayRef<int> Mask) {
10422 return isLaneCrossingShuffleMask(128, VT.getScalarSizeInBits(), Mask);
10423}
10424
10425/// Test whether a shuffle mask is equivalent within each sub-lane.
10426///
10427/// This checks a shuffle mask to see if it is performing the same
10428/// lane-relative shuffle in each sub-lane. This trivially implies
10429/// that it is also not lane-crossing. It may however involve a blend from the
10430/// same lane of a second vector.
10431///
10432/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
10433/// non-trivial to compute in the face of undef lanes. The representation is
10434/// suitable for use with existing 128-bit shuffles as entries from the second
10435/// vector have been remapped to [LaneSize, 2*LaneSize).
10436static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
10437 ArrayRef<int> Mask,
10438 SmallVectorImpl<int> &RepeatedMask) {
10439 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
10440 RepeatedMask.assign(LaneSize, -1);
10441 int Size = Mask.size();
10442 for (int i = 0; i < Size; ++i) {
10443 assert(Mask[i] == SM_SentinelUndef || Mask[i] >= 0)((Mask[i] == SM_SentinelUndef || Mask[i] >= 0) ? static_cast
<void> (0) : __assert_fail ("Mask[i] == SM_SentinelUndef || Mask[i] >= 0"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10443, __PRETTY_FUNCTION__));
10444 if (Mask[i] < 0)
10445 continue;
10446 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
10447 // This entry crosses lanes, so there is no way to model this shuffle.
10448 return false;
10449
10450 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
10451 // Adjust second vector indices to start at LaneSize instead of Size.
10452 int LocalM = Mask[i] < Size ? Mask[i] % LaneSize
10453 : Mask[i] % LaneSize + LaneSize;
10454 if (RepeatedMask[i % LaneSize] < 0)
10455 // This is the first non-undef entry in this slot of a 128-bit lane.
10456 RepeatedMask[i % LaneSize] = LocalM;
10457 else if (RepeatedMask[i % LaneSize] != LocalM)
10458 // Found a mismatch with the repeated mask.
10459 return false;
10460 }
10461 return true;
10462}
10463
10464/// Test whether a shuffle mask is equivalent within each 128-bit lane.
10465static bool
10466is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
10467 SmallVectorImpl<int> &RepeatedMask) {
10468 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
10469}
10470
10471static bool
10472is128BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask) {
10473 SmallVector<int, 32> RepeatedMask;
10474 return isRepeatedShuffleMask(128, VT, Mask, RepeatedMask);
10475}
10476
10477/// Test whether a shuffle mask is equivalent within each 256-bit lane.
10478static bool
10479is256BitLaneRepeatedShuffleMask(MVT VT, ArrayRef<int> Mask,
10480 SmallVectorImpl<int> &RepeatedMask) {
10481 return isRepeatedShuffleMask(256, VT, Mask, RepeatedMask);
10482}
10483
10484/// Test whether a target shuffle mask is equivalent within each sub-lane.
10485/// Unlike isRepeatedShuffleMask we must respect SM_SentinelZero.
10486static bool isRepeatedTargetShuffleMask(unsigned LaneSizeInBits, MVT VT,
10487 ArrayRef<int> Mask,
10488 SmallVectorImpl<int> &RepeatedMask) {
10489 int LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
10490 RepeatedMask.assign(LaneSize, SM_SentinelUndef);
10491 int Size = Mask.size();
10492 for (int i = 0; i < Size; ++i) {
10493 assert(isUndefOrZero(Mask[i]) || (Mask[i] >= 0))((isUndefOrZero(Mask[i]) || (Mask[i] >= 0)) ? static_cast<
void> (0) : __assert_fail ("isUndefOrZero(Mask[i]) || (Mask[i] >= 0)"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10493, __PRETTY_FUNCTION__));
10494 if (Mask[i] == SM_SentinelUndef)
10495 continue;
10496 if (Mask[i] == SM_SentinelZero) {
10497 if (!isUndefOrZero(RepeatedMask[i % LaneSize]))
10498 return false;
10499 RepeatedMask[i % LaneSize] = SM_SentinelZero;
10500 continue;
10501 }
10502 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
10503 // This entry crosses lanes, so there is no way to model this shuffle.
10504 return false;
10505
10506 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
10507 // Adjust second vector indices to start at LaneSize instead of Size.
10508 int LocalM =
10509 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
10510 if (RepeatedMask[i % LaneSize] == SM_SentinelUndef)
10511 // This is the first non-undef entry in this slot of a 128-bit lane.
10512 RepeatedMask[i % LaneSize] = LocalM;
10513 else if (RepeatedMask[i % LaneSize] != LocalM)
10514 // Found a mismatch with the repeated mask.
10515 return false;
10516 }
10517 return true;
10518}
10519
10520/// Checks whether a shuffle mask is equivalent to an explicit list of
10521/// arguments.
10522///
10523/// This is a fast way to test a shuffle mask against a fixed pattern:
10524///
10525/// if (isShuffleEquivalent(Mask, 3, 2, {1, 0})) { ... }
10526///
10527/// It returns true if the mask is exactly as wide as the argument list, and
10528/// each element of the mask is either -1 (signifying undef) or the value given
10529/// in the argument.
10530static bool isShuffleEquivalent(SDValue V1, SDValue V2, ArrayRef<int> Mask,
10531 ArrayRef<int> ExpectedMask) {
10532 if (Mask.size() != ExpectedMask.size())
10533 return false;
10534
10535 int Size = Mask.size();
10536
10537 // If the values are build vectors, we can look through them to find
10538 // equivalent inputs that make the shuffles equivalent.
10539 auto *BV1 = dyn_cast<BuildVectorSDNode>(V1);
10540 auto *BV2 = dyn_cast<BuildVectorSDNode>(V2);
10541
10542 for (int i = 0; i < Size; ++i) {
10543 assert(Mask[i] >= -1 && "Out of bound mask element!")((Mask[i] >= -1 && "Out of bound mask element!") ?
static_cast<void> (0) : __assert_fail ("Mask[i] >= -1 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10543, __PRETTY_FUNCTION__));
10544 if (Mask[i] >= 0 && Mask[i] != ExpectedMask[i]) {
10545 auto *MaskBV = Mask[i] < Size ? BV1 : BV2;
10546 auto *ExpectedBV = ExpectedMask[i] < Size ? BV1 : BV2;
10547 if (!MaskBV || !ExpectedBV ||
10548 MaskBV->getOperand(Mask[i] % Size) !=
10549 ExpectedBV->getOperand(ExpectedMask[i] % Size))
10550 return false;
10551 }
10552 }
10553
10554 return true;
10555}
10556
10557/// Checks whether a target shuffle mask is equivalent to an explicit pattern.
10558///
10559/// The masks must be exactly the same width.
10560///
10561/// If an element in Mask matches SM_SentinelUndef (-1) then the corresponding
10562/// value in ExpectedMask is always accepted. Otherwise the indices must match.
10563///
10564/// SM_SentinelZero is accepted as a valid negative index but must match in
10565/// both.
10566static bool isTargetShuffleEquivalent(ArrayRef<int> Mask,
10567 ArrayRef<int> ExpectedMask,
10568 SDValue V1 = SDValue(),
10569 SDValue V2 = SDValue()) {
10570 int Size = Mask.size();
10571 if (Size != (int)ExpectedMask.size())
10572 return false;
10573 assert(isUndefOrZeroOrInRange(ExpectedMask, 0, 2 * Size) &&((isUndefOrZeroOrInRange(ExpectedMask, 0, 2 * Size) &&
"Illegal target shuffle mask") ? static_cast<void> (0)
: __assert_fail ("isUndefOrZeroOrInRange(ExpectedMask, 0, 2 * Size) && \"Illegal target shuffle mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10574, __PRETTY_FUNCTION__))
10574 "Illegal target shuffle mask")((isUndefOrZeroOrInRange(ExpectedMask, 0, 2 * Size) &&
"Illegal target shuffle mask") ? static_cast<void> (0)
: __assert_fail ("isUndefOrZeroOrInRange(ExpectedMask, 0, 2 * Size) && \"Illegal target shuffle mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10574, __PRETTY_FUNCTION__));
10575
10576 // Check for out-of-range target shuffle mask indices.
10577 if (!isUndefOrZeroOrInRange(Mask, 0, 2 * Size))
10578 return false;
10579
10580 // If the values are build vectors, we can look through them to find
10581 // equivalent inputs that make the shuffles equivalent.
10582 auto *BV1 = dyn_cast_or_null<BuildVectorSDNode>(V1);
10583 auto *BV2 = dyn_cast_or_null<BuildVectorSDNode>(V2);
10584 BV1 = ((BV1 && Size != (int)BV1->getNumOperands()) ? nullptr : BV1);
10585 BV2 = ((BV2 && Size != (int)BV2->getNumOperands()) ? nullptr : BV2);
10586
10587 for (int i = 0; i < Size; ++i) {
10588 if (Mask[i] == SM_SentinelUndef || Mask[i] == ExpectedMask[i])
10589 continue;
10590 if (0 <= Mask[i] && 0 <= ExpectedMask[i]) {
10591 auto *MaskBV = Mask[i] < Size ? BV1 : BV2;
10592 auto *ExpectedBV = ExpectedMask[i] < Size ? BV1 : BV2;
10593 if (MaskBV && ExpectedBV &&
10594 MaskBV->getOperand(Mask[i] % Size) ==
10595 ExpectedBV->getOperand(ExpectedMask[i] % Size))
10596 continue;
10597 }
10598 // TODO - handle SM_Sentinel equivalences.
10599 return false;
10600 }
10601 return true;
10602}
10603
10604// Attempt to create a shuffle mask from a VSELECT condition mask.
10605static bool createShuffleMaskFromVSELECT(SmallVectorImpl<int> &Mask,
10606 SDValue Cond) {
10607 if (!ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
10608 return false;
10609
10610 unsigned Size = Cond.getValueType().getVectorNumElements();
10611 Mask.resize(Size, SM_SentinelUndef);
10612
10613 for (int i = 0; i != (int)Size; ++i) {
10614 SDValue CondElt = Cond.getOperand(i);
10615 Mask[i] = i;
10616 // Arbitrarily choose from the 2nd operand if the select condition element
10617 // is undef.
10618 // TODO: Can we do better by matching patterns such as even/odd?
10619 if (CondElt.isUndef() || isNullConstant(CondElt))
10620 Mask[i] += Size;
10621 }
10622
10623 return true;
10624}
10625
10626// Check if the shuffle mask is suitable for the AVX vpunpcklwd or vpunpckhwd
10627// instructions.
10628static bool isUnpackWdShuffleMask(ArrayRef<int> Mask, MVT VT) {
10629 if (VT != MVT::v8i32 && VT != MVT::v8f32)
10630 return false;
10631
10632 SmallVector<int, 8> Unpcklwd;
10633 createUnpackShuffleMask(MVT::v8i16, Unpcklwd, /* Lo = */ true,
10634 /* Unary = */ false);
10635 SmallVector<int, 8> Unpckhwd;
10636 createUnpackShuffleMask(MVT::v8i16, Unpckhwd, /* Lo = */ false,
10637 /* Unary = */ false);
10638 bool IsUnpackwdMask = (isTargetShuffleEquivalent(Mask, Unpcklwd) ||
10639 isTargetShuffleEquivalent(Mask, Unpckhwd));
10640 return IsUnpackwdMask;
10641}
10642
10643static bool is128BitUnpackShuffleMask(ArrayRef<int> Mask) {
10644 // Create 128-bit vector type based on mask size.
10645 MVT EltVT = MVT::getIntegerVT(128 / Mask.size());
10646 MVT VT = MVT::getVectorVT(EltVT, Mask.size());
10647
10648 // We can't assume a canonical shuffle mask, so try the commuted version too.
10649 SmallVector<int, 4> CommutedMask(Mask.begin(), Mask.end());
10650 ShuffleVectorSDNode::commuteMask(CommutedMask);
10651
10652 // Match any of unary/binary or low/high.
10653 for (unsigned i = 0; i != 4; ++i) {
10654 SmallVector<int, 16> UnpackMask;
10655 createUnpackShuffleMask(VT, UnpackMask, (i >> 1) % 2, i % 2);
10656 if (isTargetShuffleEquivalent(Mask, UnpackMask) ||
10657 isTargetShuffleEquivalent(CommutedMask, UnpackMask))
10658 return true;
10659 }
10660 return false;
10661}
10662
10663/// Return true if a shuffle mask chooses elements identically in its top and
10664/// bottom halves. For example, any splat mask has the same top and bottom
10665/// halves. If an element is undefined in only one half of the mask, the halves
10666/// are not considered identical.
10667static bool hasIdenticalHalvesShuffleMask(ArrayRef<int> Mask) {
10668 assert(Mask.size() % 2 == 0 && "Expecting even number of elements in mask")((Mask.size() % 2 == 0 && "Expecting even number of elements in mask"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() % 2 == 0 && \"Expecting even number of elements in mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10668, __PRETTY_FUNCTION__));
10669 unsigned HalfSize = Mask.size() / 2;
10670 for (unsigned i = 0; i != HalfSize; ++i) {
10671 if (Mask[i] != Mask[i + HalfSize])
10672 return false;
10673 }
10674 return true;
10675}
10676
10677/// Get a 4-lane 8-bit shuffle immediate for a mask.
10678///
10679/// This helper function produces an 8-bit shuffle immediate corresponding to
10680/// the ubiquitous shuffle encoding scheme used in x86 instructions for
10681/// shuffling 4 lanes. It can be used with most of the PSHUF instructions for
10682/// example.
10683///
10684/// NB: We rely heavily on "undef" masks preserving the input lane.
10685static unsigned getV4X86ShuffleImm(ArrayRef<int> Mask) {
10686 assert(Mask.size() == 4 && "Only 4-lane shuffle masks")((Mask.size() == 4 && "Only 4-lane shuffle masks") ? static_cast
<void> (0) : __assert_fail ("Mask.size() == 4 && \"Only 4-lane shuffle masks\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10686, __PRETTY_FUNCTION__));
10687 assert(Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!")((Mask[0] >= -1 && Mask[0] < 4 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[0] >= -1 && Mask[0] < 4 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10687, __PRETTY_FUNCTION__));
10688 assert(Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!")((Mask[1] >= -1 && Mask[1] < 4 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[1] >= -1 && Mask[1] < 4 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10688, __PRETTY_FUNCTION__));
10689 assert(Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!")((Mask[2] >= -1 && Mask[2] < 4 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[2] >= -1 && Mask[2] < 4 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10689, __PRETTY_FUNCTION__));
10690 assert(Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!")((Mask[3] >= -1 && Mask[3] < 4 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[3] >= -1 && Mask[3] < 4 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10690, __PRETTY_FUNCTION__));
10691
10692 unsigned Imm = 0;
10693 Imm |= (Mask[0] < 0 ? 0 : Mask[0]) << 0;
10694 Imm |= (Mask[1] < 0 ? 1 : Mask[1]) << 2;
10695 Imm |= (Mask[2] < 0 ? 2 : Mask[2]) << 4;
10696 Imm |= (Mask[3] < 0 ? 3 : Mask[3]) << 6;
10697 return Imm;
10698}
10699
10700static SDValue getV4X86ShuffleImm8ForMask(ArrayRef<int> Mask, const SDLoc &DL,
10701 SelectionDAG &DAG) {
10702 return DAG.getTargetConstant(getV4X86ShuffleImm(Mask), DL, MVT::i8);
10703}
10704
10705// The Shuffle result is as follow:
10706// 0*a[0]0*a[1]...0*a[n] , n >=0 where a[] elements in a ascending order.
10707// Each Zeroable's element correspond to a particular Mask's element.
10708// As described in computeZeroableShuffleElements function.
10709//
10710// The function looks for a sub-mask that the nonzero elements are in
10711// increasing order. If such sub-mask exist. The function returns true.
10712static bool isNonZeroElementsInOrder(const APInt &Zeroable,
10713 ArrayRef<int> Mask, const EVT &VectorType,
10714 bool &IsZeroSideLeft) {
10715 int NextElement = -1;
10716 // Check if the Mask's nonzero elements are in increasing order.
10717 for (int i = 0, e = Mask.size(); i < e; i++) {
10718 // Checks if the mask's zeros elements are built from only zeros.
10719 assert(Mask[i] >= -1 && "Out of bound mask element!")((Mask[i] >= -1 && "Out of bound mask element!") ?
static_cast<void> (0) : __assert_fail ("Mask[i] >= -1 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10719, __PRETTY_FUNCTION__));
10720 if (Mask[i] < 0)
10721 return false;
10722 if (Zeroable[i])
10723 continue;
10724 // Find the lowest non zero element
10725 if (NextElement < 0) {
10726 NextElement = Mask[i] != 0 ? VectorType.getVectorNumElements() : 0;
10727 IsZeroSideLeft = NextElement != 0;
10728 }
10729 // Exit if the mask's non zero elements are not in increasing order.
10730 if (NextElement != Mask[i])
10731 return false;
10732 NextElement++;
10733 }
10734 return true;
10735}
10736
10737/// Try to lower a shuffle with a single PSHUFB of V1 or V2.
10738static SDValue lowerShuffleWithPSHUFB(const SDLoc &DL, MVT VT,
10739 ArrayRef<int> Mask, SDValue V1,
10740 SDValue V2, const APInt &Zeroable,
10741 const X86Subtarget &Subtarget,
10742 SelectionDAG &DAG) {
10743 int Size = Mask.size();
10744 int LaneSize = 128 / VT.getScalarSizeInBits();
10745 const int NumBytes = VT.getSizeInBits() / 8;
10746 const int NumEltBytes = VT.getScalarSizeInBits() / 8;
10747
10748 assert((Subtarget.hasSSSE3() && VT.is128BitVector()) ||(((Subtarget.hasSSSE3() && VT.is128BitVector()) || (Subtarget
.hasAVX2() && VT.is256BitVector()) || (Subtarget.hasBWI
() && VT.is512BitVector())) ? static_cast<void>
(0) : __assert_fail ("(Subtarget.hasSSSE3() && VT.is128BitVector()) || (Subtarget.hasAVX2() && VT.is256BitVector()) || (Subtarget.hasBWI() && VT.is512BitVector())"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10750, __PRETTY_FUNCTION__))
10749 (Subtarget.hasAVX2() && VT.is256BitVector()) ||(((Subtarget.hasSSSE3() && VT.is128BitVector()) || (Subtarget
.hasAVX2() && VT.is256BitVector()) || (Subtarget.hasBWI
() && VT.is512BitVector())) ? static_cast<void>
(0) : __assert_fail ("(Subtarget.hasSSSE3() && VT.is128BitVector()) || (Subtarget.hasAVX2() && VT.is256BitVector()) || (Subtarget.hasBWI() && VT.is512BitVector())"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10750, __PRETTY_FUNCTION__))
10750 (Subtarget.hasBWI() && VT.is512BitVector()))(((Subtarget.hasSSSE3() && VT.is128BitVector()) || (Subtarget
.hasAVX2() && VT.is256BitVector()) || (Subtarget.hasBWI
() && VT.is512BitVector())) ? static_cast<void>
(0) : __assert_fail ("(Subtarget.hasSSSE3() && VT.is128BitVector()) || (Subtarget.hasAVX2() && VT.is256BitVector()) || (Subtarget.hasBWI() && VT.is512BitVector())"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10750, __PRETTY_FUNCTION__));
10751
10752 SmallVector<SDValue, 64> PSHUFBMask(NumBytes);
10753 // Sign bit set in i8 mask means zero element.
10754 SDValue ZeroMask = DAG.getConstant(0x80, DL, MVT::i8);
10755
10756 SDValue V;
10757 for (int i = 0; i < NumBytes; ++i) {
10758 int M = Mask[i / NumEltBytes];
10759 if (M < 0) {
10760 PSHUFBMask[i] = DAG.getUNDEF(MVT::i8);
10761 continue;
10762 }
10763 if (Zeroable[i / NumEltBytes]) {
10764 PSHUFBMask[i] = ZeroMask;
10765 continue;
10766 }
10767
10768 // We can only use a single input of V1 or V2.
10769 SDValue SrcV = (M >= Size ? V2 : V1);
10770 if (V && V != SrcV)
10771 return SDValue();
10772 V = SrcV;
10773 M %= Size;
10774
10775 // PSHUFB can't cross lanes, ensure this doesn't happen.
10776 if ((M / LaneSize) != ((i / NumEltBytes) / LaneSize))
10777 return SDValue();
10778
10779 M = M % LaneSize;
10780 M = M * NumEltBytes + (i % NumEltBytes);
10781 PSHUFBMask[i] = DAG.getConstant(M, DL, MVT::i8);
10782 }
10783 assert(V && "Failed to find a source input")((V && "Failed to find a source input") ? static_cast
<void> (0) : __assert_fail ("V && \"Failed to find a source input\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10783, __PRETTY_FUNCTION__));
10784
10785 MVT I8VT = MVT::getVectorVT(MVT::i8, NumBytes);
10786 return DAG.getBitcast(
10787 VT, DAG.getNode(X86ISD::PSHUFB, DL, I8VT, DAG.getBitcast(I8VT, V),
10788 DAG.getBuildVector(I8VT, DL, PSHUFBMask)));
10789}
10790
10791static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
10792 const X86Subtarget &Subtarget, SelectionDAG &DAG,
10793 const SDLoc &dl);
10794
10795// X86 has dedicated shuffle that can be lowered to VEXPAND
10796static SDValue lowerShuffleToEXPAND(const SDLoc &DL, MVT VT,
10797 const APInt &Zeroable,
10798 ArrayRef<int> Mask, SDValue &V1,
10799 SDValue &V2, SelectionDAG &DAG,
10800 const X86Subtarget &Subtarget) {
10801 bool IsLeftZeroSide = true;
10802 if (!isNonZeroElementsInOrder(Zeroable, Mask, V1.getValueType(),
10803 IsLeftZeroSide))
10804 return SDValue();
10805 unsigned VEXPANDMask = (~Zeroable).getZExtValue();
10806 MVT IntegerType =
10807 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));
10808 SDValue MaskNode = DAG.getConstant(VEXPANDMask, DL, IntegerType);
10809 unsigned NumElts = VT.getVectorNumElements();
10810 assert((NumElts == 4 || NumElts == 8 || NumElts == 16) &&(((NumElts == 4 || NumElts == 8 || NumElts == 16) && "Unexpected number of vector elements"
) ? static_cast<void> (0) : __assert_fail ("(NumElts == 4 || NumElts == 8 || NumElts == 16) && \"Unexpected number of vector elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10811, __PRETTY_FUNCTION__))
10811 "Unexpected number of vector elements")(((NumElts == 4 || NumElts == 8 || NumElts == 16) && "Unexpected number of vector elements"
) ? static_cast<void> (0) : __assert_fail ("(NumElts == 4 || NumElts == 8 || NumElts == 16) && \"Unexpected number of vector elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10811, __PRETTY_FUNCTION__));
10812 SDValue VMask = getMaskNode(MaskNode, MVT::getVectorVT(MVT::i1, NumElts),
10813 Subtarget, DAG, DL);
10814 SDValue ZeroVector = getZeroVector(VT, Subtarget, DAG, DL);
10815 SDValue ExpandedVector = IsLeftZeroSide ? V2 : V1;
10816 return DAG.getNode(X86ISD::EXPAND, DL, VT, ExpandedVector, ZeroVector, VMask);
10817}
10818
10819static bool matchShuffleWithUNPCK(MVT VT, SDValue &V1, SDValue &V2,
10820 unsigned &UnpackOpcode, bool IsUnary,
10821 ArrayRef<int> TargetMask, const SDLoc &DL,
10822 SelectionDAG &DAG,
10823 const X86Subtarget &Subtarget) {
10824 int NumElts = VT.getVectorNumElements();
10825
10826 bool Undef1 = true, Undef2 = true, Zero1 = true, Zero2 = true;
10827 for (int i = 0; i != NumElts; i += 2) {
10828 int M1 = TargetMask[i + 0];
10829 int M2 = TargetMask[i + 1];
10830 Undef1 &= (SM_SentinelUndef == M1);
10831 Undef2 &= (SM_SentinelUndef == M2);
10832 Zero1 &= isUndefOrZero(M1);
10833 Zero2 &= isUndefOrZero(M2);
10834 }
10835 assert(!((Undef1 || Zero1) && (Undef2 || Zero2)) &&((!((Undef1 || Zero1) && (Undef2 || Zero2)) &&
"Zeroable shuffle detected") ? static_cast<void> (0) :
__assert_fail ("!((Undef1 || Zero1) && (Undef2 || Zero2)) && \"Zeroable shuffle detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10836, __PRETTY_FUNCTION__))
10836 "Zeroable shuffle detected")((!((Undef1 || Zero1) && (Undef2 || Zero2)) &&
"Zeroable shuffle detected") ? static_cast<void> (0) :
__assert_fail ("!((Undef1 || Zero1) && (Undef2 || Zero2)) && \"Zeroable shuffle detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 10836, __PRETTY_FUNCTION__));
10837
10838 // Attempt to match the target mask against the unpack lo/hi mask patterns.
10839 SmallVector<int, 64> Unpckl, Unpckh;
10840 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, IsUnary);
10841 if (isTargetShuffleEquivalent(TargetMask, Unpckl)) {
10842 UnpackOpcode = X86ISD::UNPCKL;
10843 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
10844 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
10845 return true;
10846 }
10847
10848 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, IsUnary);
10849 if (isTargetShuffleEquivalent(TargetMask, Unpckh)) {
10850 UnpackOpcode = X86ISD::UNPCKH;
10851 V2 = (Undef2 ? DAG.getUNDEF(VT) : (IsUnary ? V1 : V2));
10852 V1 = (Undef1 ? DAG.getUNDEF(VT) : V1);
10853 return true;
10854 }
10855
10856 // If an unary shuffle, attempt to match as an unpack lo/hi with zero.
10857 if (IsUnary && (Zero1 || Zero2)) {
10858 // Don't bother if we can blend instead.
10859 if ((Subtarget.hasSSE41() || VT == MVT::v2i64 || VT == MVT::v2f64) &&
10860 isSequentialOrUndefOrZeroInRange(TargetMask, 0, NumElts, 0))
10861 return false;
10862
10863 bool MatchLo = true, MatchHi = true;
10864 for (int i = 0; (i != NumElts) && (MatchLo || MatchHi); ++i) {
10865 int M = TargetMask[i];
10866
10867 // Ignore if the input is known to be zero or the index is undef.
10868 if ((((i & 1) == 0) && Zero1) || (((i & 1) == 1) && Zero2) ||
10869 (M == SM_SentinelUndef))
10870 continue;
10871
10872 MatchLo &= (M == Unpckl[i]);
10873 MatchHi &= (M == Unpckh[i]);
10874 }
10875
10876 if (MatchLo || MatchHi) {
10877 UnpackOpcode = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
10878 V2 = Zero2 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
10879 V1 = Zero1 ? getZeroVector(VT, Subtarget, DAG, DL) : V1;
10880 return true;
10881 }
10882 }
10883
10884 // If a binary shuffle, commute and try again.
10885 if (!IsUnary) {
10886 ShuffleVectorSDNode::commuteMask(Unpckl);
10887 if (isTargetShuffleEquivalent(TargetMask, Unpckl)) {
10888 UnpackOpcode = X86ISD::UNPCKL;
10889 std::swap(V1, V2);
10890 return true;
10891 }
10892
10893 ShuffleVectorSDNode::commuteMask(Unpckh);
10894 if (isTargetShuffleEquivalent(TargetMask, Unpckh)) {
10895 UnpackOpcode = X86ISD::UNPCKH;
10896 std::swap(V1, V2);
10897 return true;
10898 }
10899 }
10900
10901 return false;
10902}
10903
10904// X86 has dedicated unpack instructions that can handle specific blend
10905// operations: UNPCKH and UNPCKL.
10906static SDValue lowerShuffleWithUNPCK(const SDLoc &DL, MVT VT,
10907 ArrayRef<int> Mask, SDValue V1, SDValue V2,
10908 SelectionDAG &DAG) {
10909 SmallVector<int, 8> Unpckl;
10910 createUnpackShuffleMask(VT, Unpckl, /* Lo = */ true, /* Unary = */ false);
10911 if (isShuffleEquivalent(V1, V2, Mask, Unpckl))
10912 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V1, V2);
10913
10914 SmallVector<int, 8> Unpckh;
10915 createUnpackShuffleMask(VT, Unpckh, /* Lo = */ false, /* Unary = */ false);
10916 if (isShuffleEquivalent(V1, V2, Mask, Unpckh))
10917 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V1, V2);
10918
10919 // Commute and try again.
10920 ShuffleVectorSDNode::commuteMask(Unpckl);
10921 if (isShuffleEquivalent(V1, V2, Mask, Unpckl))
10922 return DAG.getNode(X86ISD::UNPCKL, DL, VT, V2, V1);
10923
10924 ShuffleVectorSDNode::commuteMask(Unpckh);
10925 if (isShuffleEquivalent(V1, V2, Mask, Unpckh))
10926 return DAG.getNode(X86ISD::UNPCKH, DL, VT, V2, V1);
10927
10928 return SDValue();
10929}
10930
10931static bool matchShuffleAsVPMOV(ArrayRef<int> Mask, bool SwappedOps,
10932 int Delta) {
10933 int Size = (int)Mask.size();
10934 int Split = Size / Delta;
10935 int TruncatedVectorStart = SwappedOps ? Size : 0;
10936
10937 // Match for mask starting with e.g.: <8, 10, 12, 14,... or <0, 2, 4, 6,...
10938 if (!isSequentialOrUndefInRange(Mask, 0, Split, TruncatedVectorStart, Delta))
10939 return false;
10940
10941 // The rest of the mask should not refer to the truncated vector's elements.
10942 if (isAnyInRange(Mask.slice(Split, Size - Split), TruncatedVectorStart,
10943 TruncatedVectorStart + Size))
10944 return false;
10945
10946 return true;
10947}
10948
10949// Try to lower trunc+vector_shuffle to a vpmovdb or a vpmovdw instruction.
10950//
10951// An example is the following:
10952//
10953// t0: ch = EntryToken
10954// t2: v4i64,ch = CopyFromReg t0, Register:v4i64 %0
10955// t25: v4i32 = truncate t2
10956// t41: v8i16 = bitcast t25
10957// t21: v8i16 = BUILD_VECTOR undef:i16, undef:i16, undef:i16, undef:i16,
10958// Constant:i16<0>, Constant:i16<0>, Constant:i16<0>, Constant:i16<0>
10959// t51: v8i16 = vector_shuffle<0,2,4,6,12,13,14,15> t41, t21
10960// t18: v2i64 = bitcast t51
10961//
10962// Without avx512vl, this is lowered to:
10963//
10964// vpmovqd %zmm0, %ymm0
10965// vpshufb {{.*#+}} xmm0 =
10966// xmm0[0,1,4,5,8,9,12,13],zero,zero,zero,zero,zero,zero,zero,zero
10967//
10968// But when avx512vl is available, one can just use a single vpmovdw
10969// instruction.
10970static SDValue lowerShuffleWithVPMOV(const SDLoc &DL, ArrayRef<int> Mask,
10971 MVT VT, SDValue V1, SDValue V2,
10972 SelectionDAG &DAG,
10973 const X86Subtarget &Subtarget) {
10974 if (VT != MVT::v16i8 && VT != MVT::v8i16)
10975 return SDValue();
10976
10977 if (Mask.size() != VT.getVectorNumElements())
10978 return SDValue();
10979
10980 bool SwappedOps = false;
10981
10982 if (!ISD::isBuildVectorAllZeros(V2.getNode())) {
10983 if (!ISD::isBuildVectorAllZeros(V1.getNode()))
10984 return SDValue();
10985
10986 std::swap(V1, V2);
10987 SwappedOps = true;
10988 }
10989
10990 // Look for:
10991 //
10992 // bitcast (truncate <8 x i32> %vec to <8 x i16>) to <16 x i8>
10993 // bitcast (truncate <4 x i64> %vec to <4 x i32>) to <8 x i16>
10994 //
10995 // and similar ones.
10996 if (V1.getOpcode() != ISD::BITCAST)
10997 return SDValue();
10998 if (V1.getOperand(0).getOpcode() != ISD::TRUNCATE)
10999 return SDValue();
11000
11001 SDValue Src = V1.getOperand(0).getOperand(0);
11002 MVT SrcVT = Src.getSimpleValueType();
11003
11004 // The vptrunc** instructions truncating 128 bit and 256 bit vectors
11005 // are only available with avx512vl.
11006 if (!SrcVT.is512BitVector() && !Subtarget.hasVLX())
11007 return SDValue();
11008
11009 // Down Convert Word to Byte is only available with avx512bw. The case with
11010 // 256-bit output doesn't contain a shuffle and is therefore not handled here.
11011 if (SrcVT.getVectorElementType() == MVT::i16 && VT == MVT::v16i8 &&
11012 !Subtarget.hasBWI())
11013 return SDValue();
11014
11015 // The first half/quarter of the mask should refer to every second/fourth
11016 // element of the vector truncated and bitcasted.
11017 if (!matchShuffleAsVPMOV(Mask, SwappedOps, 2) &&
11018 !matchShuffleAsVPMOV(Mask, SwappedOps, 4))
11019 return SDValue();
11020
11021 return DAG.getNode(X86ISD::VTRUNC, DL, VT, Src);
11022}
11023
11024// X86 has dedicated pack instructions that can handle specific truncation
11025// operations: PACKSS and PACKUS.
11026static bool matchShuffleWithPACK(MVT VT, MVT &SrcVT, SDValue &V1, SDValue &V2,
11027 unsigned &PackOpcode, ArrayRef<int> TargetMask,
11028 SelectionDAG &DAG,
11029 const X86Subtarget &Subtarget) {
11030 unsigned NumElts = VT.getVectorNumElements();
11031 unsigned BitSize = VT.getScalarSizeInBits();
11032 MVT PackSVT = MVT::getIntegerVT(BitSize * 2);
11033 MVT PackVT = MVT::getVectorVT(PackSVT, NumElts / 2);
11034
11035 auto MatchPACK = [&](SDValue N1, SDValue N2) {
11036 SDValue VV1 = DAG.getBitcast(PackVT, N1);
11037 SDValue VV2 = DAG.getBitcast(PackVT, N2);
11038 if (Subtarget.hasSSE41() || PackSVT == MVT::i16) {
11039 APInt ZeroMask = APInt::getHighBitsSet(BitSize * 2, BitSize);
11040 if ((N1.isUndef() || DAG.MaskedValueIsZero(VV1, ZeroMask)) &&
11041 (N2.isUndef() || DAG.MaskedValueIsZero(VV2, ZeroMask))) {
11042 V1 = VV1;
11043 V2 = VV2;
11044 SrcVT = PackVT;
11045 PackOpcode = X86ISD::PACKUS;
11046 return true;
11047 }
11048 }
11049 if ((N1.isUndef() || DAG.ComputeNumSignBits(VV1) > BitSize) &&
11050 (N2.isUndef() || DAG.ComputeNumSignBits(VV2) > BitSize)) {
11051 V1 = VV1;
11052 V2 = VV2;
11053 SrcVT = PackVT;
11054 PackOpcode = X86ISD::PACKSS;
11055 return true;
11056 }
11057 return false;
11058 };
11059
11060 // Try binary shuffle.
11061 SmallVector<int, 32> BinaryMask;
11062 createPackShuffleMask(VT, BinaryMask, false);
11063 if (isTargetShuffleEquivalent(TargetMask, BinaryMask, V1, V2))
11064 if (MatchPACK(V1, V2))
11065 return true;
11066
11067 // Try unary shuffle.
11068 SmallVector<int, 32> UnaryMask;
11069 createPackShuffleMask(VT, UnaryMask, true);
11070 if (isTargetShuffleEquivalent(TargetMask, UnaryMask, V1))
11071 if (MatchPACK(V1, V1))
11072 return true;
11073
11074 return false;
11075}
11076
11077static SDValue lowerShuffleWithPACK(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
11078 SDValue V1, SDValue V2, SelectionDAG &DAG,
11079 const X86Subtarget &Subtarget) {
11080 MVT PackVT;
11081 unsigned PackOpcode;
11082 if (matchShuffleWithPACK(VT, PackVT, V1, V2, PackOpcode, Mask, DAG,
11083 Subtarget))
11084 return DAG.getNode(PackOpcode, DL, VT, DAG.getBitcast(PackVT, V1),
11085 DAG.getBitcast(PackVT, V2));
11086
11087 return SDValue();
11088}
11089
11090/// Try to emit a bitmask instruction for a shuffle.
11091///
11092/// This handles cases where we can model a blend exactly as a bitmask due to
11093/// one of the inputs being zeroable.
11094static SDValue lowerShuffleAsBitMask(const SDLoc &DL, MVT VT, SDValue V1,
11095 SDValue V2, ArrayRef<int> Mask,
11096 const APInt &Zeroable,
11097 const X86Subtarget &Subtarget,
11098 SelectionDAG &DAG) {
11099 MVT MaskVT = VT;
11100 MVT EltVT = VT.getVectorElementType();
11101 SDValue Zero, AllOnes;
11102 // Use f64 if i64 isn't legal.
11103 if (EltVT == MVT::i64 && !Subtarget.is64Bit()) {
11104 EltVT = MVT::f64;
11105 MaskVT = MVT::getVectorVT(EltVT, Mask.size());
11106 }
11107
11108 MVT LogicVT = VT;
11109 if (EltVT == MVT::f32 || EltVT == MVT::f64) {
11110 Zero = DAG.getConstantFP(0.0, DL, EltVT);
11111 AllOnes = DAG.getConstantFP(
11112 APFloat::getAllOnesValue(EltVT.getSizeInBits(), true), DL, EltVT);
11113 LogicVT =
11114 MVT::getVectorVT(EltVT == MVT::f64 ? MVT::i64 : MVT::i32, Mask.size());
11115 } else {
11116 Zero = DAG.getConstant(0, DL, EltVT);
11117 AllOnes = DAG.getAllOnesConstant(DL, EltVT);
11118 }
11119
11120 SmallVector<SDValue, 16> VMaskOps(Mask.size(), Zero);
11121 SDValue V;
11122 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
11123 if (Zeroable[i])
11124 continue;
11125 if (Mask[i] % Size != i)
11126 return SDValue(); // Not a blend.
11127 if (!V)
11128 V = Mask[i] < Size ? V1 : V2;
11129 else if (V != (Mask[i] < Size ? V1 : V2))
11130 return SDValue(); // Can only let one input through the mask.
11131
11132 VMaskOps[i] = AllOnes;
11133 }
11134 if (!V)
11135 return SDValue(); // No non-zeroable elements!
11136
11137 SDValue VMask = DAG.getBuildVector(MaskVT, DL, VMaskOps);
11138 VMask = DAG.getBitcast(LogicVT, VMask);
11139 V = DAG.getBitcast(LogicVT, V);
11140 SDValue And = DAG.getNode(ISD::AND, DL, LogicVT, V, VMask);
11141 return DAG.getBitcast(VT, And);
11142}
11143
11144/// Try to emit a blend instruction for a shuffle using bit math.
11145///
11146/// This is used as a fallback approach when first class blend instructions are
11147/// unavailable. Currently it is only suitable for integer vectors, but could
11148/// be generalized for floating point vectors if desirable.
11149static SDValue lowerShuffleAsBitBlend(const SDLoc &DL, MVT VT, SDValue V1,
11150 SDValue V2, ArrayRef<int> Mask,
11151 SelectionDAG &DAG) {
11152 assert(VT.isInteger() && "Only supports integer vector types!")((VT.isInteger() && "Only supports integer vector types!"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"Only supports integer vector types!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11152, __PRETTY_FUNCTION__));
11153 MVT EltVT = VT.getVectorElementType();
11154 SDValue Zero = DAG.getConstant(0, DL, EltVT);
11155 SDValue AllOnes = DAG.getAllOnesConstant(DL, EltVT);
11156 SmallVector<SDValue, 16> MaskOps;
11157 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
11158 if (Mask[i] >= 0 && Mask[i] != i && Mask[i] != i + Size)
11159 return SDValue(); // Shuffled input!
11160 MaskOps.push_back(Mask[i] < Size ? AllOnes : Zero);
11161 }
11162
11163 SDValue V1Mask = DAG.getBuildVector(VT, DL, MaskOps);
11164 V1 = DAG.getNode(ISD::AND, DL, VT, V1, V1Mask);
11165 V2 = DAG.getNode(X86ISD::ANDNP, DL, VT, V1Mask, V2);
11166 return DAG.getNode(ISD::OR, DL, VT, V1, V2);
11167}
11168
11169static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
11170 SDValue PreservedSrc,
11171 const X86Subtarget &Subtarget,
11172 SelectionDAG &DAG);
11173
11174static bool matchShuffleAsBlend(SDValue V1, SDValue V2,
11175 MutableArrayRef<int> Mask,
11176 const APInt &Zeroable, bool &ForceV1Zero,
11177 bool &ForceV2Zero, uint64_t &BlendMask) {
11178 bool V1IsZeroOrUndef =
11179 V1.isUndef() || ISD::isBuildVectorAllZeros(V1.getNode());
11180 bool V2IsZeroOrUndef =
11181 V2.isUndef() || ISD::isBuildVectorAllZeros(V2.getNode());
11182
11183 BlendMask = 0;
11184 ForceV1Zero = false, ForceV2Zero = false;
11185 assert(Mask.size() <= 64 && "Shuffle mask too big for blend mask")((Mask.size() <= 64 && "Shuffle mask too big for blend mask"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() <= 64 && \"Shuffle mask too big for blend mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11185, __PRETTY_FUNCTION__));
11186
11187 // Attempt to generate the binary blend mask. If an input is zero then
11188 // we can use any lane.
11189 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
11190 int M = Mask[i];
11191 if (M == SM_SentinelUndef)
11192 continue;
11193 if (M == i)
11194 continue;
11195 if (M == i + Size) {
11196 BlendMask |= 1ull << i;
11197 continue;
11198 }
11199 if (Zeroable[i]) {
11200 if (V1IsZeroOrUndef) {
11201 ForceV1Zero = true;
11202 Mask[i] = i;
11203 continue;
11204 }
11205 if (V2IsZeroOrUndef) {
11206 ForceV2Zero = true;
11207 BlendMask |= 1ull << i;
11208 Mask[i] = i + Size;
11209 continue;
11210 }
11211 }
11212 return false;
11213 }
11214 return true;
11215}
11216
11217static uint64_t scaleVectorShuffleBlendMask(uint64_t BlendMask, int Size,
11218 int Scale) {
11219 uint64_t ScaledMask = 0;
11220 for (int i = 0; i != Size; ++i)
11221 if (BlendMask & (1ull << i))
11222 ScaledMask |= ((1ull << Scale) - 1) << (i * Scale);
11223 return ScaledMask;
11224}
11225
11226/// Try to emit a blend instruction for a shuffle.
11227///
11228/// This doesn't do any checks for the availability of instructions for blending
11229/// these values. It relies on the availability of the X86ISD::BLENDI pattern to
11230/// be matched in the backend with the type given. What it does check for is
11231/// that the shuffle mask is a blend, or convertible into a blend with zero.
11232static SDValue lowerShuffleAsBlend(const SDLoc &DL, MVT VT, SDValue V1,
11233 SDValue V2, ArrayRef<int> Original,
11234 const APInt &Zeroable,
11235 const X86Subtarget &Subtarget,
11236 SelectionDAG &DAG) {
11237 uint64_t BlendMask = 0;
11238 bool ForceV1Zero = false, ForceV2Zero = false;
11239 SmallVector<int, 64> Mask(Original.begin(), Original.end());
11240 if (!matchShuffleAsBlend(V1, V2, Mask, Zeroable, ForceV1Zero, ForceV2Zero,
11241 BlendMask))
11242 return SDValue();
11243
11244 // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
11245 if (ForceV1Zero)
11246 V1 = getZeroVector(VT, Subtarget, DAG, DL);
11247 if (ForceV2Zero)
11248 V2 = getZeroVector(VT, Subtarget, DAG, DL);
11249
11250 switch (VT.SimpleTy) {
11251 case MVT::v4i64:
11252 case MVT::v8i32:
11253 assert(Subtarget.hasAVX2() && "256-bit integer blends require AVX2!")((Subtarget.hasAVX2() && "256-bit integer blends require AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"256-bit integer blends require AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11253, __PRETTY_FUNCTION__));
11254 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11255 case MVT::v4f64:
11256 case MVT::v8f32:
11257 assert(Subtarget.hasAVX() && "256-bit float blends require AVX!")((Subtarget.hasAVX() && "256-bit float blends require AVX!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX() && \"256-bit float blends require AVX!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11257, __PRETTY_FUNCTION__));
11258 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11259 case MVT::v2f64:
11260 case MVT::v2i64:
11261 case MVT::v4f32:
11262 case MVT::v4i32:
11263 case MVT::v8i16:
11264 assert(Subtarget.hasSSE41() && "128-bit blends require SSE41!")((Subtarget.hasSSE41() && "128-bit blends require SSE41!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE41() && \"128-bit blends require SSE41!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11264, __PRETTY_FUNCTION__));
11265 return DAG.getNode(X86ISD::BLENDI, DL, VT, V1, V2,
11266 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
11267 case MVT::v16i16: {
11268 assert(Subtarget.hasAVX2() && "v16i16 blends require AVX2!")((Subtarget.hasAVX2() && "v16i16 blends require AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"v16i16 blends require AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11268, __PRETTY_FUNCTION__));
11269 SmallVector<int, 8> RepeatedMask;
11270 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
11271 // We can lower these with PBLENDW which is mirrored across 128-bit lanes.
11272 assert(RepeatedMask.size() == 8 && "Repeated mask size doesn't match!")((RepeatedMask.size() == 8 && "Repeated mask size doesn't match!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 8 && \"Repeated mask size doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11272, __PRETTY_FUNCTION__));
11273 BlendMask = 0;
11274 for (int i = 0; i < 8; ++i)
11275 if (RepeatedMask[i] >= 8)
11276 BlendMask |= 1ull << i;
11277 return DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11278 DAG.getTargetConstant(BlendMask, DL, MVT::i8));
11279 }
11280 // Use PBLENDW for lower/upper lanes and then blend lanes.
11281 // TODO - we should allow 2 PBLENDW here and leave shuffle combine to
11282 // merge to VSELECT where useful.
11283 uint64_t LoMask = BlendMask & 0xFF;
11284 uint64_t HiMask = (BlendMask >> 8) & 0xFF;
11285 if (LoMask == 0 || LoMask == 255 || HiMask == 0 || HiMask == 255) {
11286 SDValue Lo = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11287 DAG.getTargetConstant(LoMask, DL, MVT::i8));
11288 SDValue Hi = DAG.getNode(X86ISD::BLENDI, DL, MVT::v16i16, V1, V2,
11289 DAG.getTargetConstant(HiMask, DL, MVT::i8));
11290 return DAG.getVectorShuffle(
11291 MVT::v16i16, DL, Lo, Hi,
11292 {0, 1, 2, 3, 4, 5, 6, 7, 24, 25, 26, 27, 28, 29, 30, 31});
11293 }
11294 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11295 }
11296 case MVT::v32i8:
11297 assert(Subtarget.hasAVX2() && "256-bit byte-blends require AVX2!")((Subtarget.hasAVX2() && "256-bit byte-blends require AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"256-bit byte-blends require AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11297, __PRETTY_FUNCTION__));
11298 LLVM_FALLTHROUGH[[gnu::fallthrough]];
11299 case MVT::v16i8: {
11300 assert(Subtarget.hasSSE41() && "128-bit byte-blends require SSE41!")((Subtarget.hasSSE41() && "128-bit byte-blends require SSE41!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE41() && \"128-bit byte-blends require SSE41!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11300, __PRETTY_FUNCTION__));
11301
11302 // Attempt to lower to a bitmask if we can. VPAND is faster than VPBLENDVB.
11303 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11304 Subtarget, DAG))
11305 return Masked;
11306
11307 if (Subtarget.hasBWI() && Subtarget.hasVLX()) {
11308 MVT IntegerType =
11309 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));
11310 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11311 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
11312 }
11313
11314 // Scale the blend by the number of bytes per element.
11315 int Scale = VT.getScalarSizeInBits() / 8;
11316
11317 // This form of blend is always done on bytes. Compute the byte vector
11318 // type.
11319 MVT BlendVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
11320
11321 // x86 allows load folding with blendvb from the 2nd source operand. But
11322 // we are still using LLVM select here (see comment below), so that's V1.
11323 // If V2 can be load-folded and V1 cannot be load-folded, then commute to
11324 // allow that load-folding possibility.
11325 if (!ISD::isNormalLoad(V1.getNode()) && ISD::isNormalLoad(V2.getNode())) {
11326 ShuffleVectorSDNode::commuteMask(Mask);
11327 std::swap(V1, V2);
11328 }
11329
11330 // Compute the VSELECT mask. Note that VSELECT is really confusing in the
11331 // mix of LLVM's code generator and the x86 backend. We tell the code
11332 // generator that boolean values in the elements of an x86 vector register
11333 // are -1 for true and 0 for false. We then use the LLVM semantics of 'true'
11334 // mapping a select to operand #1, and 'false' mapping to operand #2. The
11335 // reality in x86 is that vector masks (pre-AVX-512) use only the high bit
11336 // of the element (the remaining are ignored) and 0 in that high bit would
11337 // mean operand #1 while 1 in the high bit would mean operand #2. So while
11338 // the LLVM model for boolean values in vector elements gets the relevant
11339 // bit set, it is set backwards and over constrained relative to x86's
11340 // actual model.
11341 SmallVector<SDValue, 32> VSELECTMask;
11342 for (int i = 0, Size = Mask.size(); i < Size; ++i)
11343 for (int j = 0; j < Scale; ++j)
11344 VSELECTMask.push_back(
11345 Mask[i] < 0 ? DAG.getUNDEF(MVT::i8)
11346 : DAG.getConstant(Mask[i] < Size ? -1 : 0, DL,
11347 MVT::i8));
11348
11349 V1 = DAG.getBitcast(BlendVT, V1);
11350 V2 = DAG.getBitcast(BlendVT, V2);
11351 return DAG.getBitcast(
11352 VT,
11353 DAG.getSelect(DL, BlendVT, DAG.getBuildVector(BlendVT, DL, VSELECTMask),
11354 V1, V2));
11355 }
11356 case MVT::v16f32:
11357 case MVT::v8f64:
11358 case MVT::v8i64:
11359 case MVT::v16i32:
11360 case MVT::v32i16:
11361 case MVT::v64i8: {
11362 // Attempt to lower to a bitmask if we can. Only if not optimizing for size.
11363 bool OptForSize = DAG.shouldOptForSize();
11364 if (!OptForSize) {
11365 if (SDValue Masked = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
11366 Subtarget, DAG))
11367 return Masked;
11368 }
11369
11370 // Otherwise load an immediate into a GPR, cast to k-register, and use a
11371 // masked move.
11372 MVT IntegerType =
11373 MVT::getIntegerVT(std::max((int)VT.getVectorNumElements(), 8));
11374 SDValue MaskNode = DAG.getConstant(BlendMask, DL, IntegerType);
11375 return getVectorMaskingNode(V2, MaskNode, V1, Subtarget, DAG);
11376 }
11377 default:
11378 llvm_unreachable("Not a supported integer vector type!")::llvm::llvm_unreachable_internal("Not a supported integer vector type!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11378);
11379 }
11380}
11381
11382/// Try to lower as a blend of elements from two inputs followed by
11383/// a single-input permutation.
11384///
11385/// This matches the pattern where we can blend elements from two inputs and
11386/// then reduce the shuffle to a single-input permutation.
11387static SDValue lowerShuffleAsBlendAndPermute(const SDLoc &DL, MVT VT,
11388 SDValue V1, SDValue V2,
11389 ArrayRef<int> Mask,
11390 SelectionDAG &DAG,
11391 bool ImmBlends = false) {
11392 // We build up the blend mask while checking whether a blend is a viable way
11393 // to reduce the shuffle.
11394 SmallVector<int, 32> BlendMask(Mask.size(), -1);
11395 SmallVector<int, 32> PermuteMask(Mask.size(), -1);
11396
11397 for (int i = 0, Size = Mask.size(); i < Size; ++i) {
11398 if (Mask[i] < 0)
11399 continue;
11400
11401 assert(Mask[i] < Size * 2 && "Shuffle input is out of bounds.")((Mask[i] < Size * 2 && "Shuffle input is out of bounds."
) ? static_cast<void> (0) : __assert_fail ("Mask[i] < Size * 2 && \"Shuffle input is out of bounds.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11401, __PRETTY_FUNCTION__));
11402
11403 if (BlendMask[Mask[i] % Size] < 0)
11404 BlendMask[Mask[i] % Size] = Mask[i];
11405 else if (BlendMask[Mask[i] % Size] != Mask[i])
11406 return SDValue(); // Can't blend in the needed input!
11407
11408 PermuteMask[i] = Mask[i] % Size;
11409 }
11410
11411 // If only immediate blends, then bail if the blend mask can't be widened to
11412 // i16.
11413 unsigned EltSize = VT.getScalarSizeInBits();
11414 if (ImmBlends && EltSize == 8 && !canWidenShuffleElements(BlendMask))
11415 return SDValue();
11416
11417 SDValue V = DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
11418 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), PermuteMask);
11419}
11420
11421/// Try to lower as an unpack of elements from two inputs followed by
11422/// a single-input permutation.
11423///
11424/// This matches the pattern where we can unpack elements from two inputs and
11425/// then reduce the shuffle to a single-input (wider) permutation.
11426static SDValue lowerShuffleAsUNPCKAndPermute(const SDLoc &DL, MVT VT,
11427 SDValue V1, SDValue V2,
11428 ArrayRef<int> Mask,
11429 SelectionDAG &DAG) {
11430 int NumElts = Mask.size();
11431 int NumLanes = VT.getSizeInBits() / 128;
11432 int NumLaneElts = NumElts / NumLanes;
11433 int NumHalfLaneElts = NumLaneElts / 2;
11434
11435 bool MatchLo = true, MatchHi = true;
11436 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
11437
11438 // Determine UNPCKL/UNPCKH type and operand order.
11439 for (int Lane = 0; Lane != NumElts; Lane += NumLaneElts) {
11440 for (int Elt = 0; Elt != NumLaneElts; ++Elt) {
11441 int M = Mask[Lane + Elt];
11442 if (M < 0)
11443 continue;
11444
11445 SDValue &Op = Ops[Elt & 1];
11446 if (M < NumElts && (Op.isUndef() || Op == V1))
11447 Op = V1;
11448 else if (NumElts <= M && (Op.isUndef() || Op == V2))
11449 Op = V2;
11450 else
11451 return SDValue();
11452
11453 int Lo = Lane, Mid = Lane + NumHalfLaneElts, Hi = Lane + NumLaneElts;
11454 MatchLo &= isUndefOrInRange(M, Lo, Mid) ||
11455 isUndefOrInRange(M, NumElts + Lo, NumElts + Mid);
11456 MatchHi &= isUndefOrInRange(M, Mid, Hi) ||
11457 isUndefOrInRange(M, NumElts + Mid, NumElts + Hi);
11458 if (!MatchLo && !MatchHi)
11459 return SDValue();
11460 }
11461 }
11462 assert((MatchLo ^ MatchHi) && "Failed to match UNPCKLO/UNPCKHI")(((MatchLo ^ MatchHi) && "Failed to match UNPCKLO/UNPCKHI"
) ? static_cast<void> (0) : __assert_fail ("(MatchLo ^ MatchHi) && \"Failed to match UNPCKLO/UNPCKHI\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11462, __PRETTY_FUNCTION__));
11463
11464 // Now check that each pair of elts come from the same unpack pair
11465 // and set the permute mask based on each pair.
11466 // TODO - Investigate cases where we permute individual elements.
11467 SmallVector<int, 32> PermuteMask(NumElts, -1);
11468 for (int Lane = 0; Lane != NumElts; Lane += NumLaneElts) {
11469 for (int Elt = 0; Elt != NumLaneElts; Elt += 2) {
11470 int M0 = Mask[Lane + Elt + 0];
11471 int M1 = Mask[Lane + Elt + 1];
11472 if (0 <= M0 && 0 <= M1 &&
11473 (M0 % NumHalfLaneElts) != (M1 % NumHalfLaneElts))
11474 return SDValue();
11475 if (0 <= M0)
11476 PermuteMask[Lane + Elt + 0] = Lane + (2 * (M0 % NumHalfLaneElts));
11477 if (0 <= M1)
11478 PermuteMask[Lane + Elt + 1] = Lane + (2 * (M1 % NumHalfLaneElts)) + 1;
11479 }
11480 }
11481
11482 unsigned UnpckOp = MatchLo ? X86ISD::UNPCKL : X86ISD::UNPCKH;
11483 SDValue Unpck = DAG.getNode(UnpckOp, DL, VT, Ops);
11484 return DAG.getVectorShuffle(VT, DL, Unpck, DAG.getUNDEF(VT), PermuteMask);
11485}
11486
11487/// Helper to form a PALIGNR-based rotate+permute, merging 2 inputs and then
11488/// permuting the elements of the result in place.
11489static SDValue lowerShuffleAsByteRotateAndPermute(
11490 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11491 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11492 if ((VT.is128BitVector() && !Subtarget.hasSSSE3()) ||
11493 (VT.is256BitVector() && !Subtarget.hasAVX2()) ||
11494 (VT.is512BitVector() && !Subtarget.hasBWI()))
11495 return SDValue();
11496
11497 // We don't currently support lane crossing permutes.
11498 if (is128BitLaneCrossingShuffleMask(VT, Mask))
11499 return SDValue();
11500
11501 int Scale = VT.getScalarSizeInBits() / 8;
11502 int NumLanes = VT.getSizeInBits() / 128;
11503 int NumElts = VT.getVectorNumElements();
11504 int NumEltsPerLane = NumElts / NumLanes;
11505
11506 // Determine range of mask elts.
11507 bool Blend1 = true;
11508 bool Blend2 = true;
11509 std::pair<int, int> Range1 = std::make_pair(INT_MAX2147483647, INT_MIN(-2147483647 -1));
11510 std::pair<int, int> Range2 = std::make_pair(INT_MAX2147483647, INT_MIN(-2147483647 -1));
11511 for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
11512 for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
11513 int M = Mask[Lane + Elt];
11514 if (M < 0)
11515 continue;
11516 if (M < NumElts) {
11517 Blend1 &= (M == (Lane + Elt));
11518 assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask")((Lane <= M && M < (Lane + NumEltsPerLane) &&
"Out of range mask") ? static_cast<void> (0) : __assert_fail
("Lane <= M && M < (Lane + NumEltsPerLane) && \"Out of range mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11518, __PRETTY_FUNCTION__));
11519 M = M % NumEltsPerLane;
11520 Range1.first = std::min(Range1.first, M);
11521 Range1.second = std::max(Range1.second, M);
11522 } else {
11523 M -= NumElts;
11524 Blend2 &= (M == (Lane + Elt));
11525 assert(Lane <= M && M < (Lane + NumEltsPerLane) && "Out of range mask")((Lane <= M && M < (Lane + NumEltsPerLane) &&
"Out of range mask") ? static_cast<void> (0) : __assert_fail
("Lane <= M && M < (Lane + NumEltsPerLane) && \"Out of range mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11525, __PRETTY_FUNCTION__));
11526 M = M % NumEltsPerLane;
11527 Range2.first = std::min(Range2.first, M);
11528 Range2.second = std::max(Range2.second, M);
11529 }
11530 }
11531 }
11532
11533 // Bail if we don't need both elements.
11534 // TODO - it might be worth doing this for unary shuffles if the permute
11535 // can be widened.
11536 if (!(0 <= Range1.first && Range1.second < NumEltsPerLane) ||
11537 !(0 <= Range2.first && Range2.second < NumEltsPerLane))
11538 return SDValue();
11539
11540 if (VT.getSizeInBits() > 128 && (Blend1 || Blend2))
11541 return SDValue();
11542
11543 // Rotate the 2 ops so we can access both ranges, then permute the result.
11544 auto RotateAndPermute = [&](SDValue Lo, SDValue Hi, int RotAmt, int Ofs) {
11545 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
11546 SDValue Rotate = DAG.getBitcast(
11547 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, DAG.getBitcast(ByteVT, Hi),
11548 DAG.getBitcast(ByteVT, Lo),
11549 DAG.getTargetConstant(Scale * RotAmt, DL, MVT::i8)));
11550 SmallVector<int, 64> PermMask(NumElts, SM_SentinelUndef);
11551 for (int Lane = 0; Lane != NumElts; Lane += NumEltsPerLane) {
11552 for (int Elt = 0; Elt != NumEltsPerLane; ++Elt) {
11553 int M = Mask[Lane + Elt];
11554 if (M < 0)
11555 continue;
11556 if (M < NumElts)
11557 PermMask[Lane + Elt] = Lane + ((M + Ofs - RotAmt) % NumEltsPerLane);
11558 else
11559 PermMask[Lane + Elt] = Lane + ((M - Ofs - RotAmt) % NumEltsPerLane);
11560 }
11561 }
11562 return DAG.getVectorShuffle(VT, DL, Rotate, DAG.getUNDEF(VT), PermMask);
11563 };
11564
11565 // Check if the ranges are small enough to rotate from either direction.
11566 if (Range2.second < Range1.first)
11567 return RotateAndPermute(V1, V2, Range1.first, 0);
11568 if (Range1.second < Range2.first)
11569 return RotateAndPermute(V2, V1, Range2.first, NumElts);
11570 return SDValue();
11571}
11572
11573/// Generic routine to decompose a shuffle and blend into independent
11574/// blends and permutes.
11575///
11576/// This matches the extremely common pattern for handling combined
11577/// shuffle+blend operations on newer X86 ISAs where we have very fast blend
11578/// operations. It will try to pick the best arrangement of shuffles and
11579/// blends.
11580static SDValue lowerShuffleAsDecomposedShuffleBlend(
11581 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
11582 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
11583 // Shuffle the input elements into the desired positions in V1 and V2 and
11584 // blend them together.
11585 SmallVector<int, 32> V1Mask(Mask.size(), -1);
11586 SmallVector<int, 32> V2Mask(Mask.size(), -1);
11587 SmallVector<int, 32> BlendMask(Mask.size(), -1);
11588 for (int i = 0, Size = Mask.size(); i < Size; ++i)
11589 if (Mask[i] >= 0 && Mask[i] < Size) {
11590 V1Mask[i] = Mask[i];
11591 BlendMask[i] = i;
11592 } else if (Mask[i] >= Size) {
11593 V2Mask[i] = Mask[i] - Size;
11594 BlendMask[i] = i + Size;
11595 }
11596
11597 // Try to lower with the simpler initial blend/unpack/rotate strategies unless
11598 // one of the input shuffles would be a no-op. We prefer to shuffle inputs as
11599 // the shuffle may be able to fold with a load or other benefit. However, when
11600 // we'll have to do 2x as many shuffles in order to achieve this, a 2-input
11601 // pre-shuffle first is a better strategy.
11602 if (!isNoopShuffleMask(V1Mask) && !isNoopShuffleMask(V2Mask)) {
11603 // Only prefer immediate blends to unpack/rotate.
11604 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11605 DAG, true))
11606 return BlendPerm;
11607 if (SDValue UnpackPerm = lowerShuffleAsUNPCKAndPermute(DL, VT, V1, V2, Mask,
11608 DAG))
11609 return UnpackPerm;
11610 if (SDValue RotatePerm = lowerShuffleAsByteRotateAndPermute(
11611 DL, VT, V1, V2, Mask, Subtarget, DAG))
11612 return RotatePerm;
11613 // Unpack/rotate failed - try again with variable blends.
11614 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, VT, V1, V2, Mask,
11615 DAG))
11616 return BlendPerm;
11617 }
11618
11619 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
11620 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
11621 return DAG.getVectorShuffle(VT, DL, V1, V2, BlendMask);
11622}
11623
11624/// Try to lower a vector shuffle as a rotation.
11625///
11626/// This is used for support PALIGNR for SSSE3 or VALIGND/Q for AVX512.
11627static int matchShuffleAsRotate(SDValue &V1, SDValue &V2, ArrayRef<int> Mask) {
11628 int NumElts = Mask.size();
11629
11630 // We need to detect various ways of spelling a rotation:
11631 // [11, 12, 13, 14, 15, 0, 1, 2]
11632 // [-1, 12, 13, 14, -1, -1, 1, -1]
11633 // [-1, -1, -1, -1, -1, -1, 1, 2]
11634 // [ 3, 4, 5, 6, 7, 8, 9, 10]
11635 // [-1, 4, 5, 6, -1, -1, 9, -1]
11636 // [-1, 4, 5, 6, -1, -1, -1, -1]
11637 int Rotation = 0;
11638 SDValue Lo, Hi;
11639 for (int i = 0; i < NumElts; ++i) {
11640 int M = Mask[i];
11641 assert((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) &&(((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts
))) && "Unexpected mask index.") ? static_cast<void
> (0) : __assert_fail ("(M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) && \"Unexpected mask index.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11642, __PRETTY_FUNCTION__))
11642 "Unexpected mask index.")(((M == SM_SentinelUndef || (0 <= M && M < (2*NumElts
))) && "Unexpected mask index.") ? static_cast<void
> (0) : __assert_fail ("(M == SM_SentinelUndef || (0 <= M && M < (2*NumElts))) && \"Unexpected mask index.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11642, __PRETTY_FUNCTION__));
11643 if (M < 0)
11644 continue;
11645
11646 // Determine where a rotated vector would have started.
11647 int StartIdx = i - (M % NumElts);
11648 if (StartIdx == 0)
11649 // The identity rotation isn't interesting, stop.
11650 return -1;
11651
11652 // If we found the tail of a vector the rotation must be the missing
11653 // front. If we found the head of a vector, it must be how much of the
11654 // head.
11655 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
11656
11657 if (Rotation == 0)
11658 Rotation = CandidateRotation;
11659 else if (Rotation != CandidateRotation)
11660 // The rotations don't match, so we can't match this mask.
11661 return -1;
11662
11663 // Compute which value this mask is pointing at.
11664 SDValue MaskV = M < NumElts ? V1 : V2;
11665
11666 // Compute which of the two target values this index should be assigned
11667 // to. This reflects whether the high elements are remaining or the low
11668 // elements are remaining.
11669 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
11670
11671 // Either set up this value if we've not encountered it before, or check
11672 // that it remains consistent.
11673 if (!TargetV)
11674 TargetV = MaskV;
11675 else if (TargetV != MaskV)
11676 // This may be a rotation, but it pulls from the inputs in some
11677 // unsupported interleaving.
11678 return -1;
11679 }
11680
11681 // Check that we successfully analyzed the mask, and normalize the results.
11682 assert(Rotation != 0 && "Failed to locate a viable rotation!")((Rotation != 0 && "Failed to locate a viable rotation!"
) ? static_cast<void> (0) : __assert_fail ("Rotation != 0 && \"Failed to locate a viable rotation!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11682, __PRETTY_FUNCTION__));
11683 assert((Lo || Hi) && "Failed to find a rotated input vector!")(((Lo || Hi) && "Failed to find a rotated input vector!"
) ? static_cast<void> (0) : __assert_fail ("(Lo || Hi) && \"Failed to find a rotated input vector!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11683, __PRETTY_FUNCTION__));
11684 if (!Lo)
11685 Lo = Hi;
11686 else if (!Hi)
11687 Hi = Lo;
11688
11689 V1 = Lo;
11690 V2 = Hi;
11691
11692 return Rotation;
11693}
11694
11695/// Try to lower a vector shuffle as a byte rotation.
11696///
11697/// SSSE3 has a generic PALIGNR instruction in x86 that will do an arbitrary
11698/// byte-rotation of the concatenation of two vectors; pre-SSSE3 can use
11699/// a PSRLDQ/PSLLDQ/POR pattern to get a similar effect. This routine will
11700/// try to generically lower a vector shuffle through such an pattern. It
11701/// does not check for the profitability of lowering either as PALIGNR or
11702/// PSRLDQ/PSLLDQ/POR, only whether the mask is valid to lower in that form.
11703/// This matches shuffle vectors that look like:
11704///
11705/// v8i16 [11, 12, 13, 14, 15, 0, 1, 2]
11706///
11707/// Essentially it concatenates V1 and V2, shifts right by some number of
11708/// elements, and takes the low elements as the result. Note that while this is
11709/// specified as a *right shift* because x86 is little-endian, it is a *left
11710/// rotate* of the vector lanes.
11711static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2,
11712 ArrayRef<int> Mask) {
11713 // Don't accept any shuffles with zero elements.
11714 if (any_of(Mask, [](int M) { return M == SM_SentinelZero; }))
11715 return -1;
11716
11717 // PALIGNR works on 128-bit lanes.
11718 SmallVector<int, 16> RepeatedMask;
11719 if (!is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedMask))
11720 return -1;
11721
11722 int Rotation = matchShuffleAsRotate(V1, V2, RepeatedMask);
11723 if (Rotation <= 0)
11724 return -1;
11725
11726 // PALIGNR rotates bytes, so we need to scale the
11727 // rotation based on how many bytes are in the vector lane.
11728 int NumElts = RepeatedMask.size();
11729 int Scale = 16 / NumElts;
11730 return Rotation * Scale;
11731}
11732
11733static SDValue lowerShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue V1,
11734 SDValue V2, ArrayRef<int> Mask,
11735 const X86Subtarget &Subtarget,
11736 SelectionDAG &DAG) {
11737 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!")((!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"
) ? static_cast<void> (0) : __assert_fail ("!isNoopShuffleMask(Mask) && \"We shouldn't lower no-op shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11737, __PRETTY_FUNCTION__));
11738
11739 SDValue Lo = V1, Hi = V2;
11740 int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask);
11741 if (ByteRotation <= 0)
11742 return SDValue();
11743
11744 // Cast the inputs to i8 vector of correct length to match PALIGNR or
11745 // PSLLDQ/PSRLDQ.
11746 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
11747 Lo = DAG.getBitcast(ByteVT, Lo);
11748 Hi = DAG.getBitcast(ByteVT, Hi);
11749
11750 // SSSE3 targets can use the palignr instruction.
11751 if (Subtarget.hasSSSE3()) {
11752 assert((!VT.is512BitVector() || Subtarget.hasBWI()) &&(((!VT.is512BitVector() || Subtarget.hasBWI()) && "512-bit PALIGNR requires BWI instructions"
) ? static_cast<void> (0) : __assert_fail ("(!VT.is512BitVector() || Subtarget.hasBWI()) && \"512-bit PALIGNR requires BWI instructions\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11753, __PRETTY_FUNCTION__))
11753 "512-bit PALIGNR requires BWI instructions")(((!VT.is512BitVector() || Subtarget.hasBWI()) && "512-bit PALIGNR requires BWI instructions"
) ? static_cast<void> (0) : __assert_fail ("(!VT.is512BitVector() || Subtarget.hasBWI()) && \"512-bit PALIGNR requires BWI instructions\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11753, __PRETTY_FUNCTION__));
11754 return DAG.getBitcast(
11755 VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi,
11756 DAG.getTargetConstant(ByteRotation, DL, MVT::i8)));
11757 }
11758
11759 assert(VT.is128BitVector() &&((VT.is128BitVector() && "Rotate-based lowering only supports 128-bit lowering!"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Rotate-based lowering only supports 128-bit lowering!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11760, __PRETTY_FUNCTION__))
11760 "Rotate-based lowering only supports 128-bit lowering!")((VT.is128BitVector() && "Rotate-based lowering only supports 128-bit lowering!"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Rotate-based lowering only supports 128-bit lowering!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11760, __PRETTY_FUNCTION__));
11761 assert(Mask.size() <= 16 &&((Mask.size() <= 16 && "Can shuffle at most 16 bytes in a 128-bit vector!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() <= 16 && \"Can shuffle at most 16 bytes in a 128-bit vector!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11762, __PRETTY_FUNCTION__))
11762 "Can shuffle at most 16 bytes in a 128-bit vector!")((Mask.size() <= 16 && "Can shuffle at most 16 bytes in a 128-bit vector!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() <= 16 && \"Can shuffle at most 16 bytes in a 128-bit vector!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11762, __PRETTY_FUNCTION__));
11763 assert(ByteVT == MVT::v16i8 &&((ByteVT == MVT::v16i8 && "SSE2 rotate lowering only needed for v16i8!"
) ? static_cast<void> (0) : __assert_fail ("ByteVT == MVT::v16i8 && \"SSE2 rotate lowering only needed for v16i8!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11764, __PRETTY_FUNCTION__))
11764 "SSE2 rotate lowering only needed for v16i8!")((ByteVT == MVT::v16i8 && "SSE2 rotate lowering only needed for v16i8!"
) ? static_cast<void> (0) : __assert_fail ("ByteVT == MVT::v16i8 && \"SSE2 rotate lowering only needed for v16i8!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11764, __PRETTY_FUNCTION__));
11765
11766 // Default SSE2 implementation
11767 int LoByteShift = 16 - ByteRotation;
11768 int HiByteShift = ByteRotation;
11769
11770 SDValue LoShift =
11771 DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo,
11772 DAG.getTargetConstant(LoByteShift, DL, MVT::i8));
11773 SDValue HiShift =
11774 DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Hi,
11775 DAG.getTargetConstant(HiByteShift, DL, MVT::i8));
11776 return DAG.getBitcast(VT,
11777 DAG.getNode(ISD::OR, DL, MVT::v16i8, LoShift, HiShift));
11778}
11779
11780/// Try to lower a vector shuffle as a dword/qword rotation.
11781///
11782/// AVX512 has a VALIGND/VALIGNQ instructions that will do an arbitrary
11783/// rotation of the concatenation of two vectors; This routine will
11784/// try to generically lower a vector shuffle through such an pattern.
11785///
11786/// Essentially it concatenates V1 and V2, shifts right by some number of
11787/// elements, and takes the low elements as the result. Note that while this is
11788/// specified as a *right shift* because x86 is little-endian, it is a *left
11789/// rotate* of the vector lanes.
11790static SDValue lowerShuffleAsRotate(const SDLoc &DL, MVT VT, SDValue V1,
11791 SDValue V2, ArrayRef<int> Mask,
11792 const X86Subtarget &Subtarget,
11793 SelectionDAG &DAG) {
11794 assert((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) &&(((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT
::i64) && "Only 32-bit and 64-bit elements are supported!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) && \"Only 32-bit and 64-bit elements are supported!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11795, __PRETTY_FUNCTION__))
11795 "Only 32-bit and 64-bit elements are supported!")(((VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT
::i64) && "Only 32-bit and 64-bit elements are supported!"
) ? static_cast<void> (0) : __assert_fail ("(VT.getScalarType() == MVT::i32 || VT.getScalarType() == MVT::i64) && \"Only 32-bit and 64-bit elements are supported!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11795, __PRETTY_FUNCTION__));
11796
11797 // 128/256-bit vectors are only supported with VLX.
11798 assert((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector()))(((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT
.is256BitVector())) && "VLX required for 128/256-bit vectors"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector())) && \"VLX required for 128/256-bit vectors\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11799, __PRETTY_FUNCTION__))
11799 && "VLX required for 128/256-bit vectors")(((Subtarget.hasVLX() || (!VT.is128BitVector() && !VT
.is256BitVector())) && "VLX required for 128/256-bit vectors"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasVLX() || (!VT.is128BitVector() && !VT.is256BitVector())) && \"VLX required for 128/256-bit vectors\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11799, __PRETTY_FUNCTION__));
11800
11801 SDValue Lo = V1, Hi = V2;
11802 int Rotation = matchShuffleAsRotate(Lo, Hi, Mask);
11803 if (Rotation <= 0)
11804 return SDValue();
11805
11806 return DAG.getNode(X86ISD::VALIGN, DL, VT, Lo, Hi,
11807 DAG.getTargetConstant(Rotation, DL, MVT::i8));
11808}
11809
11810/// Try to lower a vector shuffle as a byte shift sequence.
11811static SDValue lowerShuffleAsByteShiftMask(const SDLoc &DL, MVT VT, SDValue V1,
11812 SDValue V2, ArrayRef<int> Mask,
11813 const APInt &Zeroable,
11814 const X86Subtarget &Subtarget,
11815 SelectionDAG &DAG) {
11816 assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!")((!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"
) ? static_cast<void> (0) : __assert_fail ("!isNoopShuffleMask(Mask) && \"We shouldn't lower no-op shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11816, __PRETTY_FUNCTION__));
11817 assert(VT.is128BitVector() && "Only 128-bit vectors supported")((VT.is128BitVector() && "Only 128-bit vectors supported"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Only 128-bit vectors supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11817, __PRETTY_FUNCTION__));
11818
11819 // We need a shuffle that has zeros at one/both ends and a sequential
11820 // shuffle from one source within.
11821 unsigned ZeroLo = Zeroable.countTrailingOnes();
11822 unsigned ZeroHi = Zeroable.countLeadingOnes();
11823 if (!ZeroLo && !ZeroHi)
11824 return SDValue();
11825
11826 unsigned NumElts = Mask.size();
11827 unsigned Len = NumElts - (ZeroLo + ZeroHi);
11828 if (!isSequentialOrUndefInRange(Mask, ZeroLo, Len, Mask[ZeroLo]))
11829 return SDValue();
11830
11831 unsigned Scale = VT.getScalarSizeInBits() / 8;
11832 ArrayRef<int> StubMask = Mask.slice(ZeroLo, Len);
11833 if (!isUndefOrInRange(StubMask, 0, NumElts) &&
11834 !isUndefOrInRange(StubMask, NumElts, 2 * NumElts))
11835 return SDValue();
11836
11837 SDValue Res = Mask[ZeroLo] < (int)NumElts ? V1 : V2;
11838 Res = DAG.getBitcast(MVT::v16i8, Res);
11839
11840 // Use VSHLDQ/VSRLDQ ops to zero the ends of a vector and leave an
11841 // inner sequential set of elements, possibly offset:
11842 // 01234567 --> zzzzzz01 --> 1zzzzzzz
11843 // 01234567 --> 4567zzzz --> zzzzz456
11844 // 01234567 --> z0123456 --> 3456zzzz --> zz3456zz
11845 if (ZeroLo == 0) {
11846 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11847 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11848 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11849 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11850 DAG.getTargetConstant(Scale * ZeroHi, DL, MVT::i8));
11851 } else if (ZeroHi == 0) {
11852 unsigned Shift = Mask[ZeroLo] % NumElts;
11853 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11854 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11855 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11856 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11857 } else if (!Subtarget.hasSSSE3()) {
11858 // If we don't have PSHUFB then its worth avoiding an AND constant mask
11859 // by performing 3 byte shifts. Shuffle combining can kick in above that.
11860 // TODO: There may be some cases where VSH{LR}DQ+PAND is still better.
11861 unsigned Shift = (NumElts - 1) - (Mask[ZeroLo + Len - 1] % NumElts);
11862 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11863 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11864 Shift += Mask[ZeroLo] % NumElts;
11865 Res = DAG.getNode(X86ISD::VSRLDQ, DL, MVT::v16i8, Res,
11866 DAG.getTargetConstant(Scale * Shift, DL, MVT::i8));
11867 Res = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Res,
11868 DAG.getTargetConstant(Scale * ZeroLo, DL, MVT::i8));
11869 } else
11870 return SDValue();
11871
11872 return DAG.getBitcast(VT, Res);
11873}
11874
11875/// Try to lower a vector shuffle as a bit shift (shifts in zeros).
11876///
11877/// Attempts to match a shuffle mask against the PSLL(W/D/Q/DQ) and
11878/// PSRL(W/D/Q/DQ) SSE2 and AVX2 logical bit-shift instructions. The function
11879/// matches elements from one of the input vectors shuffled to the left or
11880/// right with zeroable elements 'shifted in'. It handles both the strictly
11881/// bit-wise element shifts and the byte shift across an entire 128-bit double
11882/// quad word lane.
11883///
11884/// PSHL : (little-endian) left bit shift.
11885/// [ zz, 0, zz, 2 ]
11886/// [ -1, 4, zz, -1 ]
11887/// PSRL : (little-endian) right bit shift.
11888/// [ 1, zz, 3, zz]
11889/// [ -1, -1, 7, zz]
11890/// PSLLDQ : (little-endian) left byte shift
11891/// [ zz, 0, 1, 2, 3, 4, 5, 6]
11892/// [ zz, zz, -1, -1, 2, 3, 4, -1]
11893/// [ zz, zz, zz, zz, zz, zz, -1, 1]
11894/// PSRLDQ : (little-endian) right byte shift
11895/// [ 5, 6, 7, zz, zz, zz, zz, zz]
11896/// [ -1, 5, 6, 7, zz, zz, zz, zz]
11897/// [ 1, 2, -1, -1, -1, -1, zz, zz]
11898static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
11899 unsigned ScalarSizeInBits, ArrayRef<int> Mask,
11900 int MaskOffset, const APInt &Zeroable,
11901 const X86Subtarget &Subtarget) {
11902 int Size = Mask.size();
11903 unsigned SizeInBits = Size * ScalarSizeInBits;
11904
11905 auto CheckZeros = [&](int Shift, int Scale, bool Left) {
11906 for (int i = 0; i < Size; i += Scale)
11907 for (int j = 0; j < Shift; ++j)
11908 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
11909 return false;
11910
11911 return true;
11912 };
11913
11914 auto MatchShift = [&](int Shift, int Scale, bool Left) {
11915 for (int i = 0; i != Size; i += Scale) {
11916 unsigned Pos = Left ? i + Shift : i;
11917 unsigned Low = Left ? i : i + Shift;
11918 unsigned Len = Scale - Shift;
11919 if (!isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset))
11920 return -1;
11921 }
11922
11923 int ShiftEltBits = ScalarSizeInBits * Scale;
11924 bool ByteShift = ShiftEltBits > 64;
11925 Opcode = Left ? (ByteShift ? X86ISD::VSHLDQ : X86ISD::VSHLI)
11926 : (ByteShift ? X86ISD::VSRLDQ : X86ISD::VSRLI);
11927 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
11928
11929 // Normalize the scale for byte shifts to still produce an i64 element
11930 // type.
11931 Scale = ByteShift ? Scale / 2 : Scale;
11932
11933 // We need to round trip through the appropriate type for the shift.
11934 MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale);
11935 ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8)
11936 : MVT::getVectorVT(ShiftSVT, Size / Scale);
11937 return (int)ShiftAmt;
11938 };
11939
11940 // SSE/AVX supports logical shifts up to 64-bit integers - so we can just
11941 // keep doubling the size of the integer elements up to that. We can
11942 // then shift the elements of the integer vector by whole multiples of
11943 // their width within the elements of the larger integer vector. Test each
11944 // multiple to see if we can find a match with the moved element indices
11945 // and that the shifted in elements are all zeroable.
11946 unsigned MaxWidth = ((SizeInBits == 512) && !Subtarget.hasBWI() ? 64 : 128);
11947 for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
11948 for (int Shift = 1; Shift != Scale; ++Shift)
11949 for (bool Left : {true, false})
11950 if (CheckZeros(Shift, Scale, Left)) {
11951 int ShiftAmt = MatchShift(Shift, Scale, Left);
11952 if (0 < ShiftAmt)
11953 return ShiftAmt;
11954 }
11955
11956 // no match
11957 return -1;
11958}
11959
11960static SDValue lowerShuffleAsShift(const SDLoc &DL, MVT VT, SDValue V1,
11961 SDValue V2, ArrayRef<int> Mask,
11962 const APInt &Zeroable,
11963 const X86Subtarget &Subtarget,
11964 SelectionDAG &DAG) {
11965 int Size = Mask.size();
11966 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size")((Size == (int)VT.getVectorNumElements() && "Unexpected mask size"
) ? static_cast<void> (0) : __assert_fail ("Size == (int)VT.getVectorNumElements() && \"Unexpected mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11966, __PRETTY_FUNCTION__));
11967
11968 MVT ShiftVT;
11969 SDValue V = V1;
11970 unsigned Opcode;
11971
11972 // Try to match shuffle against V1 shift.
11973 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11974 Mask, 0, Zeroable, Subtarget);
11975
11976 // If V1 failed, try to match shuffle against V2 shift.
11977 if (ShiftAmt < 0) {
11978 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
11979 Mask, Size, Zeroable, Subtarget);
11980 V = V2;
11981 }
11982
11983 if (ShiftAmt < 0)
11984 return SDValue();
11985
11986 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&((DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
"Illegal integer vector type") ? static_cast<void> (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) && \"Illegal integer vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11987, __PRETTY_FUNCTION__))
11987 "Illegal integer vector type")((DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
"Illegal integer vector type") ? static_cast<void> (0)
: __assert_fail ("DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) && \"Illegal integer vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 11987, __PRETTY_FUNCTION__));
11988 V = DAG.getBitcast(ShiftVT, V);
11989 V = DAG.getNode(Opcode, DL, ShiftVT, V,
11990 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
11991 return DAG.getBitcast(VT, V);
11992}
11993
11994// EXTRQ: Extract Len elements from lower half of source, starting at Idx.
11995// Remainder of lower half result is zero and upper half is all undef.
11996static bool matchShuffleAsEXTRQ(MVT VT, SDValue &V1, SDValue &V2,
11997 ArrayRef<int> Mask, uint64_t &BitLen,
11998 uint64_t &BitIdx, const APInt &Zeroable) {
11999 int Size = Mask.size();
12000 int HalfSize = Size / 2;
12001 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size")((Size == (int)VT.getVectorNumElements() && "Unexpected mask size"
) ? static_cast<void> (0) : __assert_fail ("Size == (int)VT.getVectorNumElements() && \"Unexpected mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12001, __PRETTY_FUNCTION__));
12002 assert(!Zeroable.isAllOnesValue() && "Fully zeroable shuffle mask")((!Zeroable.isAllOnesValue() && "Fully zeroable shuffle mask"
) ? static_cast<void> (0) : __assert_fail ("!Zeroable.isAllOnesValue() && \"Fully zeroable shuffle mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12002, __PRETTY_FUNCTION__));
12003
12004 // Upper half must be undefined.
12005 if (!isUndefUpperHalf(Mask))
12006 return false;
12007
12008 // Determine the extraction length from the part of the
12009 // lower half that isn't zeroable.
12010 int Len = HalfSize;
12011 for (; Len > 0; --Len)
12012 if (!Zeroable[Len - 1])
12013 break;
12014 assert(Len > 0 && "Zeroable shuffle mask")((Len > 0 && "Zeroable shuffle mask") ? static_cast
<void> (0) : __assert_fail ("Len > 0 && \"Zeroable shuffle mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12014, __PRETTY_FUNCTION__));
12015
12016 // Attempt to match first Len sequential elements from the lower half.
12017 SDValue Src;
12018 int Idx = -1;
12019 for (int i = 0; i != Len; ++i) {
12020 int M = Mask[i];
12021 if (M == SM_SentinelUndef)
12022 continue;
12023 SDValue &V = (M < Size ? V1 : V2);
12024 M = M % Size;
12025
12026 // The extracted elements must start at a valid index and all mask
12027 // elements must be in the lower half.
12028 if (i > M || M >= HalfSize)
12029 return false;
12030
12031 if (Idx < 0 || (Src == V && Idx == (M - i))) {
12032 Src = V;
12033 Idx = M - i;
12034 continue;
12035 }
12036 return false;
12037 }
12038
12039 if (!Src || Idx < 0)
12040 return false;
12041
12042 assert((Idx + Len) <= HalfSize && "Illegal extraction mask")(((Idx + Len) <= HalfSize && "Illegal extraction mask"
) ? static_cast<void> (0) : __assert_fail ("(Idx + Len) <= HalfSize && \"Illegal extraction mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12042, __PRETTY_FUNCTION__));
12043 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
12044 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
12045 V1 = Src;
12046 return true;
12047}
12048
12049// INSERTQ: Extract lowest Len elements from lower half of second source and
12050// insert over first source, starting at Idx.
12051// { A[0], .., A[Idx-1], B[0], .., B[Len-1], A[Idx+Len], .., UNDEF, ... }
12052static bool matchShuffleAsINSERTQ(MVT VT, SDValue &V1, SDValue &V2,
12053 ArrayRef<int> Mask, uint64_t &BitLen,
12054 uint64_t &BitIdx) {
12055 int Size = Mask.size();
12056 int HalfSize = Size / 2;
12057 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size")((Size == (int)VT.getVectorNumElements() && "Unexpected mask size"
) ? static_cast<void> (0) : __assert_fail ("Size == (int)VT.getVectorNumElements() && \"Unexpected mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12057, __PRETTY_FUNCTION__));
12058
12059 // Upper half must be undefined.
12060 if (!isUndefUpperHalf(Mask))
12061 return false;
12062
12063 for (int Idx = 0; Idx != HalfSize; ++Idx) {
12064 SDValue Base;
12065
12066 // Attempt to match first source from mask before insertion point.
12067 if (isUndefInRange(Mask, 0, Idx)) {
12068 /* EMPTY */
12069 } else if (isSequentialOrUndefInRange(Mask, 0, Idx, 0)) {
12070 Base = V1;
12071 } else if (isSequentialOrUndefInRange(Mask, 0, Idx, Size)) {
12072 Base = V2;
12073 } else {
12074 continue;
12075 }
12076
12077 // Extend the extraction length looking to match both the insertion of
12078 // the second source and the remaining elements of the first.
12079 for (int Hi = Idx + 1; Hi <= HalfSize; ++Hi) {
12080 SDValue Insert;
12081 int Len = Hi - Idx;
12082
12083 // Match insertion.
12084 if (isSequentialOrUndefInRange(Mask, Idx, Len, 0)) {
12085 Insert = V1;
12086 } else if (isSequentialOrUndefInRange(Mask, Idx, Len, Size)) {
12087 Insert = V2;
12088 } else {
12089 continue;
12090 }
12091
12092 // Match the remaining elements of the lower half.
12093 if (isUndefInRange(Mask, Hi, HalfSize - Hi)) {
12094 /* EMPTY */
12095 } else if ((!Base || (Base == V1)) &&
12096 isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi, Hi)) {
12097 Base = V1;
12098 } else if ((!Base || (Base == V2)) &&
12099 isSequentialOrUndefInRange(Mask, Hi, HalfSize - Hi,
12100 Size + Hi)) {
12101 Base = V2;
12102 } else {
12103 continue;
12104 }
12105
12106 BitLen = (Len * VT.getScalarSizeInBits()) & 0x3f;
12107 BitIdx = (Idx * VT.getScalarSizeInBits()) & 0x3f;
12108 V1 = Base;
12109 V2 = Insert;
12110 return true;
12111 }
12112 }
12113
12114 return false;
12115}
12116
12117/// Try to lower a vector shuffle using SSE4a EXTRQ/INSERTQ.
12118static SDValue lowerShuffleWithSSE4A(const SDLoc &DL, MVT VT, SDValue V1,
12119 SDValue V2, ArrayRef<int> Mask,
12120 const APInt &Zeroable, SelectionDAG &DAG) {
12121 uint64_t BitLen, BitIdx;
12122 if (matchShuffleAsEXTRQ(VT, V1, V2, Mask, BitLen, BitIdx, Zeroable))
12123 return DAG.getNode(X86ISD::EXTRQI, DL, VT, V1,
12124 DAG.getTargetConstant(BitLen, DL, MVT::i8),
12125 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
12126
12127 if (matchShuffleAsINSERTQ(VT, V1, V2, Mask, BitLen, BitIdx))
12128 return DAG.getNode(X86ISD::INSERTQI, DL, VT, V1 ? V1 : DAG.getUNDEF(VT),
12129 V2 ? V2 : DAG.getUNDEF(VT),
12130 DAG.getTargetConstant(BitLen, DL, MVT::i8),
12131 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
12132
12133 return SDValue();
12134}
12135
12136/// Lower a vector shuffle as a zero or any extension.
12137///
12138/// Given a specific number of elements, element bit width, and extension
12139/// stride, produce either a zero or any extension based on the available
12140/// features of the subtarget. The extended elements are consecutive and
12141/// begin and can start from an offsetted element index in the input; to
12142/// avoid excess shuffling the offset must either being in the bottom lane
12143/// or at the start of a higher lane. All extended elements must be from
12144/// the same lane.
12145static SDValue lowerShuffleAsSpecificZeroOrAnyExtend(
12146 const SDLoc &DL, MVT VT, int Scale, int Offset, bool AnyExt, SDValue InputV,
12147 ArrayRef<int> Mask, const X86Subtarget &Subtarget, SelectionDAG &DAG) {
12148 assert(Scale > 1 && "Need a scale to extend.")((Scale > 1 && "Need a scale to extend.") ? static_cast
<void> (0) : __assert_fail ("Scale > 1 && \"Need a scale to extend.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12148, __PRETTY_FUNCTION__));
12149 int EltBits = VT.getScalarSizeInBits();
12150 int NumElements = VT.getVectorNumElements();
12151 int NumEltsPerLane = 128 / EltBits;
12152 int OffsetLane = Offset / NumEltsPerLane;
12153 assert((EltBits == 8 || EltBits == 16 || EltBits == 32) &&(((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
"Only 8, 16, and 32 bit elements can be extended.") ? static_cast
<void> (0) : __assert_fail ("(EltBits == 8 || EltBits == 16 || EltBits == 32) && \"Only 8, 16, and 32 bit elements can be extended.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12154, __PRETTY_FUNCTION__))
12154 "Only 8, 16, and 32 bit elements can be extended.")(((EltBits == 8 || EltBits == 16 || EltBits == 32) &&
"Only 8, 16, and 32 bit elements can be extended.") ? static_cast
<void> (0) : __assert_fail ("(EltBits == 8 || EltBits == 16 || EltBits == 32) && \"Only 8, 16, and 32 bit elements can be extended.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12154, __PRETTY_FUNCTION__));
12155 assert(Scale * EltBits <= 64 && "Cannot zero extend past 64 bits.")((Scale * EltBits <= 64 && "Cannot zero extend past 64 bits."
) ? static_cast<void> (0) : __assert_fail ("Scale * EltBits <= 64 && \"Cannot zero extend past 64 bits.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12155, __PRETTY_FUNCTION__));
12156 assert(0 <= Offset && "Extension offset must be positive.")((0 <= Offset && "Extension offset must be positive."
) ? static_cast<void> (0) : __assert_fail ("0 <= Offset && \"Extension offset must be positive.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12156, __PRETTY_FUNCTION__));
12157 assert((Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0) &&(((Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0
) && "Extension offset must be in the first lane or start an upper lane."
) ? static_cast<void> (0) : __assert_fail ("(Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0) && \"Extension offset must be in the first lane or start an upper lane.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12158, __PRETTY_FUNCTION__))
12158 "Extension offset must be in the first lane or start an upper lane.")(((Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0
) && "Extension offset must be in the first lane or start an upper lane."
) ? static_cast<void> (0) : __assert_fail ("(Offset < NumEltsPerLane || Offset % NumEltsPerLane == 0) && \"Extension offset must be in the first lane or start an upper lane.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12158, __PRETTY_FUNCTION__));
12159
12160 // Check that an index is in same lane as the base offset.
12161 auto SafeOffset = [&](int Idx) {
12162 return OffsetLane == (Idx / NumEltsPerLane);
12163 };
12164
12165 // Shift along an input so that the offset base moves to the first element.
12166 auto ShuffleOffset = [&](SDValue V) {
12167 if (!Offset)
12168 return V;
12169
12170 SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
12171 for (int i = 0; i * Scale < NumElements; ++i) {
12172 int SrcIdx = i + Offset;
12173 ShMask[i] = SafeOffset(SrcIdx) ? SrcIdx : -1;
12174 }
12175 return DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), ShMask);
12176 };
12177
12178 // Found a valid a/zext mask! Try various lowering strategies based on the
12179 // input type and available ISA extensions.
12180 if (Subtarget.hasSSE41()) {
12181 // Not worth offsetting 128-bit vectors if scale == 2, a pattern using
12182 // PUNPCK will catch this in a later shuffle match.
12183 if (Offset && Scale == 2 && VT.is128BitVector())
12184 return SDValue();
12185 MVT ExtVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits * Scale),
12186 NumElements / Scale);
12187 InputV = ShuffleOffset(InputV);
12188 InputV = getExtendInVec(AnyExt ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND, DL,
12189 ExtVT, InputV, DAG);
12190 return DAG.getBitcast(VT, InputV);
12191 }
12192
12193 assert(VT.is128BitVector() && "Only 128-bit vectors can be extended.")((VT.is128BitVector() && "Only 128-bit vectors can be extended."
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Only 128-bit vectors can be extended.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12193, __PRETTY_FUNCTION__));
12194
12195 // For any extends we can cheat for larger element sizes and use shuffle
12196 // instructions that can fold with a load and/or copy.
12197 if (AnyExt && EltBits == 32) {
12198 int PSHUFDMask[4] = {Offset, -1, SafeOffset(Offset + 1) ? Offset + 1 : -1,
12199 -1};
12200 return DAG.getBitcast(
12201 VT, DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
12202 DAG.getBitcast(MVT::v4i32, InputV),
12203 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
12204 }
12205 if (AnyExt && EltBits == 16 && Scale > 2) {
12206 int PSHUFDMask[4] = {Offset / 2, -1,
12207 SafeOffset(Offset + 1) ? (Offset + 1) / 2 : -1, -1};
12208 InputV = DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32,
12209 DAG.getBitcast(MVT::v4i32, InputV),
12210 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
12211 int PSHUFWMask[4] = {1, -1, -1, -1};
12212 unsigned OddEvenOp = (Offset & 1) ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
12213 return DAG.getBitcast(
12214 VT, DAG.getNode(OddEvenOp, DL, MVT::v8i16,
12215 DAG.getBitcast(MVT::v8i16, InputV),
12216 getV4X86ShuffleImm8ForMask(PSHUFWMask, DL, DAG)));
12217 }
12218
12219 // The SSE4A EXTRQ instruction can efficiently extend the first 2 lanes
12220 // to 64-bits.
12221 if ((Scale * EltBits) == 64 && EltBits < 32 && Subtarget.hasSSE4A()) {
12222 assert(NumElements == (int)Mask.size() && "Unexpected shuffle mask size!")((NumElements == (int)Mask.size() && "Unexpected shuffle mask size!"
) ? static_cast<void> (0) : __assert_fail ("NumElements == (int)Mask.size() && \"Unexpected shuffle mask size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12222, __PRETTY_FUNCTION__));
12223 assert(VT.is128BitVector() && "Unexpected vector width!")((VT.is128BitVector() && "Unexpected vector width!") ?
static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Unexpected vector width!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12223, __PRETTY_FUNCTION__));
12224
12225 int LoIdx = Offset * EltBits;
12226 SDValue Lo = DAG.getBitcast(
12227 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
12228 DAG.getTargetConstant(EltBits, DL, MVT::i8),
12229 DAG.getTargetConstant(LoIdx, DL, MVT::i8)));
12230
12231 if (isUndefUpperHalf(Mask) || !SafeOffset(Offset + 1))
12232 return DAG.getBitcast(VT, Lo);
12233
12234 int HiIdx = (Offset + 1) * EltBits;
12235 SDValue Hi = DAG.getBitcast(
12236 MVT::v2i64, DAG.getNode(X86ISD::EXTRQI, DL, VT, InputV,
12237 DAG.getTargetConstant(EltBits, DL, MVT::i8),
12238 DAG.getTargetConstant(HiIdx, DL, MVT::i8)));
12239 return DAG.getBitcast(VT,
12240 DAG.getNode(X86ISD::UNPCKL, DL, MVT::v2i64, Lo, Hi));
12241 }
12242
12243 // If this would require more than 2 unpack instructions to expand, use
12244 // pshufb when available. We can only use more than 2 unpack instructions
12245 // when zero extending i8 elements which also makes it easier to use pshufb.
12246 if (Scale > 4 && EltBits == 8 && Subtarget.hasSSSE3()) {
12247 assert(NumElements == 16 && "Unexpected byte vector width!")((NumElements == 16 && "Unexpected byte vector width!"
) ? static_cast<void> (0) : __assert_fail ("NumElements == 16 && \"Unexpected byte vector width!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12247, __PRETTY_FUNCTION__));
12248 SDValue PSHUFBMask[16];
12249 for (int i = 0; i < 16; ++i) {
12250 int Idx = Offset + (i / Scale);
12251 if ((i % Scale == 0 && SafeOffset(Idx))) {
12252 PSHUFBMask[i] = DAG.getConstant(Idx, DL, MVT::i8);
12253 continue;
12254 }
12255 PSHUFBMask[i] =
12256 AnyExt ? DAG.getUNDEF(MVT::i8) : DAG.getConstant(0x80, DL, MVT::i8);
12257 }
12258 InputV = DAG.getBitcast(MVT::v16i8, InputV);
12259 return DAG.getBitcast(
12260 VT, DAG.getNode(X86ISD::PSHUFB, DL, MVT::v16i8, InputV,
12261 DAG.getBuildVector(MVT::v16i8, DL, PSHUFBMask)));
12262 }
12263
12264 // If we are extending from an offset, ensure we start on a boundary that
12265 // we can unpack from.
12266 int AlignToUnpack = Offset % (NumElements / Scale);
12267 if (AlignToUnpack) {
12268 SmallVector<int, 8> ShMask((unsigned)NumElements, -1);
12269 for (int i = AlignToUnpack; i < NumElements; ++i)
12270 ShMask[i - AlignToUnpack] = i;
12271 InputV = DAG.getVectorShuffle(VT, DL, InputV, DAG.getUNDEF(VT), ShMask);
12272 Offset -= AlignToUnpack;
12273 }
12274
12275 // Otherwise emit a sequence of unpacks.
12276 do {
12277 unsigned UnpackLoHi = X86ISD::UNPCKL;
12278 if (Offset >= (NumElements / 2)) {
12279 UnpackLoHi = X86ISD::UNPCKH;
12280 Offset -= (NumElements / 2);
12281 }
12282
12283 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
12284 SDValue Ext = AnyExt ? DAG.getUNDEF(InputVT)
12285 : getZeroVector(InputVT, Subtarget, DAG, DL);
12286 InputV = DAG.getBitcast(InputVT, InputV);
12287 InputV = DAG.getNode(UnpackLoHi, DL, InputVT, InputV, Ext);
12288 Scale /= 2;
12289 EltBits *= 2;
12290 NumElements /= 2;
12291 } while (Scale > 1);
12292 return DAG.getBitcast(VT, InputV);
12293}
12294
12295/// Try to lower a vector shuffle as a zero extension on any microarch.
12296///
12297/// This routine will try to do everything in its power to cleverly lower
12298/// a shuffle which happens to match the pattern of a zero extend. It doesn't
12299/// check for the profitability of this lowering, it tries to aggressively
12300/// match this pattern. It will use all of the micro-architectural details it
12301/// can to emit an efficient lowering. It handles both blends with all-zero
12302/// inputs to explicitly zero-extend and undef-lanes (sometimes undef due to
12303/// masking out later).
12304///
12305/// The reason we have dedicated lowering for zext-style shuffles is that they
12306/// are both incredibly common and often quite performance sensitive.
12307static SDValue lowerShuffleAsZeroOrAnyExtend(
12308 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12309 const APInt &Zeroable, const X86Subtarget &Subtarget,
12310 SelectionDAG &DAG) {
12311 int Bits = VT.getSizeInBits();
12312 int NumLanes = Bits / 128;
12313 int NumElements = VT.getVectorNumElements();
12314 int NumEltsPerLane = NumElements / NumLanes;
12315 assert(VT.getScalarSizeInBits() <= 32 &&((VT.getScalarSizeInBits() <= 32 && "Exceeds 32-bit integer zero extension limit"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() <= 32 && \"Exceeds 32-bit integer zero extension limit\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12316, __PRETTY_FUNCTION__))
12316 "Exceeds 32-bit integer zero extension limit")((VT.getScalarSizeInBits() <= 32 && "Exceeds 32-bit integer zero extension limit"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() <= 32 && \"Exceeds 32-bit integer zero extension limit\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12316, __PRETTY_FUNCTION__));
12317 assert((int)Mask.size() == NumElements && "Unexpected shuffle mask size")(((int)Mask.size() == NumElements && "Unexpected shuffle mask size"
) ? static_cast<void> (0) : __assert_fail ("(int)Mask.size() == NumElements && \"Unexpected shuffle mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12317, __PRETTY_FUNCTION__));
12318
12319 // Define a helper function to check a particular ext-scale and lower to it if
12320 // valid.
12321 auto Lower = [&](int Scale) -> SDValue {
12322 SDValue InputV;
12323 bool AnyExt = true;
12324 int Offset = 0;
12325 int Matches = 0;
12326 for (int i = 0; i < NumElements; ++i) {
12327 int M = Mask[i];
12328 if (M < 0)
12329 continue; // Valid anywhere but doesn't tell us anything.
12330 if (i % Scale != 0) {
12331 // Each of the extended elements need to be zeroable.
12332 if (!Zeroable[i])
12333 return SDValue();
12334
12335 // We no longer are in the anyext case.
12336 AnyExt = false;
12337 continue;
12338 }
12339
12340 // Each of the base elements needs to be consecutive indices into the
12341 // same input vector.
12342 SDValue V = M < NumElements ? V1 : V2;
12343 M = M % NumElements;
12344 if (!InputV) {
12345 InputV = V;
12346 Offset = M - (i / Scale);
12347 } else if (InputV != V)
12348 return SDValue(); // Flip-flopping inputs.
12349
12350 // Offset must start in the lowest 128-bit lane or at the start of an
12351 // upper lane.
12352 // FIXME: Is it ever worth allowing a negative base offset?
12353 if (!((0 <= Offset && Offset < NumEltsPerLane) ||
12354 (Offset % NumEltsPerLane) == 0))
12355 return SDValue();
12356
12357 // If we are offsetting, all referenced entries must come from the same
12358 // lane.
12359 if (Offset && (Offset / NumEltsPerLane) != (M / NumEltsPerLane))
12360 return SDValue();
12361
12362 if ((M % NumElements) != (Offset + (i / Scale)))
12363 return SDValue(); // Non-consecutive strided elements.
12364 Matches++;
12365 }
12366
12367 // If we fail to find an input, we have a zero-shuffle which should always
12368 // have already been handled.
12369 // FIXME: Maybe handle this here in case during blending we end up with one?
12370 if (!InputV)
12371 return SDValue();
12372
12373 // If we are offsetting, don't extend if we only match a single input, we
12374 // can always do better by using a basic PSHUF or PUNPCK.
12375 if (Offset != 0 && Matches < 2)
12376 return SDValue();
12377
12378 return lowerShuffleAsSpecificZeroOrAnyExtend(DL, VT, Scale, Offset, AnyExt,
12379 InputV, Mask, Subtarget, DAG);
12380 };
12381
12382 // The widest scale possible for extending is to a 64-bit integer.
12383 assert(Bits % 64 == 0 &&((Bits % 64 == 0 && "The number of bits in a vector must be divisible by 64 on x86!"
) ? static_cast<void> (0) : __assert_fail ("Bits % 64 == 0 && \"The number of bits in a vector must be divisible by 64 on x86!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12384, __PRETTY_FUNCTION__))
12384 "The number of bits in a vector must be divisible by 64 on x86!")((Bits % 64 == 0 && "The number of bits in a vector must be divisible by 64 on x86!"
) ? static_cast<void> (0) : __assert_fail ("Bits % 64 == 0 && \"The number of bits in a vector must be divisible by 64 on x86!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12384, __PRETTY_FUNCTION__));
12385 int NumExtElements = Bits / 64;
12386
12387 // Each iteration, try extending the elements half as much, but into twice as
12388 // many elements.
12389 for (; NumExtElements < NumElements; NumExtElements *= 2) {
12390 assert(NumElements % NumExtElements == 0 &&((NumElements % NumExtElements == 0 && "The input vector size must be divisible by the extended size."
) ? static_cast<void> (0) : __assert_fail ("NumElements % NumExtElements == 0 && \"The input vector size must be divisible by the extended size.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12391, __PRETTY_FUNCTION__))
12391 "The input vector size must be divisible by the extended size.")((NumElements % NumExtElements == 0 && "The input vector size must be divisible by the extended size."
) ? static_cast<void> (0) : __assert_fail ("NumElements % NumExtElements == 0 && \"The input vector size must be divisible by the extended size.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12391, __PRETTY_FUNCTION__));
12392 if (SDValue V = Lower(NumElements / NumExtElements))
12393 return V;
12394 }
12395
12396 // General extends failed, but 128-bit vectors may be able to use MOVQ.
12397 if (Bits != 128)
12398 return SDValue();
12399
12400 // Returns one of the source operands if the shuffle can be reduced to a
12401 // MOVQ, copying the lower 64-bits and zero-extending to the upper 64-bits.
12402 auto CanZExtLowHalf = [&]() {
12403 for (int i = NumElements / 2; i != NumElements; ++i)
12404 if (!Zeroable[i])
12405 return SDValue();
12406 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, 0))
12407 return V1;
12408 if (isSequentialOrUndefInRange(Mask, 0, NumElements / 2, NumElements))
12409 return V2;
12410 return SDValue();
12411 };
12412
12413 if (SDValue V = CanZExtLowHalf()) {
12414 V = DAG.getBitcast(MVT::v2i64, V);
12415 V = DAG.getNode(X86ISD::VZEXT_MOVL, DL, MVT::v2i64, V);
12416 return DAG.getBitcast(VT, V);
12417 }
12418
12419 // No viable ext lowering found.
12420 return SDValue();
12421}
12422
12423/// Try to get a scalar value for a specific element of a vector.
12424///
12425/// Looks through BUILD_VECTOR and SCALAR_TO_VECTOR nodes to find a scalar.
12426static SDValue getScalarValueForVectorElement(SDValue V, int Idx,
12427 SelectionDAG &DAG) {
12428 MVT VT = V.getSimpleValueType();
12429 MVT EltVT = VT.getVectorElementType();
12430 V = peekThroughBitcasts(V);
12431
12432 // If the bitcasts shift the element size, we can't extract an equivalent
12433 // element from it.
12434 MVT NewVT = V.getSimpleValueType();
12435 if (!NewVT.isVector() || NewVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
12436 return SDValue();
12437
12438 if (V.getOpcode() == ISD::BUILD_VECTOR ||
12439 (Idx == 0 && V.getOpcode() == ISD::SCALAR_TO_VECTOR)) {
12440 // Ensure the scalar operand is the same size as the destination.
12441 // FIXME: Add support for scalar truncation where possible.
12442 SDValue S = V.getOperand(Idx);
12443 if (EltVT.getSizeInBits() == S.getSimpleValueType().getSizeInBits())
12444 return DAG.getBitcast(EltVT, S);
12445 }
12446
12447 return SDValue();
12448}
12449
12450/// Helper to test for a load that can be folded with x86 shuffles.
12451///
12452/// This is particularly important because the set of instructions varies
12453/// significantly based on whether the operand is a load or not.
12454static bool isShuffleFoldableLoad(SDValue V) {
12455 V = peekThroughBitcasts(V);
12456 return ISD::isNON_EXTLoad(V.getNode());
12457}
12458
12459/// Try to lower insertion of a single element into a zero vector.
12460///
12461/// This is a common pattern that we have especially efficient patterns to lower
12462/// across all subtarget feature sets.
12463static SDValue lowerShuffleAsElementInsertion(
12464 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12465 const APInt &Zeroable, const X86Subtarget &Subtarget,
12466 SelectionDAG &DAG) {
12467 MVT ExtVT = VT;
12468 MVT EltVT = VT.getVectorElementType();
12469
12470 int V2Index =
12471 find_if(Mask, [&Mask](int M) { return M >= (int)Mask.size(); }) -
12472 Mask.begin();
12473 bool IsV1Zeroable = true;
12474 for (int i = 0, Size = Mask.size(); i < Size; ++i)
12475 if (i != V2Index && !Zeroable[i]) {
12476 IsV1Zeroable = false;
12477 break;
12478 }
12479
12480 // Check for a single input from a SCALAR_TO_VECTOR node.
12481 // FIXME: All of this should be canonicalized into INSERT_VECTOR_ELT and
12482 // all the smarts here sunk into that routine. However, the current
12483 // lowering of BUILD_VECTOR makes that nearly impossible until the old
12484 // vector shuffle lowering is dead.
12485 SDValue V2S = getScalarValueForVectorElement(V2, Mask[V2Index] - Mask.size(),
12486 DAG);
12487 if (V2S && DAG.getTargetLoweringInfo().isTypeLegal(V2S.getValueType())) {
12488 // We need to zext the scalar if it is smaller than an i32.
12489 V2S = DAG.getBitcast(EltVT, V2S);
12490 if (EltVT == MVT::i8 || EltVT == MVT::i16) {
12491 // Using zext to expand a narrow element won't work for non-zero
12492 // insertions.
12493 if (!IsV1Zeroable)
12494 return SDValue();
12495
12496 // Zero-extend directly to i32.
12497 ExtVT = MVT::getVectorVT(MVT::i32, ExtVT.getSizeInBits() / 32);
12498 V2S = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i32, V2S);
12499 }
12500 V2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ExtVT, V2S);
12501 } else if (Mask[V2Index] != (int)Mask.size() || EltVT == MVT::i8 ||
12502 EltVT == MVT::i16) {
12503 // Either not inserting from the low element of the input or the input
12504 // element size is too small to use VZEXT_MOVL to clear the high bits.
12505 return SDValue();
12506 }
12507
12508 if (!IsV1Zeroable) {
12509 // If V1 can't be treated as a zero vector we have fewer options to lower
12510 // this. We can't support integer vectors or non-zero targets cheaply, and
12511 // the V1 elements can't be permuted in any way.
12512 assert(VT == ExtVT && "Cannot change extended type when non-zeroable!")((VT == ExtVT && "Cannot change extended type when non-zeroable!"
) ? static_cast<void> (0) : __assert_fail ("VT == ExtVT && \"Cannot change extended type when non-zeroable!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12512, __PRETTY_FUNCTION__));
12513 if (!VT.isFloatingPoint() || V2Index != 0)
12514 return SDValue();
12515 SmallVector<int, 8> V1Mask(Mask.begin(), Mask.end());
12516 V1Mask[V2Index] = -1;
12517 if (!isNoopShuffleMask(V1Mask))
12518 return SDValue();
12519 if (!VT.is128BitVector())
12520 return SDValue();
12521
12522 // Otherwise, use MOVSD or MOVSS.
12523 assert((EltVT == MVT::f32 || EltVT == MVT::f64) &&(((EltVT == MVT::f32 || EltVT == MVT::f64) && "Only two types of floating point element types to handle!"
) ? static_cast<void> (0) : __assert_fail ("(EltVT == MVT::f32 || EltVT == MVT::f64) && \"Only two types of floating point element types to handle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12524, __PRETTY_FUNCTION__))
12524 "Only two types of floating point element types to handle!")(((EltVT == MVT::f32 || EltVT == MVT::f64) && "Only two types of floating point element types to handle!"
) ? static_cast<void> (0) : __assert_fail ("(EltVT == MVT::f32 || EltVT == MVT::f64) && \"Only two types of floating point element types to handle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12524, __PRETTY_FUNCTION__));
12525 return DAG.getNode(EltVT == MVT::f32 ? X86ISD::MOVSS : X86ISD::MOVSD, DL,
12526 ExtVT, V1, V2);
12527 }
12528
12529 // This lowering only works for the low element with floating point vectors.
12530 if (VT.isFloatingPoint() && V2Index != 0)
12531 return SDValue();
12532
12533 V2 = DAG.getNode(X86ISD::VZEXT_MOVL, DL, ExtVT, V2);
12534 if (ExtVT != VT)
12535 V2 = DAG.getBitcast(VT, V2);
12536
12537 if (V2Index != 0) {
12538 // If we have 4 or fewer lanes we can cheaply shuffle the element into
12539 // the desired position. Otherwise it is more efficient to do a vector
12540 // shift left. We know that we can do a vector shift left because all
12541 // the inputs are zero.
12542 if (VT.isFloatingPoint() || VT.getVectorNumElements() <= 4) {
12543 SmallVector<int, 4> V2Shuffle(Mask.size(), 1);
12544 V2Shuffle[V2Index] = 0;
12545 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Shuffle);
12546 } else {
12547 V2 = DAG.getBitcast(MVT::v16i8, V2);
12548 V2 = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, V2,
12549 DAG.getTargetConstant(
12550 V2Index * EltVT.getSizeInBits() / 8, DL, MVT::i8));
12551 V2 = DAG.getBitcast(VT, V2);
12552 }
12553 }
12554 return V2;
12555}
12556
12557/// Try to lower broadcast of a single - truncated - integer element,
12558/// coming from a scalar_to_vector/build_vector node \p V0 with larger elements.
12559///
12560/// This assumes we have AVX2.
12561static SDValue lowerShuffleAsTruncBroadcast(const SDLoc &DL, MVT VT, SDValue V0,
12562 int BroadcastIdx,
12563 const X86Subtarget &Subtarget,
12564 SelectionDAG &DAG) {
12565 assert(Subtarget.hasAVX2() &&((Subtarget.hasAVX2() && "We can only lower integer broadcasts with AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"We can only lower integer broadcasts with AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12566, __PRETTY_FUNCTION__))
12566 "We can only lower integer broadcasts with AVX2!")((Subtarget.hasAVX2() && "We can only lower integer broadcasts with AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"We can only lower integer broadcasts with AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12566, __PRETTY_FUNCTION__));
12567
12568 EVT EltVT = VT.getVectorElementType();
12569 EVT V0VT = V0.getValueType();
12570
12571 assert(VT.isInteger() && "Unexpected non-integer trunc broadcast!")((VT.isInteger() && "Unexpected non-integer trunc broadcast!"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"Unexpected non-integer trunc broadcast!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12571, __PRETTY_FUNCTION__));
12572 assert(V0VT.isVector() && "Unexpected non-vector vector-sized value!")((V0VT.isVector() && "Unexpected non-vector vector-sized value!"
) ? static_cast<void> (0) : __assert_fail ("V0VT.isVector() && \"Unexpected non-vector vector-sized value!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12572, __PRETTY_FUNCTION__));
12573
12574 EVT V0EltVT = V0VT.getVectorElementType();
12575 if (!V0EltVT.isInteger())
12576 return SDValue();
12577
12578 const unsigned EltSize = EltVT.getSizeInBits();
12579 const unsigned V0EltSize = V0EltVT.getSizeInBits();
12580
12581 // This is only a truncation if the original element type is larger.
12582 if (V0EltSize <= EltSize)
12583 return SDValue();
12584
12585 assert(((V0EltSize % EltSize) == 0) &&((((V0EltSize % EltSize) == 0) && "Scalar type sizes must all be powers of 2 on x86!"
) ? static_cast<void> (0) : __assert_fail ("((V0EltSize % EltSize) == 0) && \"Scalar type sizes must all be powers of 2 on x86!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12586, __PRETTY_FUNCTION__))
12586 "Scalar type sizes must all be powers of 2 on x86!")((((V0EltSize % EltSize) == 0) && "Scalar type sizes must all be powers of 2 on x86!"
) ? static_cast<void> (0) : __assert_fail ("((V0EltSize % EltSize) == 0) && \"Scalar type sizes must all be powers of 2 on x86!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12586, __PRETTY_FUNCTION__));
12587
12588 const unsigned V0Opc = V0.getOpcode();
12589 const unsigned Scale = V0EltSize / EltSize;
12590 const unsigned V0BroadcastIdx = BroadcastIdx / Scale;
12591
12592 if ((V0Opc != ISD::SCALAR_TO_VECTOR || V0BroadcastIdx != 0) &&
12593 V0Opc != ISD::BUILD_VECTOR)
12594 return SDValue();
12595
12596 SDValue Scalar = V0.getOperand(V0BroadcastIdx);
12597
12598 // If we're extracting non-least-significant bits, shift so we can truncate.
12599 // Hopefully, we can fold away the trunc/srl/load into the broadcast.
12600 // Even if we can't (and !isShuffleFoldableLoad(Scalar)), prefer
12601 // vpbroadcast+vmovd+shr to vpshufb(m)+vmovd.
12602 if (const int OffsetIdx = BroadcastIdx % Scale)
12603 Scalar = DAG.getNode(ISD::SRL, DL, Scalar.getValueType(), Scalar,
12604 DAG.getConstant(OffsetIdx * EltSize, DL, MVT::i8));
12605
12606 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
12607 DAG.getNode(ISD::TRUNCATE, DL, EltVT, Scalar));
12608}
12609
12610/// Test whether this can be lowered with a single SHUFPS instruction.
12611///
12612/// This is used to disable more specialized lowerings when the shufps lowering
12613/// will happen to be efficient.
12614static bool isSingleSHUFPSMask(ArrayRef<int> Mask) {
12615 // This routine only handles 128-bit shufps.
12616 assert(Mask.size() == 4 && "Unsupported mask size!")((Mask.size() == 4 && "Unsupported mask size!") ? static_cast
<void> (0) : __assert_fail ("Mask.size() == 4 && \"Unsupported mask size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12616, __PRETTY_FUNCTION__));
12617 assert(Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!")((Mask[0] >= -1 && Mask[0] < 8 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[0] >= -1 && Mask[0] < 8 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12617, __PRETTY_FUNCTION__));
12618 assert(Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!")((Mask[1] >= -1 && Mask[1] < 8 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[1] >= -1 && Mask[1] < 8 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12618, __PRETTY_FUNCTION__));
12619 assert(Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!")((Mask[2] >= -1 && Mask[2] < 8 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[2] >= -1 && Mask[2] < 8 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12619, __PRETTY_FUNCTION__));
12620 assert(Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!")((Mask[3] >= -1 && Mask[3] < 8 && "Out of bound mask element!"
) ? static_cast<void> (0) : __assert_fail ("Mask[3] >= -1 && Mask[3] < 8 && \"Out of bound mask element!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12620, __PRETTY_FUNCTION__));
12621
12622 // To lower with a single SHUFPS we need to have the low half and high half
12623 // each requiring a single input.
12624 if (Mask[0] >= 0 && Mask[1] >= 0 && (Mask[0] < 4) != (Mask[1] < 4))
12625 return false;
12626 if (Mask[2] >= 0 && Mask[3] >= 0 && (Mask[2] < 4) != (Mask[3] < 4))
12627 return false;
12628
12629 return true;
12630}
12631
12632/// If we are extracting two 128-bit halves of a vector and shuffling the
12633/// result, match that to a 256-bit AVX2 vperm* instruction to avoid a
12634/// multi-shuffle lowering.
12635static SDValue lowerShuffleOfExtractsAsVperm(const SDLoc &DL, SDValue N0,
12636 SDValue N1, ArrayRef<int> Mask,
12637 SelectionDAG &DAG) {
12638 EVT VT = N0.getValueType();
12639 assert((VT.is128BitVector() &&(((VT.is128BitVector() && (VT.getScalarSizeInBits() ==
32 || VT.getScalarSizeInBits() == 64)) && "VPERM* family of shuffles requires 32-bit or 64-bit elements"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() && (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) && \"VPERM* family of shuffles requires 32-bit or 64-bit elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12641, __PRETTY_FUNCTION__))
12640 (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) &&(((VT.is128BitVector() && (VT.getScalarSizeInBits() ==
32 || VT.getScalarSizeInBits() == 64)) && "VPERM* family of shuffles requires 32-bit or 64-bit elements"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() && (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) && \"VPERM* family of shuffles requires 32-bit or 64-bit elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12641, __PRETTY_FUNCTION__))
12641 "VPERM* family of shuffles requires 32-bit or 64-bit elements")(((VT.is128BitVector() && (VT.getScalarSizeInBits() ==
32 || VT.getScalarSizeInBits() == 64)) && "VPERM* family of shuffles requires 32-bit or 64-bit elements"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() && (VT.getScalarSizeInBits() == 32 || VT.getScalarSizeInBits() == 64)) && \"VPERM* family of shuffles requires 32-bit or 64-bit elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12641, __PRETTY_FUNCTION__));
12642
12643 // Check that both sources are extracts of the same source vector.
12644 if (!N0.hasOneUse() || !N1.hasOneUse() ||
12645 N0.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12646 N1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
12647 N0.getOperand(0) != N1.getOperand(0))
12648 return SDValue();
12649
12650 SDValue WideVec = N0.getOperand(0);
12651 EVT WideVT = WideVec.getValueType();
12652 if (!WideVT.is256BitVector() || !isa<ConstantSDNode>(N0.getOperand(1)) ||
12653 !isa<ConstantSDNode>(N1.getOperand(1)))
12654 return SDValue();
12655
12656 // Match extracts of each half of the wide source vector. Commute the shuffle
12657 // if the extract of the low half is N1.
12658 unsigned NumElts = VT.getVectorNumElements();
12659 SmallVector<int, 4> NewMask(Mask.begin(), Mask.end());
12660 const APInt &ExtIndex0 = N0.getConstantOperandAPInt(1);
12661 const APInt &ExtIndex1 = N1.getConstantOperandAPInt(1);
12662 if (ExtIndex1 == 0 && ExtIndex0 == NumElts)
12663 ShuffleVectorSDNode::commuteMask(NewMask);
12664 else if (ExtIndex0 != 0 || ExtIndex1 != NumElts)
12665 return SDValue();
12666
12667 // Final bailout: if the mask is simple, we are better off using an extract
12668 // and a simple narrow shuffle. Prefer extract+unpack(h/l)ps to vpermps
12669 // because that avoids a constant load from memory.
12670 if (NumElts == 4 &&
12671 (isSingleSHUFPSMask(NewMask) || is128BitUnpackShuffleMask(NewMask)))
12672 return SDValue();
12673
12674 // Extend the shuffle mask with undef elements.
12675 NewMask.append(NumElts, -1);
12676
12677 // shuf (extract X, 0), (extract X, 4), M --> extract (shuf X, undef, M'), 0
12678 SDValue Shuf = DAG.getVectorShuffle(WideVT, DL, WideVec, DAG.getUNDEF(WideVT),
12679 NewMask);
12680 // This is free: ymm -> xmm.
12681 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Shuf,
12682 DAG.getIntPtrConstant(0, DL));
12683}
12684
12685/// Try to lower broadcast of a single element.
12686///
12687/// For convenience, this code also bundles all of the subtarget feature set
12688/// filtering. While a little annoying to re-dispatch on type here, there isn't
12689/// a convenient way to factor it out.
12690static SDValue lowerShuffleAsBroadcast(const SDLoc &DL, MVT VT, SDValue V1,
12691 SDValue V2, ArrayRef<int> Mask,
12692 const X86Subtarget &Subtarget,
12693 SelectionDAG &DAG) {
12694 if (!((Subtarget.hasSSE3() && VT == MVT::v2f64) ||
12695 (Subtarget.hasAVX() && VT.isFloatingPoint()) ||
12696 (Subtarget.hasAVX2() && VT.isInteger())))
12697 return SDValue();
12698
12699 // With MOVDDUP (v2f64) we can broadcast from a register or a load, otherwise
12700 // we can only broadcast from a register with AVX2.
12701 unsigned NumElts = Mask.size();
12702 unsigned NumEltBits = VT.getScalarSizeInBits();
12703 unsigned Opcode = (VT == MVT::v2f64 && !Subtarget.hasAVX2())
12704 ? X86ISD::MOVDDUP
12705 : X86ISD::VBROADCAST;
12706 bool BroadcastFromReg = (Opcode == X86ISD::MOVDDUP) || Subtarget.hasAVX2();
12707
12708 // Check that the mask is a broadcast.
12709 int BroadcastIdx = -1;
12710 for (int i = 0; i != (int)NumElts; ++i) {
12711 SmallVector<int, 8> BroadcastMask(NumElts, i);
12712 if (isShuffleEquivalent(V1, V2, Mask, BroadcastMask)) {
12713 BroadcastIdx = i;
12714 break;
12715 }
12716 }
12717
12718 if (BroadcastIdx < 0)
12719 return SDValue();
12720 assert(BroadcastIdx < (int)Mask.size() && "We only expect to be called with "((BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
"a sorted mask where the broadcast " "comes from V1.") ? static_cast
<void> (0) : __assert_fail ("BroadcastIdx < (int)Mask.size() && \"We only expect to be called with \" \"a sorted mask where the broadcast \" \"comes from V1.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12722, __PRETTY_FUNCTION__))
12721 "a sorted mask where the broadcast "((BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
"a sorted mask where the broadcast " "comes from V1.") ? static_cast
<void> (0) : __assert_fail ("BroadcastIdx < (int)Mask.size() && \"We only expect to be called with \" \"a sorted mask where the broadcast \" \"comes from V1.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12722, __PRETTY_FUNCTION__))
12722 "comes from V1.")((BroadcastIdx < (int)Mask.size() && "We only expect to be called with "
"a sorted mask where the broadcast " "comes from V1.") ? static_cast
<void> (0) : __assert_fail ("BroadcastIdx < (int)Mask.size() && \"We only expect to be called with \" \"a sorted mask where the broadcast \" \"comes from V1.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12722, __PRETTY_FUNCTION__));
12723
12724 // Go up the chain of (vector) values to find a scalar load that we can
12725 // combine with the broadcast.
12726 int BitOffset = BroadcastIdx * NumEltBits;
12727 SDValue V = V1;
12728 for (;;) {
12729 switch (V.getOpcode()) {
12730 case ISD::BITCAST: {
12731 V = V.getOperand(0);
12732 continue;
12733 }
12734 case ISD::CONCAT_VECTORS: {
12735 int OpBitWidth = V.getOperand(0).getValueSizeInBits();
12736 int OpIdx = BitOffset / OpBitWidth;
12737 V = V.getOperand(OpIdx);
12738 BitOffset %= OpBitWidth;
12739 continue;
12740 }
12741 case ISD::INSERT_SUBVECTOR: {
12742 SDValue VOuter = V.getOperand(0), VInner = V.getOperand(1);
12743 auto ConstantIdx = dyn_cast<ConstantSDNode>(V.getOperand(2));
12744 if (!ConstantIdx)
12745 break;
12746
12747 int EltBitWidth = VOuter.getScalarValueSizeInBits();
12748 int Idx = (int)ConstantIdx->getZExtValue();
12749 int NumSubElts = (int)VInner.getSimpleValueType().getVectorNumElements();
12750 int BeginOffset = Idx * EltBitWidth;
12751 int EndOffset = BeginOffset + NumSubElts * EltBitWidth;
12752 if (BeginOffset <= BitOffset && BitOffset < EndOffset) {
12753 BitOffset -= BeginOffset;
12754 V = VInner;
12755 } else {
12756 V = VOuter;
12757 }
12758 continue;
12759 }
12760 }
12761 break;
12762 }
12763 assert((BitOffset % NumEltBits) == 0 && "Illegal bit-offset")(((BitOffset % NumEltBits) == 0 && "Illegal bit-offset"
) ? static_cast<void> (0) : __assert_fail ("(BitOffset % NumEltBits) == 0 && \"Illegal bit-offset\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12763, __PRETTY_FUNCTION__));
12764 BroadcastIdx = BitOffset / NumEltBits;
12765
12766 // Do we need to bitcast the source to retrieve the original broadcast index?
12767 bool BitCastSrc = V.getScalarValueSizeInBits() != NumEltBits;
12768
12769 // Check if this is a broadcast of a scalar. We special case lowering
12770 // for scalars so that we can more effectively fold with loads.
12771 // If the original value has a larger element type than the shuffle, the
12772 // broadcast element is in essence truncated. Make that explicit to ease
12773 // folding.
12774 if (BitCastSrc && VT.isInteger())
12775 if (SDValue TruncBroadcast = lowerShuffleAsTruncBroadcast(
12776 DL, VT, V, BroadcastIdx, Subtarget, DAG))
12777 return TruncBroadcast;
12778
12779 MVT BroadcastVT = VT;
12780
12781 // Also check the simpler case, where we can directly reuse the scalar.
12782 if (!BitCastSrc &&
12783 ((V.getOpcode() == ISD::BUILD_VECTOR && V.hasOneUse()) ||
12784 (V.getOpcode() == ISD::SCALAR_TO_VECTOR && BroadcastIdx == 0))) {
12785 V = V.getOperand(BroadcastIdx);
12786
12787 // If we can't broadcast from a register, check that the input is a load.
12788 if (!BroadcastFromReg && !isShuffleFoldableLoad(V))
12789 return SDValue();
12790 } else if (MayFoldLoad(V) && cast<LoadSDNode>(V)->isSimple()) {
12791 // 32-bit targets need to load i64 as a f64 and then bitcast the result.
12792 if (!Subtarget.is64Bit() && VT.getScalarType() == MVT::i64) {
12793 BroadcastVT = MVT::getVectorVT(MVT::f64, VT.getVectorNumElements());
12794 Opcode = (BroadcastVT.is128BitVector() && !Subtarget.hasAVX2())
12795 ? X86ISD::MOVDDUP
12796 : Opcode;
12797 }
12798
12799 // If we are broadcasting a load that is only used by the shuffle
12800 // then we can reduce the vector load to the broadcasted scalar load.
12801 LoadSDNode *Ld = cast<LoadSDNode>(V);
12802 SDValue BaseAddr = Ld->getOperand(1);
12803 EVT SVT = BroadcastVT.getScalarType();
12804 unsigned Offset = BroadcastIdx * SVT.getStoreSize();
12805 assert((int)(Offset * 8) == BitOffset && "Unexpected bit-offset")(((int)(Offset * 8) == BitOffset && "Unexpected bit-offset"
) ? static_cast<void> (0) : __assert_fail ("(int)(Offset * 8) == BitOffset && \"Unexpected bit-offset\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12805, __PRETTY_FUNCTION__));
12806 SDValue NewAddr = DAG.getMemBasePlusOffset(BaseAddr, Offset, DL);
12807 V = DAG.getLoad(SVT, DL, Ld->getChain(), NewAddr,
12808 DAG.getMachineFunction().getMachineMemOperand(
12809 Ld->getMemOperand(), Offset, SVT.getStoreSize()));
12810 DAG.makeEquivalentMemoryOrdering(Ld, V);
12811 } else if (!BroadcastFromReg) {
12812 // We can't broadcast from a vector register.
12813 return SDValue();
12814 } else if (BitOffset != 0) {
12815 // We can only broadcast from the zero-element of a vector register,
12816 // but it can be advantageous to broadcast from the zero-element of a
12817 // subvector.
12818 if (!VT.is256BitVector() && !VT.is512BitVector())
12819 return SDValue();
12820
12821 // VPERMQ/VPERMPD can perform the cross-lane shuffle directly.
12822 if (VT == MVT::v4f64 || VT == MVT::v4i64)
12823 return SDValue();
12824
12825 // Only broadcast the zero-element of a 128-bit subvector.
12826 if ((BitOffset % 128) != 0)
12827 return SDValue();
12828
12829 assert((BitOffset % V.getScalarValueSizeInBits()) == 0 &&(((BitOffset % V.getScalarValueSizeInBits()) == 0 && "Unexpected bit-offset"
) ? static_cast<void> (0) : __assert_fail ("(BitOffset % V.getScalarValueSizeInBits()) == 0 && \"Unexpected bit-offset\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12830, __PRETTY_FUNCTION__))
12830 "Unexpected bit-offset")(((BitOffset % V.getScalarValueSizeInBits()) == 0 && "Unexpected bit-offset"
) ? static_cast<void> (0) : __assert_fail ("(BitOffset % V.getScalarValueSizeInBits()) == 0 && \"Unexpected bit-offset\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12830, __PRETTY_FUNCTION__));
12831 assert((V.getValueSizeInBits() == 256 || V.getValueSizeInBits() == 512) &&(((V.getValueSizeInBits() == 256 || V.getValueSizeInBits() ==
512) && "Unexpected vector size") ? static_cast<void
> (0) : __assert_fail ("(V.getValueSizeInBits() == 256 || V.getValueSizeInBits() == 512) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12832, __PRETTY_FUNCTION__))
12832 "Unexpected vector size")(((V.getValueSizeInBits() == 256 || V.getValueSizeInBits() ==
512) && "Unexpected vector size") ? static_cast<void
> (0) : __assert_fail ("(V.getValueSizeInBits() == 256 || V.getValueSizeInBits() == 512) && \"Unexpected vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12832, __PRETTY_FUNCTION__));
12833 unsigned ExtractIdx = BitOffset / V.getScalarValueSizeInBits();
12834 V = extract128BitVector(V, ExtractIdx, DAG, DL);
12835 }
12836
12837 if (Opcode == X86ISD::MOVDDUP && !V.getValueType().isVector())
12838 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
12839 DAG.getBitcast(MVT::f64, V));
12840
12841 // Bitcast back to the same scalar type as BroadcastVT.
12842 if (V.getValueType().getScalarType() != BroadcastVT.getScalarType()) {
12843 assert(NumEltBits == BroadcastVT.getScalarSizeInBits() &&((NumEltBits == BroadcastVT.getScalarSizeInBits() && "Unexpected vector element size"
) ? static_cast<void> (0) : __assert_fail ("NumEltBits == BroadcastVT.getScalarSizeInBits() && \"Unexpected vector element size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12844, __PRETTY_FUNCTION__))
12844 "Unexpected vector element size")((NumEltBits == BroadcastVT.getScalarSizeInBits() && "Unexpected vector element size"
) ? static_cast<void> (0) : __assert_fail ("NumEltBits == BroadcastVT.getScalarSizeInBits() && \"Unexpected vector element size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12844, __PRETTY_FUNCTION__));
12845 MVT ExtVT;
12846 if (V.getValueType().isVector()) {
12847 unsigned NumSrcElts = V.getValueSizeInBits() / NumEltBits;
12848 ExtVT = MVT::getVectorVT(BroadcastVT.getScalarType(), NumSrcElts);
12849 } else {
12850 ExtVT = BroadcastVT.getScalarType();
12851 }
12852 V = DAG.getBitcast(ExtVT, V);
12853 }
12854
12855 // 32-bit targets need to load i64 as a f64 and then bitcast the result.
12856 if (!Subtarget.is64Bit() && V.getValueType() == MVT::i64) {
12857 V = DAG.getBitcast(MVT::f64, V);
12858 unsigned NumBroadcastElts = BroadcastVT.getVectorNumElements();
12859 BroadcastVT = MVT::getVectorVT(MVT::f64, NumBroadcastElts);
12860 }
12861
12862 // We only support broadcasting from 128-bit vectors to minimize the
12863 // number of patterns we need to deal with in isel. So extract down to
12864 // 128-bits, removing as many bitcasts as possible.
12865 if (V.getValueSizeInBits() > 128) {
12866 MVT ExtVT = V.getSimpleValueType().getScalarType();
12867 ExtVT = MVT::getVectorVT(ExtVT, 128 / ExtVT.getScalarSizeInBits());
12868 V = extract128BitVector(peekThroughBitcasts(V), 0, DAG, DL);
12869 V = DAG.getBitcast(ExtVT, V);
12870 }
12871
12872 return DAG.getBitcast(VT, DAG.getNode(Opcode, DL, BroadcastVT, V));
12873}
12874
12875// Check for whether we can use INSERTPS to perform the shuffle. We only use
12876// INSERTPS when the V1 elements are already in the correct locations
12877// because otherwise we can just always use two SHUFPS instructions which
12878// are much smaller to encode than a SHUFPS and an INSERTPS. We can also
12879// perform INSERTPS if a single V1 element is out of place and all V2
12880// elements are zeroable.
12881static bool matchShuffleAsInsertPS(SDValue &V1, SDValue &V2,
12882 unsigned &InsertPSMask,
12883 const APInt &Zeroable,
12884 ArrayRef<int> Mask, SelectionDAG &DAG) {
12885 assert(V1.getSimpleValueType().is128BitVector() && "Bad operand type!")((V1.getSimpleValueType().is128BitVector() && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType().is128BitVector() && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12885, __PRETTY_FUNCTION__));
12886 assert(V2.getSimpleValueType().is128BitVector() && "Bad operand type!")((V2.getSimpleValueType().is128BitVector() && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType().is128BitVector() && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12886, __PRETTY_FUNCTION__));
12887 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!")((Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 4 && \"Unexpected mask size for v4 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12887, __PRETTY_FUNCTION__));
12888
12889 // Attempt to match INSERTPS with one element from VA or VB being
12890 // inserted into VA (or undef). If successful, V1, V2 and InsertPSMask
12891 // are updated.
12892 auto matchAsInsertPS = [&](SDValue VA, SDValue VB,
12893 ArrayRef<int> CandidateMask) {
12894 unsigned ZMask = 0;
12895 int VADstIndex = -1;
12896 int VBDstIndex = -1;
12897 bool VAUsedInPlace = false;
12898
12899 for (int i = 0; i < 4; ++i) {
12900 // Synthesize a zero mask from the zeroable elements (includes undefs).
12901 if (Zeroable[i]) {
12902 ZMask |= 1 << i;
12903 continue;
12904 }
12905
12906 // Flag if we use any VA inputs in place.
12907 if (i == CandidateMask[i]) {
12908 VAUsedInPlace = true;
12909 continue;
12910 }
12911
12912 // We can only insert a single non-zeroable element.
12913 if (VADstIndex >= 0 || VBDstIndex >= 0)
12914 return false;
12915
12916 if (CandidateMask[i] < 4) {
12917 // VA input out of place for insertion.
12918 VADstIndex = i;
12919 } else {
12920 // VB input for insertion.
12921 VBDstIndex = i;
12922 }
12923 }
12924
12925 // Don't bother if we have no (non-zeroable) element for insertion.
12926 if (VADstIndex < 0 && VBDstIndex < 0)
12927 return false;
12928
12929 // Determine element insertion src/dst indices. The src index is from the
12930 // start of the inserted vector, not the start of the concatenated vector.
12931 unsigned VBSrcIndex = 0;
12932 if (VADstIndex >= 0) {
12933 // If we have a VA input out of place, we use VA as the V2 element
12934 // insertion and don't use the original V2 at all.
12935 VBSrcIndex = CandidateMask[VADstIndex];
12936 VBDstIndex = VADstIndex;
12937 VB = VA;
12938 } else {
12939 VBSrcIndex = CandidateMask[VBDstIndex] - 4;
12940 }
12941
12942 // If no V1 inputs are used in place, then the result is created only from
12943 // the zero mask and the V2 insertion - so remove V1 dependency.
12944 if (!VAUsedInPlace)
12945 VA = DAG.getUNDEF(MVT::v4f32);
12946
12947 // Update V1, V2 and InsertPSMask accordingly.
12948 V1 = VA;
12949 V2 = VB;
12950
12951 // Insert the V2 element into the desired position.
12952 InsertPSMask = VBSrcIndex << 6 | VBDstIndex << 4 | ZMask;
12953 assert((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!")(((InsertPSMask & ~0xFFu) == 0 && "Invalid mask!"
) ? static_cast<void> (0) : __assert_fail ("(InsertPSMask & ~0xFFu) == 0 && \"Invalid mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12953, __PRETTY_FUNCTION__));
12954 return true;
12955 };
12956
12957 if (matchAsInsertPS(V1, V2, Mask))
12958 return true;
12959
12960 // Commute and try again.
12961 SmallVector<int, 4> CommutedMask(Mask.begin(), Mask.end());
12962 ShuffleVectorSDNode::commuteMask(CommutedMask);
12963 if (matchAsInsertPS(V2, V1, CommutedMask))
12964 return true;
12965
12966 return false;
12967}
12968
12969static SDValue lowerShuffleAsInsertPS(const SDLoc &DL, SDValue V1, SDValue V2,
12970 ArrayRef<int> Mask, const APInt &Zeroable,
12971 SelectionDAG &DAG) {
12972 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v4f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12972, __PRETTY_FUNCTION__));
12973 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v4f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12973, __PRETTY_FUNCTION__));
12974
12975 // Attempt to match the insertps pattern.
12976 unsigned InsertPSMask;
12977 if (!matchShuffleAsInsertPS(V1, V2, InsertPSMask, Zeroable, Mask, DAG))
12978 return SDValue();
12979
12980 // Insert the V2 element into the desired position.
12981 return DAG.getNode(X86ISD::INSERTPS, DL, MVT::v4f32, V1, V2,
12982 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
12983}
12984
12985/// Try to lower a shuffle as a permute of the inputs followed by an
12986/// UNPCK instruction.
12987///
12988/// This specifically targets cases where we end up with alternating between
12989/// the two inputs, and so can permute them into something that feeds a single
12990/// UNPCK instruction. Note that this routine only targets integer vectors
12991/// because for floating point vectors we have a generalized SHUFPS lowering
12992/// strategy that handles everything that doesn't *exactly* match an unpack,
12993/// making this clever lowering unnecessary.
12994static SDValue lowerShuffleAsPermuteAndUnpack(
12995 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
12996 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
12997 assert(!VT.isFloatingPoint() &&((!VT.isFloatingPoint() && "This routine only supports integer vectors."
) ? static_cast<void> (0) : __assert_fail ("!VT.isFloatingPoint() && \"This routine only supports integer vectors.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12998, __PRETTY_FUNCTION__))
12998 "This routine only supports integer vectors.")((!VT.isFloatingPoint() && "This routine only supports integer vectors."
) ? static_cast<void> (0) : __assert_fail ("!VT.isFloatingPoint() && \"This routine only supports integer vectors.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 12998, __PRETTY_FUNCTION__));
12999 assert(VT.is128BitVector() &&((VT.is128BitVector() && "This routine only works on 128-bit vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"This routine only works on 128-bit vectors.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13000, __PRETTY_FUNCTION__))
13000 "This routine only works on 128-bit vectors.")((VT.is128BitVector() && "This routine only works on 128-bit vectors."
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"This routine only works on 128-bit vectors.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13000, __PRETTY_FUNCTION__));
13001 assert(!V2.isUndef() &&((!V2.isUndef() && "This routine should only be used when blending two inputs."
) ? static_cast<void> (0) : __assert_fail ("!V2.isUndef() && \"This routine should only be used when blending two inputs.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13002, __PRETTY_FUNCTION__))
13002 "This routine should only be used when blending two inputs.")((!V2.isUndef() && "This routine should only be used when blending two inputs."
) ? static_cast<void> (0) : __assert_fail ("!V2.isUndef() && \"This routine should only be used when blending two inputs.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13002, __PRETTY_FUNCTION__));
13003 assert(Mask.size() >= 2 && "Single element masks are invalid.")((Mask.size() >= 2 && "Single element masks are invalid."
) ? static_cast<void> (0) : __assert_fail ("Mask.size() >= 2 && \"Single element masks are invalid.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13003, __PRETTY_FUNCTION__));
13004
13005 int Size = Mask.size();
13006
13007 int NumLoInputs =
13008 count_if(Mask, [Size](int M) { return M >= 0 && M % Size < Size / 2; });
13009 int NumHiInputs =
13010 count_if(Mask, [Size](int M) { return M % Size >= Size / 2; });
13011
13012 bool UnpackLo = NumLoInputs >= NumHiInputs;
13013
13014 auto TryUnpack = [&](int ScalarSize, int Scale) {
13015 SmallVector<int, 16> V1Mask((unsigned)Size, -1);
13016 SmallVector<int, 16> V2Mask((unsigned)Size, -1);
13017
13018 for (int i = 0; i < Size; ++i) {
13019 if (Mask[i] < 0)
13020 continue;
13021
13022 // Each element of the unpack contains Scale elements from this mask.
13023 int UnpackIdx = i / Scale;
13024
13025 // We only handle the case where V1 feeds the first slots of the unpack.
13026 // We rely on canonicalization to ensure this is the case.
13027 if ((UnpackIdx % 2 == 0) != (Mask[i] < Size))
13028 return SDValue();
13029
13030 // Setup the mask for this input. The indexing is tricky as we have to
13031 // handle the unpack stride.
13032 SmallVectorImpl<int> &VMask = (UnpackIdx % 2 == 0) ? V1Mask : V2Mask;
13033 VMask[(UnpackIdx / 2) * Scale + i % Scale + (UnpackLo ? 0 : Size / 2)] =
13034 Mask[i] % Size;
13035 }
13036
13037 // If we will have to shuffle both inputs to use the unpack, check whether
13038 // we can just unpack first and shuffle the result. If so, skip this unpack.
13039 if ((NumLoInputs == 0 || NumHiInputs == 0) && !isNoopShuffleMask(V1Mask) &&
13040 !isNoopShuffleMask(V2Mask))
13041 return SDValue();
13042
13043 // Shuffle the inputs into place.
13044 V1 = DAG.getVectorShuffle(VT, DL, V1, DAG.getUNDEF(VT), V1Mask);
13045 V2 = DAG.getVectorShuffle(VT, DL, V2, DAG.getUNDEF(VT), V2Mask);
13046
13047 // Cast the inputs to the type we will use to unpack them.
13048 MVT UnpackVT = MVT::getVectorVT(MVT::getIntegerVT(ScalarSize), Size / Scale);
13049 V1 = DAG.getBitcast(UnpackVT, V1);
13050 V2 = DAG.getBitcast(UnpackVT, V2);
13051
13052 // Unpack the inputs and cast the result back to the desired type.
13053 return DAG.getBitcast(
13054 VT, DAG.getNode(UnpackLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
13055 UnpackVT, V1, V2));
13056 };
13057
13058 // We try each unpack from the largest to the smallest to try and find one
13059 // that fits this mask.
13060 int OrigScalarSize = VT.getScalarSizeInBits();
13061 for (int ScalarSize = 64; ScalarSize >= OrigScalarSize; ScalarSize /= 2)
13062 if (SDValue Unpack = TryUnpack(ScalarSize, ScalarSize / OrigScalarSize))
13063 return Unpack;
13064
13065 // If we're shuffling with a zero vector then we're better off not doing
13066 // VECTOR_SHUFFLE(UNPCK()) as we lose track of those zero elements.
13067 if (ISD::isBuildVectorAllZeros(V1.getNode()) ||
13068 ISD::isBuildVectorAllZeros(V2.getNode()))
13069 return SDValue();
13070
13071 // If none of the unpack-rooted lowerings worked (or were profitable) try an
13072 // initial unpack.
13073 if (NumLoInputs == 0 || NumHiInputs == 0) {
13074 assert((NumLoInputs > 0 || NumHiInputs > 0) &&(((NumLoInputs > 0 || NumHiInputs > 0) && "We have to have *some* inputs!"
) ? static_cast<void> (0) : __assert_fail ("(NumLoInputs > 0 || NumHiInputs > 0) && \"We have to have *some* inputs!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13075, __PRETTY_FUNCTION__))
13075 "We have to have *some* inputs!")(((NumLoInputs > 0 || NumHiInputs > 0) && "We have to have *some* inputs!"
) ? static_cast<void> (0) : __assert_fail ("(NumLoInputs > 0 || NumHiInputs > 0) && \"We have to have *some* inputs!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13075, __PRETTY_FUNCTION__));
13076 int HalfOffset = NumLoInputs == 0 ? Size / 2 : 0;
13077
13078 // FIXME: We could consider the total complexity of the permute of each
13079 // possible unpacking. Or at the least we should consider how many
13080 // half-crossings are created.
13081 // FIXME: We could consider commuting the unpacks.
13082
13083 SmallVector<int, 32> PermMask((unsigned)Size, -1);
13084 for (int i = 0; i < Size; ++i) {
13085 if (Mask[i] < 0)
13086 continue;
13087
13088 assert(Mask[i] % Size >= HalfOffset && "Found input from wrong half!")((Mask[i] % Size >= HalfOffset && "Found input from wrong half!"
) ? static_cast<void> (0) : __assert_fail ("Mask[i] % Size >= HalfOffset && \"Found input from wrong half!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13088, __PRETTY_FUNCTION__));
13089
13090 PermMask[i] =
13091 2 * ((Mask[i] % Size) - HalfOffset) + (Mask[i] < Size ? 0 : 1);
13092 }
13093 return DAG.getVectorShuffle(
13094 VT, DL, DAG.getNode(NumLoInputs == 0 ? X86ISD::UNPCKH : X86ISD::UNPCKL,
13095 DL, VT, V1, V2),
13096 DAG.getUNDEF(VT), PermMask);
13097 }
13098
13099 return SDValue();
13100}
13101
13102/// Handle lowering of 2-lane 64-bit floating point shuffles.
13103///
13104/// This is the basis function for the 2-lane 64-bit shuffles as we have full
13105/// support for floating point shuffles but not integer shuffles. These
13106/// instructions will incur a domain crossing penalty on some chips though so
13107/// it is better to avoid lowering through this for integer vectors where
13108/// possible.
13109static SDValue lowerV2F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13110 const APInt &Zeroable, SDValue V1, SDValue V2,
13111 const X86Subtarget &Subtarget,
13112 SelectionDAG &DAG) {
13113 assert(V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v2f64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13113, __PRETTY_FUNCTION__));
13114 assert(V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v2f64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v2f64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13114, __PRETTY_FUNCTION__));
13115 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!")((Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 2 && \"Unexpected mask size for v2 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13115, __PRETTY_FUNCTION__));
13116
13117 if (V2.isUndef()) {
13118 // Check for being able to broadcast a single element.
13119 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2f64, V1, V2,
13120 Mask, Subtarget, DAG))
13121 return Broadcast;
13122
13123 // Straight shuffle of a single input vector. Simulate this by using the
13124 // single input as both of the "inputs" to this instruction..
13125 unsigned SHUFPDMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1);
13126
13127 if (Subtarget.hasAVX()) {
13128 // If we have AVX, we can use VPERMILPS which will allow folding a load
13129 // into the shuffle.
13130 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v2f64, V1,
13131 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13132 }
13133
13134 return DAG.getNode(
13135 X86ISD::SHUFP, DL, MVT::v2f64,
13136 Mask[0] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
13137 Mask[1] == SM_SentinelUndef ? DAG.getUNDEF(MVT::v2f64) : V1,
13138 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13139 }
13140 assert(Mask[0] >= 0 && "No undef lanes in multi-input v2 shuffles!")((Mask[0] >= 0 && "No undef lanes in multi-input v2 shuffles!"
) ? static_cast<void> (0) : __assert_fail ("Mask[0] >= 0 && \"No undef lanes in multi-input v2 shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13140, __PRETTY_FUNCTION__));
13141 assert(Mask[1] >= 0 && "No undef lanes in multi-input v2 shuffles!")((Mask[1] >= 0 && "No undef lanes in multi-input v2 shuffles!"
) ? static_cast<void> (0) : __assert_fail ("Mask[1] >= 0 && \"No undef lanes in multi-input v2 shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13141, __PRETTY_FUNCTION__));
13142 assert(Mask[0] < 2 && "We sort V1 to be the first input.")((Mask[0] < 2 && "We sort V1 to be the first input."
) ? static_cast<void> (0) : __assert_fail ("Mask[0] < 2 && \"We sort V1 to be the first input.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13142, __PRETTY_FUNCTION__));
13143 assert(Mask[1] >= 2 && "We sort V2 to be the second input.")((Mask[1] >= 2 && "We sort V2 to be the second input."
) ? static_cast<void> (0) : __assert_fail ("Mask[1] >= 2 && \"We sort V2 to be the second input.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13143, __PRETTY_FUNCTION__));
13144
13145 if (Subtarget.hasAVX2())
13146 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13147 return Extract;
13148
13149 // When loading a scalar and then shuffling it into a vector we can often do
13150 // the insertion cheaply.
13151 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13152 DL, MVT::v2f64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13153 return Insertion;
13154 // Try inverting the insertion since for v2 masks it is easy to do and we
13155 // can't reliably sort the mask one way or the other.
13156 int InverseMask[2] = {Mask[0] < 0 ? -1 : (Mask[0] ^ 2),
13157 Mask[1] < 0 ? -1 : (Mask[1] ^ 2)};
13158 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13159 DL, MVT::v2f64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13160 return Insertion;
13161
13162 // Try to use one of the special instruction patterns to handle two common
13163 // blend patterns if a zero-blend above didn't work.
13164 if (isShuffleEquivalent(V1, V2, Mask, {0, 3}) ||
13165 isShuffleEquivalent(V1, V2, Mask, {1, 3}))
13166 if (SDValue V1S = getScalarValueForVectorElement(V1, Mask[0], DAG))
13167 // We can either use a special instruction to load over the low double or
13168 // to move just the low double.
13169 return DAG.getNode(
13170 X86ISD::MOVSD, DL, MVT::v2f64, V2,
13171 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, V1S));
13172
13173 if (Subtarget.hasSSE41())
13174 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2f64, V1, V2, Mask,
13175 Zeroable, Subtarget, DAG))
13176 return Blend;
13177
13178 // Use dedicated unpack instructions for masks that match their pattern.
13179 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2f64, Mask, V1, V2, DAG))
13180 return V;
13181
13182 unsigned SHUFPDMask = (Mask[0] == 1) | (((Mask[1] - 2) == 1) << 1);
13183 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v2f64, V1, V2,
13184 DAG.getTargetConstant(SHUFPDMask, DL, MVT::i8));
13185}
13186
13187/// Handle lowering of 2-lane 64-bit integer shuffles.
13188///
13189/// Tries to lower a 2-lane 64-bit shuffle using shuffle operations provided by
13190/// the integer unit to minimize domain crossing penalties. However, for blends
13191/// it falls back to the floating point shuffle operation with appropriate bit
13192/// casting.
13193static SDValue lowerV2I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13194 const APInt &Zeroable, SDValue V1, SDValue V2,
13195 const X86Subtarget &Subtarget,
13196 SelectionDAG &DAG) {
13197 assert(V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v2i64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13197, __PRETTY_FUNCTION__));
13198 assert(V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v2i64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v2i64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13198, __PRETTY_FUNCTION__));
13199 assert(Mask.size() == 2 && "Unexpected mask size for v2 shuffle!")((Mask.size() == 2 && "Unexpected mask size for v2 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 2 && \"Unexpected mask size for v2 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13199, __PRETTY_FUNCTION__));
13200
13201 if (V2.isUndef()) {
13202 // Check for being able to broadcast a single element.
13203 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v2i64, V1, V2,
13204 Mask, Subtarget, DAG))
13205 return Broadcast;
13206
13207 // Straight shuffle of a single input vector. For everything from SSE2
13208 // onward this has a single fast instruction with no scary immediates.
13209 // We have to map the mask as it is actually a v4i32 shuffle instruction.
13210 V1 = DAG.getBitcast(MVT::v4i32, V1);
13211 int WidenedMask[4] = {
13212 std::max(Mask[0], 0) * 2, std::max(Mask[0], 0) * 2 + 1,
13213 std::max(Mask[1], 0) * 2, std::max(Mask[1], 0) * 2 + 1};
13214 return DAG.getBitcast(
13215 MVT::v2i64,
13216 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13217 getV4X86ShuffleImm8ForMask(WidenedMask, DL, DAG)));
13218 }
13219 assert(Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!")((Mask[0] != -1 && "No undef lanes in multi-input v2 shuffles!"
) ? static_cast<void> (0) : __assert_fail ("Mask[0] != -1 && \"No undef lanes in multi-input v2 shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13219, __PRETTY_FUNCTION__));
13220 assert(Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!")((Mask[1] != -1 && "No undef lanes in multi-input v2 shuffles!"
) ? static_cast<void> (0) : __assert_fail ("Mask[1] != -1 && \"No undef lanes in multi-input v2 shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13220, __PRETTY_FUNCTION__));
13221 assert(Mask[0] < 2 && "We sort V1 to be the first input.")((Mask[0] < 2 && "We sort V1 to be the first input."
) ? static_cast<void> (0) : __assert_fail ("Mask[0] < 2 && \"We sort V1 to be the first input.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13221, __PRETTY_FUNCTION__));
13222 assert(Mask[1] >= 2 && "We sort V2 to be the second input.")((Mask[1] >= 2 && "We sort V2 to be the second input."
) ? static_cast<void> (0) : __assert_fail ("Mask[1] >= 2 && \"We sort V2 to be the second input.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13222, __PRETTY_FUNCTION__));
13223
13224 if (Subtarget.hasAVX2())
13225 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13226 return Extract;
13227
13228 // Try to use shift instructions.
13229 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v2i64, V1, V2, Mask,
13230 Zeroable, Subtarget, DAG))
13231 return Shift;
13232
13233 // When loading a scalar and then shuffling it into a vector we can often do
13234 // the insertion cheaply.
13235 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13236 DL, MVT::v2i64, V1, V2, Mask, Zeroable, Subtarget, DAG))
13237 return Insertion;
13238 // Try inverting the insertion since for v2 masks it is easy to do and we
13239 // can't reliably sort the mask one way or the other.
13240 int InverseMask[2] = {Mask[0] ^ 2, Mask[1] ^ 2};
13241 if (SDValue Insertion = lowerShuffleAsElementInsertion(
13242 DL, MVT::v2i64, V2, V1, InverseMask, Zeroable, Subtarget, DAG))
13243 return Insertion;
13244
13245 // We have different paths for blend lowering, but they all must use the
13246 // *exact* same predicate.
13247 bool IsBlendSupported = Subtarget.hasSSE41();
13248 if (IsBlendSupported)
13249 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v2i64, V1, V2, Mask,
13250 Zeroable, Subtarget, DAG))
13251 return Blend;
13252
13253 // Use dedicated unpack instructions for masks that match their pattern.
13254 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v2i64, Mask, V1, V2, DAG))
13255 return V;
13256
13257 // Try to use byte rotation instructions.
13258 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
13259 if (Subtarget.hasSSSE3()) {
13260 if (Subtarget.hasVLX())
13261 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v2i64, V1, V2, Mask,
13262 Subtarget, DAG))
13263 return Rotate;
13264
13265 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v2i64, V1, V2, Mask,
13266 Subtarget, DAG))
13267 return Rotate;
13268 }
13269
13270 // If we have direct support for blends, we should lower by decomposing into
13271 // a permute. That will be faster than the domain cross.
13272 if (IsBlendSupported)
13273 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v2i64, V1, V2, Mask,
13274 Subtarget, DAG);
13275
13276 // We implement this with SHUFPD which is pretty lame because it will likely
13277 // incur 2 cycles of stall for integer vectors on Nehalem and older chips.
13278 // However, all the alternatives are still more cycles and newer chips don't
13279 // have this problem. It would be really nice if x86 had better shuffles here.
13280 V1 = DAG.getBitcast(MVT::v2f64, V1);
13281 V2 = DAG.getBitcast(MVT::v2f64, V2);
13282 return DAG.getBitcast(MVT::v2i64,
13283 DAG.getVectorShuffle(MVT::v2f64, DL, V1, V2, Mask));
13284}
13285
13286/// Lower a vector shuffle using the SHUFPS instruction.
13287///
13288/// This is a helper routine dedicated to lowering vector shuffles using SHUFPS.
13289/// It makes no assumptions about whether this is the *best* lowering, it simply
13290/// uses it.
13291static SDValue lowerShuffleWithSHUFPS(const SDLoc &DL, MVT VT,
13292 ArrayRef<int> Mask, SDValue V1,
13293 SDValue V2, SelectionDAG &DAG) {
13294 SDValue LowV = V1, HighV = V2;
13295 int NewMask[4] = {Mask[0], Mask[1], Mask[2], Mask[3]};
13296
13297 int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
13298
13299 if (NumV2Elements == 1) {
13300 int V2Index = find_if(Mask, [](int M) { return M >= 4; }) - Mask.begin();
13301
13302 // Compute the index adjacent to V2Index and in the same half by toggling
13303 // the low bit.
13304 int V2AdjIndex = V2Index ^ 1;
13305
13306 if (Mask[V2AdjIndex] < 0) {
13307 // Handles all the cases where we have a single V2 element and an undef.
13308 // This will only ever happen in the high lanes because we commute the
13309 // vector otherwise.
13310 if (V2Index < 2)
13311 std::swap(LowV, HighV);
13312 NewMask[V2Index] -= 4;
13313 } else {
13314 // Handle the case where the V2 element ends up adjacent to a V1 element.
13315 // To make this work, blend them together as the first step.
13316 int V1Index = V2AdjIndex;
13317 int BlendMask[4] = {Mask[V2Index] - 4, 0, Mask[V1Index], 0};
13318 V2 = DAG.getNode(X86ISD::SHUFP, DL, VT, V2, V1,
13319 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13320
13321 // Now proceed to reconstruct the final blend as we have the necessary
13322 // high or low half formed.
13323 if (V2Index < 2) {
13324 LowV = V2;
13325 HighV = V1;
13326 } else {
13327 HighV = V2;
13328 }
13329 NewMask[V1Index] = 2; // We put the V1 element in V2[2].
13330 NewMask[V2Index] = 0; // We shifted the V2 element into V2[0].
13331 }
13332 } else if (NumV2Elements == 2) {
13333 if (Mask[0] < 4 && Mask[1] < 4) {
13334 // Handle the easy case where we have V1 in the low lanes and V2 in the
13335 // high lanes.
13336 NewMask[2] -= 4;
13337 NewMask[3] -= 4;
13338 } else if (Mask[2] < 4 && Mask[3] < 4) {
13339 // We also handle the reversed case because this utility may get called
13340 // when we detect a SHUFPS pattern but can't easily commute the shuffle to
13341 // arrange things in the right direction.
13342 NewMask[0] -= 4;
13343 NewMask[1] -= 4;
13344 HighV = V1;
13345 LowV = V2;
13346 } else {
13347 // We have a mixture of V1 and V2 in both low and high lanes. Rather than
13348 // trying to place elements directly, just blend them and set up the final
13349 // shuffle to place them.
13350
13351 // The first two blend mask elements are for V1, the second two are for
13352 // V2.
13353 int BlendMask[4] = {Mask[0] < 4 ? Mask[0] : Mask[1],
13354 Mask[2] < 4 ? Mask[2] : Mask[3],
13355 (Mask[0] >= 4 ? Mask[0] : Mask[1]) - 4,
13356 (Mask[2] >= 4 ? Mask[2] : Mask[3]) - 4};
13357 V1 = DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
13358 getV4X86ShuffleImm8ForMask(BlendMask, DL, DAG));
13359
13360 // Now we do a normal shuffle of V1 by giving V1 as both operands to
13361 // a blend.
13362 LowV = HighV = V1;
13363 NewMask[0] = Mask[0] < 4 ? 0 : 2;
13364 NewMask[1] = Mask[0] < 4 ? 2 : 0;
13365 NewMask[2] = Mask[2] < 4 ? 1 : 3;
13366 NewMask[3] = Mask[2] < 4 ? 3 : 1;
13367 }
13368 }
13369 return DAG.getNode(X86ISD::SHUFP, DL, VT, LowV, HighV,
13370 getV4X86ShuffleImm8ForMask(NewMask, DL, DAG));
13371}
13372
13373/// Lower 4-lane 32-bit floating point shuffles.
13374///
13375/// Uses instructions exclusively from the floating point unit to minimize
13376/// domain crossing penalties, as these are sufficient to implement all v4f32
13377/// shuffles.
13378static SDValue lowerV4F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13379 const APInt &Zeroable, SDValue V1, SDValue V2,
13380 const X86Subtarget &Subtarget,
13381 SelectionDAG &DAG) {
13382 assert(V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v4f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13382, __PRETTY_FUNCTION__));
13383 assert(V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v4f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v4f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13383, __PRETTY_FUNCTION__));
13384 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!")((Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 4 && \"Unexpected mask size for v4 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13384, __PRETTY_FUNCTION__));
13385
13386 int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
13387
13388 if (NumV2Elements == 0) {
13389 // Check for being able to broadcast a single element.
13390 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f32, V1, V2,
13391 Mask, Subtarget, DAG))
13392 return Broadcast;
13393
13394 // Use even/odd duplicate instructions for masks that match their pattern.
13395 if (Subtarget.hasSSE3()) {
13396 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2}))
13397 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v4f32, V1);
13398 if (isShuffleEquivalent(V1, V2, Mask, {1, 1, 3, 3}))
13399 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v4f32, V1);
13400 }
13401
13402 if (Subtarget.hasAVX()) {
13403 // If we have AVX, we can use VPERMILPS which will allow folding a load
13404 // into the shuffle.
13405 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f32, V1,
13406 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13407 }
13408
13409 // Use MOVLHPS/MOVHLPS to simulate unary shuffles. These are only valid
13410 // in SSE1 because otherwise they are widened to v2f64 and never get here.
13411 if (!Subtarget.hasSSE2()) {
13412 if (isShuffleEquivalent(V1, V2, Mask, {0, 1, 0, 1}))
13413 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V1);
13414 if (isShuffleEquivalent(V1, V2, Mask, {2, 3, 2, 3}))
13415 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V1, V1);
13416 }
13417
13418 // Otherwise, use a straight shuffle of a single input vector. We pass the
13419 // input vector to both operands to simulate this with a SHUFPS.
13420 return DAG.getNode(X86ISD::SHUFP, DL, MVT::v4f32, V1, V1,
13421 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13422 }
13423
13424 if (Subtarget.hasAVX2())
13425 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13426 return Extract;
13427
13428 // There are special ways we can lower some single-element blends. However, we
13429 // have custom ways we can lower more complex single-element blends below that
13430 // we defer to if both this and BLENDPS fail to match, so restrict this to
13431 // when the V2 input is targeting element 0 of the mask -- that is the fast
13432 // case here.
13433 if (NumV2Elements == 1 && Mask[0] >= 4)
13434 if (SDValue V = lowerShuffleAsElementInsertion(
13435 DL, MVT::v4f32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13436 return V;
13437
13438 if (Subtarget.hasSSE41()) {
13439 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f32, V1, V2, Mask,
13440 Zeroable, Subtarget, DAG))
13441 return Blend;
13442
13443 // Use INSERTPS if we can complete the shuffle efficiently.
13444 if (SDValue V = lowerShuffleAsInsertPS(DL, V1, V2, Mask, Zeroable, DAG))
13445 return V;
13446
13447 if (!isSingleSHUFPSMask(Mask))
13448 if (SDValue BlendPerm = lowerShuffleAsBlendAndPermute(DL, MVT::v4f32, V1,
13449 V2, Mask, DAG))
13450 return BlendPerm;
13451 }
13452
13453 // Use low/high mov instructions. These are only valid in SSE1 because
13454 // otherwise they are widened to v2f64 and never get here.
13455 if (!Subtarget.hasSSE2()) {
13456 if (isShuffleEquivalent(V1, V2, Mask, {0, 1, 4, 5}))
13457 return DAG.getNode(X86ISD::MOVLHPS, DL, MVT::v4f32, V1, V2);
13458 if (isShuffleEquivalent(V1, V2, Mask, {2, 3, 6, 7}))
13459 return DAG.getNode(X86ISD::MOVHLPS, DL, MVT::v4f32, V2, V1);
13460 }
13461
13462 // Use dedicated unpack instructions for masks that match their pattern.
13463 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f32, Mask, V1, V2, DAG))
13464 return V;
13465
13466 // Otherwise fall back to a SHUFPS lowering strategy.
13467 return lowerShuffleWithSHUFPS(DL, MVT::v4f32, Mask, V1, V2, DAG);
13468}
13469
13470/// Lower 4-lane i32 vector shuffles.
13471///
13472/// We try to handle these with integer-domain shuffles where we can, but for
13473/// blends we use the floating point domain blend instructions.
13474static SDValue lowerV4I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
13475 const APInt &Zeroable, SDValue V1, SDValue V2,
13476 const X86Subtarget &Subtarget,
13477 SelectionDAG &DAG) {
13478 assert(V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v4i32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13478, __PRETTY_FUNCTION__));
13479 assert(V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v4i32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v4i32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13479, __PRETTY_FUNCTION__));
13480 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!")((Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 4 && \"Unexpected mask size for v4 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13480, __PRETTY_FUNCTION__));
13481
13482 // Whenever we can lower this as a zext, that instruction is strictly faster
13483 // than any alternative. It also allows us to fold memory operands into the
13484 // shuffle in many cases.
13485 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v4i32, V1, V2, Mask,
13486 Zeroable, Subtarget, DAG))
13487 return ZExt;
13488
13489 int NumV2Elements = count_if(Mask, [](int M) { return M >= 4; });
13490
13491 if (NumV2Elements == 0) {
13492 // Try to use broadcast unless the mask only has one non-undef element.
13493 if (count_if(Mask, [](int M) { return M >= 0 && M < 4; }) > 1) {
13494 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i32, V1, V2,
13495 Mask, Subtarget, DAG))
13496 return Broadcast;
13497 }
13498
13499 // Straight shuffle of a single input vector. For everything from SSE2
13500 // onward this has a single fast instruction with no scary immediates.
13501 // We coerce the shuffle pattern to be compatible with UNPCK instructions
13502 // but we aren't actually going to use the UNPCK instruction because doing
13503 // so prevents folding a load into this instruction or making a copy.
13504 const int UnpackLoMask[] = {0, 0, 1, 1};
13505 const int UnpackHiMask[] = {2, 2, 3, 3};
13506 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 1, 1}))
13507 Mask = UnpackLoMask;
13508 else if (isShuffleEquivalent(V1, V2, Mask, {2, 2, 3, 3}))
13509 Mask = UnpackHiMask;
13510
13511 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v4i32, V1,
13512 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
13513 }
13514
13515 if (Subtarget.hasAVX2())
13516 if (SDValue Extract = lowerShuffleOfExtractsAsVperm(DL, V1, V2, Mask, DAG))
13517 return Extract;
13518
13519 // Try to use shift instructions.
13520 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i32, V1, V2, Mask,
13521 Zeroable, Subtarget, DAG))
13522 return Shift;
13523
13524 // There are special ways we can lower some single-element blends.
13525 if (NumV2Elements == 1)
13526 if (SDValue V = lowerShuffleAsElementInsertion(
13527 DL, MVT::v4i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
13528 return V;
13529
13530 // We have different paths for blend lowering, but they all must use the
13531 // *exact* same predicate.
13532 bool IsBlendSupported = Subtarget.hasSSE41();
13533 if (IsBlendSupported)
13534 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i32, V1, V2, Mask,
13535 Zeroable, Subtarget, DAG))
13536 return Blend;
13537
13538 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v4i32, V1, V2, Mask,
13539 Zeroable, Subtarget, DAG))
13540 return Masked;
13541
13542 // Use dedicated unpack instructions for masks that match their pattern.
13543 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i32, Mask, V1, V2, DAG))
13544 return V;
13545
13546 // Try to use byte rotation instructions.
13547 // Its more profitable for pre-SSSE3 to use shuffles/unpacks.
13548 if (Subtarget.hasSSSE3()) {
13549 if (Subtarget.hasVLX())
13550 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v4i32, V1, V2, Mask,
13551 Subtarget, DAG))
13552 return Rotate;
13553
13554 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i32, V1, V2, Mask,
13555 Subtarget, DAG))
13556 return Rotate;
13557 }
13558
13559 // Assume that a single SHUFPS is faster than an alternative sequence of
13560 // multiple instructions (even if the CPU has a domain penalty).
13561 // If some CPU is harmed by the domain switch, we can fix it in a later pass.
13562 if (!isSingleSHUFPSMask(Mask)) {
13563 // If we have direct support for blends, we should lower by decomposing into
13564 // a permute. That will be faster than the domain cross.
13565 if (IsBlendSupported)
13566 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i32, V1, V2, Mask,
13567 Subtarget, DAG);
13568
13569 // Try to lower by permuting the inputs into an unpack instruction.
13570 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v4i32, V1, V2,
13571 Mask, Subtarget, DAG))
13572 return Unpack;
13573 }
13574
13575 // We implement this with SHUFPS because it can blend from two vectors.
13576 // Because we're going to eventually use SHUFPS, we use SHUFPS even to build
13577 // up the inputs, bypassing domain shift penalties that we would incur if we
13578 // directly used PSHUFD on Nehalem and older. For newer chips, this isn't
13579 // relevant.
13580 SDValue CastV1 = DAG.getBitcast(MVT::v4f32, V1);
13581 SDValue CastV2 = DAG.getBitcast(MVT::v4f32, V2);
13582 SDValue ShufPS = DAG.getVectorShuffle(MVT::v4f32, DL, CastV1, CastV2, Mask);
13583 return DAG.getBitcast(MVT::v4i32, ShufPS);
13584}
13585
13586/// Lowering of single-input v8i16 shuffles is the cornerstone of SSE2
13587/// shuffle lowering, and the most complex part.
13588///
13589/// The lowering strategy is to try to form pairs of input lanes which are
13590/// targeted at the same half of the final vector, and then use a dword shuffle
13591/// to place them onto the right half, and finally unpack the paired lanes into
13592/// their final position.
13593///
13594/// The exact breakdown of how to form these dword pairs and align them on the
13595/// correct sides is really tricky. See the comments within the function for
13596/// more of the details.
13597///
13598/// This code also handles repeated 128-bit lanes of v8i16 shuffles, but each
13599/// lane must shuffle the *exact* same way. In fact, you must pass a v8 Mask to
13600/// this routine for it to work correctly. To shuffle a 256-bit or 512-bit i16
13601/// vector, form the analogous 128-bit 8-element Mask.
13602static SDValue lowerV8I16GeneralSingleInputShuffle(
13603 const SDLoc &DL, MVT VT, SDValue V, MutableArrayRef<int> Mask,
13604 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
13605 assert(VT.getVectorElementType() == MVT::i16 && "Bad input type!")((VT.getVectorElementType() == MVT::i16 && "Bad input type!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == MVT::i16 && \"Bad input type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13605, __PRETTY_FUNCTION__));
13606 MVT PSHUFDVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
13607
13608 assert(Mask.size() == 8 && "Shuffle mask length doesn't match!")((Mask.size() == 8 && "Shuffle mask length doesn't match!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 8 && \"Shuffle mask length doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13608, __PRETTY_FUNCTION__));
13609 MutableArrayRef<int> LoMask = Mask.slice(0, 4);
13610 MutableArrayRef<int> HiMask = Mask.slice(4, 4);
13611
13612 // Attempt to directly match PSHUFLW or PSHUFHW.
13613 if (isUndefOrInRange(LoMask, 0, 4) &&
13614 isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
13615 return DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
13616 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
13617 }
13618 if (isUndefOrInRange(HiMask, 4, 8) &&
13619 isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
13620 for (int i = 0; i != 4; ++i)
13621 HiMask[i] = (HiMask[i] < 0 ? HiMask[i] : (HiMask[i] - 4));
13622 return DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
13623 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
13624 }
13625
13626 SmallVector<int, 4> LoInputs;
13627 copy_if(LoMask, std::back_inserter(LoInputs), [](int M) { return M >= 0; });
13628 array_pod_sort(LoInputs.begin(), LoInputs.end());
13629 LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()), LoInputs.end());
13630 SmallVector<int, 4> HiInputs;
13631 copy_if(HiMask, std::back_inserter(HiInputs), [](int M) { return M >= 0; });
13632 array_pod_sort(HiInputs.begin(), HiInputs.end());
13633 HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()), HiInputs.end());
13634 int NumLToL = llvm::lower_bound(LoInputs, 4) - LoInputs.begin();
13635 int NumHToL = LoInputs.size() - NumLToL;
13636 int NumLToH = llvm::lower_bound(HiInputs, 4) - HiInputs.begin();
13637 int NumHToH = HiInputs.size() - NumLToH;
13638 MutableArrayRef<int> LToLInputs(LoInputs.data(), NumLToL);
13639 MutableArrayRef<int> LToHInputs(HiInputs.data(), NumLToH);
13640 MutableArrayRef<int> HToLInputs(LoInputs.data() + NumLToL, NumHToL);
13641 MutableArrayRef<int> HToHInputs(HiInputs.data() + NumLToH, NumHToH);
13642
13643 // If we are shuffling values from one half - check how many different DWORD
13644 // pairs we need to create. If only 1 or 2 then we can perform this as a
13645 // PSHUFLW/PSHUFHW + PSHUFD instead of the PSHUFD+PSHUFLW+PSHUFHW chain below.
13646 auto ShuffleDWordPairs = [&](ArrayRef<int> PSHUFHalfMask,
13647 ArrayRef<int> PSHUFDMask, unsigned ShufWOp) {
13648 V = DAG.getNode(ShufWOp, DL, VT, V,
13649 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13650 V = DAG.getBitcast(PSHUFDVT, V);
13651 V = DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, V,
13652 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG));
13653 return DAG.getBitcast(VT, V);
13654 };
13655
13656 if ((NumHToL + NumHToH) == 0 || (NumLToL + NumLToH) == 0) {
13657 int PSHUFDMask[4] = { -1, -1, -1, -1 };
13658 SmallVector<std::pair<int, int>, 4> DWordPairs;
13659 int DOffset = ((NumHToL + NumHToH) == 0 ? 0 : 2);
13660
13661 // Collect the different DWORD pairs.
13662 for (int DWord = 0; DWord != 4; ++DWord) {
13663 int M0 = Mask[2 * DWord + 0];
13664 int M1 = Mask[2 * DWord + 1];
13665 M0 = (M0 >= 0 ? M0 % 4 : M0);
13666 M1 = (M1 >= 0 ? M1 % 4 : M1);
13667 if (M0 < 0 && M1 < 0)
13668 continue;
13669
13670 bool Match = false;
13671 for (int j = 0, e = DWordPairs.size(); j < e; ++j) {
13672 auto &DWordPair = DWordPairs[j];
13673 if ((M0 < 0 || isUndefOrEqual(DWordPair.first, M0)) &&
13674 (M1 < 0 || isUndefOrEqual(DWordPair.second, M1))) {
13675 DWordPair.first = (M0 >= 0 ? M0 : DWordPair.first);
13676 DWordPair.second = (M1 >= 0 ? M1 : DWordPair.second);
13677 PSHUFDMask[DWord] = DOffset + j;
13678 Match = true;
13679 break;
13680 }
13681 }
13682 if (!Match) {
13683 PSHUFDMask[DWord] = DOffset + DWordPairs.size();
13684 DWordPairs.push_back(std::make_pair(M0, M1));
13685 }
13686 }
13687
13688 if (DWordPairs.size() <= 2) {
13689 DWordPairs.resize(2, std::make_pair(-1, -1));
13690 int PSHUFHalfMask[4] = {DWordPairs[0].first, DWordPairs[0].second,
13691 DWordPairs[1].first, DWordPairs[1].second};
13692 if ((NumHToL + NumHToH) == 0)
13693 return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFLW);
13694 if ((NumLToL + NumLToH) == 0)
13695 return ShuffleDWordPairs(PSHUFHalfMask, PSHUFDMask, X86ISD::PSHUFHW);
13696 }
13697 }
13698
13699 // Simplify the 1-into-3 and 3-into-1 cases with a single pshufd. For all
13700 // such inputs we can swap two of the dwords across the half mark and end up
13701 // with <=2 inputs to each half in each half. Once there, we can fall through
13702 // to the generic code below. For example:
13703 //
13704 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13705 // Mask: [0, 1, 2, 7, 4, 5, 6, 3] -----------------> [0, 1, 4, 7, 2, 3, 6, 5]
13706 //
13707 // However in some very rare cases we have a 1-into-3 or 3-into-1 on one half
13708 // and an existing 2-into-2 on the other half. In this case we may have to
13709 // pre-shuffle the 2-into-2 half to avoid turning it into a 3-into-1 or
13710 // 1-into-3 which could cause us to cycle endlessly fixing each side in turn.
13711 // Fortunately, we don't have to handle anything but a 2-into-2 pattern
13712 // because any other situation (including a 3-into-1 or 1-into-3 in the other
13713 // half than the one we target for fixing) will be fixed when we re-enter this
13714 // path. We will also combine away any sequence of PSHUFD instructions that
13715 // result into a single instruction. Here is an example of the tricky case:
13716 //
13717 // Input: [a, b, c, d, e, f, g, h] -PSHUFD[0,2,1,3]-> [a, b, e, f, c, d, g, h]
13718 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -THIS-IS-BAD!!!!-> [5, 7, 1, 0, 4, 7, 5, 3]
13719 //
13720 // This now has a 1-into-3 in the high half! Instead, we do two shuffles:
13721 //
13722 // Input: [a, b, c, d, e, f, g, h] PSHUFHW[0,2,1,3]-> [a, b, c, d, e, g, f, h]
13723 // Mask: [3, 7, 1, 0, 2, 7, 3, 5] -----------------> [3, 7, 1, 0, 2, 7, 3, 6]
13724 //
13725 // Input: [a, b, c, d, e, g, f, h] -PSHUFD[0,2,1,3]-> [a, b, e, g, c, d, f, h]
13726 // Mask: [3, 7, 1, 0, 2, 7, 3, 6] -----------------> [5, 7, 1, 0, 4, 7, 5, 6]
13727 //
13728 // The result is fine to be handled by the generic logic.
13729 auto balanceSides = [&](ArrayRef<int> AToAInputs, ArrayRef<int> BToAInputs,
13730 ArrayRef<int> BToBInputs, ArrayRef<int> AToBInputs,
13731 int AOffset, int BOffset) {
13732 assert((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&(((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&
"Must call this with A having 3 or 1 inputs from the A half."
) ? static_cast<void> (0) : __assert_fail ("(AToAInputs.size() == 3 || AToAInputs.size() == 1) && \"Must call this with A having 3 or 1 inputs from the A half.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13733, __PRETTY_FUNCTION__))
13733 "Must call this with A having 3 or 1 inputs from the A half.")(((AToAInputs.size() == 3 || AToAInputs.size() == 1) &&
"Must call this with A having 3 or 1 inputs from the A half."
) ? static_cast<void> (0) : __assert_fail ("(AToAInputs.size() == 3 || AToAInputs.size() == 1) && \"Must call this with A having 3 or 1 inputs from the A half.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13733, __PRETTY_FUNCTION__));
13734 assert((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&(((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&
"Must call this with B having 1 or 3 inputs from the B half."
) ? static_cast<void> (0) : __assert_fail ("(BToAInputs.size() == 1 || BToAInputs.size() == 3) && \"Must call this with B having 1 or 3 inputs from the B half.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13735, __PRETTY_FUNCTION__))
13735 "Must call this with B having 1 or 3 inputs from the B half.")(((BToAInputs.size() == 1 || BToAInputs.size() == 3) &&
"Must call this with B having 1 or 3 inputs from the B half."
) ? static_cast<void> (0) : __assert_fail ("(BToAInputs.size() == 1 || BToAInputs.size() == 3) && \"Must call this with B having 1 or 3 inputs from the B half.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13735, __PRETTY_FUNCTION__));
13736 assert(AToAInputs.size() + BToAInputs.size() == 4 &&((AToAInputs.size() + BToAInputs.size() == 4 && "Must call this with either 3:1 or 1:3 inputs (summing to 4)."
) ? static_cast<void> (0) : __assert_fail ("AToAInputs.size() + BToAInputs.size() == 4 && \"Must call this with either 3:1 or 1:3 inputs (summing to 4).\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13737, __PRETTY_FUNCTION__))
13737 "Must call this with either 3:1 or 1:3 inputs (summing to 4).")((AToAInputs.size() + BToAInputs.size() == 4 && "Must call this with either 3:1 or 1:3 inputs (summing to 4)."
) ? static_cast<void> (0) : __assert_fail ("AToAInputs.size() + BToAInputs.size() == 4 && \"Must call this with either 3:1 or 1:3 inputs (summing to 4).\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13737, __PRETTY_FUNCTION__));
13738
13739 bool ThreeAInputs = AToAInputs.size() == 3;
13740
13741 // Compute the index of dword with only one word among the three inputs in
13742 // a half by taking the sum of the half with three inputs and subtracting
13743 // the sum of the actual three inputs. The difference is the remaining
13744 // slot.
13745 int ADWord = 0, BDWord = 0;
13746 int &TripleDWord = ThreeAInputs ? ADWord : BDWord;
13747 int &OneInputDWord = ThreeAInputs ? BDWord : ADWord;
13748 int TripleInputOffset = ThreeAInputs ? AOffset : BOffset;
13749 ArrayRef<int> TripleInputs = ThreeAInputs ? AToAInputs : BToAInputs;
13750 int OneInput = ThreeAInputs ? BToAInputs[0] : AToAInputs[0];
13751 int TripleInputSum = 0 + 1 + 2 + 3 + (4 * TripleInputOffset);
13752 int TripleNonInputIdx =
13753 TripleInputSum - std::accumulate(TripleInputs.begin(), TripleInputs.end(), 0);
13754 TripleDWord = TripleNonInputIdx / 2;
13755
13756 // We use xor with one to compute the adjacent DWord to whichever one the
13757 // OneInput is in.
13758 OneInputDWord = (OneInput / 2) ^ 1;
13759
13760 // Check for one tricky case: We're fixing a 3<-1 or a 1<-3 shuffle for AToA
13761 // and BToA inputs. If there is also such a problem with the BToB and AToB
13762 // inputs, we don't try to fix it necessarily -- we'll recurse and see it in
13763 // the next pass. However, if we have a 2<-2 in the BToB and AToB inputs, it
13764 // is essential that we don't *create* a 3<-1 as then we might oscillate.
13765 if (BToBInputs.size() == 2 && AToBInputs.size() == 2) {
13766 // Compute how many inputs will be flipped by swapping these DWords. We
13767 // need
13768 // to balance this to ensure we don't form a 3-1 shuffle in the other
13769 // half.
13770 int NumFlippedAToBInputs =
13771 std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord) +
13772 std::count(AToBInputs.begin(), AToBInputs.end(), 2 * ADWord + 1);
13773 int NumFlippedBToBInputs =
13774 std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord) +
13775 std::count(BToBInputs.begin(), BToBInputs.end(), 2 * BDWord + 1);
13776 if ((NumFlippedAToBInputs == 1 &&
13777 (NumFlippedBToBInputs == 0 || NumFlippedBToBInputs == 2)) ||
13778 (NumFlippedBToBInputs == 1 &&
13779 (NumFlippedAToBInputs == 0 || NumFlippedAToBInputs == 2))) {
13780 // We choose whether to fix the A half or B half based on whether that
13781 // half has zero flipped inputs. At zero, we may not be able to fix it
13782 // with that half. We also bias towards fixing the B half because that
13783 // will more commonly be the high half, and we have to bias one way.
13784 auto FixFlippedInputs = [&V, &DL, &Mask, &DAG](int PinnedIdx, int DWord,
13785 ArrayRef<int> Inputs) {
13786 int FixIdx = PinnedIdx ^ 1; // The adjacent slot to the pinned slot.
13787 bool IsFixIdxInput = is_contained(Inputs, PinnedIdx ^ 1);
13788 // Determine whether the free index is in the flipped dword or the
13789 // unflipped dword based on where the pinned index is. We use this bit
13790 // in an xor to conditionally select the adjacent dword.
13791 int FixFreeIdx = 2 * (DWord ^ (PinnedIdx / 2 == DWord));
13792 bool IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13793 if (IsFixIdxInput == IsFixFreeIdxInput)
13794 FixFreeIdx += 1;
13795 IsFixFreeIdxInput = is_contained(Inputs, FixFreeIdx);
13796 assert(IsFixIdxInput != IsFixFreeIdxInput &&((IsFixIdxInput != IsFixFreeIdxInput && "We need to be changing the number of flipped inputs!"
) ? static_cast<void> (0) : __assert_fail ("IsFixIdxInput != IsFixFreeIdxInput && \"We need to be changing the number of flipped inputs!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13797, __PRETTY_FUNCTION__))
13797 "We need to be changing the number of flipped inputs!")((IsFixIdxInput != IsFixFreeIdxInput && "We need to be changing the number of flipped inputs!"
) ? static_cast<void> (0) : __assert_fail ("IsFixIdxInput != IsFixFreeIdxInput && \"We need to be changing the number of flipped inputs!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13797, __PRETTY_FUNCTION__));
13798 int PSHUFHalfMask[] = {0, 1, 2, 3};
13799 std::swap(PSHUFHalfMask[FixFreeIdx % 4], PSHUFHalfMask[FixIdx % 4]);
13800 V = DAG.getNode(
13801 FixIdx < 4 ? X86ISD::PSHUFLW : X86ISD::PSHUFHW, DL,
13802 MVT::getVectorVT(MVT::i16, V.getValueSizeInBits() / 16), V,
13803 getV4X86ShuffleImm8ForMask(PSHUFHalfMask, DL, DAG));
13804
13805 for (int &M : Mask)
13806 if (M >= 0 && M == FixIdx)
13807 M = FixFreeIdx;
13808 else if (M >= 0 && M == FixFreeIdx)
13809 M = FixIdx;
13810 };
13811 if (NumFlippedBToBInputs != 0) {
13812 int BPinnedIdx =
13813 BToAInputs.size() == 3 ? TripleNonInputIdx : OneInput;
13814 FixFlippedInputs(BPinnedIdx, BDWord, BToBInputs);
13815 } else {
13816 assert(NumFlippedAToBInputs != 0 && "Impossible given predicates!")((NumFlippedAToBInputs != 0 && "Impossible given predicates!"
) ? static_cast<void> (0) : __assert_fail ("NumFlippedAToBInputs != 0 && \"Impossible given predicates!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13816, __PRETTY_FUNCTION__));
13817 int APinnedIdx = ThreeAInputs ? TripleNonInputIdx : OneInput;
13818 FixFlippedInputs(APinnedIdx, ADWord, AToBInputs);
13819 }
13820 }
13821 }
13822
13823 int PSHUFDMask[] = {0, 1, 2, 3};
13824 PSHUFDMask[ADWord] = BDWord;
13825 PSHUFDMask[BDWord] = ADWord;
13826 V = DAG.getBitcast(
13827 VT,
13828 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
13829 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
13830
13831 // Adjust the mask to match the new locations of A and B.
13832 for (int &M : Mask)
13833 if (M >= 0 && M/2 == ADWord)
13834 M = 2 * BDWord + M % 2;
13835 else if (M >= 0 && M/2 == BDWord)
13836 M = 2 * ADWord + M % 2;
13837
13838 // Recurse back into this routine to re-compute state now that this isn't
13839 // a 3 and 1 problem.
13840 return lowerV8I16GeneralSingleInputShuffle(DL, VT, V, Mask, Subtarget, DAG);
13841 };
13842 if ((NumLToL == 3 && NumHToL == 1) || (NumLToL == 1 && NumHToL == 3))
13843 return balanceSides(LToLInputs, HToLInputs, HToHInputs, LToHInputs, 0, 4);
13844 if ((NumHToH == 3 && NumLToH == 1) || (NumHToH == 1 && NumLToH == 3))
13845 return balanceSides(HToHInputs, LToHInputs, LToLInputs, HToLInputs, 4, 0);
13846
13847 // At this point there are at most two inputs to the low and high halves from
13848 // each half. That means the inputs can always be grouped into dwords and
13849 // those dwords can then be moved to the correct half with a dword shuffle.
13850 // We use at most one low and one high word shuffle to collect these paired
13851 // inputs into dwords, and finally a dword shuffle to place them.
13852 int PSHUFLMask[4] = {-1, -1, -1, -1};
13853 int PSHUFHMask[4] = {-1, -1, -1, -1};
13854 int PSHUFDMask[4] = {-1, -1, -1, -1};
13855
13856 // First fix the masks for all the inputs that are staying in their
13857 // original halves. This will then dictate the targets of the cross-half
13858 // shuffles.
13859 auto fixInPlaceInputs =
13860 [&PSHUFDMask](ArrayRef<int> InPlaceInputs, ArrayRef<int> IncomingInputs,
13861 MutableArrayRef<int> SourceHalfMask,
13862 MutableArrayRef<int> HalfMask, int HalfOffset) {
13863 if (InPlaceInputs.empty())
13864 return;
13865 if (InPlaceInputs.size() == 1) {
13866 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13867 InPlaceInputs[0] - HalfOffset;
13868 PSHUFDMask[InPlaceInputs[0] / 2] = InPlaceInputs[0] / 2;
13869 return;
13870 }
13871 if (IncomingInputs.empty()) {
13872 // Just fix all of the in place inputs.
13873 for (int Input : InPlaceInputs) {
13874 SourceHalfMask[Input - HalfOffset] = Input - HalfOffset;
13875 PSHUFDMask[Input / 2] = Input / 2;
13876 }
13877 return;
13878 }
13879
13880 assert(InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!")((InPlaceInputs.size() == 2 && "Cannot handle 3 or 4 inputs!"
) ? static_cast<void> (0) : __assert_fail ("InPlaceInputs.size() == 2 && \"Cannot handle 3 or 4 inputs!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13880, __PRETTY_FUNCTION__));
13881 SourceHalfMask[InPlaceInputs[0] - HalfOffset] =
13882 InPlaceInputs[0] - HalfOffset;
13883 // Put the second input next to the first so that they are packed into
13884 // a dword. We find the adjacent index by toggling the low bit.
13885 int AdjIndex = InPlaceInputs[0] ^ 1;
13886 SourceHalfMask[AdjIndex - HalfOffset] = InPlaceInputs[1] - HalfOffset;
13887 std::replace(HalfMask.begin(), HalfMask.end(), InPlaceInputs[1], AdjIndex);
13888 PSHUFDMask[AdjIndex / 2] = AdjIndex / 2;
13889 };
13890 fixInPlaceInputs(LToLInputs, HToLInputs, PSHUFLMask, LoMask, 0);
13891 fixInPlaceInputs(HToHInputs, LToHInputs, PSHUFHMask, HiMask, 4);
13892
13893 // Now gather the cross-half inputs and place them into a free dword of
13894 // their target half.
13895 // FIXME: This operation could almost certainly be simplified dramatically to
13896 // look more like the 3-1 fixing operation.
13897 auto moveInputsToRightHalf = [&PSHUFDMask](
13898 MutableArrayRef<int> IncomingInputs, ArrayRef<int> ExistingInputs,
13899 MutableArrayRef<int> SourceHalfMask, MutableArrayRef<int> HalfMask,
13900 MutableArrayRef<int> FinalSourceHalfMask, int SourceOffset,
13901 int DestOffset) {
13902 auto isWordClobbered = [](ArrayRef<int> SourceHalfMask, int Word) {
13903 return SourceHalfMask[Word] >= 0 && SourceHalfMask[Word] != Word;
13904 };
13905 auto isDWordClobbered = [&isWordClobbered](ArrayRef<int> SourceHalfMask,
13906 int Word) {
13907 int LowWord = Word & ~1;
13908 int HighWord = Word | 1;
13909 return isWordClobbered(SourceHalfMask, LowWord) ||
13910 isWordClobbered(SourceHalfMask, HighWord);
13911 };
13912
13913 if (IncomingInputs.empty())
13914 return;
13915
13916 if (ExistingInputs.empty()) {
13917 // Map any dwords with inputs from them into the right half.
13918 for (int Input : IncomingInputs) {
13919 // If the source half mask maps over the inputs, turn those into
13920 // swaps and use the swapped lane.
13921 if (isWordClobbered(SourceHalfMask, Input - SourceOffset)) {
13922 if (SourceHalfMask[SourceHalfMask[Input - SourceOffset]] < 0) {
13923 SourceHalfMask[SourceHalfMask[Input - SourceOffset]] =
13924 Input - SourceOffset;
13925 // We have to swap the uses in our half mask in one sweep.
13926 for (int &M : HalfMask)
13927 if (M == SourceHalfMask[Input - SourceOffset] + SourceOffset)
13928 M = Input;
13929 else if (M == Input)
13930 M = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13931 } else {
13932 assert(SourceHalfMask[SourceHalfMask[Input - SourceOffset]] ==((SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == Input
- SourceOffset && "Previous placement doesn't match!"
) ? static_cast<void> (0) : __assert_fail ("SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == Input - SourceOffset && \"Previous placement doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13934, __PRETTY_FUNCTION__))
13933 Input - SourceOffset &&((SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == Input
- SourceOffset && "Previous placement doesn't match!"
) ? static_cast<void> (0) : __assert_fail ("SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == Input - SourceOffset && \"Previous placement doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13934, __PRETTY_FUNCTION__))
13934 "Previous placement doesn't match!")((SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == Input
- SourceOffset && "Previous placement doesn't match!"
) ? static_cast<void> (0) : __assert_fail ("SourceHalfMask[SourceHalfMask[Input - SourceOffset]] == Input - SourceOffset && \"Previous placement doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13934, __PRETTY_FUNCTION__));
13935 }
13936 // Note that this correctly re-maps both when we do a swap and when
13937 // we observe the other side of the swap above. We rely on that to
13938 // avoid swapping the members of the input list directly.
13939 Input = SourceHalfMask[Input - SourceOffset] + SourceOffset;
13940 }
13941
13942 // Map the input's dword into the correct half.
13943 if (PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] < 0)
13944 PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] = Input / 2;
13945 else
13946 assert(PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] ==((PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == Input
/ 2 && "Previous placement doesn't match!") ? static_cast
<void> (0) : __assert_fail ("PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == Input / 2 && \"Previous placement doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13948, __PRETTY_FUNCTION__))
13947 Input / 2 &&((PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == Input
/ 2 && "Previous placement doesn't match!") ? static_cast
<void> (0) : __assert_fail ("PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == Input / 2 && \"Previous placement doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13948, __PRETTY_FUNCTION__))
13948 "Previous placement doesn't match!")((PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == Input
/ 2 && "Previous placement doesn't match!") ? static_cast
<void> (0) : __assert_fail ("PSHUFDMask[(Input - SourceOffset + DestOffset) / 2] == Input / 2 && \"Previous placement doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13948, __PRETTY_FUNCTION__));
13949 }
13950
13951 // And just directly shift any other-half mask elements to be same-half
13952 // as we will have mirrored the dword containing the element into the
13953 // same position within that half.
13954 for (int &M : HalfMask)
13955 if (M >= SourceOffset && M < SourceOffset + 4) {
13956 M = M - SourceOffset + DestOffset;
13957 assert(M >= 0 && "This should never wrap below zero!")((M >= 0 && "This should never wrap below zero!") ?
static_cast<void> (0) : __assert_fail ("M >= 0 && \"This should never wrap below zero!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 13957, __PRETTY_FUNCTION__));
13958 }
13959 return;
13960 }
13961
13962 // Ensure we have the input in a viable dword of its current half. This
13963 // is particularly tricky because the original position may be clobbered
13964 // by inputs being moved and *staying* in that half.
13965 if (IncomingInputs.size() == 1) {
13966 if (isWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13967 int InputFixed = find(SourceHalfMask, -1) - std::begin(SourceHalfMask) +
13968 SourceOffset;
13969 SourceHalfMask[InputFixed - SourceOffset] =
13970 IncomingInputs[0] - SourceOffset;
13971 std::replace(HalfMask.begin(), HalfMask.end(), IncomingInputs[0],
13972 InputFixed);
13973 IncomingInputs[0] = InputFixed;
13974 }
13975 } else if (IncomingInputs.size() == 2) {
13976 if (IncomingInputs[0] / 2 != IncomingInputs[1] / 2 ||
13977 isDWordClobbered(SourceHalfMask, IncomingInputs[0] - SourceOffset)) {
13978 // We have two non-adjacent or clobbered inputs we need to extract from
13979 // the source half. To do this, we need to map them into some adjacent
13980 // dword slot in the source mask.
13981 int InputsFixed[2] = {IncomingInputs[0] - SourceOffset,
13982 IncomingInputs[1] - SourceOffset};
13983
13984 // If there is a free slot in the source half mask adjacent to one of
13985 // the inputs, place the other input in it. We use (Index XOR 1) to
13986 // compute an adjacent index.
13987 if (!isWordClobbered(SourceHalfMask, InputsFixed[0]) &&
13988 SourceHalfMask[InputsFixed[0] ^ 1] < 0) {
13989 SourceHalfMask[InputsFixed[0]] = InputsFixed[0];
13990 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
13991 InputsFixed[1] = InputsFixed[0] ^ 1;
13992 } else if (!isWordClobbered(SourceHalfMask, InputsFixed[1]) &&
13993 SourceHalfMask[InputsFixed[1] ^ 1] < 0) {
13994 SourceHalfMask[InputsFixed[1]] = InputsFixed[1];
13995 SourceHalfMask[InputsFixed[1] ^ 1] = InputsFixed[0];
13996 InputsFixed[0] = InputsFixed[1] ^ 1;
13997 } else if (SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] < 0 &&
13998 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] < 0) {
13999 // The two inputs are in the same DWord but it is clobbered and the
14000 // adjacent DWord isn't used at all. Move both inputs to the free
14001 // slot.
14002 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1)] = InputsFixed[0];
14003 SourceHalfMask[2 * ((InputsFixed[0] / 2) ^ 1) + 1] = InputsFixed[1];
14004 InputsFixed[0] = 2 * ((InputsFixed[0] / 2) ^ 1);
14005 InputsFixed[1] = 2 * ((InputsFixed[0] / 2) ^ 1) + 1;
14006 } else {
14007 // The only way we hit this point is if there is no clobbering
14008 // (because there are no off-half inputs to this half) and there is no
14009 // free slot adjacent to one of the inputs. In this case, we have to
14010 // swap an input with a non-input.
14011 for (int i = 0; i < 4; ++i)
14012 assert((SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) &&(((SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) &&
"We can't handle any clobbers here!") ? static_cast<void>
(0) : __assert_fail ("(SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) && \"We can't handle any clobbers here!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14013, __PRETTY_FUNCTION__))
14013 "We can't handle any clobbers here!")(((SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) &&
"We can't handle any clobbers here!") ? static_cast<void>
(0) : __assert_fail ("(SourceHalfMask[i] < 0 || SourceHalfMask[i] == i) && \"We can't handle any clobbers here!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14013, __PRETTY_FUNCTION__));
14014 assert(InputsFixed[1] != (InputsFixed[0] ^ 1) &&((InputsFixed[1] != (InputsFixed[0] ^ 1) && "Cannot have adjacent inputs here!"
) ? static_cast<void> (0) : __assert_fail ("InputsFixed[1] != (InputsFixed[0] ^ 1) && \"Cannot have adjacent inputs here!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14015, __PRETTY_FUNCTION__))
14015 "Cannot have adjacent inputs here!")((InputsFixed[1] != (InputsFixed[0] ^ 1) && "Cannot have adjacent inputs here!"
) ? static_cast<void> (0) : __assert_fail ("InputsFixed[1] != (InputsFixed[0] ^ 1) && \"Cannot have adjacent inputs here!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14015, __PRETTY_FUNCTION__));
14016
14017 SourceHalfMask[InputsFixed[0] ^ 1] = InputsFixed[1];
14018 SourceHalfMask[InputsFixed[1]] = InputsFixed[0] ^ 1;
14019
14020 // We also have to update the final source mask in this case because
14021 // it may need to undo the above swap.
14022 for (int &M : FinalSourceHalfMask)
14023 if (M == (InputsFixed[0] ^ 1) + SourceOffset)
14024 M = InputsFixed[1] + SourceOffset;
14025 else if (M == InputsFixed[1] + SourceOffset)
14026 M = (InputsFixed[0] ^ 1) + SourceOffset;
14027
14028 InputsFixed[1] = InputsFixed[0] ^ 1;
14029 }
14030
14031 // Point everything at the fixed inputs.
14032 for (int &M : HalfMask)
14033 if (M == IncomingInputs[0])
14034 M = InputsFixed[0] + SourceOffset;
14035 else if (M == IncomingInputs[1])
14036 M = InputsFixed[1] + SourceOffset;
14037
14038 IncomingInputs[0] = InputsFixed[0] + SourceOffset;
14039 IncomingInputs[1] = InputsFixed[1] + SourceOffset;
14040 }
14041 } else {
14042 llvm_unreachable("Unhandled input size!")::llvm::llvm_unreachable_internal("Unhandled input size!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14042);
14043 }
14044
14045 // Now hoist the DWord down to the right half.
14046 int FreeDWord = (PSHUFDMask[DestOffset / 2] < 0 ? 0 : 1) + DestOffset / 2;
14047 assert(PSHUFDMask[FreeDWord] < 0 && "DWord not free")((PSHUFDMask[FreeDWord] < 0 && "DWord not free") ?
static_cast<void> (0) : __assert_fail ("PSHUFDMask[FreeDWord] < 0 && \"DWord not free\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14047, __PRETTY_FUNCTION__));
14048 PSHUFDMask[FreeDWord] = IncomingInputs[0] / 2;
14049 for (int &M : HalfMask)
14050 for (int Input : IncomingInputs)
14051 if (M == Input)
14052 M = FreeDWord * 2 + Input % 2;
14053 };
14054 moveInputsToRightHalf(HToLInputs, LToLInputs, PSHUFHMask, LoMask, HiMask,
14055 /*SourceOffset*/ 4, /*DestOffset*/ 0);
14056 moveInputsToRightHalf(LToHInputs, HToHInputs, PSHUFLMask, HiMask, LoMask,
14057 /*SourceOffset*/ 0, /*DestOffset*/ 4);
14058
14059 // Now enact all the shuffles we've computed to move the inputs into their
14060 // target half.
14061 if (!isNoopShuffleMask(PSHUFLMask))
14062 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
14063 getV4X86ShuffleImm8ForMask(PSHUFLMask, DL, DAG));
14064 if (!isNoopShuffleMask(PSHUFHMask))
14065 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
14066 getV4X86ShuffleImm8ForMask(PSHUFHMask, DL, DAG));
14067 if (!isNoopShuffleMask(PSHUFDMask))
14068 V = DAG.getBitcast(
14069 VT,
14070 DAG.getNode(X86ISD::PSHUFD, DL, PSHUFDVT, DAG.getBitcast(PSHUFDVT, V),
14071 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
14072
14073 // At this point, each half should contain all its inputs, and we can then
14074 // just shuffle them into their final position.
14075 assert(count_if(LoMask, [](int M) { return M >= 4; }) == 0 &&((count_if(LoMask, [](int M) { return M >= 4; }) == 0 &&
"Failed to lift all the high half inputs to the low mask!") ?
static_cast<void> (0) : __assert_fail ("count_if(LoMask, [](int M) { return M >= 4; }) == 0 && \"Failed to lift all the high half inputs to the low mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14076, __PRETTY_FUNCTION__))
14076 "Failed to lift all the high half inputs to the low mask!")((count_if(LoMask, [](int M) { return M >= 4; }) == 0 &&
"Failed to lift all the high half inputs to the low mask!") ?
static_cast<void> (0) : __assert_fail ("count_if(LoMask, [](int M) { return M >= 4; }) == 0 && \"Failed to lift all the high half inputs to the low mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14076, __PRETTY_FUNCTION__));
14077 assert(count_if(HiMask, [](int M) { return M >= 0 && M < 4; }) == 0 &&((count_if(HiMask, [](int M) { return M >= 0 && M <
4; }) == 0 && "Failed to lift all the low half inputs to the high mask!"
) ? static_cast<void> (0) : __assert_fail ("count_if(HiMask, [](int M) { return M >= 0 && M < 4; }) == 0 && \"Failed to lift all the low half inputs to the high mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14078, __PRETTY_FUNCTION__))
14078 "Failed to lift all the low half inputs to the high mask!")((count_if(HiMask, [](int M) { return M >= 0 && M <
4; }) == 0 && "Failed to lift all the low half inputs to the high mask!"
) ? static_cast<void> (0) : __assert_fail ("count_if(HiMask, [](int M) { return M >= 0 && M < 4; }) == 0 && \"Failed to lift all the low half inputs to the high mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14078, __PRETTY_FUNCTION__));
14079
14080 // Do a half shuffle for the low mask.
14081 if (!isNoopShuffleMask(LoMask))
14082 V = DAG.getNode(X86ISD::PSHUFLW, DL, VT, V,
14083 getV4X86ShuffleImm8ForMask(LoMask, DL, DAG));
14084
14085 // Do a half shuffle with the high mask after shifting its values down.
14086 for (int &M : HiMask)
14087 if (M >= 0)
14088 M -= 4;
14089 if (!isNoopShuffleMask(HiMask))
14090 V = DAG.getNode(X86ISD::PSHUFHW, DL, VT, V,
14091 getV4X86ShuffleImm8ForMask(HiMask, DL, DAG));
14092
14093 return V;
14094}
14095
14096/// Helper to form a PSHUFB-based shuffle+blend, opportunistically avoiding the
14097/// blend if only one input is used.
14098static SDValue lowerShuffleAsBlendOfPSHUFBs(
14099 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14100 const APInt &Zeroable, SelectionDAG &DAG, bool &V1InUse, bool &V2InUse) {
14101 assert(!is128BitLaneCrossingShuffleMask(VT, Mask) &&((!is128BitLaneCrossingShuffleMask(VT, Mask) && "Lane crossing shuffle masks not supported"
) ? static_cast<void> (0) : __assert_fail ("!is128BitLaneCrossingShuffleMask(VT, Mask) && \"Lane crossing shuffle masks not supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14102, __PRETTY_FUNCTION__))
14102 "Lane crossing shuffle masks not supported")((!is128BitLaneCrossingShuffleMask(VT, Mask) && "Lane crossing shuffle masks not supported"
) ? static_cast<void> (0) : __assert_fail ("!is128BitLaneCrossingShuffleMask(VT, Mask) && \"Lane crossing shuffle masks not supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14102, __PRETTY_FUNCTION__));
14103
14104 int NumBytes = VT.getSizeInBits() / 8;
14105 int Size = Mask.size();
14106 int Scale = NumBytes / Size;
14107
14108 SmallVector<SDValue, 64> V1Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14109 SmallVector<SDValue, 64> V2Mask(NumBytes, DAG.getUNDEF(MVT::i8));
14110 V1InUse = false;
14111 V2InUse = false;
14112
14113 for (int i = 0; i < NumBytes; ++i) {
14114 int M = Mask[i / Scale];
14115 if (M < 0)
14116 continue;
14117
14118 const int ZeroMask = 0x80;
14119 int V1Idx = M < Size ? M * Scale + i % Scale : ZeroMask;
14120 int V2Idx = M < Size ? ZeroMask : (M - Size) * Scale + i % Scale;
14121 if (Zeroable[i / Scale])
14122 V1Idx = V2Idx = ZeroMask;
14123
14124 V1Mask[i] = DAG.getConstant(V1Idx, DL, MVT::i8);
14125 V2Mask[i] = DAG.getConstant(V2Idx, DL, MVT::i8);
14126 V1InUse |= (ZeroMask != V1Idx);
14127 V2InUse |= (ZeroMask != V2Idx);
14128 }
14129
14130 MVT ShufVT = MVT::getVectorVT(MVT::i8, NumBytes);
14131 if (V1InUse)
14132 V1 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V1),
14133 DAG.getBuildVector(ShufVT, DL, V1Mask));
14134 if (V2InUse)
14135 V2 = DAG.getNode(X86ISD::PSHUFB, DL, ShufVT, DAG.getBitcast(ShufVT, V2),
14136 DAG.getBuildVector(ShufVT, DL, V2Mask));
14137
14138 // If we need shuffled inputs from both, blend the two.
14139 SDValue V;
14140 if (V1InUse && V2InUse)
14141 V = DAG.getNode(ISD::OR, DL, ShufVT, V1, V2);
14142 else
14143 V = V1InUse ? V1 : V2;
14144
14145 // Cast the result back to the correct type.
14146 return DAG.getBitcast(VT, V);
14147}
14148
14149/// Generic lowering of 8-lane i16 shuffles.
14150///
14151/// This handles both single-input shuffles and combined shuffle/blends with
14152/// two inputs. The single input shuffles are immediately delegated to
14153/// a dedicated lowering routine.
14154///
14155/// The blends are lowered in one of three fundamental ways. If there are few
14156/// enough inputs, it delegates to a basic UNPCK-based strategy. If the shuffle
14157/// of the input is significantly cheaper when lowered as an interleaving of
14158/// the two inputs, try to interleave them. Otherwise, blend the low and high
14159/// halves of the inputs separately (making them have relatively few inputs)
14160/// and then concatenate them.
14161static SDValue lowerV8I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14162 const APInt &Zeroable, SDValue V1, SDValue V2,
14163 const X86Subtarget &Subtarget,
14164 SelectionDAG &DAG) {
14165 assert(V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v8i16 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14165, __PRETTY_FUNCTION__));
14166 assert(V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v8i16 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v8i16 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14166, __PRETTY_FUNCTION__));
14167 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!")((Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 8 && \"Unexpected mask size for v8 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14167, __PRETTY_FUNCTION__));
14168
14169 // Whenever we can lower this as a zext, that instruction is strictly faster
14170 // than any alternative.
14171 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i16, V1, V2, Mask,
14172 Zeroable, Subtarget, DAG))
14173 return ZExt;
14174
14175 int NumV2Inputs = count_if(Mask, [](int M) { return M >= 8; });
14176
14177 if (NumV2Inputs == 0) {
14178 // Try to use shift instructions.
14179 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V1, Mask,
14180 Zeroable, Subtarget, DAG))
14181 return Shift;
14182
14183 // Check for being able to broadcast a single element.
14184 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i16, V1, V2,
14185 Mask, Subtarget, DAG))
14186 return Broadcast;
14187
14188 // Use dedicated unpack instructions for masks that match their pattern.
14189 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
14190 return V;
14191
14192 // Use dedicated pack instructions for masks that match their pattern.
14193 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
14194 Subtarget))
14195 return V;
14196
14197 // Try to use byte rotation instructions.
14198 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V1, Mask,
14199 Subtarget, DAG))
14200 return Rotate;
14201
14202 // Make a copy of the mask so it can be modified.
14203 SmallVector<int, 8> MutableMask(Mask.begin(), Mask.end());
14204 return lowerV8I16GeneralSingleInputShuffle(DL, MVT::v8i16, V1, MutableMask,
14205 Subtarget, DAG);
14206 }
14207
14208 assert(llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) &&((llvm::any_of(Mask, [](int M) { return M >= 0 && M
< 8; }) && "All single-input shuffles should be canonicalized to be V1-input "
"shuffles.") ? static_cast<void> (0) : __assert_fail (
"llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) && \"All single-input shuffles should be canonicalized to be V1-input \" \"shuffles.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14210, __PRETTY_FUNCTION__))
14209 "All single-input shuffles should be canonicalized to be V1-input "((llvm::any_of(Mask, [](int M) { return M >= 0 && M
< 8; }) && "All single-input shuffles should be canonicalized to be V1-input "
"shuffles.") ? static_cast<void> (0) : __assert_fail (
"llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) && \"All single-input shuffles should be canonicalized to be V1-input \" \"shuffles.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14210, __PRETTY_FUNCTION__))
14210 "shuffles.")((llvm::any_of(Mask, [](int M) { return M >= 0 && M
< 8; }) && "All single-input shuffles should be canonicalized to be V1-input "
"shuffles.") ? static_cast<void> (0) : __assert_fail (
"llvm::any_of(Mask, [](int M) { return M >= 0 && M < 8; }) && \"All single-input shuffles should be canonicalized to be V1-input \" \"shuffles.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14210, __PRETTY_FUNCTION__));
14211
14212 // Try to use shift instructions.
14213 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i16, V1, V2, Mask,
14214 Zeroable, Subtarget, DAG))
14215 return Shift;
14216
14217 // See if we can use SSE4A Extraction / Insertion.
14218 if (Subtarget.hasSSE4A())
14219 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v8i16, V1, V2, Mask,
14220 Zeroable, DAG))
14221 return V;
14222
14223 // There are special ways we can lower some single-element blends.
14224 if (NumV2Inputs == 1)
14225 if (SDValue V = lowerShuffleAsElementInsertion(
14226 DL, MVT::v8i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
14227 return V;
14228
14229 // We have different paths for blend lowering, but they all must use the
14230 // *exact* same predicate.
14231 bool IsBlendSupported = Subtarget.hasSSE41();
14232 if (IsBlendSupported)
14233 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i16, V1, V2, Mask,
14234 Zeroable, Subtarget, DAG))
14235 return Blend;
14236
14237 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v8i16, V1, V2, Mask,
14238 Zeroable, Subtarget, DAG))
14239 return Masked;
14240
14241 // Use dedicated unpack instructions for masks that match their pattern.
14242 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i16, Mask, V1, V2, DAG))
14243 return V;
14244
14245 // Use dedicated pack instructions for masks that match their pattern.
14246 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v8i16, Mask, V1, V2, DAG,
14247 Subtarget))
14248 return V;
14249
14250 // Try to use byte rotation instructions.
14251 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i16, V1, V2, Mask,
14252 Subtarget, DAG))
14253 return Rotate;
14254
14255 if (SDValue BitBlend =
14256 lowerShuffleAsBitBlend(DL, MVT::v8i16, V1, V2, Mask, DAG))
14257 return BitBlend;
14258
14259 // Try to use byte shift instructions to mask.
14260 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v8i16, V1, V2, Mask,
14261 Zeroable, Subtarget, DAG))
14262 return V;
14263
14264 // Try to lower by permuting the inputs into an unpack instruction.
14265 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(DL, MVT::v8i16, V1, V2,
14266 Mask, Subtarget, DAG))
14267 return Unpack;
14268
14269 // If we can't directly blend but can use PSHUFB, that will be better as it
14270 // can both shuffle and set up the inefficient blend.
14271 if (!IsBlendSupported && Subtarget.hasSSSE3()) {
14272 bool V1InUse, V2InUse;
14273 return lowerShuffleAsBlendOfPSHUFBs(DL, MVT::v8i16, V1, V2, Mask,
14274 Zeroable, DAG, V1InUse, V2InUse);
14275 }
14276
14277 // We can always bit-blend if we have to so the fallback strategy is to
14278 // decompose into single-input permutes and blends.
14279 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i16, V1, V2,
14280 Mask, Subtarget, DAG);
14281}
14282
14283/// Check whether a compaction lowering can be done by dropping even
14284/// elements and compute how many times even elements must be dropped.
14285///
14286/// This handles shuffles which take every Nth element where N is a power of
14287/// two. Example shuffle masks:
14288///
14289/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 0, 2, 4, 6, 8, 10, 12, 14
14290/// N = 1: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30
14291/// N = 2: 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12
14292/// N = 2: 0, 4, 8, 12, 16, 20, 24, 28, 0, 4, 8, 12, 16, 20, 24, 28
14293/// N = 3: 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8
14294/// N = 3: 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24, 0, 8, 16, 24
14295///
14296/// Any of these lanes can of course be undef.
14297///
14298/// This routine only supports N <= 3.
14299/// FIXME: Evaluate whether either AVX or AVX-512 have any opportunities here
14300/// for larger N.
14301///
14302/// \returns N above, or the number of times even elements must be dropped if
14303/// there is such a number. Otherwise returns zero.
14304static int canLowerByDroppingEvenElements(ArrayRef<int> Mask,
14305 bool IsSingleInput) {
14306 // The modulus for the shuffle vector entries is based on whether this is
14307 // a single input or not.
14308 int ShuffleModulus = Mask.size() * (IsSingleInput ? 1 : 2);
14309 assert(isPowerOf2_32((uint32_t)ShuffleModulus) &&((isPowerOf2_32((uint32_t)ShuffleModulus) && "We should only be called with masks with a power-of-2 size!"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32((uint32_t)ShuffleModulus) && \"We should only be called with masks with a power-of-2 size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14310, __PRETTY_FUNCTION__))
14310 "We should only be called with masks with a power-of-2 size!")((isPowerOf2_32((uint32_t)ShuffleModulus) && "We should only be called with masks with a power-of-2 size!"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32((uint32_t)ShuffleModulus) && \"We should only be called with masks with a power-of-2 size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14310, __PRETTY_FUNCTION__));
14311
14312 uint64_t ModMask = (uint64_t)ShuffleModulus - 1;
14313
14314 // We track whether the input is viable for all power-of-2 strides 2^1, 2^2,
14315 // and 2^3 simultaneously. This is because we may have ambiguity with
14316 // partially undef inputs.
14317 bool ViableForN[3] = {true, true, true};
14318
14319 for (int i = 0, e = Mask.size(); i < e; ++i) {
14320 // Ignore undef lanes, we'll optimistically collapse them to the pattern we
14321 // want.
14322 if (Mask[i] < 0)
14323 continue;
14324
14325 bool IsAnyViable = false;
14326 for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
14327 if (ViableForN[j]) {
14328 uint64_t N = j + 1;
14329
14330 // The shuffle mask must be equal to (i * 2^N) % M.
14331 if ((uint64_t)Mask[i] == (((uint64_t)i << N) & ModMask))
14332 IsAnyViable = true;
14333 else
14334 ViableForN[j] = false;
14335 }
14336 // Early exit if we exhaust the possible powers of two.
14337 if (!IsAnyViable)
14338 break;
14339 }
14340
14341 for (unsigned j = 0; j != array_lengthof(ViableForN); ++j)
14342 if (ViableForN[j])
14343 return j + 1;
14344
14345 // Return 0 as there is no viable power of two.
14346 return 0;
14347}
14348
14349static SDValue lowerShuffleWithPERMV(const SDLoc &DL, MVT VT,
14350 ArrayRef<int> Mask, SDValue V1,
14351 SDValue V2, SelectionDAG &DAG) {
14352 MVT MaskEltVT = MVT::getIntegerVT(VT.getScalarSizeInBits());
14353 MVT MaskVecVT = MVT::getVectorVT(MaskEltVT, VT.getVectorNumElements());
14354
14355 SDValue MaskNode = getConstVector(Mask, MaskVecVT, DAG, DL, true);
14356 if (V2.isUndef())
14357 return DAG.getNode(X86ISD::VPERMV, DL, VT, MaskNode, V1);
14358
14359 return DAG.getNode(X86ISD::VPERMV3, DL, VT, V1, MaskNode, V2);
14360}
14361
14362/// Generic lowering of v16i8 shuffles.
14363///
14364/// This is a hybrid strategy to lower v16i8 vectors. It first attempts to
14365/// detect any complexity reducing interleaving. If that doesn't help, it uses
14366/// UNPCK to spread the i8 elements across two i16-element vectors, and uses
14367/// the existing lowering for v8i16 blends on each half, finally PACK-ing them
14368/// back together.
14369static SDValue lowerV16I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
14370 const APInt &Zeroable, SDValue V1, SDValue V2,
14371 const X86Subtarget &Subtarget,
14372 SelectionDAG &DAG) {
14373 assert(V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v16i8 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14373, __PRETTY_FUNCTION__));
14374 assert(V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v16i8 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v16i8 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14374, __PRETTY_FUNCTION__));
14375 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!")((Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 16 && \"Unexpected mask size for v16 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14375, __PRETTY_FUNCTION__));
14376
14377 // Try to use shift instructions.
14378 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i8, V1, V2, Mask,
14379 Zeroable, Subtarget, DAG))
14380 return Shift;
14381
14382 // Try to use byte rotation instructions.
14383 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i8, V1, V2, Mask,
14384 Subtarget, DAG))
14385 return Rotate;
14386
14387 // Use dedicated pack instructions for masks that match their pattern.
14388 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i8, Mask, V1, V2, DAG,
14389 Subtarget))
14390 return V;
14391
14392 // Try to use a zext lowering.
14393 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v16i8, V1, V2, Mask,
14394 Zeroable, Subtarget, DAG))
14395 return ZExt;
14396
14397 // See if we can use SSE4A Extraction / Insertion.
14398 if (Subtarget.hasSSE4A())
14399 if (SDValue V = lowerShuffleWithSSE4A(DL, MVT::v16i8, V1, V2, Mask,
14400 Zeroable, DAG))
14401 return V;
14402
14403 int NumV2Elements = count_if(Mask, [](int M) { return M >= 16; });
14404
14405 // For single-input shuffles, there are some nicer lowering tricks we can use.
14406 if (NumV2Elements == 0) {
14407 // Check for being able to broadcast a single element.
14408 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i8, V1, V2,
14409 Mask, Subtarget, DAG))
14410 return Broadcast;
14411
14412 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14413 return V;
14414
14415 // Check whether we can widen this to an i16 shuffle by duplicating bytes.
14416 // Notably, this handles splat and partial-splat shuffles more efficiently.
14417 // However, it only makes sense if the pre-duplication shuffle simplifies
14418 // things significantly. Currently, this means we need to be able to
14419 // express the pre-duplication shuffle as an i16 shuffle.
14420 //
14421 // FIXME: We should check for other patterns which can be widened into an
14422 // i16 shuffle as well.
14423 auto canWidenViaDuplication = [](ArrayRef<int> Mask) {
14424 for (int i = 0; i < 16; i += 2)
14425 if (Mask[i] >= 0 && Mask[i + 1] >= 0 && Mask[i] != Mask[i + 1])
14426 return false;
14427
14428 return true;
14429 };
14430 auto tryToWidenViaDuplication = [&]() -> SDValue {
14431 if (!canWidenViaDuplication(Mask))
14432 return SDValue();
14433 SmallVector<int, 4> LoInputs;
14434 copy_if(Mask, std::back_inserter(LoInputs),
14435 [](int M) { return M >= 0 && M < 8; });
14436 array_pod_sort(LoInputs.begin(), LoInputs.end());
14437 LoInputs.erase(std::unique(LoInputs.begin(), LoInputs.end()),
14438 LoInputs.end());
14439 SmallVector<int, 4> HiInputs;
14440 copy_if(Mask, std::back_inserter(HiInputs), [](int M) { return M >= 8; });
14441 array_pod_sort(HiInputs.begin(), HiInputs.end());
14442 HiInputs.erase(std::unique(HiInputs.begin(), HiInputs.end()),
14443 HiInputs.end());
14444
14445 bool TargetLo = LoInputs.size() >= HiInputs.size();
14446 ArrayRef<int> InPlaceInputs = TargetLo ? LoInputs : HiInputs;
14447 ArrayRef<int> MovingInputs = TargetLo ? HiInputs : LoInputs;
14448
14449 int PreDupI16Shuffle[] = {-1, -1, -1, -1, -1, -1, -1, -1};
14450 SmallDenseMap<int, int, 8> LaneMap;
14451 for (int I : InPlaceInputs) {
14452 PreDupI16Shuffle[I/2] = I/2;
14453 LaneMap[I] = I;
14454 }
14455 int j = TargetLo ? 0 : 4, je = j + 4;
14456 for (int i = 0, ie = MovingInputs.size(); i < ie; ++i) {
14457 // Check if j is already a shuffle of this input. This happens when
14458 // there are two adjacent bytes after we move the low one.
14459 if (PreDupI16Shuffle[j] != MovingInputs[i] / 2) {
14460 // If we haven't yet mapped the input, search for a slot into which
14461 // we can map it.
14462 while (j < je && PreDupI16Shuffle[j] >= 0)
14463 ++j;
14464
14465 if (j == je)
14466 // We can't place the inputs into a single half with a simple i16 shuffle, so bail.
14467 return SDValue();
14468
14469 // Map this input with the i16 shuffle.
14470 PreDupI16Shuffle[j] = MovingInputs[i] / 2;
14471 }
14472
14473 // Update the lane map based on the mapping we ended up with.
14474 LaneMap[MovingInputs[i]] = 2 * j + MovingInputs[i] % 2;
14475 }
14476 V1 = DAG.getBitcast(
14477 MVT::v16i8,
14478 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14479 DAG.getUNDEF(MVT::v8i16), PreDupI16Shuffle));
14480
14481 // Unpack the bytes to form the i16s that will be shuffled into place.
14482 bool EvenInUse = false, OddInUse = false;
14483 for (int i = 0; i < 16; i += 2) {
14484 EvenInUse |= (Mask[i + 0] >= 0);
14485 OddInUse |= (Mask[i + 1] >= 0);
14486 if (EvenInUse && OddInUse)
14487 break;
14488 }
14489 V1 = DAG.getNode(TargetLo ? X86ISD::UNPCKL : X86ISD::UNPCKH, DL,
14490 MVT::v16i8, EvenInUse ? V1 : DAG.getUNDEF(MVT::v16i8),
14491 OddInUse ? V1 : DAG.getUNDEF(MVT::v16i8));
14492
14493 int PostDupI16Shuffle[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
14494 for (int i = 0; i < 16; ++i)
14495 if (Mask[i] >= 0) {
14496 int MappedMask = LaneMap[Mask[i]] - (TargetLo ? 0 : 8);
14497 assert(MappedMask < 8 && "Invalid v8 shuffle mask!")((MappedMask < 8 && "Invalid v8 shuffle mask!") ? static_cast
<void> (0) : __assert_fail ("MappedMask < 8 && \"Invalid v8 shuffle mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14497, __PRETTY_FUNCTION__));
14498 if (PostDupI16Shuffle[i / 2] < 0)
14499 PostDupI16Shuffle[i / 2] = MappedMask;
14500 else
14501 assert(PostDupI16Shuffle[i / 2] == MappedMask &&((PostDupI16Shuffle[i / 2] == MappedMask && "Conflicting entries in the original shuffle!"
) ? static_cast<void> (0) : __assert_fail ("PostDupI16Shuffle[i / 2] == MappedMask && \"Conflicting entries in the original shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14502, __PRETTY_FUNCTION__))
14502 "Conflicting entries in the original shuffle!")((PostDupI16Shuffle[i / 2] == MappedMask && "Conflicting entries in the original shuffle!"
) ? static_cast<void> (0) : __assert_fail ("PostDupI16Shuffle[i / 2] == MappedMask && \"Conflicting entries in the original shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14502, __PRETTY_FUNCTION__));
14503 }
14504 return DAG.getBitcast(
14505 MVT::v16i8,
14506 DAG.getVectorShuffle(MVT::v8i16, DL, DAG.getBitcast(MVT::v8i16, V1),
14507 DAG.getUNDEF(MVT::v8i16), PostDupI16Shuffle));
14508 };
14509 if (SDValue V = tryToWidenViaDuplication())
14510 return V;
14511 }
14512
14513 if (SDValue Masked = lowerShuffleAsBitMask(DL, MVT::v16i8, V1, V2, Mask,
14514 Zeroable, Subtarget, DAG))
14515 return Masked;
14516
14517 // Use dedicated unpack instructions for masks that match their pattern.
14518 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i8, Mask, V1, V2, DAG))
14519 return V;
14520
14521 // Try to use byte shift instructions to mask.
14522 if (SDValue V = lowerShuffleAsByteShiftMask(DL, MVT::v16i8, V1, V2, Mask,
14523 Zeroable, Subtarget, DAG))
14524 return V;
14525
14526 // Check for SSSE3 which lets us lower all v16i8 shuffles much more directly
14527 // with PSHUFB. It is important to do this before we attempt to generate any
14528 // blends but after all of the single-input lowerings. If the single input
14529 // lowerings can find an instruction sequence that is faster than a PSHUFB, we
14530 // want to preserve that and we can DAG combine any longer sequences into
14531 // a PSHUFB in the end. But once we start blending from multiple inputs,
14532 // the complexity of DAG combining bad patterns back into PSHUFB is too high,
14533 // and there are *very* few patterns that would actually be faster than the
14534 // PSHUFB approach because of its ability to zero lanes.
14535 //
14536 // FIXME: The only exceptions to the above are blends which are exact
14537 // interleavings with direct instructions supporting them. We currently don't
14538 // handle those well here.
14539 if (Subtarget.hasSSSE3()) {
14540 bool V1InUse = false;
14541 bool V2InUse = false;
14542
14543 SDValue PSHUFB = lowerShuffleAsBlendOfPSHUFBs(
14544 DL, MVT::v16i8, V1, V2, Mask, Zeroable, DAG, V1InUse, V2InUse);
14545
14546 // If both V1 and V2 are in use and we can use a direct blend or an unpack,
14547 // do so. This avoids using them to handle blends-with-zero which is
14548 // important as a single pshufb is significantly faster for that.
14549 if (V1InUse && V2InUse) {
14550 if (Subtarget.hasSSE41())
14551 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i8, V1, V2, Mask,
14552 Zeroable, Subtarget, DAG))
14553 return Blend;
14554
14555 // We can use an unpack to do the blending rather than an or in some
14556 // cases. Even though the or may be (very minorly) more efficient, we
14557 // preference this lowering because there are common cases where part of
14558 // the complexity of the shuffles goes away when we do the final blend as
14559 // an unpack.
14560 // FIXME: It might be worth trying to detect if the unpack-feeding
14561 // shuffles will both be pshufb, in which case we shouldn't bother with
14562 // this.
14563 if (SDValue Unpack = lowerShuffleAsPermuteAndUnpack(
14564 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14565 return Unpack;
14566
14567 // If we have VBMI we can use one VPERM instead of multiple PSHUFBs.
14568 if (Subtarget.hasVBMI() && Subtarget.hasVLX())
14569 return lowerShuffleWithPERMV(DL, MVT::v16i8, Mask, V1, V2, DAG);
14570
14571 // Use PALIGNR+Permute if possible - permute might become PSHUFB but the
14572 // PALIGNR will be cheaper than the second PSHUFB+OR.
14573 if (SDValue V = lowerShuffleAsByteRotateAndPermute(
14574 DL, MVT::v16i8, V1, V2, Mask, Subtarget, DAG))
14575 return V;
14576 }
14577
14578 return PSHUFB;
14579 }
14580
14581 // There are special ways we can lower some single-element blends.
14582 if (NumV2Elements == 1)
14583 if (SDValue V = lowerShuffleAsElementInsertion(
14584 DL, MVT::v16i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
14585 return V;
14586
14587 if (SDValue Blend = lowerShuffleAsBitBlend(DL, MVT::v16i8, V1, V2, Mask, DAG))
14588 return Blend;
14589
14590 // Check whether a compaction lowering can be done. This handles shuffles
14591 // which take every Nth element for some even N. See the helper function for
14592 // details.
14593 //
14594 // We special case these as they can be particularly efficiently handled with
14595 // the PACKUSB instruction on x86 and they show up in common patterns of
14596 // rearranging bytes to truncate wide elements.
14597 bool IsSingleInput = V2.isUndef();
14598 if (int NumEvenDrops = canLowerByDroppingEvenElements(Mask, IsSingleInput)) {
14599 // NumEvenDrops is the power of two stride of the elements. Another way of
14600 // thinking about it is that we need to drop the even elements this many
14601 // times to get the original input.
14602
14603 // First we need to zero all the dropped bytes.
14604 assert(NumEvenDrops <= 3 &&((NumEvenDrops <= 3 && "No support for dropping even elements more than 3 times."
) ? static_cast<void> (0) : __assert_fail ("NumEvenDrops <= 3 && \"No support for dropping even elements more than 3 times.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14605, __PRETTY_FUNCTION__))
14605 "No support for dropping even elements more than 3 times.")((NumEvenDrops <= 3 && "No support for dropping even elements more than 3 times."
) ? static_cast<void> (0) : __assert_fail ("NumEvenDrops <= 3 && \"No support for dropping even elements more than 3 times.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14605, __PRETTY_FUNCTION__));
14606 SmallVector<SDValue, 16> ByteClearOps(16, DAG.getConstant(0, DL, MVT::i8));
14607 for (unsigned i = 0; i != 16; i += 1 << NumEvenDrops)
14608 ByteClearOps[i] = DAG.getConstant(0xFF, DL, MVT::i8);
14609 SDValue ByteClearMask = DAG.getBuildVector(MVT::v16i8, DL, ByteClearOps);
14610 V1 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V1, ByteClearMask);
14611 if (!IsSingleInput)
14612 V2 = DAG.getNode(ISD::AND, DL, MVT::v16i8, V2, ByteClearMask);
14613
14614 // Now pack things back together.
14615 V1 = DAG.getBitcast(MVT::v8i16, V1);
14616 V2 = IsSingleInput ? V1 : DAG.getBitcast(MVT::v8i16, V2);
14617 SDValue Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, V1, V2);
14618 for (int i = 1; i < NumEvenDrops; ++i) {
14619 Result = DAG.getBitcast(MVT::v8i16, Result);
14620 Result = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Result, Result);
14621 }
14622
14623 return Result;
14624 }
14625
14626 // Handle multi-input cases by blending single-input shuffles.
14627 if (NumV2Elements > 0)
14628 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v16i8, V1, V2, Mask,
14629 Subtarget, DAG);
14630
14631 // The fallback path for single-input shuffles widens this into two v8i16
14632 // vectors with unpacks, shuffles those, and then pulls them back together
14633 // with a pack.
14634 SDValue V = V1;
14635
14636 std::array<int, 8> LoBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14637 std::array<int, 8> HiBlendMask = {{-1, -1, -1, -1, -1, -1, -1, -1}};
14638 for (int i = 0; i < 16; ++i)
14639 if (Mask[i] >= 0)
14640 (i < 8 ? LoBlendMask[i] : HiBlendMask[i % 8]) = Mask[i];
14641
14642 SDValue VLoHalf, VHiHalf;
14643 // Check if any of the odd lanes in the v16i8 are used. If not, we can mask
14644 // them out and avoid using UNPCK{L,H} to extract the elements of V as
14645 // i16s.
14646 if (none_of(LoBlendMask, [](int M) { return M >= 0 && M % 2 == 1; }) &&
14647 none_of(HiBlendMask, [](int M) { return M >= 0 && M % 2 == 1; })) {
14648 // Use a mask to drop the high bytes.
14649 VLoHalf = DAG.getBitcast(MVT::v8i16, V);
14650 VLoHalf = DAG.getNode(ISD::AND, DL, MVT::v8i16, VLoHalf,
14651 DAG.getConstant(0x00FF, DL, MVT::v8i16));
14652
14653 // This will be a single vector shuffle instead of a blend so nuke VHiHalf.
14654 VHiHalf = DAG.getUNDEF(MVT::v8i16);
14655
14656 // Squash the masks to point directly into VLoHalf.
14657 for (int &M : LoBlendMask)
14658 if (M >= 0)
14659 M /= 2;
14660 for (int &M : HiBlendMask)
14661 if (M >= 0)
14662 M /= 2;
14663 } else {
14664 // Otherwise just unpack the low half of V into VLoHalf and the high half into
14665 // VHiHalf so that we can blend them as i16s.
14666 SDValue Zero = getZeroVector(MVT::v16i8, Subtarget, DAG, DL);
14667
14668 VLoHalf = DAG.getBitcast(
14669 MVT::v8i16, DAG.getNode(X86ISD::UNPCKL, DL, MVT::v16i8, V, Zero));
14670 VHiHalf = DAG.getBitcast(
14671 MVT::v8i16, DAG.getNode(X86ISD::UNPCKH, DL, MVT::v16i8, V, Zero));
14672 }
14673
14674 SDValue LoV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, LoBlendMask);
14675 SDValue HiV = DAG.getVectorShuffle(MVT::v8i16, DL, VLoHalf, VHiHalf, HiBlendMask);
14676
14677 return DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, LoV, HiV);
14678}
14679
14680/// Dispatching routine to lower various 128-bit x86 vector shuffles.
14681///
14682/// This routine breaks down the specific type of 128-bit shuffle and
14683/// dispatches to the lowering routines accordingly.
14684static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
14685 MVT VT, SDValue V1, SDValue V2,
14686 const APInt &Zeroable,
14687 const X86Subtarget &Subtarget,
14688 SelectionDAG &DAG) {
14689 switch (VT.SimpleTy) {
14690 case MVT::v2i64:
14691 return lowerV2I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14692 case MVT::v2f64:
14693 return lowerV2F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14694 case MVT::v4i32:
14695 return lowerV4I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14696 case MVT::v4f32:
14697 return lowerV4F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14698 case MVT::v8i16:
14699 return lowerV8I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14700 case MVT::v16i8:
14701 return lowerV16I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
14702
14703 default:
14704 llvm_unreachable("Unimplemented!")::llvm::llvm_unreachable_internal("Unimplemented!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14704);
14705 }
14706}
14707
14708/// Generic routine to split vector shuffle into half-sized shuffles.
14709///
14710/// This routine just extracts two subvectors, shuffles them independently, and
14711/// then concatenates them back together. This should work effectively with all
14712/// AVX vector shuffle types.
14713static SDValue splitAndLowerShuffle(const SDLoc &DL, MVT VT, SDValue V1,
14714 SDValue V2, ArrayRef<int> Mask,
14715 SelectionDAG &DAG) {
14716 assert(VT.getSizeInBits() >= 256 &&((VT.getSizeInBits() >= 256 && "Only for 256-bit or wider vector shuffles!"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() >= 256 && \"Only for 256-bit or wider vector shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14717, __PRETTY_FUNCTION__))
14717 "Only for 256-bit or wider vector shuffles!")((VT.getSizeInBits() >= 256 && "Only for 256-bit or wider vector shuffles!"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() >= 256 && \"Only for 256-bit or wider vector shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14717, __PRETTY_FUNCTION__));
14718 assert(V1.getSimpleValueType() == VT && "Bad operand type!")((V1.getSimpleValueType() == VT && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == VT && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14718, __PRETTY_FUNCTION__));
14719 assert(V2.getSimpleValueType() == VT && "Bad operand type!")((V2.getSimpleValueType() == VT && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == VT && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14719, __PRETTY_FUNCTION__));
14720
14721 ArrayRef<int> LoMask = Mask.slice(0, Mask.size() / 2);
14722 ArrayRef<int> HiMask = Mask.slice(Mask.size() / 2);
14723
14724 int NumElements = VT.getVectorNumElements();
14725 int SplitNumElements = NumElements / 2;
14726 MVT ScalarVT = VT.getVectorElementType();
14727 MVT SplitVT = MVT::getVectorVT(ScalarVT, NumElements / 2);
14728
14729 // Rather than splitting build-vectors, just build two narrower build
14730 // vectors. This helps shuffling with splats and zeros.
14731 auto SplitVector = [&](SDValue V) {
14732 V = peekThroughBitcasts(V);
14733
14734 MVT OrigVT = V.getSimpleValueType();
14735 int OrigNumElements = OrigVT.getVectorNumElements();
14736 int OrigSplitNumElements = OrigNumElements / 2;
14737 MVT OrigScalarVT = OrigVT.getVectorElementType();
14738 MVT OrigSplitVT = MVT::getVectorVT(OrigScalarVT, OrigNumElements / 2);
14739
14740 SDValue LoV, HiV;
14741
14742 auto *BV = dyn_cast<BuildVectorSDNode>(V);
14743 if (!BV) {
14744 LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
14745 DAG.getIntPtrConstant(0, DL));
14746 HiV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, OrigSplitVT, V,
14747 DAG.getIntPtrConstant(OrigSplitNumElements, DL));
14748 } else {
14749
14750 SmallVector<SDValue, 16> LoOps, HiOps;
14751 for (int i = 0; i < OrigSplitNumElements; ++i) {
14752 LoOps.push_back(BV->getOperand(i));
14753 HiOps.push_back(BV->getOperand(i + OrigSplitNumElements));
14754 }
14755 LoV = DAG.getBuildVector(OrigSplitVT, DL, LoOps);
14756 HiV = DAG.getBuildVector(OrigSplitVT, DL, HiOps);
14757 }
14758 return std::make_pair(DAG.getBitcast(SplitVT, LoV),
14759 DAG.getBitcast(SplitVT, HiV));
14760 };
14761
14762 SDValue LoV1, HiV1, LoV2, HiV2;
14763 std::tie(LoV1, HiV1) = SplitVector(V1);
14764 std::tie(LoV2, HiV2) = SplitVector(V2);
14765
14766 // Now create two 4-way blends of these half-width vectors.
14767 auto HalfBlend = [&](ArrayRef<int> HalfMask) {
14768 bool UseLoV1 = false, UseHiV1 = false, UseLoV2 = false, UseHiV2 = false;
14769 SmallVector<int, 32> V1BlendMask((unsigned)SplitNumElements, -1);
14770 SmallVector<int, 32> V2BlendMask((unsigned)SplitNumElements, -1);
14771 SmallVector<int, 32> BlendMask((unsigned)SplitNumElements, -1);
14772 for (int i = 0; i < SplitNumElements; ++i) {
14773 int M = HalfMask[i];
14774 if (M >= NumElements) {
14775 if (M >= NumElements + SplitNumElements)
14776 UseHiV2 = true;
14777 else
14778 UseLoV2 = true;
14779 V2BlendMask[i] = M - NumElements;
14780 BlendMask[i] = SplitNumElements + i;
14781 } else if (M >= 0) {
14782 if (M >= SplitNumElements)
14783 UseHiV1 = true;
14784 else
14785 UseLoV1 = true;
14786 V1BlendMask[i] = M;
14787 BlendMask[i] = i;
14788 }
14789 }
14790
14791 // Because the lowering happens after all combining takes place, we need to
14792 // manually combine these blend masks as much as possible so that we create
14793 // a minimal number of high-level vector shuffle nodes.
14794
14795 // First try just blending the halves of V1 or V2.
14796 if (!UseLoV1 && !UseHiV1 && !UseLoV2 && !UseHiV2)
14797 return DAG.getUNDEF(SplitVT);
14798 if (!UseLoV2 && !UseHiV2)
14799 return DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14800 if (!UseLoV1 && !UseHiV1)
14801 return DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14802
14803 SDValue V1Blend, V2Blend;
14804 if (UseLoV1 && UseHiV1) {
14805 V1Blend =
14806 DAG.getVectorShuffle(SplitVT, DL, LoV1, HiV1, V1BlendMask);
14807 } else {
14808 // We only use half of V1 so map the usage down into the final blend mask.
14809 V1Blend = UseLoV1 ? LoV1 : HiV1;
14810 for (int i = 0; i < SplitNumElements; ++i)
14811 if (BlendMask[i] >= 0 && BlendMask[i] < SplitNumElements)
14812 BlendMask[i] = V1BlendMask[i] - (UseLoV1 ? 0 : SplitNumElements);
14813 }
14814 if (UseLoV2 && UseHiV2) {
14815 V2Blend =
14816 DAG.getVectorShuffle(SplitVT, DL, LoV2, HiV2, V2BlendMask);
14817 } else {
14818 // We only use half of V2 so map the usage down into the final blend mask.
14819 V2Blend = UseLoV2 ? LoV2 : HiV2;
14820 for (int i = 0; i < SplitNumElements; ++i)
14821 if (BlendMask[i] >= SplitNumElements)
14822 BlendMask[i] = V2BlendMask[i] + (UseLoV2 ? SplitNumElements : 0);
14823 }
14824 return DAG.getVectorShuffle(SplitVT, DL, V1Blend, V2Blend, BlendMask);
14825 };
14826 SDValue Lo = HalfBlend(LoMask);
14827 SDValue Hi = HalfBlend(HiMask);
14828 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
14829}
14830
14831/// Either split a vector in halves or decompose the shuffles and the
14832/// blend.
14833///
14834/// This is provided as a good fallback for many lowerings of non-single-input
14835/// shuffles with more than one 128-bit lane. In those cases, we want to select
14836/// between splitting the shuffle into 128-bit components and stitching those
14837/// back together vs. extracting the single-input shuffles and blending those
14838/// results.
14839static SDValue lowerShuffleAsSplitOrBlend(const SDLoc &DL, MVT VT, SDValue V1,
14840 SDValue V2, ArrayRef<int> Mask,
14841 const X86Subtarget &Subtarget,
14842 SelectionDAG &DAG) {
14843 assert(!V2.isUndef() && "This routine must not be used to lower single-input "((!V2.isUndef() && "This routine must not be used to lower single-input "
"shuffles as it could then recurse on itself.") ? static_cast
<void> (0) : __assert_fail ("!V2.isUndef() && \"This routine must not be used to lower single-input \" \"shuffles as it could then recurse on itself.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14844, __PRETTY_FUNCTION__))
14844 "shuffles as it could then recurse on itself.")((!V2.isUndef() && "This routine must not be used to lower single-input "
"shuffles as it could then recurse on itself.") ? static_cast
<void> (0) : __assert_fail ("!V2.isUndef() && \"This routine must not be used to lower single-input \" \"shuffles as it could then recurse on itself.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14844, __PRETTY_FUNCTION__));
14845 int Size = Mask.size();
14846
14847 // If this can be modeled as a broadcast of two elements followed by a blend,
14848 // prefer that lowering. This is especially important because broadcasts can
14849 // often fold with memory operands.
14850 auto DoBothBroadcast = [&] {
14851 int V1BroadcastIdx = -1, V2BroadcastIdx = -1;
14852 for (int M : Mask)
14853 if (M >= Size) {
14854 if (V2BroadcastIdx < 0)
14855 V2BroadcastIdx = M - Size;
14856 else if (M - Size != V2BroadcastIdx)
14857 return false;
14858 } else if (M >= 0) {
14859 if (V1BroadcastIdx < 0)
14860 V1BroadcastIdx = M;
14861 else if (M != V1BroadcastIdx)
14862 return false;
14863 }
14864 return true;
14865 };
14866 if (DoBothBroadcast())
14867 return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask,
14868 Subtarget, DAG);
14869
14870 // If the inputs all stem from a single 128-bit lane of each input, then we
14871 // split them rather than blending because the split will decompose to
14872 // unusually few instructions.
14873 int LaneCount = VT.getSizeInBits() / 128;
14874 int LaneSize = Size / LaneCount;
14875 SmallBitVector LaneInputs[2];
14876 LaneInputs[0].resize(LaneCount, false);
14877 LaneInputs[1].resize(LaneCount, false);
14878 for (int i = 0; i < Size; ++i)
14879 if (Mask[i] >= 0)
14880 LaneInputs[Mask[i] / Size][(Mask[i] % Size) / LaneSize] = true;
14881 if (LaneInputs[0].count() <= 1 && LaneInputs[1].count() <= 1)
14882 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
14883
14884 // Otherwise, just fall back to decomposed shuffles and a blend. This requires
14885 // that the decomposed single-input shuffles don't end up here.
14886 return lowerShuffleAsDecomposedShuffleBlend(DL, VT, V1, V2, Mask, Subtarget,
14887 DAG);
14888}
14889
14890/// Lower a vector shuffle crossing multiple 128-bit lanes as
14891/// a lane permutation followed by a per-lane permutation.
14892///
14893/// This is mainly for cases where we can have non-repeating permutes
14894/// in each lane.
14895///
14896/// TODO: This is very similar to lowerShuffleAsLanePermuteAndRepeatedMask,
14897/// we should investigate merging them.
14898static SDValue lowerShuffleAsLanePermuteAndPermute(
14899 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14900 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14901 int NumElts = VT.getVectorNumElements();
14902 int NumLanes = VT.getSizeInBits() / 128;
14903 int NumEltsPerLane = NumElts / NumLanes;
14904
14905 SmallVector<int, 4> SrcLaneMask(NumLanes, SM_SentinelUndef);
14906 SmallVector<int, 16> PermMask(NumElts, SM_SentinelUndef);
14907
14908 for (int i = 0; i != NumElts; ++i) {
14909 int M = Mask[i];
14910 if (M < 0)
14911 continue;
14912
14913 // Ensure that each lane comes from a single source lane.
14914 int SrcLane = M / NumEltsPerLane;
14915 int DstLane = i / NumEltsPerLane;
14916 if (!isUndefOrEqual(SrcLaneMask[DstLane], SrcLane))
14917 return SDValue();
14918 SrcLaneMask[DstLane] = SrcLane;
14919
14920 PermMask[i] = (DstLane * NumEltsPerLane) + (M % NumEltsPerLane);
14921 }
14922
14923 // Make sure we set all elements of the lane mask, to avoid undef propagation.
14924 SmallVector<int, 16> LaneMask(NumElts, SM_SentinelUndef);
14925 for (int DstLane = 0; DstLane != NumLanes; ++DstLane) {
14926 int SrcLane = SrcLaneMask[DstLane];
14927 if (0 <= SrcLane)
14928 for (int j = 0; j != NumEltsPerLane; ++j) {
14929 LaneMask[(DstLane * NumEltsPerLane) + j] =
14930 (SrcLane * NumEltsPerLane) + j;
14931 }
14932 }
14933
14934 // If we're only shuffling a single lowest lane and the rest are identity
14935 // then don't bother.
14936 // TODO - isShuffleMaskInputInPlace could be extended to something like this.
14937 int NumIdentityLanes = 0;
14938 bool OnlyShuffleLowestLane = true;
14939 for (int i = 0; i != NumLanes; ++i) {
14940 if (isSequentialOrUndefInRange(PermMask, i * NumEltsPerLane, NumEltsPerLane,
14941 i * NumEltsPerLane))
14942 NumIdentityLanes++;
14943 else if (SrcLaneMask[i] != 0 && SrcLaneMask[i] != NumLanes)
14944 OnlyShuffleLowestLane = false;
14945 }
14946 if (OnlyShuffleLowestLane && NumIdentityLanes == (NumLanes - 1))
14947 return SDValue();
14948
14949 SDValue LanePermute = DAG.getVectorShuffle(VT, DL, V1, V2, LaneMask);
14950 return DAG.getVectorShuffle(VT, DL, LanePermute, DAG.getUNDEF(VT), PermMask);
14951}
14952
14953/// Lower a vector shuffle crossing multiple 128-bit lanes by shuffling one
14954/// source with a lane permutation.
14955///
14956/// This lowering strategy results in four instructions in the worst case for a
14957/// single-input cross lane shuffle which is lower than any other fully general
14958/// cross-lane shuffle strategy I'm aware of. Special cases for each particular
14959/// shuffle pattern should be handled prior to trying this lowering.
14960static SDValue lowerShuffleAsLanePermuteAndShuffle(
14961 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
14962 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
14963 // FIXME: This should probably be generalized for 512-bit vectors as well.
14964 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!")((VT.is256BitVector() && "Only for 256-bit vector shuffles!"
) ? static_cast<void> (0) : __assert_fail ("VT.is256BitVector() && \"Only for 256-bit vector shuffles!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14964, __PRETTY_FUNCTION__));
18
←
'?' condition is true
→
14965 int Size = Mask.size();
14966 int LaneSize = Size / 2;
14967
14968 // If there are only inputs from one 128-bit lane, splitting will in fact be
14969 // less expensive. The flags track whether the given lane contains an element
14970 // that crosses to another lane.
14971 if (!Subtarget.hasAVX2()) {
19
←
Calling 'X86Subtarget::hasAVX2'
→
22
←
Returning from 'X86Subtarget::hasAVX2'
→
23
←
Taking true branch
→
14972 bool LaneCrossing[2] = {false, false};
14973 for (int i = 0; i < Size; ++i)
24
←
Assuming 'i' is < 'Size'
→
25
←
Loop condition is true. Entering loop body
→
29
←
Assuming 'i' is >= 'Size'
→
30
←
Loop condition is false. Execution continues on line 14976
→
14974 if (Mask[i] >= 0 && (Mask[i] % Size) / LaneSize != i / LaneSize)
26
←
Assuming the condition is true
→
27
←
Assuming the condition is true
→
28
←
Taking true branch
→
14975 LaneCrossing[(Mask[i] % Size) / LaneSize] = true;
14976 if (!LaneCrossing[0] || !LaneCrossing[1])
31
←
Assuming the condition is false
→
32
←
Assuming the condition is false
→
33
←
Taking false branch
→
14977 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
14978 } else {
14979 bool LaneUsed[2] = {false, false};
14980 for (int i = 0; i < Size; ++i)
14981 if (Mask[i] >= 0)
14982 LaneUsed[(Mask[i] / LaneSize)] = true;
14983 if (!LaneUsed[0] || !LaneUsed[1])
14984 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
14985 }
14986
14987 // TODO - we could support shuffling V2 in the Flipped input.
14988 assert(V2.isUndef() &&((V2.isUndef() && "This last part of this routine only works on single input shuffles"
) ? static_cast<void> (0) : __assert_fail ("V2.isUndef() && \"This last part of this routine only works on single input shuffles\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14989, __PRETTY_FUNCTION__))
34
←
'?' condition is true
→
14989 "This last part of this routine only works on single input shuffles")((V2.isUndef() && "This last part of this routine only works on single input shuffles"
) ? static_cast<void> (0) : __assert_fail ("V2.isUndef() && \"This last part of this routine only works on single input shuffles\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 14989, __PRETTY_FUNCTION__));
14990
14991 SmallVector<int, 32> InLaneMask(Mask.begin(), Mask.end());
14992 for (int i = 0; i < Size; ++i) {
35
←
Loop condition is true. Entering loop body
→
14993 int &M = InLaneMask[i];
14994 if (M < 0)
36
←
Assuming 'M' is >= 0
→
37
←
Taking false branch
→
14995 continue;
14996 if (((M % Size) / LaneSize) != (i / LaneSize))
38
←
The result of the '/' expression is undefined
14997 M = (M % LaneSize) + ((i / LaneSize) * LaneSize) + Size;
14998 }
14999 assert(!is128BitLaneCrossingShuffleMask(VT, InLaneMask) &&((!is128BitLaneCrossingShuffleMask(VT, InLaneMask) &&
"In-lane shuffle mask expected") ? static_cast<void> (
0) : __assert_fail ("!is128BitLaneCrossingShuffleMask(VT, InLaneMask) && \"In-lane shuffle mask expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15000, __PRETTY_FUNCTION__))
15000 "In-lane shuffle mask expected")((!is128BitLaneCrossingShuffleMask(VT, InLaneMask) &&
"In-lane shuffle mask expected") ? static_cast<void> (
0) : __assert_fail ("!is128BitLaneCrossingShuffleMask(VT, InLaneMask) && \"In-lane shuffle mask expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15000, __PRETTY_FUNCTION__));
15001
15002 // Flip the lanes, and shuffle the results which should now be in-lane.
15003 MVT PVT = VT.isFloatingPoint() ? MVT::v4f64 : MVT::v4i64;
15004 SDValue Flipped = DAG.getBitcast(PVT, V1);
15005 Flipped =
15006 DAG.getVectorShuffle(PVT, DL, Flipped, DAG.getUNDEF(PVT), {2, 3, 0, 1});
15007 Flipped = DAG.getBitcast(VT, Flipped);
15008 return DAG.getVectorShuffle(VT, DL, V1, Flipped, InLaneMask);
15009}
15010
15011/// Handle lowering 2-lane 128-bit shuffles.
15012static SDValue lowerV2X128Shuffle(const SDLoc &DL, MVT VT, SDValue V1,
15013 SDValue V2, ArrayRef<int> Mask,
15014 const APInt &Zeroable,
15015 const X86Subtarget &Subtarget,
15016 SelectionDAG &DAG) {
15017 // With AVX2, use VPERMQ/VPERMPD for unary shuffles to allow memory folding.
15018 if (Subtarget.hasAVX2() && V2.isUndef())
15019 return SDValue();
15020
15021 bool V2IsZero = !V2.isUndef() && ISD::isBuildVectorAllZeros(V2.getNode());
15022
15023 SmallVector<int, 4> WidenedMask;
15024 if (!canWidenShuffleElements(Mask, Zeroable, V2IsZero, WidenedMask))
15025 return SDValue();
15026
15027 bool IsLowZero = (Zeroable & 0x3) == 0x3;
15028 bool IsHighZero = (Zeroable & 0xc) == 0xc;
15029
15030 // Try to use an insert into a zero vector.
15031 if (WidenedMask[0] == 0 && IsHighZero) {
15032 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
15033 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
15034 DAG.getIntPtrConstant(0, DL));
15035 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
15036 getZeroVector(VT, Subtarget, DAG, DL), LoV,
15037 DAG.getIntPtrConstant(0, DL));
15038 }
15039
15040 // TODO: If minimizing size and one of the inputs is a zero vector and the
15041 // the zero vector has only one use, we could use a VPERM2X128 to save the
15042 // instruction bytes needed to explicitly generate the zero vector.
15043
15044 // Blends are faster and handle all the non-lane-crossing cases.
15045 if (SDValue Blend = lowerShuffleAsBlend(DL, VT, V1, V2, Mask, Zeroable,
15046 Subtarget, DAG))
15047 return Blend;
15048
15049 // If either input operand is a zero vector, use VPERM2X128 because its mask
15050 // allows us to replace the zero input with an implicit zero.
15051 if (!IsLowZero && !IsHighZero) {
15052 // Check for patterns which can be matched with a single insert of a 128-bit
15053 // subvector.
15054 bool OnlyUsesV1 = isShuffleEquivalent(V1, V2, Mask, {0, 1, 0, 1});
15055 if (OnlyUsesV1 || isShuffleEquivalent(V1, V2, Mask, {0, 1, 4, 5})) {
15056
15057 // With AVX1, use vperm2f128 (below) to allow load folding. Otherwise,
15058 // this will likely become vinsertf128 which can't fold a 256-bit memop.
15059 if (!isa<LoadSDNode>(peekThroughBitcasts(V1))) {
15060 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
15061 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
15062 OnlyUsesV1 ? V1 : V2,
15063 DAG.getIntPtrConstant(0, DL));
15064 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
15065 DAG.getIntPtrConstant(2, DL));
15066 }
15067 }
15068
15069 // Try to use SHUF128 if possible.
15070 if (Subtarget.hasVLX()) {
15071 if (WidenedMask[0] < 2 && WidenedMask[1] >= 2) {
15072 unsigned PermMask = ((WidenedMask[0] % 2) << 0) |
15073 ((WidenedMask[1] % 2) << 1);
15074 return DAG.getNode(X86ISD::SHUF128, DL, VT, V1, V2,
15075 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15076 }
15077 }
15078 }
15079
15080 // Otherwise form a 128-bit permutation. After accounting for undefs,
15081 // convert the 64-bit shuffle mask selection values into 128-bit
15082 // selection bits by dividing the indexes by 2 and shifting into positions
15083 // defined by a vperm2*128 instruction's immediate control byte.
15084
15085 // The immediate permute control byte looks like this:
15086 // [1:0] - select 128 bits from sources for low half of destination
15087 // [2] - ignore
15088 // [3] - zero low half of destination
15089 // [5:4] - select 128 bits from sources for high half of destination
15090 // [6] - ignore
15091 // [7] - zero high half of destination
15092
15093 assert((WidenedMask[0] >= 0 || IsLowZero) &&(((WidenedMask[0] >= 0 || IsLowZero) && (WidenedMask
[1] >= 0 || IsHighZero) && "Undef half?") ? static_cast
<void> (0) : __assert_fail ("(WidenedMask[0] >= 0 || IsLowZero) && (WidenedMask[1] >= 0 || IsHighZero) && \"Undef half?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15094, __PRETTY_FUNCTION__))
15094 (WidenedMask[1] >= 0 || IsHighZero) && "Undef half?")(((WidenedMask[0] >= 0 || IsLowZero) && (WidenedMask
[1] >= 0 || IsHighZero) && "Undef half?") ? static_cast
<void> (0) : __assert_fail ("(WidenedMask[0] >= 0 || IsLowZero) && (WidenedMask[1] >= 0 || IsHighZero) && \"Undef half?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15094, __PRETTY_FUNCTION__));
15095
15096 unsigned PermMask = 0;
15097 PermMask |= IsLowZero ? 0x08 : (WidenedMask[0] << 0);
15098 PermMask |= IsHighZero ? 0x80 : (WidenedMask[1] << 4);
15099
15100 // Check the immediate mask and replace unused sources with undef.
15101 if ((PermMask & 0x0a) != 0x00 && (PermMask & 0xa0) != 0x00)
15102 V1 = DAG.getUNDEF(VT);
15103 if ((PermMask & 0x0a) != 0x02 && (PermMask & 0xa0) != 0x20)
15104 V2 = DAG.getUNDEF(VT);
15105
15106 return DAG.getNode(X86ISD::VPERM2X128, DL, VT, V1, V2,
15107 DAG.getTargetConstant(PermMask, DL, MVT::i8));
15108}
15109
15110/// Lower a vector shuffle by first fixing the 128-bit lanes and then
15111/// shuffling each lane.
15112///
15113/// This attempts to create a repeated lane shuffle where each lane uses one
15114/// or two of the lanes of the inputs. The lanes of the input vectors are
15115/// shuffled in one or two independent shuffles to get the lanes into the
15116/// position needed by the final shuffle.
15117static SDValue lowerShuffleAsLanePermuteAndRepeatedMask(
15118 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15119 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15120 assert(!V2.isUndef() && "This is only useful with multiple inputs.")((!V2.isUndef() && "This is only useful with multiple inputs."
) ? static_cast<void> (0) : __assert_fail ("!V2.isUndef() && \"This is only useful with multiple inputs.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15120, __PRETTY_FUNCTION__));
15121
15122 if (is128BitLaneRepeatedShuffleMask(VT, Mask))
15123 return SDValue();
15124
15125 int NumElts = Mask.size();
15126 int NumLanes = VT.getSizeInBits() / 128;
15127 int NumLaneElts = 128 / VT.getScalarSizeInBits();
15128 SmallVector<int, 16> RepeatMask(NumLaneElts, -1);
15129 SmallVector<std::array<int, 2>, 2> LaneSrcs(NumLanes, {{-1, -1}});
15130
15131 // First pass will try to fill in the RepeatMask from lanes that need two
15132 // sources.
15133 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15134 int Srcs[2] = {-1, -1};
15135 SmallVector<int, 16> InLaneMask(NumLaneElts, -1);
15136 for (int i = 0; i != NumLaneElts; ++i) {
15137 int M = Mask[(Lane * NumLaneElts) + i];
15138 if (M < 0)
15139 continue;
15140 // Determine which of the possible input lanes (NumLanes from each source)
15141 // this element comes from. Assign that as one of the sources for this
15142 // lane. We can assign up to 2 sources for this lane. If we run out
15143 // sources we can't do anything.
15144 int LaneSrc = M / NumLaneElts;
15145 int Src;
15146 if (Srcs[0] < 0 || Srcs[0] == LaneSrc)
15147 Src = 0;
15148 else if (Srcs[1] < 0 || Srcs[1] == LaneSrc)
15149 Src = 1;
15150 else
15151 return SDValue();
15152
15153 Srcs[Src] = LaneSrc;
15154 InLaneMask[i] = (M % NumLaneElts) + Src * NumElts;
15155 }
15156
15157 // If this lane has two sources, see if it fits with the repeat mask so far.
15158 if (Srcs[1] < 0)
15159 continue;
15160
15161 LaneSrcs[Lane][0] = Srcs[0];
15162 LaneSrcs[Lane][1] = Srcs[1];
15163
15164 auto MatchMasks = [](ArrayRef<int> M1, ArrayRef<int> M2) {
15165 assert(M1.size() == M2.size() && "Unexpected mask size")((M1.size() == M2.size() && "Unexpected mask size") ?
static_cast<void> (0) : __assert_fail ("M1.size() == M2.size() && \"Unexpected mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15165, __PRETTY_FUNCTION__));
15166 for (int i = 0, e = M1.size(); i != e; ++i)
15167 if (M1[i] >= 0 && M2[i] >= 0 && M1[i] != M2[i])
15168 return false;
15169 return true;
15170 };
15171
15172 auto MergeMasks = [](ArrayRef<int> Mask, MutableArrayRef<int> MergedMask) {
15173 assert(Mask.size() == MergedMask.size() && "Unexpected mask size")((Mask.size() == MergedMask.size() && "Unexpected mask size"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == MergedMask.size() && \"Unexpected mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15173, __PRETTY_FUNCTION__));
15174 for (int i = 0, e = MergedMask.size(); i != e; ++i) {
15175 int M = Mask[i];
15176 if (M < 0)
15177 continue;
15178 assert((MergedMask[i] < 0 || MergedMask[i] == M) &&(((MergedMask[i] < 0 || MergedMask[i] == M) && "Unexpected mask element"
) ? static_cast<void> (0) : __assert_fail ("(MergedMask[i] < 0 || MergedMask[i] == M) && \"Unexpected mask element\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15179, __PRETTY_FUNCTION__))
15179 "Unexpected mask element")(((MergedMask[i] < 0 || MergedMask[i] == M) && "Unexpected mask element"
) ? static_cast<void> (0) : __assert_fail ("(MergedMask[i] < 0 || MergedMask[i] == M) && \"Unexpected mask element\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15179, __PRETTY_FUNCTION__));
15180 MergedMask[i] = M;
15181 }
15182 };
15183
15184 if (MatchMasks(InLaneMask, RepeatMask)) {
15185 // Merge this lane mask into the final repeat mask.
15186 MergeMasks(InLaneMask, RepeatMask);
15187 continue;
15188 }
15189
15190 // Didn't find a match. Swap the operands and try again.
15191 std::swap(LaneSrcs[Lane][0], LaneSrcs[Lane][1]);
15192 ShuffleVectorSDNode::commuteMask(InLaneMask);
15193
15194 if (MatchMasks(InLaneMask, RepeatMask)) {
15195 // Merge this lane mask into the final repeat mask.
15196 MergeMasks(InLaneMask, RepeatMask);
15197 continue;
15198 }
15199
15200 // Couldn't find a match with the operands in either order.
15201 return SDValue();
15202 }
15203
15204 // Now handle any lanes with only one source.
15205 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15206 // If this lane has already been processed, skip it.
15207 if (LaneSrcs[Lane][0] >= 0)
15208 continue;
15209
15210 for (int i = 0; i != NumLaneElts; ++i) {
15211 int M = Mask[(Lane * NumLaneElts) + i];
15212 if (M < 0)
15213 continue;
15214
15215 // If RepeatMask isn't defined yet we can define it ourself.
15216 if (RepeatMask[i] < 0)
15217 RepeatMask[i] = M % NumLaneElts;
15218
15219 if (RepeatMask[i] < NumElts) {
15220 if (RepeatMask[i] != M % NumLaneElts)
15221 return SDValue();
15222 LaneSrcs[Lane][0] = M / NumLaneElts;
15223 } else {
15224 if (RepeatMask[i] != ((M % NumLaneElts) + NumElts))
15225 return SDValue();
15226 LaneSrcs[Lane][1] = M / NumLaneElts;
15227 }
15228 }
15229
15230 if (LaneSrcs[Lane][0] < 0 && LaneSrcs[Lane][1] < 0)
15231 return SDValue();
15232 }
15233
15234 SmallVector<int, 16> NewMask(NumElts, -1);
15235 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15236 int Src = LaneSrcs[Lane][0];
15237 for (int i = 0; i != NumLaneElts; ++i) {
15238 int M = -1;
15239 if (Src >= 0)
15240 M = Src * NumLaneElts + i;
15241 NewMask[Lane * NumLaneElts + i] = M;
15242 }
15243 }
15244 SDValue NewV1 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15245 // Ensure we didn't get back the shuffle we started with.
15246 // FIXME: This is a hack to make up for some splat handling code in
15247 // getVectorShuffle.
15248 if (isa<ShuffleVectorSDNode>(NewV1) &&
15249 cast<ShuffleVectorSDNode>(NewV1)->getMask() == Mask)
15250 return SDValue();
15251
15252 for (int Lane = 0; Lane != NumLanes; ++Lane) {
15253 int Src = LaneSrcs[Lane][1];
15254 for (int i = 0; i != NumLaneElts; ++i) {
15255 int M = -1;
15256 if (Src >= 0)
15257 M = Src * NumLaneElts + i;
15258 NewMask[Lane * NumLaneElts + i] = M;
15259 }
15260 }
15261 SDValue NewV2 = DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
15262 // Ensure we didn't get back the shuffle we started with.
15263 // FIXME: This is a hack to make up for some splat handling code in
15264 // getVectorShuffle.
15265 if (isa<ShuffleVectorSDNode>(NewV2) &&
15266 cast<ShuffleVectorSDNode>(NewV2)->getMask() == Mask)
15267 return SDValue();
15268
15269 for (int i = 0; i != NumElts; ++i) {
15270 NewMask[i] = RepeatMask[i % NumLaneElts];
15271 if (NewMask[i] < 0)
15272 continue;
15273
15274 NewMask[i] += (i / NumLaneElts) * NumLaneElts;
15275 }
15276 return DAG.getVectorShuffle(VT, DL, NewV1, NewV2, NewMask);
15277}
15278
15279/// If the input shuffle mask results in a vector that is undefined in all upper
15280/// or lower half elements and that mask accesses only 2 halves of the
15281/// shuffle's operands, return true. A mask of half the width with mask indexes
15282/// adjusted to access the extracted halves of the original shuffle operands is
15283/// returned in HalfMask. HalfIdx1 and HalfIdx2 return whether the upper or
15284/// lower half of each input operand is accessed.
15285static bool
15286getHalfShuffleMask(ArrayRef<int> Mask, MutableArrayRef<int> HalfMask,
15287 int &HalfIdx1, int &HalfIdx2) {
15288 assert((Mask.size() == HalfMask.size() * 2) &&(((Mask.size() == HalfMask.size() * 2) && "Expected input mask to be twice as long as output"
) ? static_cast<void> (0) : __assert_fail ("(Mask.size() == HalfMask.size() * 2) && \"Expected input mask to be twice as long as output\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15289, __PRETTY_FUNCTION__))
15289 "Expected input mask to be twice as long as output")(((Mask.size() == HalfMask.size() * 2) && "Expected input mask to be twice as long as output"
) ? static_cast<void> (0) : __assert_fail ("(Mask.size() == HalfMask.size() * 2) && \"Expected input mask to be twice as long as output\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15289, __PRETTY_FUNCTION__));
15290
15291 // Exactly one half of the result must be undef to allow narrowing.
15292 bool UndefLower = isUndefLowerHalf(Mask);
15293 bool UndefUpper = isUndefUpperHalf(Mask);
15294 if (UndefLower == UndefUpper)
15295 return false;
15296
15297 unsigned HalfNumElts = HalfMask.size();
15298 unsigned MaskIndexOffset = UndefLower ? HalfNumElts : 0;
15299 HalfIdx1 = -1;
15300 HalfIdx2 = -1;
15301 for (unsigned i = 0; i != HalfNumElts; ++i) {
15302 int M = Mask[i + MaskIndexOffset];
15303 if (M < 0) {
15304 HalfMask[i] = M;
15305 continue;
15306 }
15307
15308 // Determine which of the 4 half vectors this element is from.
15309 // i.e. 0 = Lower V1, 1 = Upper V1, 2 = Lower V2, 3 = Upper V2.
15310 int HalfIdx = M / HalfNumElts;
15311
15312 // Determine the element index into its half vector source.
15313 int HalfElt = M % HalfNumElts;
15314
15315 // We can shuffle with up to 2 half vectors, set the new 'half'
15316 // shuffle mask accordingly.
15317 if (HalfIdx1 < 0 || HalfIdx1 == HalfIdx) {
15318 HalfMask[i] = HalfElt;
15319 HalfIdx1 = HalfIdx;
15320 continue;
15321 }
15322 if (HalfIdx2 < 0 || HalfIdx2 == HalfIdx) {
15323 HalfMask[i] = HalfElt + HalfNumElts;
15324 HalfIdx2 = HalfIdx;
15325 continue;
15326 }
15327
15328 // Too many half vectors referenced.
15329 return false;
15330 }
15331
15332 return true;
15333}
15334
15335/// Given the output values from getHalfShuffleMask(), create a half width
15336/// shuffle of extracted vectors followed by an insert back to full width.
15337static SDValue getShuffleHalfVectors(const SDLoc &DL, SDValue V1, SDValue V2,
15338 ArrayRef<int> HalfMask, int HalfIdx1,
15339 int HalfIdx2, bool UndefLower,
15340 SelectionDAG &DAG, bool UseConcat = false) {
15341 assert(V1.getValueType() == V2.getValueType() && "Different sized vectors?")((V1.getValueType() == V2.getValueType() && "Different sized vectors?"
) ? static_cast<void> (0) : __assert_fail ("V1.getValueType() == V2.getValueType() && \"Different sized vectors?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15341, __PRETTY_FUNCTION__));
15342 assert(V1.getValueType().isSimple() && "Expecting only simple types")((V1.getValueType().isSimple() && "Expecting only simple types"
) ? static_cast<void> (0) : __assert_fail ("V1.getValueType().isSimple() && \"Expecting only simple types\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15342, __PRETTY_FUNCTION__));
15343
15344 MVT VT = V1.getSimpleValueType();
15345 MVT HalfVT = VT.getHalfNumVectorElementsVT();
15346 unsigned HalfNumElts = HalfVT.getVectorNumElements();
15347
15348 auto getHalfVector = [&](int HalfIdx) {
15349 if (HalfIdx < 0)
15350 return DAG.getUNDEF(HalfVT);
15351 SDValue V = (HalfIdx < 2 ? V1 : V2);
15352 HalfIdx = (HalfIdx % 2) * HalfNumElts;
15353 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V,
15354 DAG.getIntPtrConstant(HalfIdx, DL));
15355 };
15356
15357 // ins undef, (shuf (ext V1, HalfIdx1), (ext V2, HalfIdx2), HalfMask), Offset
15358 SDValue Half1 = getHalfVector(HalfIdx1);
15359 SDValue Half2 = getHalfVector(HalfIdx2);
15360 SDValue V = DAG.getVectorShuffle(HalfVT, DL, Half1, Half2, HalfMask);
15361 if (UseConcat) {
15362 SDValue Op0 = V;
15363 SDValue Op1 = DAG.getUNDEF(HalfVT);
15364 if (UndefLower)
15365 std::swap(Op0, Op1);
15366 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Op0, Op1);
15367 }
15368
15369 unsigned Offset = UndefLower ? HalfNumElts : 0;
15370 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), V,
15371 DAG.getIntPtrConstant(Offset, DL));
15372}
15373
15374/// Lower shuffles where an entire half of a 256 or 512-bit vector is UNDEF.
15375/// This allows for fast cases such as subvector extraction/insertion
15376/// or shuffling smaller vector types which can lower more efficiently.
15377static SDValue lowerShuffleWithUndefHalf(const SDLoc &DL, MVT VT, SDValue V1,
15378 SDValue V2, ArrayRef<int> Mask,
15379 const X86Subtarget &Subtarget,
15380 SelectionDAG &DAG) {
15381 assert((VT.is256BitVector() || VT.is512BitVector()) &&(((VT.is256BitVector() || VT.is512BitVector()) && "Expected 256-bit or 512-bit vector"
) ? static_cast<void> (0) : __assert_fail ("(VT.is256BitVector() || VT.is512BitVector()) && \"Expected 256-bit or 512-bit vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15382, __PRETTY_FUNCTION__))
15382 "Expected 256-bit or 512-bit vector")(((VT.is256BitVector() || VT.is512BitVector()) && "Expected 256-bit or 512-bit vector"
) ? static_cast<void> (0) : __assert_fail ("(VT.is256BitVector() || VT.is512BitVector()) && \"Expected 256-bit or 512-bit vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15382, __PRETTY_FUNCTION__));
15383
15384 bool UndefLower = isUndefLowerHalf(Mask);
15385 if (!UndefLower && !isUndefUpperHalf(Mask))
15386 return SDValue();
15387
15388 assert((!UndefLower || !isUndefUpperHalf(Mask)) &&(((!UndefLower || !isUndefUpperHalf(Mask)) && "Completely undef shuffle mask should have been simplified already"
) ? static_cast<void> (0) : __assert_fail ("(!UndefLower || !isUndefUpperHalf(Mask)) && \"Completely undef shuffle mask should have been simplified already\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15389, __PRETTY_FUNCTION__))
15389 "Completely undef shuffle mask should have been simplified already")(((!UndefLower || !isUndefUpperHalf(Mask)) && "Completely undef shuffle mask should have been simplified already"
) ? static_cast<void> (0) : __assert_fail ("(!UndefLower || !isUndefUpperHalf(Mask)) && \"Completely undef shuffle mask should have been simplified already\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15389, __PRETTY_FUNCTION__));
15390
15391 // Upper half is undef and lower half is whole upper subvector.
15392 // e.g. vector_shuffle <4, 5, 6, 7, u, u, u, u> or <2, 3, u, u>
15393 MVT HalfVT = VT.getHalfNumVectorElementsVT();
15394 unsigned HalfNumElts = HalfVT.getVectorNumElements();
15395 if (!UndefLower &&
15396 isSequentialOrUndefInRange(Mask, 0, HalfNumElts, HalfNumElts)) {
15397 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15398 DAG.getIntPtrConstant(HalfNumElts, DL));
15399 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15400 DAG.getIntPtrConstant(0, DL));
15401 }
15402
15403 // Lower half is undef and upper half is whole lower subvector.
15404 // e.g. vector_shuffle <u, u, u, u, 0, 1, 2, 3> or <u, u, 0, 1>
15405 if (UndefLower &&
15406 isSequentialOrUndefInRange(Mask, HalfNumElts, HalfNumElts, 0)) {
15407 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, HalfVT, V1,
15408 DAG.getIntPtrConstant(0, DL));
15409 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), Hi,
15410 DAG.getIntPtrConstant(HalfNumElts, DL));
15411 }
15412
15413 int HalfIdx1, HalfIdx2;
15414 SmallVector<int, 8> HalfMask(HalfNumElts);
15415 if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2))
15416 return SDValue();
15417
15418 assert(HalfMask.size() == HalfNumElts && "Unexpected shuffle mask length")((HalfMask.size() == HalfNumElts && "Unexpected shuffle mask length"
) ? static_cast<void> (0) : __assert_fail ("HalfMask.size() == HalfNumElts && \"Unexpected shuffle mask length\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15418, __PRETTY_FUNCTION__));
15419
15420 // Only shuffle the halves of the inputs when useful.
15421 unsigned NumLowerHalves =
15422 (HalfIdx1 == 0 || HalfIdx1 == 2) + (HalfIdx2 == 0 || HalfIdx2 == 2);
15423 unsigned NumUpperHalves =
15424 (HalfIdx1 == 1 || HalfIdx1 == 3) + (HalfIdx2 == 1 || HalfIdx2 == 3);
15425 assert(NumLowerHalves + NumUpperHalves <= 2 && "Only 1 or 2 halves allowed")((NumLowerHalves + NumUpperHalves <= 2 && "Only 1 or 2 halves allowed"
) ? static_cast<void> (0) : __assert_fail ("NumLowerHalves + NumUpperHalves <= 2 && \"Only 1 or 2 halves allowed\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15425, __PRETTY_FUNCTION__));
15426
15427 // Determine the larger pattern of undef/halves, then decide if it's worth
15428 // splitting the shuffle based on subtarget capabilities and types.
15429 unsigned EltWidth = VT.getVectorElementType().getSizeInBits();
15430 if (!UndefLower) {
15431 // XXXXuuuu: no insert is needed.
15432 // Always extract lowers when setting lower - these are all free subreg ops.
15433 if (NumUpperHalves == 0)
15434 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15435 UndefLower, DAG);
15436
15437 if (NumUpperHalves == 1) {
15438 // AVX2 has efficient 32/64-bit element cross-lane shuffles.
15439 if (Subtarget.hasAVX2()) {
15440 // extract128 + vunpckhps/vshufps, is better than vblend + vpermps.
15441 if (EltWidth == 32 && NumLowerHalves && HalfVT.is128BitVector() &&
15442 !is128BitUnpackShuffleMask(HalfMask) &&
15443 (!isSingleSHUFPSMask(HalfMask) ||
15444 Subtarget.hasFastVariableShuffle()))
15445 return SDValue();
15446 // If this is a unary shuffle (assume that the 2nd operand is
15447 // canonicalized to undef), then we can use vpermpd. Otherwise, we
15448 // are better off extracting the upper half of 1 operand and using a
15449 // narrow shuffle.
15450 if (EltWidth == 64 && V2.isUndef())
15451 return SDValue();
15452 }
15453 // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15454 if (Subtarget.hasAVX512() && VT.is512BitVector())
15455 return SDValue();
15456 // Extract + narrow shuffle is better than the wide alternative.
15457 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15458 UndefLower, DAG);
15459 }
15460
15461 // Don't extract both uppers, instead shuffle and then extract.
15462 assert(NumUpperHalves == 2 && "Half vector count went wrong")((NumUpperHalves == 2 && "Half vector count went wrong"
) ? static_cast<void> (0) : __assert_fail ("NumUpperHalves == 2 && \"Half vector count went wrong\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15462, __PRETTY_FUNCTION__));
15463 return SDValue();
15464 }
15465
15466 // UndefLower - uuuuXXXX: an insert to high half is required if we split this.
15467 if (NumUpperHalves == 0) {
15468 // AVX2 has efficient 64-bit element cross-lane shuffles.
15469 // TODO: Refine to account for unary shuffle, splat, and other masks?
15470 if (Subtarget.hasAVX2() && EltWidth == 64)
15471 return SDValue();
15472 // AVX512 has efficient cross-lane shuffles for all legal 512-bit types.
15473 if (Subtarget.hasAVX512() && VT.is512BitVector())
15474 return SDValue();
15475 // Narrow shuffle + insert is better than the wide alternative.
15476 return getShuffleHalfVectors(DL, V1, V2, HalfMask, HalfIdx1, HalfIdx2,
15477 UndefLower, DAG);
15478 }
15479
15480 // NumUpperHalves != 0: don't bother with extract, shuffle, and then insert.
15481 return SDValue();
15482}
15483
15484/// Test whether the specified input (0 or 1) is in-place blended by the
15485/// given mask.
15486///
15487/// This returns true if the elements from a particular input are already in the
15488/// slot required by the given mask and require no permutation.
15489static bool isShuffleMaskInputInPlace(int Input, ArrayRef<int> Mask) {
15490 assert((Input == 0 || Input == 1) && "Only two inputs to shuffles.")(((Input == 0 || Input == 1) && "Only two inputs to shuffles."
) ? static_cast<void> (0) : __assert_fail ("(Input == 0 || Input == 1) && \"Only two inputs to shuffles.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15490, __PRETTY_FUNCTION__));
15491 int Size = Mask.size();
15492 for (int i = 0; i < Size; ++i)
15493 if (Mask[i] >= 0 && Mask[i] / Size == Input && Mask[i] % Size != i)
15494 return false;
15495
15496 return true;
15497}
15498
15499/// Handle case where shuffle sources are coming from the same 128-bit lane and
15500/// every lane can be represented as the same repeating mask - allowing us to
15501/// shuffle the sources with the repeating shuffle and then permute the result
15502/// to the destination lanes.
15503static SDValue lowerShuffleAsRepeatedMaskAndLanePermute(
15504 const SDLoc &DL, MVT VT, SDValue V1, SDValue V2, ArrayRef<int> Mask,
15505 const X86Subtarget &Subtarget, SelectionDAG &DAG) {
15506 int NumElts = VT.getVectorNumElements();
15507 int NumLanes = VT.getSizeInBits() / 128;
15508 int NumLaneElts = NumElts / NumLanes;
15509
15510 // On AVX2 we may be able to just shuffle the lowest elements and then
15511 // broadcast the result.
15512 if (Subtarget.hasAVX2()) {
15513 for (unsigned BroadcastSize : {16, 32, 64}) {
15514 if (BroadcastSize <= VT.getScalarSizeInBits())
15515 continue;
15516 int NumBroadcastElts = BroadcastSize / VT.getScalarSizeInBits();
15517
15518 // Attempt to match a repeating pattern every NumBroadcastElts,
15519 // accounting for UNDEFs but only references the lowest 128-bit
15520 // lane of the inputs.
15521 auto FindRepeatingBroadcastMask = [&](SmallVectorImpl<int> &RepeatMask) {
15522 for (int i = 0; i != NumElts; i += NumBroadcastElts)
15523 for (int j = 0; j != NumBroadcastElts; ++j) {
15524 int M = Mask[i + j];
15525 if (M < 0)
15526 continue;
15527 int &R = RepeatMask[j];
15528 if (0 != ((M % NumElts) / NumLaneElts))
15529 return false;
15530 if (0 <= R && R != M)
15531 return false;
15532 R = M;
15533 }
15534 return true;
15535 };
15536
15537 SmallVector<int, 8> RepeatMask((unsigned)NumElts, -1);
15538 if (!FindRepeatingBroadcastMask(RepeatMask))
15539 continue;
15540
15541 // Shuffle the (lowest) repeated elements in place for broadcast.
15542 SDValue RepeatShuf = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatMask);
15543
15544 // Shuffle the actual broadcast.
15545 SmallVector<int, 8> BroadcastMask((unsigned)NumElts, -1);
15546 for (int i = 0; i != NumElts; i += NumBroadcastElts)
15547 for (int j = 0; j != NumBroadcastElts; ++j)
15548 BroadcastMask[i + j] = j;
15549 return DAG.getVectorShuffle(VT, DL, RepeatShuf, DAG.getUNDEF(VT),
15550 BroadcastMask);
15551 }
15552 }
15553
15554 // Bail if the shuffle mask doesn't cross 128-bit lanes.
15555 if (!is128BitLaneCrossingShuffleMask(VT, Mask))
15556 return SDValue();
15557
15558 // Bail if we already have a repeated lane shuffle mask.
15559 SmallVector<int, 8> RepeatedShuffleMask;
15560 if (is128BitLaneRepeatedShuffleMask(VT, Mask, RepeatedShuffleMask))
15561 return SDValue();
15562
15563 // On AVX2 targets we can permute 256-bit vectors as 64-bit sub-lanes
15564 // (with PERMQ/PERMPD), otherwise we can only permute whole 128-bit lanes.
15565 int SubLaneScale = Subtarget.hasAVX2() && VT.is256BitVector() ? 2 : 1;
15566 int NumSubLanes = NumLanes * SubLaneScale;
15567 int NumSubLaneElts = NumLaneElts / SubLaneScale;
15568
15569 // Check that all the sources are coming from the same lane and see if we can
15570 // form a repeating shuffle mask (local to each sub-lane). At the same time,
15571 // determine the source sub-lane for each destination sub-lane.
15572 int TopSrcSubLane = -1;
15573 SmallVector<int, 8> Dst2SrcSubLanes((unsigned)NumSubLanes, -1);
15574 SmallVector<int, 8> RepeatedSubLaneMasks[2] = {
15575 SmallVector<int, 8>((unsigned)NumSubLaneElts, SM_SentinelUndef),
15576 SmallVector<int, 8>((unsigned)NumSubLaneElts, SM_SentinelUndef)};
15577
15578 for (int DstSubLane = 0; DstSubLane != NumSubLanes; ++DstSubLane) {
15579 // Extract the sub-lane mask, check that it all comes from the same lane
15580 // and normalize the mask entries to come from the first lane.
15581 int SrcLane = -1;
15582 SmallVector<int, 8> SubLaneMask((unsigned)NumSubLaneElts, -1);
15583 for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15584 int M = Mask[(DstSubLane * NumSubLaneElts) + Elt];
15585 if (M < 0)
15586 continue;
15587 int Lane = (M % NumElts) / NumLaneElts;
15588 if ((0 <= SrcLane) && (SrcLane != Lane))
15589 return SDValue();
15590 SrcLane = Lane;
15591 int LocalM = (M % NumLaneElts) + (M < NumElts ? 0 : NumElts);
15592 SubLaneMask[Elt] = LocalM;
15593 }
15594
15595 // Whole sub-lane is UNDEF.
15596 if (SrcLane < 0)
15597 continue;
15598
15599 // Attempt to match against the candidate repeated sub-lane masks.
15600 for (int SubLane = 0; SubLane != SubLaneScale; ++SubLane) {
15601 auto MatchMasks = [NumSubLaneElts](ArrayRef<int> M1, ArrayRef<int> M2) {
15602 for (int i = 0; i != NumSubLaneElts; ++i) {
15603 if (M1[i] < 0 || M2[i] < 0)
15604 continue;
15605 if (M1[i] != M2[i])
15606 return false;
15607 }
15608 return true;
15609 };
15610
15611 auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane];
15612 if (!MatchMasks(SubLaneMask, RepeatedSubLaneMask))
15613 continue;
15614
15615 // Merge the sub-lane mask into the matching repeated sub-lane mask.
15616 for (int i = 0; i != NumSubLaneElts; ++i) {
15617 int M = SubLaneMask[i];
15618 if (M < 0)
15619 continue;
15620 assert((RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] == M) &&(((RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] ==
M) && "Unexpected mask element") ? static_cast<void
> (0) : __assert_fail ("(RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] == M) && \"Unexpected mask element\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15621, __PRETTY_FUNCTION__))
15621 "Unexpected mask element")(((RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] ==
M) && "Unexpected mask element") ? static_cast<void
> (0) : __assert_fail ("(RepeatedSubLaneMask[i] < 0 || RepeatedSubLaneMask[i] == M) && \"Unexpected mask element\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15621, __PRETTY_FUNCTION__));
15622 RepeatedSubLaneMask[i] = M;
15623 }
15624
15625 // Track the top most source sub-lane - by setting the remaining to UNDEF
15626 // we can greatly simplify shuffle matching.
15627 int SrcSubLane = (SrcLane * SubLaneScale) + SubLane;
15628 TopSrcSubLane = std::max(TopSrcSubLane, SrcSubLane);
15629 Dst2SrcSubLanes[DstSubLane] = SrcSubLane;
15630 break;
15631 }
15632
15633 // Bail if we failed to find a matching repeated sub-lane mask.
15634 if (Dst2SrcSubLanes[DstSubLane] < 0)
15635 return SDValue();
15636 }
15637 assert(0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes &&((0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes
&& "Unexpected source lane") ? static_cast<void>
(0) : __assert_fail ("0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes && \"Unexpected source lane\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15638, __PRETTY_FUNCTION__))
15638 "Unexpected source lane")((0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes
&& "Unexpected source lane") ? static_cast<void>
(0) : __assert_fail ("0 <= TopSrcSubLane && TopSrcSubLane < NumSubLanes && \"Unexpected source lane\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15638, __PRETTY_FUNCTION__));
15639
15640 // Create a repeating shuffle mask for the entire vector.
15641 SmallVector<int, 8> RepeatedMask((unsigned)NumElts, -1);
15642 for (int SubLane = 0; SubLane <= TopSrcSubLane; ++SubLane) {
15643 int Lane = SubLane / SubLaneScale;
15644 auto &RepeatedSubLaneMask = RepeatedSubLaneMasks[SubLane % SubLaneScale];
15645 for (int Elt = 0; Elt != NumSubLaneElts; ++Elt) {
15646 int M = RepeatedSubLaneMask[Elt];
15647 if (M < 0)
15648 continue;
15649 int Idx = (SubLane * NumSubLaneElts) + Elt;
15650 RepeatedMask[Idx] = M + (Lane * NumLaneElts);
15651 }
15652 }
15653 SDValue RepeatedShuffle = DAG.getVectorShuffle(VT, DL, V1, V2, RepeatedMask);
15654
15655 // Shuffle each source sub-lane to its destination.
15656 SmallVector<int, 8> SubLaneMask((unsigned)NumElts, -1);
15657 for (int i = 0; i != NumElts; i += NumSubLaneElts) {
15658 int SrcSubLane = Dst2SrcSubLanes[i / NumSubLaneElts];
15659 if (SrcSubLane < 0)
15660 continue;
15661 for (int j = 0; j != NumSubLaneElts; ++j)
15662 SubLaneMask[i + j] = j + (SrcSubLane * NumSubLaneElts);
15663 }
15664
15665 return DAG.getVectorShuffle(VT, DL, RepeatedShuffle, DAG.getUNDEF(VT),
15666 SubLaneMask);
15667}
15668
15669static bool matchShuffleWithSHUFPD(MVT VT, SDValue &V1, SDValue &V2,
15670 bool &ForceV1Zero, bool &ForceV2Zero,
15671 unsigned &ShuffleImm, ArrayRef<int> Mask,
15672 const APInt &Zeroable) {
15673 int NumElts = VT.getVectorNumElements();
15674 assert(VT.getScalarSizeInBits() == 64 &&((VT.getScalarSizeInBits() == 64 && (NumElts == 2 || NumElts
== 4 || NumElts == 8) && "Unexpected data type for VSHUFPD"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() == 64 && (NumElts == 2 || NumElts == 4 || NumElts == 8) && \"Unexpected data type for VSHUFPD\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15676, __PRETTY_FUNCTION__))
15675 (NumElts == 2 || NumElts == 4 || NumElts == 8) &&((VT.getScalarSizeInBits() == 64 && (NumElts == 2 || NumElts
== 4 || NumElts == 8) && "Unexpected data type for VSHUFPD"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() == 64 && (NumElts == 2 || NumElts == 4 || NumElts == 8) && \"Unexpected data type for VSHUFPD\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15676, __PRETTY_FUNCTION__))
15676 "Unexpected data type for VSHUFPD")((VT.getScalarSizeInBits() == 64 && (NumElts == 2 || NumElts
== 4 || NumElts == 8) && "Unexpected data type for VSHUFPD"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() == 64 && (NumElts == 2 || NumElts == 4 || NumElts == 8) && \"Unexpected data type for VSHUFPD\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15676, __PRETTY_FUNCTION__));
15677 assert(isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) &&((isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) && "Illegal shuffle mask"
) ? static_cast<void> (0) : __assert_fail ("isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) && \"Illegal shuffle mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15678, __PRETTY_FUNCTION__))
15678 "Illegal shuffle mask")((isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) && "Illegal shuffle mask"
) ? static_cast<void> (0) : __assert_fail ("isUndefOrZeroOrInRange(Mask, 0, 2 * NumElts) && \"Illegal shuffle mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15678, __PRETTY_FUNCTION__));
15679
15680 bool ZeroLane[2] = { true, true };
15681 for (int i = 0; i < NumElts; ++i)
15682 ZeroLane[i & 1] &= Zeroable[i];
15683
15684 // Mask for V8F64: 0/1, 8/9, 2/3, 10/11, 4/5, ..
15685 // Mask for V4F64; 0/1, 4/5, 2/3, 6/7..
15686 ShuffleImm = 0;
15687 bool ShufpdMask = true;
15688 bool CommutableMask = true;
15689 for (int i = 0; i < NumElts; ++i) {
15690 if (Mask[i] == SM_SentinelUndef || ZeroLane[i & 1])
15691 continue;
15692 if (Mask[i] < 0)
15693 return false;
15694 int Val = (i & 6) + NumElts * (i & 1);
15695 int CommutVal = (i & 0xe) + NumElts * ((i & 1) ^ 1);
15696 if (Mask[i] < Val || Mask[i] > Val + 1)
15697 ShufpdMask = false;
15698 if (Mask[i] < CommutVal || Mask[i] > CommutVal + 1)
15699 CommutableMask = false;
15700 ShuffleImm |= (Mask[i] % 2) << i;
15701 }
15702
15703 if (!ShufpdMask && !CommutableMask)
15704 return false;
15705
15706 if (!ShufpdMask && CommutableMask)
15707 std::swap(V1, V2);
15708
15709 ForceV1Zero = ZeroLane[0];
15710 ForceV2Zero = ZeroLane[1];
15711 return true;
15712}
15713
15714static SDValue lowerShuffleWithSHUFPD(const SDLoc &DL, MVT VT, SDValue V1,
15715 SDValue V2, ArrayRef<int> Mask,
15716 const APInt &Zeroable,
15717 const X86Subtarget &Subtarget,
15718 SelectionDAG &DAG) {
15719 assert((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&(((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&
"Unexpected data type for VSHUFPD") ? static_cast<void>
(0) : __assert_fail ("(VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) && \"Unexpected data type for VSHUFPD\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15720, __PRETTY_FUNCTION__))
15720 "Unexpected data type for VSHUFPD")(((VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) &&
"Unexpected data type for VSHUFPD") ? static_cast<void>
(0) : __assert_fail ("(VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v8f64) && \"Unexpected data type for VSHUFPD\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15720, __PRETTY_FUNCTION__));
15721
15722 unsigned Immediate = 0;
15723 bool ForceV1Zero = false, ForceV2Zero = false;
15724 if (!matchShuffleWithSHUFPD(VT, V1, V2, ForceV1Zero, ForceV2Zero, Immediate,
15725 Mask, Zeroable))
15726 return SDValue();
15727
15728 // Create a REAL zero vector - ISD::isBuildVectorAllZeros allows UNDEFs.
15729 if (ForceV1Zero)
15730 V1 = getZeroVector(VT, Subtarget, DAG, DL);
15731 if (ForceV2Zero)
15732 V2 = getZeroVector(VT, Subtarget, DAG, DL);
15733
15734 return DAG.getNode(X86ISD::SHUFP, DL, VT, V1, V2,
15735 DAG.getTargetConstant(Immediate, DL, MVT::i8));
15736}
15737
15738// Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
15739// by zeroable elements in the remaining 24 elements. Turn this into two
15740// vmovqb instructions shuffled together.
15741static SDValue lowerShuffleAsVTRUNCAndUnpack(const SDLoc &DL, MVT VT,
15742 SDValue V1, SDValue V2,
15743 ArrayRef<int> Mask,
15744 const APInt &Zeroable,
15745 SelectionDAG &DAG) {
15746 assert(VT == MVT::v32i8 && "Unexpected type!")((VT == MVT::v32i8 && "Unexpected type!") ? static_cast
<void> (0) : __assert_fail ("VT == MVT::v32i8 && \"Unexpected type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15746, __PRETTY_FUNCTION__));
15747
15748 // The first 8 indices should be every 8th element.
15749 if (!isSequentialOrUndefInRange(Mask, 0, 8, 0, 8))
15750 return SDValue();
15751
15752 // Remaining elements need to be zeroable.
15753 if (Zeroable.countLeadingOnes() < (Mask.size() - 8))
15754 return SDValue();
15755
15756 V1 = DAG.getBitcast(MVT::v4i64, V1);
15757 V2 = DAG.getBitcast(MVT::v4i64, V2);
15758
15759 V1 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V1);
15760 V2 = DAG.getNode(X86ISD::VTRUNC, DL, MVT::v16i8, V2);
15761
15762 // The VTRUNCs will put 0s in the upper 12 bytes. Use them to put zeroes in
15763 // the upper bits of the result using an unpckldq.
15764 SDValue Unpack = DAG.getVectorShuffle(MVT::v16i8, DL, V1, V2,
15765 { 0, 1, 2, 3, 16, 17, 18, 19,
15766 4, 5, 6, 7, 20, 21, 22, 23 });
15767 // Insert the unpckldq into a zero vector to widen to v32i8.
15768 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v32i8,
15769 DAG.getConstant(0, DL, MVT::v32i8), Unpack,
15770 DAG.getIntPtrConstant(0, DL));
15771}
15772
15773
15774/// Handle lowering of 4-lane 64-bit floating point shuffles.
15775///
15776/// Also ends up handling lowering of 4-lane 64-bit integer shuffles when AVX2
15777/// isn't available.
15778static SDValue lowerV4F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15779 const APInt &Zeroable, SDValue V1, SDValue V2,
15780 const X86Subtarget &Subtarget,
15781 SelectionDAG &DAG) {
15782 assert(V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v4f64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15782, __PRETTY_FUNCTION__));
15783 assert(V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v4f64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v4f64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15783, __PRETTY_FUNCTION__));
15784 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!")((Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 4 && \"Unexpected mask size for v4 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15784, __PRETTY_FUNCTION__));
15785
15786 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4f64, V1, V2, Mask, Zeroable,
15787 Subtarget, DAG))
15788 return V;
15789
15790 if (V2.isUndef()) {
15791 // Check for being able to broadcast a single element.
15792 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4f64, V1, V2,
15793 Mask, Subtarget, DAG))
15794 return Broadcast;
15795
15796 // Use low duplicate instructions for masks that match their pattern.
15797 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2}))
15798 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v4f64, V1);
15799
15800 if (!is128BitLaneCrossingShuffleMask(MVT::v4f64, Mask)) {
15801 // Non-half-crossing single input shuffles can be lowered with an
15802 // interleaved permutation.
15803 unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
15804 ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3);
15805 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v4f64, V1,
15806 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
15807 }
15808
15809 // With AVX2 we have direct support for this permutation.
15810 if (Subtarget.hasAVX2())
15811 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4f64, V1,
15812 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
15813
15814 // Try to create an in-lane repeating shuffle mask and then shuffle the
15815 // results into the target lanes.
15816 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15817 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15818 return V;
15819
15820 // Try to permute the lanes and then use a per-lane permute.
15821 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(DL, MVT::v4f64, V1, V2,
15822 Mask, DAG, Subtarget))
15823 return V;
15824
15825 // Otherwise, fall back.
15826 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v4f64, V1, V2, Mask,
15827 DAG, Subtarget);
15828 }
15829
15830 // Use dedicated unpack instructions for masks that match their pattern.
15831 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4f64, Mask, V1, V2, DAG))
15832 return V;
15833
15834 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4f64, V1, V2, Mask,
15835 Zeroable, Subtarget, DAG))
15836 return Blend;
15837
15838 // Check if the blend happens to exactly fit that of SHUFPD.
15839 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v4f64, V1, V2, Mask,
15840 Zeroable, Subtarget, DAG))
15841 return Op;
15842
15843 // If we have one input in place, then we can permute the other input and
15844 // blend the result.
15845 if (isShuffleMaskInputInPlace(0, Mask) || isShuffleMaskInputInPlace(1, Mask))
15846 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, Mask,
15847 Subtarget, DAG);
15848
15849 // Try to create an in-lane repeating shuffle mask and then shuffle the
15850 // results into the target lanes.
15851 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15852 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15853 return V;
15854
15855 // Try to simplify this by merging 128-bit lanes to enable a lane-based
15856 // shuffle. However, if we have AVX2 and either inputs are already in place,
15857 // we will be able to shuffle even across lanes the other input in a single
15858 // instruction so skip this pattern.
15859 if (!(Subtarget.hasAVX2() && (isShuffleMaskInputInPlace(0, Mask) ||
15860 isShuffleMaskInputInPlace(1, Mask))))
15861 if (SDValue V = lowerShuffleAsLanePermuteAndRepeatedMask(
15862 DL, MVT::v4f64, V1, V2, Mask, Subtarget, DAG))
15863 return V;
15864
15865 // If we have VLX support, we can use VEXPAND.
15866 if (Subtarget.hasVLX())
15867 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4f64, Zeroable, Mask, V1, V2,
15868 DAG, Subtarget))
15869 return V;
15870
15871 // If we have AVX2 then we always want to lower with a blend because an v4 we
15872 // can fully permute the elements.
15873 if (Subtarget.hasAVX2())
15874 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4f64, V1, V2, Mask,
15875 Subtarget, DAG);
15876
15877 // Otherwise fall back on generic lowering.
15878 return lowerShuffleAsSplitOrBlend(DL, MVT::v4f64, V1, V2, Mask,
15879 Subtarget, DAG);
15880}
15881
15882/// Handle lowering of 4-lane 64-bit integer shuffles.
15883///
15884/// This routine is only called when we have AVX2 and thus a reasonable
15885/// instruction set for v4i64 shuffling..
15886static SDValue lowerV4I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15887 const APInt &Zeroable, SDValue V1, SDValue V2,
15888 const X86Subtarget &Subtarget,
15889 SelectionDAG &DAG) {
15890 assert(V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v4i64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15890, __PRETTY_FUNCTION__));
15891 assert(V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v4i64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v4i64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15891, __PRETTY_FUNCTION__));
15892 assert(Mask.size() == 4 && "Unexpected mask size for v4 shuffle!")((Mask.size() == 4 && "Unexpected mask size for v4 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 4 && \"Unexpected mask size for v4 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15892, __PRETTY_FUNCTION__));
15893 assert(Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!")((Subtarget.hasAVX2() && "We can only lower v4i64 with AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"We can only lower v4i64 with AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15893, __PRETTY_FUNCTION__));
15894
15895 if (SDValue V = lowerV2X128Shuffle(DL, MVT::v4i64, V1, V2, Mask, Zeroable,
15896 Subtarget, DAG))
15897 return V;
15898
15899 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v4i64, V1, V2, Mask,
15900 Zeroable, Subtarget, DAG))
15901 return Blend;
15902
15903 // Check for being able to broadcast a single element.
15904 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v4i64, V1, V2, Mask,
15905 Subtarget, DAG))
15906 return Broadcast;
15907
15908 if (V2.isUndef()) {
15909 // When the shuffle is mirrored between the 128-bit lanes of the unit, we
15910 // can use lower latency instructions that will operate on both lanes.
15911 SmallVector<int, 2> RepeatedMask;
15912 if (is128BitLaneRepeatedShuffleMask(MVT::v4i64, Mask, RepeatedMask)) {
15913 SmallVector<int, 4> PSHUFDMask;
15914 scaleShuffleMask<int>(2, RepeatedMask, PSHUFDMask);
15915 return DAG.getBitcast(
15916 MVT::v4i64,
15917 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32,
15918 DAG.getBitcast(MVT::v8i32, V1),
15919 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
15920 }
15921
15922 // AVX2 provides a direct instruction for permuting a single input across
15923 // lanes.
15924 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v4i64, V1,
15925 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
15926 }
15927
15928 // Try to use shift instructions.
15929 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v4i64, V1, V2, Mask,
15930 Zeroable, Subtarget, DAG))
15931 return Shift;
15932
15933 // If we have VLX support, we can use VALIGN or VEXPAND.
15934 if (Subtarget.hasVLX()) {
15935 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v4i64, V1, V2, Mask,
15936 Subtarget, DAG))
15937 return Rotate;
15938
15939 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v4i64, Zeroable, Mask, V1, V2,
15940 DAG, Subtarget))
15941 return V;
15942 }
15943
15944 // Try to use PALIGNR.
15945 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v4i64, V1, V2, Mask,
15946 Subtarget, DAG))
15947 return Rotate;
15948
15949 // Use dedicated unpack instructions for masks that match their pattern.
15950 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v4i64, Mask, V1, V2, DAG))
15951 return V;
15952
15953 // If we have one input in place, then we can permute the other input and
15954 // blend the result.
15955 if (isShuffleMaskInputInPlace(0, Mask) || isShuffleMaskInputInPlace(1, Mask))
15956 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, Mask,
15957 Subtarget, DAG);
15958
15959 // Try to create an in-lane repeating shuffle mask and then shuffle the
15960 // results into the target lanes.
15961 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
15962 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
15963 return V;
15964
15965 // Try to simplify this by merging 128-bit lanes to enable a lane-based
15966 // shuffle. However, if we have AVX2 and either inputs are already in place,
15967 // we will be able to shuffle even across lanes the other input in a single
15968 // instruction so skip this pattern.
15969 if (!isShuffleMaskInputInPlace(0, Mask) &&
15970 !isShuffleMaskInputInPlace(1, Mask))
15971 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
15972 DL, MVT::v4i64, V1, V2, Mask, Subtarget, DAG))
15973 return Result;
15974
15975 // Otherwise fall back on generic blend lowering.
15976 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v4i64, V1, V2, Mask,
15977 Subtarget, DAG);
15978}
15979
15980/// Handle lowering of 8-lane 32-bit floating point shuffles.
15981///
15982/// Also ends up handling lowering of 8-lane 32-bit integer shuffles when AVX2
15983/// isn't available.
15984static SDValue lowerV8F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
15985 const APInt &Zeroable, SDValue V1, SDValue V2,
15986 const X86Subtarget &Subtarget,
15987 SelectionDAG &DAG) {
15988 assert(V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v8f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15988, __PRETTY_FUNCTION__));
15989 assert(V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v8f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v8f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15989, __PRETTY_FUNCTION__));
15990 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!")((Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 8 && \"Unexpected mask size for v8 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 15990, __PRETTY_FUNCTION__));
15991
15992 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f32, V1, V2, Mask,
15993 Zeroable, Subtarget, DAG))
15994 return Blend;
15995
15996 // Check for being able to broadcast a single element.
15997 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8f32, V1, V2, Mask,
15998 Subtarget, DAG))
15999 return Broadcast;
16000
16001 // If the shuffle mask is repeated in each 128-bit lane, we have many more
16002 // options to efficiently lower the shuffle.
16003 SmallVector<int, 4> RepeatedMask;
16004 if (is128BitLaneRepeatedShuffleMask(MVT::v8f32, Mask, RepeatedMask)) {
16005 assert(RepeatedMask.size() == 4 &&((RepeatedMask.size() == 4 && "Repeated masks must be half the mask width!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 4 && \"Repeated masks must be half the mask width!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16006, __PRETTY_FUNCTION__))
16006 "Repeated masks must be half the mask width!")((RepeatedMask.size() == 4 && "Repeated masks must be half the mask width!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 4 && \"Repeated masks must be half the mask width!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16006, __PRETTY_FUNCTION__));
16007
16008 // Use even/odd duplicate instructions for masks that match their pattern.
16009 if (isShuffleEquivalent(V1, V2, RepeatedMask, {0, 0, 2, 2}))
16010 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v8f32, V1);
16011 if (isShuffleEquivalent(V1, V2, RepeatedMask, {1, 1, 3, 3}))
16012 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v8f32, V1);
16013
16014 if (V2.isUndef())
16015 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, V1,
16016 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16017
16018 // Use dedicated unpack instructions for masks that match their pattern.
16019 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8f32, Mask, V1, V2, DAG))
16020 return V;
16021
16022 // Otherwise, fall back to a SHUFPS sequence. Here it is important that we
16023 // have already handled any direct blends.
16024 return lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask, V1, V2, DAG);
16025 }
16026
16027 // Try to create an in-lane repeating shuffle mask and then shuffle the
16028 // results into the target lanes.
16029 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16030 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16031 return V;
16032
16033 // If we have a single input shuffle with different shuffle patterns in the
16034 // two 128-bit lanes use the variable mask to VPERMILPS.
16035 if (V2.isUndef()) {
16036 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16037 if (!is128BitLaneCrossingShuffleMask(MVT::v8f32, Mask))
16038 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v8f32, V1, VPermMask);
16039
16040 if (Subtarget.hasAVX2())
16041 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8f32, VPermMask, V1);
16042
16043 // Otherwise, fall back.
16044 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v8f32, V1, V2, Mask,
16045 DAG, Subtarget);
16046 }
16047
16048 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16049 // shuffle.
16050 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16051 DL, MVT::v8f32, V1, V2, Mask, Subtarget, DAG))
16052 return Result;
16053
16054 // If we have VLX support, we can use VEXPAND.
16055 if (Subtarget.hasVLX())
16056 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f32, Zeroable, Mask, V1, V2,
16057 DAG, Subtarget))
16058 return V;
16059
16060 // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16061 // since after split we get a more efficient code using vpunpcklwd and
16062 // vpunpckhwd instrs than vblend.
16063 if (!Subtarget.hasAVX512() && isUnpackWdShuffleMask(Mask, MVT::v8f32))
16064 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
16065 Subtarget, DAG))
16066 return V;
16067
16068 // If we have AVX2 then we always want to lower with a blend because at v8 we
16069 // can fully permute the elements.
16070 if (Subtarget.hasAVX2())
16071 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8f32, V1, V2, Mask,
16072 Subtarget, DAG);
16073
16074 // Otherwise fall back on generic lowering.
16075 return lowerShuffleAsSplitOrBlend(DL, MVT::v8f32, V1, V2, Mask,
16076 Subtarget, DAG);
16077}
16078
16079/// Handle lowering of 8-lane 32-bit integer shuffles.
16080///
16081/// This routine is only called when we have AVX2 and thus a reasonable
16082/// instruction set for v8i32 shuffling..
16083static SDValue lowerV8I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16084 const APInt &Zeroable, SDValue V1, SDValue V2,
16085 const X86Subtarget &Subtarget,
16086 SelectionDAG &DAG) {
16087 assert(V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v8i32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16087, __PRETTY_FUNCTION__));
16088 assert(V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v8i32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v8i32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16088, __PRETTY_FUNCTION__));
16089 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!")((Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 8 && \"Unexpected mask size for v8 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16089, __PRETTY_FUNCTION__));
16090 assert(Subtarget.hasAVX2() && "We can only lower v8i32 with AVX2!")((Subtarget.hasAVX2() && "We can only lower v8i32 with AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"We can only lower v8i32 with AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16090, __PRETTY_FUNCTION__));
16091
16092 // Whenever we can lower this as a zext, that instruction is strictly faster
16093 // than any alternative. It also allows us to fold memory operands into the
16094 // shuffle in many cases.
16095 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v8i32, V1, V2, Mask,
16096 Zeroable, Subtarget, DAG))
16097 return ZExt;
16098
16099 // For non-AVX512 if the Mask is of 16bit elements in lane then try to split
16100 // since after split we get a more efficient code than vblend by using
16101 // vpunpcklwd and vpunpckhwd instrs.
16102 if (isUnpackWdShuffleMask(Mask, MVT::v8i32) && !V2.isUndef() &&
16103 !Subtarget.hasAVX512())
16104 if (SDValue V = lowerShuffleAsSplitOrBlend(DL, MVT::v8i32, V1, V2, Mask,
16105 Subtarget, DAG))
16106 return V;
16107
16108 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i32, V1, V2, Mask,
16109 Zeroable, Subtarget, DAG))
16110 return Blend;
16111
16112 // Check for being able to broadcast a single element.
16113 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v8i32, V1, V2, Mask,
16114 Subtarget, DAG))
16115 return Broadcast;
16116
16117 // If the shuffle mask is repeated in each 128-bit lane we can use more
16118 // efficient instructions that mirror the shuffles across the two 128-bit
16119 // lanes.
16120 SmallVector<int, 4> RepeatedMask;
16121 bool Is128BitLaneRepeatedShuffle =
16122 is128BitLaneRepeatedShuffleMask(MVT::v8i32, Mask, RepeatedMask);
16123 if (Is128BitLaneRepeatedShuffle) {
16124 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!")((RepeatedMask.size() == 4 && "Unexpected repeated mask size!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 4 && \"Unexpected repeated mask size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16124, __PRETTY_FUNCTION__));
16125 if (V2.isUndef())
16126 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v8i32, V1,
16127 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16128
16129 // Use dedicated unpack instructions for masks that match their pattern.
16130 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v8i32, Mask, V1, V2, DAG))
16131 return V;
16132 }
16133
16134 // Try to use shift instructions.
16135 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i32, V1, V2, Mask,
16136 Zeroable, Subtarget, DAG))
16137 return Shift;
16138
16139 // If we have VLX support, we can use VALIGN or EXPAND.
16140 if (Subtarget.hasVLX()) {
16141 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v8i32, V1, V2, Mask,
16142 Subtarget, DAG))
16143 return Rotate;
16144
16145 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i32, Zeroable, Mask, V1, V2,
16146 DAG, Subtarget))
16147 return V;
16148 }
16149
16150 // Try to use byte rotation instructions.
16151 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i32, V1, V2, Mask,
16152 Subtarget, DAG))
16153 return Rotate;
16154
16155 // Try to create an in-lane repeating shuffle mask and then shuffle the
16156 // results into the target lanes.
16157 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16158 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16159 return V;
16160
16161 // If the shuffle patterns aren't repeated but it is a single input, directly
16162 // generate a cross-lane VPERMD instruction.
16163 if (V2.isUndef()) {
16164 SDValue VPermMask = getConstVector(Mask, MVT::v8i32, DAG, DL, true);
16165 return DAG.getNode(X86ISD::VPERMV, DL, MVT::v8i32, VPermMask, V1);
16166 }
16167
16168 // Assume that a single SHUFPS is faster than an alternative sequence of
16169 // multiple instructions (even if the CPU has a domain penalty).
16170 // If some CPU is harmed by the domain switch, we can fix it in a later pass.
16171 if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
16172 SDValue CastV1 = DAG.getBitcast(MVT::v8f32, V1);
16173 SDValue CastV2 = DAG.getBitcast(MVT::v8f32, V2);
16174 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v8f32, RepeatedMask,
16175 CastV1, CastV2, DAG);
16176 return DAG.getBitcast(MVT::v8i32, ShufPS);
16177 }
16178
16179 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16180 // shuffle.
16181 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16182 DL, MVT::v8i32, V1, V2, Mask, Subtarget, DAG))
16183 return Result;
16184
16185 // Otherwise fall back on generic blend lowering.
16186 return lowerShuffleAsDecomposedShuffleBlend(DL, MVT::v8i32, V1, V2, Mask,
16187 Subtarget, DAG);
16188}
16189
16190/// Handle lowering of 16-lane 16-bit integer shuffles.
16191///
16192/// This routine is only called when we have AVX2 and thus a reasonable
16193/// instruction set for v16i16 shuffling..
16194static SDValue lowerV16I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16195 const APInt &Zeroable, SDValue V1, SDValue V2,
16196 const X86Subtarget &Subtarget,
16197 SelectionDAG &DAG) {
16198 assert(V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v16i16 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16198, __PRETTY_FUNCTION__));
16199 assert(V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v16i16 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v16i16 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16199, __PRETTY_FUNCTION__));
16200 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!")((Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 16 && \"Unexpected mask size for v16 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16200, __PRETTY_FUNCTION__));
16201 assert(Subtarget.hasAVX2() && "We can only lower v16i16 with AVX2!")((Subtarget.hasAVX2() && "We can only lower v16i16 with AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"We can only lower v16i16 with AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16201, __PRETTY_FUNCTION__));
16202
16203 // Whenever we can lower this as a zext, that instruction is strictly faster
16204 // than any alternative. It also allows us to fold memory operands into the
16205 // shuffle in many cases.
16206 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16207 DL, MVT::v16i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16208 return ZExt;
16209
16210 // Check for being able to broadcast a single element.
16211 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v16i16, V1, V2, Mask,
16212 Subtarget, DAG))
16213 return Broadcast;
16214
16215 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i16, V1, V2, Mask,
16216 Zeroable, Subtarget, DAG))
16217 return Blend;
16218
16219 // Use dedicated unpack instructions for masks that match their pattern.
16220 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i16, Mask, V1, V2, DAG))
16221 return V;
16222
16223 // Use dedicated pack instructions for masks that match their pattern.
16224 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v16i16, Mask, V1, V2, DAG,
16225 Subtarget))
16226 return V;
16227
16228 // Try to use shift instructions.
16229 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i16, V1, V2, Mask,
16230 Zeroable, Subtarget, DAG))
16231 return Shift;
16232
16233 // Try to use byte rotation instructions.
16234 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i16, V1, V2, Mask,
16235 Subtarget, DAG))
16236 return Rotate;
16237
16238 // Try to create an in-lane repeating shuffle mask and then shuffle the
16239 // results into the target lanes.
16240 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16241 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16242 return V;
16243
16244 if (V2.isUndef()) {
16245 // There are no generalized cross-lane shuffle operations available on i16
16246 // element types.
16247 if (is128BitLaneCrossingShuffleMask(MVT::v16i16, Mask)) {
16248 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16249 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16250 return V;
16251
16252 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v16i16, V1, V2, Mask,
16253 DAG, Subtarget);
16254 }
16255
16256 SmallVector<int, 8> RepeatedMask;
16257 if (is128BitLaneRepeatedShuffleMask(MVT::v16i16, Mask, RepeatedMask)) {
16258 // As this is a single-input shuffle, the repeated mask should be
16259 // a strictly valid v8i16 mask that we can pass through to the v8i16
16260 // lowering to handle even the v16 case.
16261 return lowerV8I16GeneralSingleInputShuffle(
16262 DL, MVT::v16i16, V1, RepeatedMask, Subtarget, DAG);
16263 }
16264 }
16265
16266 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v16i16, Mask, V1, V2,
16267 Zeroable, Subtarget, DAG))
16268 return PSHUFB;
16269
16270 // AVX512BWVL can lower to VPERMW.
16271 if (Subtarget.hasBWI() && Subtarget.hasVLX())
16272 return lowerShuffleWithPERMV(DL, MVT::v16i16, Mask, V1, V2, DAG);
16273
16274 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16275 // shuffle.
16276 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16277 DL, MVT::v16i16, V1, V2, Mask, Subtarget, DAG))
16278 return Result;
16279
16280 // Try to permute the lanes and then use a per-lane permute.
16281 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16282 DL, MVT::v16i16, V1, V2, Mask, DAG, Subtarget))
16283 return V;
16284
16285 // Otherwise fall back on generic lowering.
16286 return lowerShuffleAsSplitOrBlend(DL, MVT::v16i16, V1, V2, Mask,
16287 Subtarget, DAG);
16288}
16289
16290/// Handle lowering of 32-lane 8-bit integer shuffles.
16291///
16292/// This routine is only called when we have AVX2 and thus a reasonable
16293/// instruction set for v32i8 shuffling..
16294static SDValue lowerV32I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16295 const APInt &Zeroable, SDValue V1, SDValue V2,
16296 const X86Subtarget &Subtarget,
16297 SelectionDAG &DAG) {
16298 assert(V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v32i8 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16298, __PRETTY_FUNCTION__));
1
'?' condition is true
→
16299 assert(V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v32i8 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v32i8 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16299, __PRETTY_FUNCTION__));
2
←
'?' condition is true
→
16300 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!")((Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 32 && \"Unexpected mask size for v32 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16300, __PRETTY_FUNCTION__));
3
←
Assuming the condition is true
→
4
←
'?' condition is true
→
16301 assert(Subtarget.hasAVX2() && "We can only lower v32i8 with AVX2!")((Subtarget.hasAVX2() && "We can only lower v32i8 with AVX2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"We can only lower v32i8 with AVX2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16301, __PRETTY_FUNCTION__));
5
←
'?' condition is true
→
16302
16303 // Whenever we can lower this as a zext, that instruction is strictly faster
16304 // than any alternative. It also allows us to fold memory operands into the
16305 // shuffle in many cases.
16306 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(DL, MVT::v32i8, V1, V2, Mask,
6
←
Taking false branch
→
16307 Zeroable, Subtarget, DAG))
16308 return ZExt;
16309
16310 // Check for being able to broadcast a single element.
16311 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, MVT::v32i8, V1, V2, Mask,
7
←
Taking false branch
→
16312 Subtarget, DAG))
16313 return Broadcast;
16314
16315 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i8, V1, V2, Mask,
8
←
Taking false branch
→
16316 Zeroable, Subtarget, DAG))
16317 return Blend;
16318
16319 // Use dedicated unpack instructions for masks that match their pattern.
16320 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i8, Mask, V1, V2, DAG))
9
←
Taking false branch
→
16321 return V;
16322
16323 // Use dedicated pack instructions for masks that match their pattern.
16324 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v32i8, Mask, V1, V2, DAG,
10
←
Taking false branch
→
16325 Subtarget))
16326 return V;
16327
16328 // Try to use shift instructions.
16329 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i8, V1, V2, Mask,
11
←
Taking false branch
→
16330 Zeroable, Subtarget, DAG))
16331 return Shift;
16332
16333 // Try to use byte rotation instructions.
16334 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i8, V1, V2, Mask,
12
←
Taking false branch
→
16335 Subtarget, DAG))
16336 return Rotate;
16337
16338 // Try to create an in-lane repeating shuffle mask and then shuffle the
16339 // results into the target lanes.
16340 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
13
←
Taking false branch
→
16341 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16342 return V;
16343
16344 // There are no generalized cross-lane shuffle operations available on i8
16345 // element types.
16346 if (V2.isUndef() && is128BitLaneCrossingShuffleMask(MVT::v32i8, Mask)) {
14
←
Assuming the condition is true
→
15
←
Taking true branch
→
16347 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16
←
Taking false branch
→
16348 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16349 return V;
16350
16351 return lowerShuffleAsLanePermuteAndShuffle(DL, MVT::v32i8, V1, V2, Mask,
17
←
Calling 'lowerShuffleAsLanePermuteAndShuffle'
→
16352 DAG, Subtarget);
16353 }
16354
16355 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i8, Mask, V1, V2,
16356 Zeroable, Subtarget, DAG))
16357 return PSHUFB;
16358
16359 // AVX512VBMIVL can lower to VPERMB.
16360 if (Subtarget.hasVBMI() && Subtarget.hasVLX())
16361 return lowerShuffleWithPERMV(DL, MVT::v32i8, Mask, V1, V2, DAG);
16362
16363 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16364 // shuffle.
16365 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16366 DL, MVT::v32i8, V1, V2, Mask, Subtarget, DAG))
16367 return Result;
16368
16369 // Try to permute the lanes and then use a per-lane permute.
16370 if (SDValue V = lowerShuffleAsLanePermuteAndPermute(
16371 DL, MVT::v32i8, V1, V2, Mask, DAG, Subtarget))
16372 return V;
16373
16374 // Look for {0, 8, 16, 24, 32, 40, 48, 56 } in the first 8 elements. Followed
16375 // by zeroable elements in the remaining 24 elements. Turn this into two
16376 // vmovqb instructions shuffled together.
16377 if (Subtarget.hasVLX())
16378 if (SDValue V = lowerShuffleAsVTRUNCAndUnpack(DL, MVT::v32i8, V1, V2,
16379 Mask, Zeroable, DAG))
16380 return V;
16381
16382 // Otherwise fall back on generic lowering.
16383 return lowerShuffleAsSplitOrBlend(DL, MVT::v32i8, V1, V2, Mask,
16384 Subtarget, DAG);
16385}
16386
16387/// High-level routine to lower various 256-bit x86 vector shuffles.
16388///
16389/// This routine either breaks down the specific type of a 256-bit x86 vector
16390/// shuffle or splits it into two 128-bit shuffles and fuses the results back
16391/// together based on the available instructions.
16392static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef<int> Mask, MVT VT,
16393 SDValue V1, SDValue V2, const APInt &Zeroable,
16394 const X86Subtarget &Subtarget,
16395 SelectionDAG &DAG) {
16396 // If we have a single input to the zero element, insert that into V1 if we
16397 // can do so cheaply.
16398 int NumElts = VT.getVectorNumElements();
16399 int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
16400
16401 if (NumV2Elements == 1 && Mask[0] >= NumElts)
16402 if (SDValue Insertion = lowerShuffleAsElementInsertion(
16403 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16404 return Insertion;
16405
16406 // Handle special cases where the lower or upper half is UNDEF.
16407 if (SDValue V =
16408 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
16409 return V;
16410
16411 // There is a really nice hard cut-over between AVX1 and AVX2 that means we
16412 // can check for those subtargets here and avoid much of the subtarget
16413 // querying in the per-vector-type lowering routines. With AVX1 we have
16414 // essentially *zero* ability to manipulate a 256-bit vector with integer
16415 // types. Since we'll use floating point types there eventually, just
16416 // immediately cast everything to a float and operate entirely in that domain.
16417 if (VT.isInteger() && !Subtarget.hasAVX2()) {
16418 int ElementBits = VT.getScalarSizeInBits();
16419 if (ElementBits < 32) {
16420 // No floating point type available, if we can't use the bit operations
16421 // for masking/blending then decompose into 128-bit vectors.
16422 if (SDValue V = lowerShuffleAsBitMask(DL, VT, V1, V2, Mask, Zeroable,
16423 Subtarget, DAG))
16424 return V;
16425 if (SDValue V = lowerShuffleAsBitBlend(DL, VT, V1, V2, Mask, DAG))
16426 return V;
16427 return splitAndLowerShuffle(DL, VT, V1, V2, Mask, DAG);
16428 }
16429
16430 MVT FpVT = MVT::getVectorVT(MVT::getFloatingPointVT(ElementBits),
16431 VT.getVectorNumElements());
16432 V1 = DAG.getBitcast(FpVT, V1);
16433 V2 = DAG.getBitcast(FpVT, V2);
16434 return DAG.getBitcast(VT, DAG.getVectorShuffle(FpVT, DL, V1, V2, Mask));
16435 }
16436
16437 switch (VT.SimpleTy) {
16438 case MVT::v4f64:
16439 return lowerV4F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16440 case MVT::v4i64:
16441 return lowerV4I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16442 case MVT::v8f32:
16443 return lowerV8F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16444 case MVT::v8i32:
16445 return lowerV8I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16446 case MVT::v16i16:
16447 return lowerV16I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16448 case MVT::v32i8:
16449 return lowerV32I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16450
16451 default:
16452 llvm_unreachable("Not a valid 256-bit x86 vector type!")::llvm::llvm_unreachable_internal("Not a valid 256-bit x86 vector type!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16452);
16453 }
16454}
16455
16456/// Try to lower a vector shuffle as a 128-bit shuffles.
16457static SDValue lowerV4X128Shuffle(const SDLoc &DL, MVT VT, ArrayRef<int> Mask,
16458 const APInt &Zeroable, SDValue V1, SDValue V2,
16459 const X86Subtarget &Subtarget,
16460 SelectionDAG &DAG) {
16461 assert(VT.getScalarSizeInBits() == 64 &&((VT.getScalarSizeInBits() == 64 && "Unexpected element type size for 128bit shuffle."
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() == 64 && \"Unexpected element type size for 128bit shuffle.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16462, __PRETTY_FUNCTION__))
16462 "Unexpected element type size for 128bit shuffle.")((VT.getScalarSizeInBits() == 64 && "Unexpected element type size for 128bit shuffle."
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() == 64 && \"Unexpected element type size for 128bit shuffle.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16462, __PRETTY_FUNCTION__));
16463
16464 // To handle 256 bit vector requires VLX and most probably
16465 // function lowerV2X128VectorShuffle() is better solution.
16466 assert(VT.is512BitVector() && "Unexpected vector size for 512bit shuffle.")((VT.is512BitVector() && "Unexpected vector size for 512bit shuffle."
) ? static_cast<void> (0) : __assert_fail ("VT.is512BitVector() && \"Unexpected vector size for 512bit shuffle.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16466, __PRETTY_FUNCTION__));
16467
16468 // TODO - use Zeroable like we do for lowerV2X128VectorShuffle?
16469 SmallVector<int, 4> WidenedMask;
16470 if (!canWidenShuffleElements(Mask, WidenedMask))
16471 return SDValue();
16472
16473 // Try to use an insert into a zero vector.
16474 if (WidenedMask[0] == 0 && (Zeroable & 0xf0) == 0xf0 &&
16475 (WidenedMask[1] == 1 || (Zeroable & 0x0c) == 0x0c)) {
16476 unsigned NumElts = ((Zeroable & 0x0c) == 0x0c) ? 2 : 4;
16477 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
16478 SDValue LoV = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V1,
16479 DAG.getIntPtrConstant(0, DL));
16480 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
16481 getZeroVector(VT, Subtarget, DAG, DL), LoV,
16482 DAG.getIntPtrConstant(0, DL));
16483 }
16484
16485 // Check for patterns which can be matched with a single insert of a 256-bit
16486 // subvector.
16487 bool OnlyUsesV1 = isShuffleEquivalent(V1, V2, Mask,
16488 {0, 1, 2, 3, 0, 1, 2, 3});
16489 if (OnlyUsesV1 || isShuffleEquivalent(V1, V2, Mask,
16490 {0, 1, 2, 3, 8, 9, 10, 11})) {
16491 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 4);
16492 SDValue SubVec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT,
16493 OnlyUsesV1 ? V1 : V2,
16494 DAG.getIntPtrConstant(0, DL));
16495 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, V1, SubVec,
16496 DAG.getIntPtrConstant(4, DL));
16497 }
16498
16499 assert(WidenedMask.size() == 4)((WidenedMask.size() == 4) ? static_cast<void> (0) : __assert_fail
("WidenedMask.size() == 4", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16499, __PRETTY_FUNCTION__));
16500
16501 // See if this is an insertion of the lower 128-bits of V2 into V1.
16502 bool IsInsert = true;
16503 int V2Index = -1;
16504 for (int i = 0; i < 4; ++i) {
16505 assert(WidenedMask[i] >= -1)((WidenedMask[i] >= -1) ? static_cast<void> (0) : __assert_fail
("WidenedMask[i] >= -1", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16505, __PRETTY_FUNCTION__));
16506 if (WidenedMask[i] < 0)
16507 continue;
16508
16509 // Make sure all V1 subvectors are in place.
16510 if (WidenedMask[i] < 4) {
16511 if (WidenedMask[i] != i) {
16512 IsInsert = false;
16513 break;
16514 }
16515 } else {
16516 // Make sure we only have a single V2 index and its the lowest 128-bits.
16517 if (V2Index >= 0 || WidenedMask[i] != 4) {
16518 IsInsert = false;
16519 break;
16520 }
16521 V2Index = i;
16522 }
16523 }
16524 if (IsInsert && V2Index >= 0) {
16525 MVT SubVT = MVT::getVectorVT(VT.getVectorElementType(), 2);
16526 SDValue Subvec = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, SubVT, V2,
16527 DAG.getIntPtrConstant(0, DL));
16528 return insert128BitVector(V1, Subvec, V2Index * 2, DAG, DL);
16529 }
16530
16531 // Try to lower to vshuf64x2/vshuf32x4.
16532 SDValue Ops[2] = {DAG.getUNDEF(VT), DAG.getUNDEF(VT)};
16533 unsigned PermMask = 0;
16534 // Insure elements came from the same Op.
16535 for (int i = 0; i < 4; ++i) {
16536 assert(WidenedMask[i] >= -1)((WidenedMask[i] >= -1) ? static_cast<void> (0) : __assert_fail
("WidenedMask[i] >= -1", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16536, __PRETTY_FUNCTION__));
16537 if (WidenedMask[i] < 0)
16538 continue;
16539
16540 SDValue Op = WidenedMask[i] >= 4 ? V2 : V1;
16541 unsigned OpIndex = i / 2;
16542 if (Ops[OpIndex].isUndef())
16543 Ops[OpIndex] = Op;
16544 else if (Ops[OpIndex] != Op)
16545 return SDValue();
16546
16547 // Convert the 128-bit shuffle mask selection values into 128-bit selection
16548 // bits defined by a vshuf64x2 instruction's immediate control byte.
16549 PermMask |= (WidenedMask[i] % 4) << (i * 2);
16550 }
16551
16552 return DAG.getNode(X86ISD::SHUF128, DL, VT, Ops[0], Ops[1],
16553 DAG.getTargetConstant(PermMask, DL, MVT::i8));
16554}
16555
16556/// Handle lowering of 8-lane 64-bit floating point shuffles.
16557static SDValue lowerV8F64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16558 const APInt &Zeroable, SDValue V1, SDValue V2,
16559 const X86Subtarget &Subtarget,
16560 SelectionDAG &DAG) {
16561 assert(V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v8f64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16561, __PRETTY_FUNCTION__));
16562 assert(V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v8f64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v8f64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16562, __PRETTY_FUNCTION__));
16563 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!")((Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 8 && \"Unexpected mask size for v8 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16563, __PRETTY_FUNCTION__));
16564
16565 if (V2.isUndef()) {
16566 // Use low duplicate instructions for masks that match their pattern.
16567 if (isShuffleEquivalent(V1, V2, Mask, {0, 0, 2, 2, 4, 4, 6, 6}))
16568 return DAG.getNode(X86ISD::MOVDDUP, DL, MVT::v8f64, V1);
16569
16570 if (!is128BitLaneCrossingShuffleMask(MVT::v8f64, Mask)) {
16571 // Non-half-crossing single input shuffles can be lowered with an
16572 // interleaved permutation.
16573 unsigned VPERMILPMask = (Mask[0] == 1) | ((Mask[1] == 1) << 1) |
16574 ((Mask[2] == 3) << 2) | ((Mask[3] == 3) << 3) |
16575 ((Mask[4] == 5) << 4) | ((Mask[5] == 5) << 5) |
16576 ((Mask[6] == 7) << 6) | ((Mask[7] == 7) << 7);
16577 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f64, V1,
16578 DAG.getTargetConstant(VPERMILPMask, DL, MVT::i8));
16579 }
16580
16581 SmallVector<int, 4> RepeatedMask;
16582 if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask))
16583 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8f64, V1,
16584 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16585 }
16586
16587 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8f64, Mask, Zeroable, V1,
16588 V2, Subtarget, DAG))
16589 return Shuf128;
16590
16591 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8f64, Mask, V1, V2, DAG))
16592 return Unpck;
16593
16594 // Check if the blend happens to exactly fit that of SHUFPD.
16595 if (SDValue Op = lowerShuffleWithSHUFPD(DL, MVT::v8f64, V1, V2, Mask,
16596 Zeroable, Subtarget, DAG))
16597 return Op;
16598
16599 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8f64, Zeroable, Mask, V1, V2,
16600 DAG, Subtarget))
16601 return V;
16602
16603 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8f64, V1, V2, Mask,
16604 Zeroable, Subtarget, DAG))
16605 return Blend;
16606
16607 return lowerShuffleWithPERMV(DL, MVT::v8f64, Mask, V1, V2, DAG);
16608}
16609
16610/// Handle lowering of 16-lane 32-bit floating point shuffles.
16611static SDValue lowerV16F32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16612 const APInt &Zeroable, SDValue V1, SDValue V2,
16613 const X86Subtarget &Subtarget,
16614 SelectionDAG &DAG) {
16615 assert(V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v16f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16615, __PRETTY_FUNCTION__));
16616 assert(V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v16f32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v16f32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16616, __PRETTY_FUNCTION__));
16617 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!")((Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 16 && \"Unexpected mask size for v16 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16617, __PRETTY_FUNCTION__));
16618
16619 // If the shuffle mask is repeated in each 128-bit lane, we have many more
16620 // options to efficiently lower the shuffle.
16621 SmallVector<int, 4> RepeatedMask;
16622 if (is128BitLaneRepeatedShuffleMask(MVT::v16f32, Mask, RepeatedMask)) {
16623 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!")((RepeatedMask.size() == 4 && "Unexpected repeated mask size!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 4 && \"Unexpected repeated mask size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16623, __PRETTY_FUNCTION__));
16624
16625 // Use even/odd duplicate instructions for masks that match their pattern.
16626 if (isShuffleEquivalent(V1, V2, RepeatedMask, {0, 0, 2, 2}))
16627 return DAG.getNode(X86ISD::MOVSLDUP, DL, MVT::v16f32, V1);
16628 if (isShuffleEquivalent(V1, V2, RepeatedMask, {1, 1, 3, 3}))
16629 return DAG.getNode(X86ISD::MOVSHDUP, DL, MVT::v16f32, V1);
16630
16631 if (V2.isUndef())
16632 return DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v16f32, V1,
16633 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16634
16635 // Use dedicated unpack instructions for masks that match their pattern.
16636 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16f32, Mask, V1, V2, DAG))
16637 return V;
16638
16639 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16f32, V1, V2, Mask,
16640 Zeroable, Subtarget, DAG))
16641 return Blend;
16642
16643 // Otherwise, fall back to a SHUFPS sequence.
16644 return lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask, V1, V2, DAG);
16645 }
16646
16647 // If we have a single input shuffle with different shuffle patterns in the
16648 // 128-bit lanes and don't lane cross, use variable mask VPERMILPS.
16649 if (V2.isUndef() &&
16650 !is128BitLaneCrossingShuffleMask(MVT::v16f32, Mask)) {
16651 SDValue VPermMask = getConstVector(Mask, MVT::v16i32, DAG, DL, true);
16652 return DAG.getNode(X86ISD::VPERMILPV, DL, MVT::v16f32, V1, VPermMask);
16653 }
16654
16655 // If we have AVX512F support, we can use VEXPAND.
16656 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16f32, Zeroable, Mask,
16657 V1, V2, DAG, Subtarget))
16658 return V;
16659
16660 return lowerShuffleWithPERMV(DL, MVT::v16f32, Mask, V1, V2, DAG);
16661}
16662
16663/// Handle lowering of 8-lane 64-bit integer shuffles.
16664static SDValue lowerV8I64Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16665 const APInt &Zeroable, SDValue V1, SDValue V2,
16666 const X86Subtarget &Subtarget,
16667 SelectionDAG &DAG) {
16668 assert(V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v8i64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16668, __PRETTY_FUNCTION__));
16669 assert(V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v8i64 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v8i64 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16669, __PRETTY_FUNCTION__));
16670 assert(Mask.size() == 8 && "Unexpected mask size for v8 shuffle!")((Mask.size() == 8 && "Unexpected mask size for v8 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 8 && \"Unexpected mask size for v8 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16670, __PRETTY_FUNCTION__));
16671
16672 if (V2.isUndef()) {
16673 // When the shuffle is mirrored between the 128-bit lanes of the unit, we
16674 // can use lower latency instructions that will operate on all four
16675 // 128-bit lanes.
16676 SmallVector<int, 2> Repeated128Mask;
16677 if (is128BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated128Mask)) {
16678 SmallVector<int, 4> PSHUFDMask;
16679 scaleShuffleMask<int>(2, Repeated128Mask, PSHUFDMask);
16680 return DAG.getBitcast(
16681 MVT::v8i64,
16682 DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32,
16683 DAG.getBitcast(MVT::v16i32, V1),
16684 getV4X86ShuffleImm8ForMask(PSHUFDMask, DL, DAG)));
16685 }
16686
16687 SmallVector<int, 4> Repeated256Mask;
16688 if (is256BitLaneRepeatedShuffleMask(MVT::v8i64, Mask, Repeated256Mask))
16689 return DAG.getNode(X86ISD::VPERMI, DL, MVT::v8i64, V1,
16690 getV4X86ShuffleImm8ForMask(Repeated256Mask, DL, DAG));
16691 }
16692
16693 if (SDValue Shuf128 = lowerV4X128Shuffle(DL, MVT::v8i64, Mask, Zeroable, V1,
16694 V2, Subtarget, DAG))
16695 return Shuf128;
16696
16697 // Try to use shift instructions.
16698 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v8i64, V1, V2, Mask,
16699 Zeroable, Subtarget, DAG))
16700 return Shift;
16701
16702 // Try to use VALIGN.
16703 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v8i64, V1, V2, Mask,
16704 Subtarget, DAG))
16705 return Rotate;
16706
16707 // Try to use PALIGNR.
16708 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v8i64, V1, V2, Mask,
16709 Subtarget, DAG))
16710 return Rotate;
16711
16712 if (SDValue Unpck = lowerShuffleWithUNPCK(DL, MVT::v8i64, Mask, V1, V2, DAG))
16713 return Unpck;
16714 // If we have AVX512F support, we can use VEXPAND.
16715 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v8i64, Zeroable, Mask, V1, V2,
16716 DAG, Subtarget))
16717 return V;
16718
16719 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v8i64, V1, V2, Mask,
16720 Zeroable, Subtarget, DAG))
16721 return Blend;
16722
16723 return lowerShuffleWithPERMV(DL, MVT::v8i64, Mask, V1, V2, DAG);
16724}
16725
16726/// Handle lowering of 16-lane 32-bit integer shuffles.
16727static SDValue lowerV16I32Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16728 const APInt &Zeroable, SDValue V1, SDValue V2,
16729 const X86Subtarget &Subtarget,
16730 SelectionDAG &DAG) {
16731 assert(V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v16i32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16731, __PRETTY_FUNCTION__));
16732 assert(V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v16i32 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v16i32 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16732, __PRETTY_FUNCTION__));
16733 assert(Mask.size() == 16 && "Unexpected mask size for v16 shuffle!")((Mask.size() == 16 && "Unexpected mask size for v16 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 16 && \"Unexpected mask size for v16 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16733, __PRETTY_FUNCTION__));
16734
16735 // Whenever we can lower this as a zext, that instruction is strictly faster
16736 // than any alternative. It also allows us to fold memory operands into the
16737 // shuffle in many cases.
16738 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16739 DL, MVT::v16i32, V1, V2, Mask, Zeroable, Subtarget, DAG))
16740 return ZExt;
16741
16742 // If the shuffle mask is repeated in each 128-bit lane we can use more
16743 // efficient instructions that mirror the shuffles across the four 128-bit
16744 // lanes.
16745 SmallVector<int, 4> RepeatedMask;
16746 bool Is128BitLaneRepeatedShuffle =
16747 is128BitLaneRepeatedShuffleMask(MVT::v16i32, Mask, RepeatedMask);
16748 if (Is128BitLaneRepeatedShuffle) {
16749 assert(RepeatedMask.size() == 4 && "Unexpected repeated mask size!")((RepeatedMask.size() == 4 && "Unexpected repeated mask size!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 4 && \"Unexpected repeated mask size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16749, __PRETTY_FUNCTION__));
16750 if (V2.isUndef())
16751 return DAG.getNode(X86ISD::PSHUFD, DL, MVT::v16i32, V1,
16752 getV4X86ShuffleImm8ForMask(RepeatedMask, DL, DAG));
16753
16754 // Use dedicated unpack instructions for masks that match their pattern.
16755 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v16i32, Mask, V1, V2, DAG))
16756 return V;
16757 }
16758
16759 // Try to use shift instructions.
16760 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v16i32, V1, V2, Mask,
16761 Zeroable, Subtarget, DAG))
16762 return Shift;
16763
16764 // Try to use VALIGN.
16765 if (SDValue Rotate = lowerShuffleAsRotate(DL, MVT::v16i32, V1, V2, Mask,
16766 Subtarget, DAG))
16767 return Rotate;
16768
16769 // Try to use byte rotation instructions.
16770 if (Subtarget.hasBWI())
16771 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v16i32, V1, V2, Mask,
16772 Subtarget, DAG))
16773 return Rotate;
16774
16775 // Assume that a single SHUFPS is faster than using a permv shuffle.
16776 // If some CPU is harmed by the domain switch, we can fix it in a later pass.
16777 if (Is128BitLaneRepeatedShuffle && isSingleSHUFPSMask(RepeatedMask)) {
16778 SDValue CastV1 = DAG.getBitcast(MVT::v16f32, V1);
16779 SDValue CastV2 = DAG.getBitcast(MVT::v16f32, V2);
16780 SDValue ShufPS = lowerShuffleWithSHUFPS(DL, MVT::v16f32, RepeatedMask,
16781 CastV1, CastV2, DAG);
16782 return DAG.getBitcast(MVT::v16i32, ShufPS);
16783 }
16784 // If we have AVX512F support, we can use VEXPAND.
16785 if (SDValue V = lowerShuffleToEXPAND(DL, MVT::v16i32, Zeroable, Mask, V1, V2,
16786 DAG, Subtarget))
16787 return V;
16788
16789 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v16i32, V1, V2, Mask,
16790 Zeroable, Subtarget, DAG))
16791 return Blend;
16792 return lowerShuffleWithPERMV(DL, MVT::v16i32, Mask, V1, V2, DAG);
16793}
16794
16795/// Handle lowering of 32-lane 16-bit integer shuffles.
16796static SDValue lowerV32I16Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16797 const APInt &Zeroable, SDValue V1, SDValue V2,
16798 const X86Subtarget &Subtarget,
16799 SelectionDAG &DAG) {
16800 assert(V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v32i16 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16800, __PRETTY_FUNCTION__));
16801 assert(V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v32i16 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v32i16 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16801, __PRETTY_FUNCTION__));
16802 assert(Mask.size() == 32 && "Unexpected mask size for v32 shuffle!")((Mask.size() == 32 && "Unexpected mask size for v32 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 32 && \"Unexpected mask size for v32 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16802, __PRETTY_FUNCTION__));
16803 assert(Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!")((Subtarget.hasBWI() && "We can only lower v32i16 with AVX-512-BWI!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI() && \"We can only lower v32i16 with AVX-512-BWI!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16803, __PRETTY_FUNCTION__));
16804
16805 // Whenever we can lower this as a zext, that instruction is strictly faster
16806 // than any alternative. It also allows us to fold memory operands into the
16807 // shuffle in many cases.
16808 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16809 DL, MVT::v32i16, V1, V2, Mask, Zeroable, Subtarget, DAG))
16810 return ZExt;
16811
16812 // Use dedicated unpack instructions for masks that match their pattern.
16813 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v32i16, Mask, V1, V2, DAG))
16814 return V;
16815
16816 // Try to use shift instructions.
16817 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v32i16, V1, V2, Mask,
16818 Zeroable, Subtarget, DAG))
16819 return Shift;
16820
16821 // Try to use byte rotation instructions.
16822 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v32i16, V1, V2, Mask,
16823 Subtarget, DAG))
16824 return Rotate;
16825
16826 if (V2.isUndef()) {
16827 SmallVector<int, 8> RepeatedMask;
16828 if (is128BitLaneRepeatedShuffleMask(MVT::v32i16, Mask, RepeatedMask)) {
16829 // As this is a single-input shuffle, the repeated mask should be
16830 // a strictly valid v8i16 mask that we can pass through to the v8i16
16831 // lowering to handle even the v32 case.
16832 return lowerV8I16GeneralSingleInputShuffle(
16833 DL, MVT::v32i16, V1, RepeatedMask, Subtarget, DAG);
16834 }
16835 }
16836
16837 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v32i16, V1, V2, Mask,
16838 Zeroable, Subtarget, DAG))
16839 return Blend;
16840
16841 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v32i16, Mask, V1, V2,
16842 Zeroable, Subtarget, DAG))
16843 return PSHUFB;
16844
16845 return lowerShuffleWithPERMV(DL, MVT::v32i16, Mask, V1, V2, DAG);
16846}
16847
16848/// Handle lowering of 64-lane 8-bit integer shuffles.
16849static SDValue lowerV64I8Shuffle(const SDLoc &DL, ArrayRef<int> Mask,
16850 const APInt &Zeroable, SDValue V1, SDValue V2,
16851 const X86Subtarget &Subtarget,
16852 SelectionDAG &DAG) {
16853 assert(V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!")((V1.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V1.getSimpleValueType() == MVT::v64i8 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16853, __PRETTY_FUNCTION__));
16854 assert(V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!")((V2.getSimpleValueType() == MVT::v64i8 && "Bad operand type!"
) ? static_cast<void> (0) : __assert_fail ("V2.getSimpleValueType() == MVT::v64i8 && \"Bad operand type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16854, __PRETTY_FUNCTION__));
16855 assert(Mask.size() == 64 && "Unexpected mask size for v64 shuffle!")((Mask.size() == 64 && "Unexpected mask size for v64 shuffle!"
) ? static_cast<void> (0) : __assert_fail ("Mask.size() == 64 && \"Unexpected mask size for v64 shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16855, __PRETTY_FUNCTION__));
16856 assert(Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!")((Subtarget.hasBWI() && "We can only lower v64i8 with AVX-512-BWI!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI() && \"We can only lower v64i8 with AVX-512-BWI!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16856, __PRETTY_FUNCTION__));
16857
16858 // Whenever we can lower this as a zext, that instruction is strictly faster
16859 // than any alternative. It also allows us to fold memory operands into the
16860 // shuffle in many cases.
16861 if (SDValue ZExt = lowerShuffleAsZeroOrAnyExtend(
16862 DL, MVT::v64i8, V1, V2, Mask, Zeroable, Subtarget, DAG))
16863 return ZExt;
16864
16865 // Use dedicated unpack instructions for masks that match their pattern.
16866 if (SDValue V = lowerShuffleWithUNPCK(DL, MVT::v64i8, Mask, V1, V2, DAG))
16867 return V;
16868
16869 // Use dedicated pack instructions for masks that match their pattern.
16870 if (SDValue V = lowerShuffleWithPACK(DL, MVT::v64i8, Mask, V1, V2, DAG,
16871 Subtarget))
16872 return V;
16873
16874 // Try to use shift instructions.
16875 if (SDValue Shift = lowerShuffleAsShift(DL, MVT::v64i8, V1, V2, Mask,
16876 Zeroable, Subtarget, DAG))
16877 return Shift;
16878
16879 // Try to use byte rotation instructions.
16880 if (SDValue Rotate = lowerShuffleAsByteRotate(DL, MVT::v64i8, V1, V2, Mask,
16881 Subtarget, DAG))
16882 return Rotate;
16883
16884 if (SDValue PSHUFB = lowerShuffleWithPSHUFB(DL, MVT::v64i8, Mask, V1, V2,
16885 Zeroable, Subtarget, DAG))
16886 return PSHUFB;
16887
16888 // VBMI can use VPERMV/VPERMV3 byte shuffles.
16889 if (Subtarget.hasVBMI())
16890 return lowerShuffleWithPERMV(DL, MVT::v64i8, Mask, V1, V2, DAG);
16891
16892 // Try to create an in-lane repeating shuffle mask and then shuffle the
16893 // results into the target lanes.
16894 if (SDValue V = lowerShuffleAsRepeatedMaskAndLanePermute(
16895 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
16896 return V;
16897
16898 if (SDValue Blend = lowerShuffleAsBlend(DL, MVT::v64i8, V1, V2, Mask,
16899 Zeroable, Subtarget, DAG))
16900 return Blend;
16901
16902 // Try to simplify this by merging 128-bit lanes to enable a lane-based
16903 // shuffle.
16904 if (!V2.isUndef())
16905 if (SDValue Result = lowerShuffleAsLanePermuteAndRepeatedMask(
16906 DL, MVT::v64i8, V1, V2, Mask, Subtarget, DAG))
16907 return Result;
16908
16909 // FIXME: Implement direct support for this type!
16910 return splitAndLowerShuffle(DL, MVT::v64i8, V1, V2, Mask, DAG);
16911}
16912
16913/// High-level routine to lower various 512-bit x86 vector shuffles.
16914///
16915/// This routine either breaks down the specific type of a 512-bit x86 vector
16916/// shuffle or splits it into two 256-bit shuffles and fuses the results back
16917/// together based on the available instructions.
16918static SDValue lower512BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
16919 MVT VT, SDValue V1, SDValue V2,
16920 const APInt &Zeroable,
16921 const X86Subtarget &Subtarget,
16922 SelectionDAG &DAG) {
16923 assert(Subtarget.hasAVX512() &&((Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/ basic ISA!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"Cannot lower 512-bit vectors w/ basic ISA!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16924, __PRETTY_FUNCTION__))
16924 "Cannot lower 512-bit vectors w/ basic ISA!")((Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/ basic ISA!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"Cannot lower 512-bit vectors w/ basic ISA!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16924, __PRETTY_FUNCTION__));
16925
16926 // If we have a single input to the zero element, insert that into V1 if we
16927 // can do so cheaply.
16928 int NumElts = Mask.size();
16929 int NumV2Elements = count_if(Mask, [NumElts](int M) { return M >= NumElts; });
16930
16931 if (NumV2Elements == 1 && Mask[0] >= NumElts)
16932 if (SDValue Insertion = lowerShuffleAsElementInsertion(
16933 DL, VT, V1, V2, Mask, Zeroable, Subtarget, DAG))
16934 return Insertion;
16935
16936 // Handle special cases where the lower or upper half is UNDEF.
16937 if (SDValue V =
16938 lowerShuffleWithUndefHalf(DL, VT, V1, V2, Mask, Subtarget, DAG))
16939 return V;
16940
16941 // Check for being able to broadcast a single element.
16942 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, Mask,
16943 Subtarget, DAG))
16944 return Broadcast;
16945
16946 // Dispatch to each element type for lowering. If we don't have support for
16947 // specific element type shuffles at 512 bits, immediately split them and
16948 // lower them. Each lowering routine of a given type is allowed to assume that
16949 // the requisite ISA extensions for that element type are available.
16950 switch (VT.SimpleTy) {
16951 case MVT::v8f64:
16952 return lowerV8F64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16953 case MVT::v16f32:
16954 return lowerV16F32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16955 case MVT::v8i64:
16956 return lowerV8I64Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16957 case MVT::v16i32:
16958 return lowerV16I32Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16959 case MVT::v32i16:
16960 return lowerV32I16Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16961 case MVT::v64i8:
16962 return lowerV64I8Shuffle(DL, Mask, Zeroable, V1, V2, Subtarget, DAG);
16963
16964 default:
16965 llvm_unreachable("Not a valid 512-bit x86 vector type!")::llvm::llvm_unreachable_internal("Not a valid 512-bit x86 vector type!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16965);
16966 }
16967}
16968
16969static SDValue lower1BitShuffleAsKSHIFTR(const SDLoc &DL, ArrayRef<int> Mask,
16970 MVT VT, SDValue V1, SDValue V2,
16971 const X86Subtarget &Subtarget,
16972 SelectionDAG &DAG) {
16973 // Shuffle should be unary.
16974 if (!V2.isUndef())
16975 return SDValue();
16976
16977 int ShiftAmt = -1;
16978 int NumElts = Mask.size();
16979 for (int i = 0; i != NumElts; ++i) {
16980 int M = Mask[i];
16981 assert((M == SM_SentinelUndef || (0 <= M && M < NumElts)) &&(((M == SM_SentinelUndef || (0 <= M && M < NumElts
)) && "Unexpected mask index.") ? static_cast<void
> (0) : __assert_fail ("(M == SM_SentinelUndef || (0 <= M && M < NumElts)) && \"Unexpected mask index.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16982, __PRETTY_FUNCTION__))
16982 "Unexpected mask index.")(((M == SM_SentinelUndef || (0 <= M && M < NumElts
)) && "Unexpected mask index.") ? static_cast<void
> (0) : __assert_fail ("(M == SM_SentinelUndef || (0 <= M && M < NumElts)) && \"Unexpected mask index.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16982, __PRETTY_FUNCTION__));
16983 if (M < 0)
16984 continue;
16985
16986 // The first non-undef element determines our shift amount.
16987 if (ShiftAmt < 0) {
16988 ShiftAmt = M - i;
16989 // Need to be shifting right.
16990 if (ShiftAmt <= 0)
16991 return SDValue();
16992 }
16993 // All non-undef elements must shift by the same amount.
16994 if (ShiftAmt != M - i)
16995 return SDValue();
16996 }
16997 assert(ShiftAmt >= 0 && "All undef?")((ShiftAmt >= 0 && "All undef?") ? static_cast<
void> (0) : __assert_fail ("ShiftAmt >= 0 && \"All undef?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 16997, __PRETTY_FUNCTION__));
16998
16999 // Great we found a shift right.
17000 MVT WideVT = VT;
17001 if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8)
17002 WideVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
17003 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
17004 DAG.getUNDEF(WideVT), V1,
17005 DAG.getIntPtrConstant(0, DL));
17006 Res = DAG.getNode(X86ISD::KSHIFTR, DL, WideVT, Res,
17007 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17008 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17009 DAG.getIntPtrConstant(0, DL));
17010}
17011
17012// Determine if this shuffle can be implemented with a KSHIFT instruction.
17013// Returns the shift amount if possible or -1 if not. This is a simplified
17014// version of matchShuffleAsShift.
17015static int match1BitShuffleAsKSHIFT(unsigned &Opcode, ArrayRef<int> Mask,
17016 int MaskOffset, const APInt &Zeroable) {
17017 int Size = Mask.size();
17018
17019 auto CheckZeros = [&](int Shift, bool Left) {
17020 for (int j = 0; j < Shift; ++j)
17021 if (!Zeroable[j + (Left ? 0 : (Size - Shift))])
17022 return false;
17023
17024 return true;
17025 };
17026
17027 auto MatchShift = [&](int Shift, bool Left) {
17028 unsigned Pos = Left ? Shift : 0;
17029 unsigned Low = Left ? 0 : Shift;
17030 unsigned Len = Size - Shift;
17031 return isSequentialOrUndefInRange(Mask, Pos, Len, Low + MaskOffset);
17032 };
17033
17034 for (int Shift = 1; Shift != Size; ++Shift)
17035 for (bool Left : {true, false})
17036 if (CheckZeros(Shift, Left) && MatchShift(Shift, Left)) {
17037 Opcode = Left ? X86ISD::KSHIFTL : X86ISD::KSHIFTR;
17038 return Shift;
17039 }
17040
17041 return -1;
17042}
17043
17044
17045// Lower vXi1 vector shuffles.
17046// There is no a dedicated instruction on AVX-512 that shuffles the masks.
17047// The only way to shuffle bits is to sign-extend the mask vector to SIMD
17048// vector, shuffle and then truncate it back.
17049static SDValue lower1BitShuffle(const SDLoc &DL, ArrayRef<int> Mask,
17050 MVT VT, SDValue V1, SDValue V2,
17051 const APInt &Zeroable,
17052 const X86Subtarget &Subtarget,
17053 SelectionDAG &DAG) {
17054 assert(Subtarget.hasAVX512() &&((Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/o basic ISA!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"Cannot lower 512-bit vectors w/o basic ISA!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17055, __PRETTY_FUNCTION__))
17055 "Cannot lower 512-bit vectors w/o basic ISA!")((Subtarget.hasAVX512() && "Cannot lower 512-bit vectors w/o basic ISA!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"Cannot lower 512-bit vectors w/o basic ISA!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17055, __PRETTY_FUNCTION__));
17056
17057 int NumElts = Mask.size();
17058
17059 // Try to recognize shuffles that are just padding a subvector with zeros.
17060 int SubvecElts = 0;
17061 int Src = -1;
17062 for (int i = 0; i != NumElts; ++i) {
17063 if (Mask[i] >= 0) {
17064 // Grab the source from the first valid mask. All subsequent elements need
17065 // to use this same source.
17066 if (Src < 0)
17067 Src = Mask[i] / NumElts;
17068 if (Src != (Mask[i] / NumElts) || (Mask[i] % NumElts) != i)
17069 break;
17070 }
17071
17072 ++SubvecElts;
17073 }
17074 assert(SubvecElts != NumElts && "Identity shuffle?")((SubvecElts != NumElts && "Identity shuffle?") ? static_cast
<void> (0) : __assert_fail ("SubvecElts != NumElts && \"Identity shuffle?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17074, __PRETTY_FUNCTION__));
17075
17076 // Clip to a power 2.
17077 SubvecElts = PowerOf2Floor(SubvecElts);
17078
17079 // Make sure the number of zeroable bits in the top at least covers the bits
17080 // not covered by the subvector.
17081 if ((int)Zeroable.countLeadingOnes() >= (NumElts - SubvecElts)) {
17082 assert(Src >= 0 && "Expected a source!")((Src >= 0 && "Expected a source!") ? static_cast<
void> (0) : __assert_fail ("Src >= 0 && \"Expected a source!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17082, __PRETTY_FUNCTION__));
17083 MVT ExtractVT = MVT::getVectorVT(MVT::i1, SubvecElts);
17084 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ExtractVT,
17085 Src == 0 ? V1 : V2,
17086 DAG.getIntPtrConstant(0, DL));
17087 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
17088 DAG.getConstant(0, DL, VT),
17089 Extract, DAG.getIntPtrConstant(0, DL));
17090 }
17091
17092 // Try a simple shift right with undef elements. Later we'll try with zeros.
17093 if (SDValue Shift = lower1BitShuffleAsKSHIFTR(DL, Mask, VT, V1, V2, Subtarget,
17094 DAG))
17095 return Shift;
17096
17097 // Try to match KSHIFTs.
17098 unsigned Offset = 0;
17099 for (SDValue V : { V1, V2 }) {
17100 unsigned Opcode;
17101 int ShiftAmt = match1BitShuffleAsKSHIFT(Opcode, Mask, Offset, Zeroable);
17102 if (ShiftAmt >= 0) {
17103 MVT WideVT = VT;
17104 if ((!Subtarget.hasDQI() && NumElts == 8) || NumElts < 8)
17105 WideVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
17106 SDValue Res = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideVT,
17107 DAG.getUNDEF(WideVT), V,
17108 DAG.getIntPtrConstant(0, DL));
17109 // Widened right shifts need two shifts to ensure we shift in zeroes.
17110 if (Opcode == X86ISD::KSHIFTR && WideVT != VT) {
17111 int WideElts = WideVT.getVectorNumElements();
17112 // Shift left to put the original vector in the MSBs of the new size.
17113 Res = DAG.getNode(X86ISD::KSHIFTL, DL, WideVT, Res,
17114 DAG.getTargetConstant(WideElts - NumElts, DL, MVT::i8));
17115 // Increase the shift amount to account for the left shift.
17116 ShiftAmt += WideElts - NumElts;
17117 }
17118
17119 Res = DAG.getNode(Opcode, DL, WideVT, Res,
17120 DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
17121 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
17122 DAG.getIntPtrConstant(0, DL));
17123 }
17124 Offset += NumElts; // Increment for next iteration.
17125 }
17126
17127
17128
17129 MVT ExtVT;
17130 switch (VT.SimpleTy) {
17131 default:
17132 llvm_unreachable("Expected a vector of i1 elements")::llvm::llvm_unreachable_internal("Expected a vector of i1 elements"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17132);
17133 case MVT::v2i1:
17134 ExtVT = MVT::v2i64;
17135 break;
17136 case MVT::v4i1:
17137 ExtVT = MVT::v4i32;
17138 break;
17139 case MVT::v8i1:
17140 // Take 512-bit type, more shuffles on KNL. If we have VLX use a 256-bit
17141 // shuffle.
17142 ExtVT = Subtarget.hasVLX() ? MVT::v8i32 : MVT::v8i64;
17143 break;
17144 case MVT::v16i1:
17145 // Take 512-bit type, unless we are avoiding 512-bit types and have the
17146 // 256-bit operation available.
17147 ExtVT = Subtarget.canExtendTo512DQ() ? MVT::v16i32 : MVT::v16i16;
17148 break;
17149 case MVT::v32i1:
17150 // Take 512-bit type, unless we are avoiding 512-bit types and have the
17151 // 256-bit operation available.
17152 assert(Subtarget.hasBWI() && "Expected AVX512BW support")((Subtarget.hasBWI() && "Expected AVX512BW support") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI() && \"Expected AVX512BW support\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17152, __PRETTY_FUNCTION__));
17153 ExtVT = Subtarget.canExtendTo512BW() ? MVT::v32i16 : MVT::v32i8;
17154 break;
17155 case MVT::v64i1:
17156 // Fall back to scalarization. FIXME: We can do better if the shuffle
17157 // can be partitioned cleanly.
17158 if (!Subtarget.useBWIRegs())
17159 return SDValue();
17160 ExtVT = MVT::v64i8;
17161 break;
17162 }
17163
17164 V1 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V1);
17165 V2 = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, V2);
17166
17167 SDValue Shuffle = DAG.getVectorShuffle(ExtVT, DL, V1, V2, Mask);
17168 // i1 was sign extended we can use X86ISD::CVT2MASK.
17169 int NumElems = VT.getVectorNumElements();
17170 if ((Subtarget.hasBWI() && (NumElems >= 32)) ||
17171 (Subtarget.hasDQI() && (NumElems < 32)))
17172 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, ExtVT),
17173 Shuffle, ISD::SETGT);
17174
17175 return DAG.getNode(ISD::TRUNCATE, DL, VT, Shuffle);
17176}
17177
17178/// Helper function that returns true if the shuffle mask should be
17179/// commuted to improve canonicalization.
17180static bool canonicalizeShuffleMaskWithCommute(ArrayRef<int> Mask) {
17181 int NumElements = Mask.size();
17182
17183 int NumV1Elements = 0, NumV2Elements = 0;
17184 for (int M : Mask)
17185 if (M < 0)
17186 continue;
17187 else if (M < NumElements)
17188 ++NumV1Elements;
17189 else
17190 ++NumV2Elements;
17191
17192 // Commute the shuffle as needed such that more elements come from V1 than
17193 // V2. This allows us to match the shuffle pattern strictly on how many
17194 // elements come from V1 without handling the symmetric cases.
17195 if (NumV2Elements > NumV1Elements)
17196 return true;
17197
17198 assert(NumV1Elements > 0 && "No V1 indices")((NumV1Elements > 0 && "No V1 indices") ? static_cast
<void> (0) : __assert_fail ("NumV1Elements > 0 && \"No V1 indices\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17198, __PRETTY_FUNCTION__));
17199
17200 if (NumV2Elements == 0)
17201 return false;
17202
17203 // When the number of V1 and V2 elements are the same, try to minimize the
17204 // number of uses of V2 in the low half of the vector. When that is tied,
17205 // ensure that the sum of indices for V1 is equal to or lower than the sum
17206 // indices for V2. When those are equal, try to ensure that the number of odd
17207 // indices for V1 is lower than the number of odd indices for V2.
17208 if (NumV1Elements == NumV2Elements) {
17209 int LowV1Elements = 0, LowV2Elements = 0;
17210 for (int M : Mask.slice(0, NumElements / 2))
17211 if (M >= NumElements)
17212 ++LowV2Elements;
17213 else if (M >= 0)
17214 ++LowV1Elements;
17215 if (LowV2Elements > LowV1Elements)
17216 return true;
17217 if (LowV2Elements == LowV1Elements) {
17218 int SumV1Indices = 0, SumV2Indices = 0;
17219 for (int i = 0, Size = Mask.size(); i < Size; ++i)
17220 if (Mask[i] >= NumElements)
17221 SumV2Indices += i;
17222 else if (Mask[i] >= 0)
17223 SumV1Indices += i;
17224 if (SumV2Indices < SumV1Indices)
17225 return true;
17226 if (SumV2Indices == SumV1Indices) {
17227 int NumV1OddIndices = 0, NumV2OddIndices = 0;
17228 for (int i = 0, Size = Mask.size(); i < Size; ++i)
17229 if (Mask[i] >= NumElements)
17230 NumV2OddIndices += i % 2;
17231 else if (Mask[i] >= 0)
17232 NumV1OddIndices += i % 2;
17233 if (NumV2OddIndices < NumV1OddIndices)
17234 return true;
17235 }
17236 }
17237 }
17238
17239 return false;
17240}
17241
17242/// Top-level lowering for x86 vector shuffles.
17243///
17244/// This handles decomposition, canonicalization, and lowering of all x86
17245/// vector shuffles. Most of the specific lowering strategies are encapsulated
17246/// above in helper routines. The canonicalization attempts to widen shuffles
17247/// to involve fewer lanes of wider elements, consolidate symmetric patterns
17248/// s.t. only one of the two inputs needs to be tested, etc.
17249static SDValue lowerVECTOR_SHUFFLE(SDValue Op, const X86Subtarget &Subtarget,
17250 SelectionDAG &DAG) {
17251 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
17252 ArrayRef<int> OrigMask = SVOp->getMask();
17253 SDValue V1 = Op.getOperand(0);
17254 SDValue V2 = Op.getOperand(1);
17255 MVT VT = Op.getSimpleValueType();
17256 int NumElements = VT.getVectorNumElements();
17257 SDLoc DL(Op);
17258 bool Is1BitVector = (VT.getVectorElementType() == MVT::i1);
17259
17260 assert((VT.getSizeInBits() != 64 || Is1BitVector) &&(((VT.getSizeInBits() != 64 || Is1BitVector) && "Can't lower MMX shuffles"
) ? static_cast<void> (0) : __assert_fail ("(VT.getSizeInBits() != 64 || Is1BitVector) && \"Can't lower MMX shuffles\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17261, __PRETTY_FUNCTION__))
17261 "Can't lower MMX shuffles")(((VT.getSizeInBits() != 64 || Is1BitVector) && "Can't lower MMX shuffles"
) ? static_cast<void> (0) : __assert_fail ("(VT.getSizeInBits() != 64 || Is1BitVector) && \"Can't lower MMX shuffles\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17261, __PRETTY_FUNCTION__));
17262
17263 bool V1IsUndef = V1.isUndef();
17264 bool V2IsUndef = V2.isUndef();
17265 if (V1IsUndef && V2IsUndef)
17266 return DAG.getUNDEF(VT);
17267
17268 // When we create a shuffle node we put the UNDEF node to second operand,
17269 // but in some cases the first operand may be transformed to UNDEF.
17270 // In this case we should just commute the node.
17271 if (V1IsUndef)
17272 return DAG.getCommutedVectorShuffle(*SVOp);
17273
17274 // Check for non-undef masks pointing at an undef vector and make the masks
17275 // undef as well. This makes it easier to match the shuffle based solely on
17276 // the mask.
17277 if (V2IsUndef &&
17278 any_of(OrigMask, [NumElements](int M) { return M >= NumElements; })) {
17279 SmallVector<int, 8> NewMask(OrigMask.begin(), OrigMask.end());
17280 for (int &M : NewMask)
17281 if (M >= NumElements)
17282 M = -1;
17283 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
17284 }
17285
17286 // Check for illegal shuffle mask element index values.
17287 int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
17288 (void)MaskUpperLimit;
17289 assert(llvm::all_of(OrigMask,((llvm::all_of(OrigMask, [&](int M) { return -1 <= M &&
M < MaskUpperLimit; }) && "Out of bounds shuffle index"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(OrigMask, [&](int M) { return -1 <= M && M < MaskUpperLimit; }) && \"Out of bounds shuffle index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17291, __PRETTY_FUNCTION__))
17290 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&((llvm::all_of(OrigMask, [&](int M) { return -1 <= M &&
M < MaskUpperLimit; }) && "Out of bounds shuffle index"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(OrigMask, [&](int M) { return -1 <= M && M < MaskUpperLimit; }) && \"Out of bounds shuffle index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17291, __PRETTY_FUNCTION__))
17291 "Out of bounds shuffle index")((llvm::all_of(OrigMask, [&](int M) { return -1 <= M &&
M < MaskUpperLimit; }) && "Out of bounds shuffle index"
) ? static_cast<void> (0) : __assert_fail ("llvm::all_of(OrigMask, [&](int M) { return -1 <= M && M < MaskUpperLimit; }) && \"Out of bounds shuffle index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17291, __PRETTY_FUNCTION__));
17292
17293 // We actually see shuffles that are entirely re-arrangements of a set of
17294 // zero inputs. This mostly happens while decomposing complex shuffles into
17295 // simple ones. Directly lower these as a buildvector of zeros.
17296 APInt KnownUndef, KnownZero;
17297 computeZeroableShuffleElements(OrigMask, V1, V2, KnownUndef, KnownZero);
17298
17299 APInt Zeroable = KnownUndef | KnownZero;
17300 if (Zeroable.isAllOnesValue())
17301 return getZeroVector(VT, Subtarget, DAG, DL);
17302
17303 bool V2IsZero = !V2IsUndef && ISD::isBuildVectorAllZeros(V2.getNode());
17304
17305 // Try to collapse shuffles into using a vector type with fewer elements but
17306 // wider element types. We cap this to not form integers or floating point
17307 // elements wider than 64 bits, but it might be interesting to form i128
17308 // integers to handle flipping the low and high halves of AVX 256-bit vectors.
17309 SmallVector<int, 16> WidenedMask;
17310 if (VT.getScalarSizeInBits() < 64 && !Is1BitVector &&
17311 canWidenShuffleElements(OrigMask, Zeroable, V2IsZero, WidenedMask)) {
17312 // Shuffle mask widening should not interfere with a broadcast opportunity
17313 // by obfuscating the operands with bitcasts.
17314 // TODO: Avoid lowering directly from this top-level function: make this
17315 // a query (canLowerAsBroadcast) and defer lowering to the type-based calls.
17316 if (SDValue Broadcast = lowerShuffleAsBroadcast(DL, VT, V1, V2, OrigMask,
17317 Subtarget, DAG))
17318 return Broadcast;
17319
17320 MVT NewEltVT = VT.isFloatingPoint()
17321 ? MVT::getFloatingPointVT(VT.getScalarSizeInBits() * 2)
17322 : MVT::getIntegerVT(VT.getScalarSizeInBits() * 2);
17323 int NewNumElts = NumElements / 2;
17324 MVT NewVT = MVT::getVectorVT(NewEltVT, NewNumElts);
17325 // Make sure that the new vector type is legal. For example, v2f64 isn't
17326 // legal on SSE1.
17327 if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
17328 if (V2IsZero) {
17329 // Modify the new Mask to take all zeros from the all-zero vector.
17330 // Choose indices that are blend-friendly.
17331 bool UsedZeroVector = false;
17332 assert(find(WidenedMask, SM_SentinelZero) != WidenedMask.end() &&((find(WidenedMask, SM_SentinelZero) != WidenedMask.end() &&
"V2's non-undef elements are used?!") ? static_cast<void>
(0) : __assert_fail ("find(WidenedMask, SM_SentinelZero) != WidenedMask.end() && \"V2's non-undef elements are used?!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17333, __PRETTY_FUNCTION__))
17333 "V2's non-undef elements are used?!")((find(WidenedMask, SM_SentinelZero) != WidenedMask.end() &&
"V2's non-undef elements are used?!") ? static_cast<void>
(0) : __assert_fail ("find(WidenedMask, SM_SentinelZero) != WidenedMask.end() && \"V2's non-undef elements are used?!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17333, __PRETTY_FUNCTION__));
17334 for (int i = 0; i != NewNumElts; ++i)
17335 if (WidenedMask[i] == SM_SentinelZero) {
17336 WidenedMask[i] = i + NewNumElts;
17337 UsedZeroVector = true;
17338 }
17339 // Ensure all elements of V2 are zero - isBuildVectorAllZeros permits
17340 // some elements to be undef.
17341 if (UsedZeroVector)
17342 V2 = getZeroVector(NewVT, Subtarget, DAG, DL);
17343 }
17344 V1 = DAG.getBitcast(NewVT, V1);
17345 V2 = DAG.getBitcast(NewVT, V2);
17346 return DAG.getBitcast(
17347 VT, DAG.getVectorShuffle(NewVT, DL, V1, V2, WidenedMask));
17348 }
17349 }
17350
17351 // Commute the shuffle if it will improve canonicalization.
17352 SmallVector<int, 64> Mask(OrigMask.begin(), OrigMask.end());
17353 if (canonicalizeShuffleMaskWithCommute(Mask)) {
17354 ShuffleVectorSDNode::commuteMask(Mask);
17355 std::swap(V1, V2);
17356 }
17357
17358 if (SDValue V = lowerShuffleWithVPMOV(DL, Mask, VT, V1, V2, DAG, Subtarget))
17359 return V;
17360
17361 // For each vector width, delegate to a specialized lowering routine.
17362 if (VT.is128BitVector())
17363 return lower128BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17364
17365 if (VT.is256BitVector())
17366 return lower256BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17367
17368 if (VT.is512BitVector())
17369 return lower512BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17370
17371 if (Is1BitVector)
17372 return lower1BitShuffle(DL, Mask, VT, V1, V2, Zeroable, Subtarget, DAG);
17373
17374 llvm_unreachable("Unimplemented!")::llvm::llvm_unreachable_internal("Unimplemented!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17374);
17375}
17376
17377/// Try to lower a VSELECT instruction to a vector shuffle.
17378static SDValue lowerVSELECTtoVectorShuffle(SDValue Op,
17379 const X86Subtarget &Subtarget,
17380 SelectionDAG &DAG) {
17381 SDValue Cond = Op.getOperand(0);
17382 SDValue LHS = Op.getOperand(1);
17383 SDValue RHS = Op.getOperand(2);
17384 MVT VT = Op.getSimpleValueType();
17385
17386 // Only non-legal VSELECTs reach this lowering, convert those into generic
17387 // shuffles and re-use the shuffle lowering path for blends.
17388 SmallVector<int, 32> Mask;
17389 if (createShuffleMaskFromVSELECT(Mask, Cond))
17390 return DAG.getVectorShuffle(VT, SDLoc(Op), LHS, RHS, Mask);
17391
17392 return SDValue();
17393}
17394
17395SDValue X86TargetLowering::LowerVSELECT(SDValue Op, SelectionDAG &DAG) const {
17396 SDValue Cond = Op.getOperand(0);
17397 SDValue LHS = Op.getOperand(1);
17398 SDValue RHS = Op.getOperand(2);
17399
17400 // A vselect where all conditions and data are constants can be optimized into
17401 // a single vector load by SelectionDAGLegalize::ExpandBUILD_VECTOR().
17402 if (ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()) &&
17403 ISD::isBuildVectorOfConstantSDNodes(LHS.getNode()) &&
17404 ISD::isBuildVectorOfConstantSDNodes(RHS.getNode()))
17405 return SDValue();
17406
17407 // Try to lower this to a blend-style vector shuffle. This can handle all
17408 // constant condition cases.
17409 if (SDValue BlendOp = lowerVSELECTtoVectorShuffle(Op, Subtarget, DAG))
17410 return BlendOp;
17411
17412 // If this VSELECT has a vector if i1 as a mask, it will be directly matched
17413 // with patterns on the mask registers on AVX-512.
17414 MVT CondVT = Cond.getSimpleValueType();
17415 unsigned CondEltSize = Cond.getScalarValueSizeInBits();
17416 if (CondEltSize == 1)
17417 return Op;
17418
17419 // Variable blends are only legal from SSE4.1 onward.
17420 if (!Subtarget.hasSSE41())
17421 return SDValue();
17422
17423 SDLoc dl(Op);
17424 MVT VT = Op.getSimpleValueType();
17425 unsigned EltSize = VT.getScalarSizeInBits();
17426 unsigned NumElts = VT.getVectorNumElements();
17427
17428 // If the VSELECT is on a 512-bit type, we have to convert a non-i1 condition
17429 // into an i1 condition so that we can use the mask-based 512-bit blend
17430 // instructions.
17431 if (VT.getSizeInBits() == 512) {
17432 // Build a mask by testing the condition against zero.
17433 MVT MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
17434 SDValue Mask = DAG.getSetCC(dl, MaskVT, Cond,
17435 DAG.getConstant(0, dl, CondVT),
17436 ISD::SETNE);
17437 // Now return a new VSELECT using the mask.
17438 return DAG.getSelect(dl, VT, Mask, LHS, RHS);
17439 }
17440
17441 // SEXT/TRUNC cases where the mask doesn't match the destination size.
17442 if (CondEltSize != EltSize) {
17443 // If we don't have a sign splat, rely on the expansion.
17444 if (CondEltSize != DAG.ComputeNumSignBits(Cond))
17445 return SDValue();
17446
17447 MVT NewCondSVT = MVT::getIntegerVT(EltSize);
17448 MVT NewCondVT = MVT::getVectorVT(NewCondSVT, NumElts);
17449 Cond = DAG.getSExtOrTrunc(Cond, dl, NewCondVT);
17450 return DAG.getNode(ISD::VSELECT, dl, VT, Cond, LHS, RHS);
17451 }
17452
17453 // Only some types will be legal on some subtargets. If we can emit a legal
17454 // VSELECT-matching blend, return Op, and but if we need to expand, return
17455 // a null value.
17456 switch (VT.SimpleTy) {
17457 default:
17458 // Most of the vector types have blends past SSE4.1.
17459 return Op;
17460
17461 case MVT::v32i8:
17462 // The byte blends for AVX vectors were introduced only in AVX2.
17463 if (Subtarget.hasAVX2())
17464 return Op;
17465
17466 return SDValue();
17467
17468 case MVT::v8i16:
17469 case MVT::v16i16: {
17470 // Bitcast everything to the vXi8 type and use a vXi8 vselect.
17471 MVT CastVT = MVT::getVectorVT(MVT::i8, NumElts * 2);
17472 Cond = DAG.getBitcast(CastVT, Cond);
17473 LHS = DAG.getBitcast(CastVT, LHS);
17474 RHS = DAG.getBitcast(CastVT, RHS);
17475 SDValue Select = DAG.getNode(ISD::VSELECT, dl, CastVT, Cond, LHS, RHS);
17476 return DAG.getBitcast(VT, Select);
17477 }
17478 }
17479}
17480
17481static SDValue LowerEXTRACT_VECTOR_ELT_SSE4(SDValue Op, SelectionDAG &DAG) {
17482 MVT VT = Op.getSimpleValueType();
17483 SDLoc dl(Op);
17484
17485 if (!Op.getOperand(0).getSimpleValueType().is128BitVector())
17486 return SDValue();
17487
17488 if (VT.getSizeInBits() == 8) {
17489 SDValue Extract = DAG.getNode(X86ISD::PEXTRB, dl, MVT::i32,
17490 Op.getOperand(0), Op.getOperand(1));
17491 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
17492 }
17493
17494 if (VT == MVT::f32) {
17495 // EXTRACTPS outputs to a GPR32 register which will require a movd to copy
17496 // the result back to FR32 register. It's only worth matching if the
17497 // result has a single use which is a store or a bitcast to i32. And in
17498 // the case of a store, it's not worth it if the index is a constant 0,
17499 // because a MOVSSmr can be used instead, which is smaller and faster.
17500 if (!Op.hasOneUse())
17501 return SDValue();
17502 SDNode *User = *Op.getNode()->use_begin();
17503 if ((User->getOpcode() != ISD::STORE ||
17504 isNullConstant(Op.getOperand(1))) &&
17505 (User->getOpcode() != ISD::BITCAST ||
17506 User->getValueType(0) != MVT::i32))
17507 return SDValue();
17508 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17509 DAG.getBitcast(MVT::v4i32, Op.getOperand(0)),
17510 Op.getOperand(1));
17511 return DAG.getBitcast(MVT::f32, Extract);
17512 }
17513
17514 if (VT == MVT::i32 || VT == MVT::i64) {
17515 // ExtractPS/pextrq works with constant index.
17516 if (isa<ConstantSDNode>(Op.getOperand(1)))
17517 return Op;
17518 }
17519
17520 return SDValue();
17521}
17522
17523/// Extract one bit from mask vector, like v16i1 or v8i1.
17524/// AVX-512 feature.
17525static SDValue ExtractBitFromMaskVector(SDValue Op, SelectionDAG &DAG,
17526 const X86Subtarget &Subtarget) {
17527 SDValue Vec = Op.getOperand(0);
17528 SDLoc dl(Vec);
17529 MVT VecVT = Vec.getSimpleValueType();
17530 SDValue Idx = Op.getOperand(1);
17531 MVT EltVT = Op.getSimpleValueType();
17532
17533 assert((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) &&(((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI(
)) && "Unexpected vector type in ExtractBitFromMaskVector"
) ? static_cast<void> (0) : __assert_fail ("(VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) && \"Unexpected vector type in ExtractBitFromMaskVector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17534, __PRETTY_FUNCTION__))
17534 "Unexpected vector type in ExtractBitFromMaskVector")(((VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI(
)) && "Unexpected vector type in ExtractBitFromMaskVector"
) ? static_cast<void> (0) : __assert_fail ("(VecVT.getVectorNumElements() <= 16 || Subtarget.hasBWI()) && \"Unexpected vector type in ExtractBitFromMaskVector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17534, __PRETTY_FUNCTION__));
17535
17536 // variable index can't be handled in mask registers,
17537 // extend vector to VR512/128
17538 if (!isa<ConstantSDNode>(Idx)) {
17539 unsigned NumElts = VecVT.getVectorNumElements();
17540 // Extending v8i1/v16i1 to 512-bit get better performance on KNL
17541 // than extending to 128/256bit.
17542 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
17543 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
17544 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec);
17545 SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ExtEltVT, Ext, Idx);
17546 return DAG.getNode(ISD::TRUNCATE, dl, EltVT, Elt);
17547 }
17548
17549 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
17550 if (IdxVal == 0) // the operation is legal
17551 return Op;
17552
17553 // Extend to natively supported kshift.
17554 unsigned NumElems = VecVT.getVectorNumElements();
17555 MVT WideVecVT = VecVT;
17556 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8) {
17557 WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
17558 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT,
17559 DAG.getUNDEF(WideVecVT), Vec,
17560 DAG.getIntPtrConstant(0, dl));
17561 }
17562
17563 // Use kshiftr instruction to move to the lower element.
17564 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec,
17565 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17566
17567 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
17568 DAG.getIntPtrConstant(0, dl));
17569}
17570
17571SDValue
17572X86TargetLowering::LowerEXTRACT_VECTOR_ELT(SDValue Op,
17573 SelectionDAG &DAG) const {
17574 SDLoc dl(Op);
17575 SDValue Vec = Op.getOperand(0);
17576 MVT VecVT = Vec.getSimpleValueType();
17577 SDValue Idx = Op.getOperand(1);
17578
17579 if (VecVT.getVectorElementType() == MVT::i1)
17580 return ExtractBitFromMaskVector(Op, DAG, Subtarget);
17581
17582 if (!isa<ConstantSDNode>(Idx)) {
17583 // Its more profitable to go through memory (1 cycles throughput)
17584 // than using VMOVD + VPERMV/PSHUFB sequence ( 2/3 cycles throughput)
17585 // IACA tool was used to get performance estimation
17586 // (https://software.intel.com/en-us/articles/intel-architecture-code-analyzer)
17587 //
17588 // example : extractelement <16 x i8> %a, i32 %i
17589 //
17590 // Block Throughput: 3.00 Cycles
17591 // Throughput Bottleneck: Port5
17592 //
17593 // | Num Of | Ports pressure in cycles | |
17594 // | Uops | 0 - DV | 5 | 6 | 7 | |
17595 // ---------------------------------------------
17596 // | 1 | | 1.0 | | | CP | vmovd xmm1, edi
17597 // | 1 | | 1.0 | | | CP | vpshufb xmm0, xmm0, xmm1
17598 // | 2 | 1.0 | 1.0 | | | CP | vpextrb eax, xmm0, 0x0
17599 // Total Num Of Uops: 4
17600 //
17601 //
17602 // Block Throughput: 1.00 Cycles
17603 // Throughput Bottleneck: PORT2_AGU, PORT3_AGU, Port4
17604 //
17605 // | | Ports pressure in cycles | |
17606 // |Uops| 1 | 2 - D |3 - D | 4 | 5 | |
17607 // ---------------------------------------------------------
17608 // |2^ | | 0.5 | 0.5 |1.0| |CP| vmovaps xmmword ptr [rsp-0x18], xmm0
17609 // |1 |0.5| | | |0.5| | lea rax, ptr [rsp-0x18]
17610 // |1 | |0.5, 0.5|0.5, 0.5| | |CP| mov al, byte ptr [rdi+rax*1]
17611 // Total Num Of Uops: 4
17612
17613 return SDValue();
17614 }
17615
17616 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
17617
17618 // If this is a 256-bit vector result, first extract the 128-bit vector and
17619 // then extract the element from the 128-bit vector.
17620 if (VecVT.is256BitVector() || VecVT.is512BitVector()) {
17621 // Get the 128-bit vector.
17622 Vec = extract128BitVector(Vec, IdxVal, DAG, dl);
17623 MVT EltVT = VecVT.getVectorElementType();
17624
17625 unsigned ElemsPerChunk = 128 / EltVT.getSizeInBits();
17626 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2")((isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(ElemsPerChunk) && \"Elements per chunk not power of 2\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17626, __PRETTY_FUNCTION__));
17627
17628 // Find IdxVal modulo ElemsPerChunk. Since ElemsPerChunk is a power of 2
17629 // this can be done with a mask.
17630 IdxVal &= ElemsPerChunk - 1;
17631 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, Op.getValueType(), Vec,
17632 DAG.getIntPtrConstant(IdxVal, dl));
17633 }
17634
17635 assert(VecVT.is128BitVector() && "Unexpected vector length")((VecVT.is128BitVector() && "Unexpected vector length"
) ? static_cast<void> (0) : __assert_fail ("VecVT.is128BitVector() && \"Unexpected vector length\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17635, __PRETTY_FUNCTION__));
17636
17637 MVT VT = Op.getSimpleValueType();
17638
17639 if (VT.getSizeInBits() == 16) {
17640 // If IdxVal is 0, it's cheaper to do a move instead of a pextrw, unless
17641 // we're going to zero extend the register or fold the store (SSE41 only).
17642 if (IdxVal == 0 && !MayFoldIntoZeroExtend(Op) &&
17643 !(Subtarget.hasSSE41() && MayFoldIntoStore(Op)))
17644 return DAG.getNode(ISD::TRUNCATE, dl, MVT::i16,
17645 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17646 DAG.getBitcast(MVT::v4i32, Vec), Idx));
17647
17648 // Transform it so it match pextrw which produces a 32-bit result.
17649 SDValue Extract = DAG.getNode(X86ISD::PEXTRW, dl, MVT::i32,
17650 Op.getOperand(0), Op.getOperand(1));
17651 return DAG.getNode(ISD::TRUNCATE, dl, VT, Extract);
17652 }
17653
17654 if (Subtarget.hasSSE41())
17655 if (SDValue Res = LowerEXTRACT_VECTOR_ELT_SSE4(Op, DAG))
17656 return Res;
17657
17658 // TODO: We only extract a single element from v16i8, we can probably afford
17659 // to be more aggressive here before using the default approach of spilling to
17660 // stack.
17661 if (VT.getSizeInBits() == 8 && Op->isOnlyUserOf(Vec.getNode())) {
17662 // Extract either the lowest i32 or any i16, and extract the sub-byte.
17663 int DWordIdx = IdxVal / 4;
17664 if (DWordIdx == 0) {
17665 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32,
17666 DAG.getBitcast(MVT::v4i32, Vec),
17667 DAG.getIntPtrConstant(DWordIdx, dl));
17668 int ShiftVal = (IdxVal % 4) * 8;
17669 if (ShiftVal != 0)
17670 Res = DAG.getNode(ISD::SRL, dl, MVT::i32, Res,
17671 DAG.getConstant(ShiftVal, dl, MVT::i8));
17672 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
17673 }
17674
17675 int WordIdx = IdxVal / 2;
17676 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i16,
17677 DAG.getBitcast(MVT::v8i16, Vec),
17678 DAG.getIntPtrConstant(WordIdx, dl));
17679 int ShiftVal = (IdxVal % 2) * 8;
17680 if (ShiftVal != 0)
17681 Res = DAG.getNode(ISD::SRL, dl, MVT::i16, Res,
17682 DAG.getConstant(ShiftVal, dl, MVT::i8));
17683 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
17684 }
17685
17686 if (VT.getSizeInBits() == 32) {
17687 if (IdxVal == 0)
17688 return Op;
17689
17690 // SHUFPS the element to the lowest double word, then movss.
17691 int Mask[4] = { static_cast<int>(IdxVal), -1, -1, -1 };
17692 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
17693 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
17694 DAG.getIntPtrConstant(0, dl));
17695 }
17696
17697 if (VT.getSizeInBits() == 64) {
17698 // FIXME: .td only matches this for <2 x f64>, not <2 x i64> on 32b
17699 // FIXME: seems like this should be unnecessary if mov{h,l}pd were taught
17700 // to match extract_elt for f64.
17701 if (IdxVal == 0)
17702 return Op;
17703
17704 // UNPCKHPD the element to the lowest double word, then movsd.
17705 // Note if the lower 64 bits of the result of the UNPCKHPD is then stored
17706 // to a f64mem, the whole operation is folded into a single MOVHPDmr.
17707 int Mask[2] = { 1, -1 };
17708 Vec = DAG.getVectorShuffle(VecVT, dl, Vec, DAG.getUNDEF(VecVT), Mask);
17709 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Vec,
17710 DAG.getIntPtrConstant(0, dl));
17711 }
17712
17713 return SDValue();
17714}
17715
17716/// Insert one bit to mask vector, like v16i1 or v8i1.
17717/// AVX-512 feature.
17718static SDValue InsertBitToMaskVector(SDValue Op, SelectionDAG &DAG,
17719 const X86Subtarget &Subtarget) {
17720 SDLoc dl(Op);
17721 SDValue Vec = Op.getOperand(0);
17722 SDValue Elt = Op.getOperand(1);
17723 SDValue Idx = Op.getOperand(2);
17724 MVT VecVT = Vec.getSimpleValueType();
17725
17726 if (!isa<ConstantSDNode>(Idx)) {
17727 // Non constant index. Extend source and destination,
17728 // insert element and then truncate the result.
17729 unsigned NumElts = VecVT.getVectorNumElements();
17730 MVT ExtEltVT = (NumElts <= 8) ? MVT::getIntegerVT(128 / NumElts) : MVT::i8;
17731 MVT ExtVecVT = MVT::getVectorVT(ExtEltVT, NumElts);
17732 SDValue ExtOp = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, ExtVecVT,
17733 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtVecVT, Vec),
17734 DAG.getNode(ISD::SIGN_EXTEND, dl, ExtEltVT, Elt), Idx);
17735 return DAG.getNode(ISD::TRUNCATE, dl, VecVT, ExtOp);
17736 }
17737
17738 // Copy into a k-register, extract to v1i1 and insert_subvector.
17739 SDValue EltInVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v1i1, Elt);
17740
17741 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VecVT, Vec, EltInVec,
17742 Op.getOperand(2));
17743}
17744
17745SDValue X86TargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
17746 SelectionDAG &DAG) const {
17747 MVT VT = Op.getSimpleValueType();
17748 MVT EltVT = VT.getVectorElementType();
17749 unsigned NumElts = VT.getVectorNumElements();
17750
17751 if (EltVT == MVT::i1)
17752 return InsertBitToMaskVector(Op, DAG, Subtarget);
17753
17754 SDLoc dl(Op);
17755 SDValue N0 = Op.getOperand(0);
17756 SDValue N1 = Op.getOperand(1);
17757 SDValue N2 = Op.getOperand(2);
17758
17759 auto *N2C = dyn_cast<ConstantSDNode>(N2);
17760 if (!N2C || N2C->getAPIntValue().uge(NumElts))
17761 return SDValue();
17762 uint64_t IdxVal = N2C->getZExtValue();
17763
17764 bool IsZeroElt = X86::isZeroNode(N1);
17765 bool IsAllOnesElt = VT.isInteger() && llvm::isAllOnesConstant(N1);
17766
17767 // If we are inserting a element, see if we can do this more efficiently with
17768 // a blend shuffle with a rematerializable vector than a costly integer
17769 // insertion.
17770 if ((IsZeroElt || IsAllOnesElt) && Subtarget.hasSSE41() &&
17771 16 <= EltVT.getSizeInBits()) {
17772 SmallVector<int, 8> BlendMask;
17773 for (unsigned i = 0; i != NumElts; ++i)
17774 BlendMask.push_back(i == IdxVal ? i + NumElts : i);
17775 SDValue CstVector = IsZeroElt ? getZeroVector(VT, Subtarget, DAG, dl)
17776 : getOnesVector(VT, DAG, dl);
17777 return DAG.getVectorShuffle(VT, dl, N0, CstVector, BlendMask);
17778 }
17779
17780 // If the vector is wider than 128 bits, extract the 128-bit subvector, insert
17781 // into that, and then insert the subvector back into the result.
17782 if (VT.is256BitVector() || VT.is512BitVector()) {
17783 // With a 256-bit vector, we can insert into the zero element efficiently
17784 // using a blend if we have AVX or AVX2 and the right data type.
17785 if (VT.is256BitVector() && IdxVal == 0) {
17786 // TODO: It is worthwhile to cast integer to floating point and back
17787 // and incur a domain crossing penalty if that's what we'll end up
17788 // doing anyway after extracting to a 128-bit vector.
17789 if ((Subtarget.hasAVX() && (EltVT == MVT::f64 || EltVT == MVT::f32)) ||
17790 (Subtarget.hasAVX2() && EltVT == MVT::i32)) {
17791 SDValue N1Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
17792 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1Vec,
17793 DAG.getTargetConstant(1, dl, MVT::i8));
17794 }
17795 }
17796
17797 // Get the desired 128-bit vector chunk.
17798 SDValue V = extract128BitVector(N0, IdxVal, DAG, dl);
17799
17800 // Insert the element into the desired chunk.
17801 unsigned NumEltsIn128 = 128 / EltVT.getSizeInBits();
17802 assert(isPowerOf2_32(NumEltsIn128))((isPowerOf2_32(NumEltsIn128)) ? static_cast<void> (0) :
__assert_fail ("isPowerOf2_32(NumEltsIn128)", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17802, __PRETTY_FUNCTION__));
17803 // Since NumEltsIn128 is a power of 2 we can use mask instead of modulo.
17804 unsigned IdxIn128 = IdxVal & (NumEltsIn128 - 1);
17805
17806 V = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, V.getValueType(), V, N1,
17807 DAG.getIntPtrConstant(IdxIn128, dl));
17808
17809 // Insert the changed part back into the bigger vector
17810 return insert128BitVector(N0, V, IdxVal, DAG, dl);
17811 }
17812 assert(VT.is128BitVector() && "Only 128-bit vector types should be left!")((VT.is128BitVector() && "Only 128-bit vector types should be left!"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Only 128-bit vector types should be left!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17812, __PRETTY_FUNCTION__));
17813
17814 // This will be just movd/movq/movss/movsd.
17815 if (IdxVal == 0 && ISD::isBuildVectorAllZeros(N0.getNode()) &&
17816 (EltVT == MVT::i32 || EltVT == MVT::f32 || EltVT == MVT::f64 ||
17817 EltVT == MVT::i64)) {
17818 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT, N1);
17819 return getShuffleVectorZeroOrUndef(N1, 0, true, Subtarget, DAG);
17820 }
17821
17822 // Transform it so it match pinsr{b,w} which expects a GR32 as its second
17823 // argument. SSE41 required for pinsrb.
17824 if (VT == MVT::v8i16 || (VT == MVT::v16i8 && Subtarget.hasSSE41())) {
17825 unsigned Opc;
17826 if (VT == MVT::v8i16) {
17827 assert(Subtarget.hasSSE2() && "SSE2 required for PINSRW")((Subtarget.hasSSE2() && "SSE2 required for PINSRW") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"SSE2 required for PINSRW\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17827, __PRETTY_FUNCTION__));
17828 Opc = X86ISD::PINSRW;
17829 } else {
17830 assert(VT == MVT::v16i8 && "PINSRB requires v16i8 vector")((VT == MVT::v16i8 && "PINSRB requires v16i8 vector")
? static_cast<void> (0) : __assert_fail ("VT == MVT::v16i8 && \"PINSRB requires v16i8 vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17830, __PRETTY_FUNCTION__));
17831 assert(Subtarget.hasSSE41() && "SSE41 required for PINSRB")((Subtarget.hasSSE41() && "SSE41 required for PINSRB"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE41() && \"SSE41 required for PINSRB\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17831, __PRETTY_FUNCTION__));
17832 Opc = X86ISD::PINSRB;
17833 }
17834
17835 if (N1.getValueType() != MVT::i32)
17836 N1 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, N1);
17837 if (N2.getValueType() != MVT::i32)
17838 N2 = DAG.getIntPtrConstant(IdxVal, dl);
17839 return DAG.getNode(Opc, dl, VT, N0, N1, N2);
17840 }
17841
17842 if (Subtarget.hasSSE41()) {
17843 if (EltVT == MVT::f32) {
17844 // Bits [7:6] of the constant are the source select. This will always be
17845 // zero here. The DAG Combiner may combine an extract_elt index into
17846 // these bits. For example (insert (extract, 3), 2) could be matched by
17847 // putting the '3' into bits [7:6] of X86ISD::INSERTPS.
17848 // Bits [5:4] of the constant are the destination select. This is the
17849 // value of the incoming immediate.
17850 // Bits [3:0] of the constant are the zero mask. The DAG Combiner may
17851 // combine either bitwise AND or insert of float 0.0 to set these bits.
17852
17853 bool MinSize = DAG.getMachineFunction().getFunction().hasMinSize();
17854 if (IdxVal == 0 && (!MinSize || !MayFoldLoad(N1))) {
17855 // If this is an insertion of 32-bits into the low 32-bits of
17856 // a vector, we prefer to generate a blend with immediate rather
17857 // than an insertps. Blends are simpler operations in hardware and so
17858 // will always have equal or better performance than insertps.
17859 // But if optimizing for size and there's a load folding opportunity,
17860 // generate insertps because blendps does not have a 32-bit memory
17861 // operand form.
17862 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
17863 return DAG.getNode(X86ISD::BLENDI, dl, VT, N0, N1,
17864 DAG.getTargetConstant(1, dl, MVT::i8));
17865 }
17866 // Create this as a scalar to vector..
17867 N1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4f32, N1);
17868 return DAG.getNode(X86ISD::INSERTPS, dl, VT, N0, N1,
17869 DAG.getTargetConstant(IdxVal << 4, dl, MVT::i8));
17870 }
17871
17872 // PINSR* works with constant index.
17873 if (EltVT == MVT::i32 || EltVT == MVT::i64)
17874 return Op;
17875 }
17876
17877 return SDValue();
17878}
17879
17880static SDValue LowerSCALAR_TO_VECTOR(SDValue Op, const X86Subtarget &Subtarget,
17881 SelectionDAG &DAG) {
17882 SDLoc dl(Op);
17883 MVT OpVT = Op.getSimpleValueType();
17884
17885 // It's always cheaper to replace a xor+movd with xorps and simplifies further
17886 // combines.
17887 if (X86::isZeroNode(Op.getOperand(0)))
17888 return getZeroVector(OpVT, Subtarget, DAG, dl);
17889
17890 // If this is a 256-bit vector result, first insert into a 128-bit
17891 // vector and then insert into the 256-bit vector.
17892 if (!OpVT.is128BitVector()) {
17893 // Insert into a 128-bit vector.
17894 unsigned SizeFactor = OpVT.getSizeInBits() / 128;
17895 MVT VT128 = MVT::getVectorVT(OpVT.getVectorElementType(),
17896 OpVT.getVectorNumElements() / SizeFactor);
17897
17898 Op = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VT128, Op.getOperand(0));
17899
17900 // Insert the 128-bit vector.
17901 return insert128BitVector(DAG.getUNDEF(OpVT), Op, 0, DAG, dl);
17902 }
17903 assert(OpVT.is128BitVector() && OpVT.isInteger() && OpVT != MVT::v2i64 &&((OpVT.is128BitVector() && OpVT.isInteger() &&
OpVT != MVT::v2i64 && "Expected an SSE type!") ? static_cast
<void> (0) : __assert_fail ("OpVT.is128BitVector() && OpVT.isInteger() && OpVT != MVT::v2i64 && \"Expected an SSE type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17904, __PRETTY_FUNCTION__))
17904 "Expected an SSE type!")((OpVT.is128BitVector() && OpVT.isInteger() &&
OpVT != MVT::v2i64 && "Expected an SSE type!") ? static_cast
<void> (0) : __assert_fail ("OpVT.is128BitVector() && OpVT.isInteger() && OpVT != MVT::v2i64 && \"Expected an SSE type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17904, __PRETTY_FUNCTION__));
17905
17906 // Pass through a v4i32 SCALAR_TO_VECTOR as that's what we use in tblgen.
17907 if (OpVT == MVT::v4i32)
17908 return Op;
17909
17910 SDValue AnyExt = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Op.getOperand(0));
17911 return DAG.getBitcast(
17912 OpVT, DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, AnyExt));
17913}
17914
17915// Lower a node with an INSERT_SUBVECTOR opcode. This may result in a
17916// simple superregister reference or explicit instructions to insert
17917// the upper bits of a vector.
17918static SDValue LowerINSERT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
17919 SelectionDAG &DAG) {
17920 assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1)((Op.getSimpleValueType().getVectorElementType() == MVT::i1) ?
static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().getVectorElementType() == MVT::i1"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17920, __PRETTY_FUNCTION__));
17921
17922 return insert1BitVector(Op, DAG, Subtarget);
17923}
17924
17925static SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, const X86Subtarget &Subtarget,
17926 SelectionDAG &DAG) {
17927 assert(Op.getSimpleValueType().getVectorElementType() == MVT::i1 &&((Op.getSimpleValueType().getVectorElementType() == MVT::i1 &&
"Only vXi1 extract_subvectors need custom lowering") ? static_cast
<void> (0) : __assert_fail ("Op.getSimpleValueType().getVectorElementType() == MVT::i1 && \"Only vXi1 extract_subvectors need custom lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17928, __PRETTY_FUNCTION__))
17928 "Only vXi1 extract_subvectors need custom lowering")((Op.getSimpleValueType().getVectorElementType() == MVT::i1 &&
"Only vXi1 extract_subvectors need custom lowering") ? static_cast
<void> (0) : __assert_fail ("Op.getSimpleValueType().getVectorElementType() == MVT::i1 && \"Only vXi1 extract_subvectors need custom lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 17928, __PRETTY_FUNCTION__));
17929
17930 SDLoc dl(Op);
17931 SDValue Vec = Op.getOperand(0);
17932 SDValue Idx = Op.getOperand(1);
17933
17934 if (!isa<ConstantSDNode>(Idx))
17935 return SDValue();
17936
17937 unsigned IdxVal = cast<ConstantSDNode>(Idx)->getZExtValue();
17938 if (IdxVal == 0) // the operation is legal
17939 return Op;
17940
17941 MVT VecVT = Vec.getSimpleValueType();
17942 unsigned NumElems = VecVT.getVectorNumElements();
17943
17944 // Extend to natively supported kshift.
17945 MVT WideVecVT = VecVT;
17946 if ((!Subtarget.hasDQI() && NumElems == 8) || NumElems < 8) {
17947 WideVecVT = Subtarget.hasDQI() ? MVT::v8i1 : MVT::v16i1;
17948 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVecVT,
17949 DAG.getUNDEF(WideVecVT), Vec,
17950 DAG.getIntPtrConstant(0, dl));
17951 }
17952
17953 // Shift to the LSB.
17954 Vec = DAG.getNode(X86ISD::KSHIFTR, dl, WideVecVT, Vec,
17955 DAG.getTargetConstant(IdxVal, dl, MVT::i8));
17956
17957 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, Op.getValueType(), Vec,
17958 DAG.getIntPtrConstant(0, dl));
17959}
17960
17961// Returns the appropriate wrapper opcode for a global reference.
17962unsigned X86TargetLowering::getGlobalWrapperKind(
17963 const GlobalValue *GV, const unsigned char OpFlags) const {
17964 // References to absolute symbols are never PC-relative.
17965 if (GV && GV->isAbsoluteSymbolRef())
17966 return X86ISD::Wrapper;
17967
17968 CodeModel::Model M = getTargetMachine().getCodeModel();
17969 if (Subtarget.isPICStyleRIPRel() &&
17970 (M == CodeModel::Small || M == CodeModel::Kernel))
17971 return X86ISD::WrapperRIP;
17972
17973 // GOTPCREL references must always use RIP.
17974 if (OpFlags == X86II::MO_GOTPCREL)
17975 return X86ISD::WrapperRIP;
17976
17977 return X86ISD::Wrapper;
17978}
17979
17980// ConstantPool, JumpTable, GlobalAddress, and ExternalSymbol are lowered as
17981// their target counterpart wrapped in the X86ISD::Wrapper node. Suppose N is
17982// one of the above mentioned nodes. It has to be wrapped because otherwise
17983// Select(N) returns N. So the raw TargetGlobalAddress nodes, etc. can only
17984// be used to form addressing mode. These wrapped nodes will be selected
17985// into MOV32ri.
17986SDValue
17987X86TargetLowering::LowerConstantPool(SDValue Op, SelectionDAG &DAG) const {
17988 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Op);
17989
17990 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
17991 // global base reg.
17992 unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
17993
17994 auto PtrVT = getPointerTy(DAG.getDataLayout());
17995 SDValue Result = DAG.getTargetConstantPool(
17996 CP->getConstVal(), PtrVT, CP->getAlignment(), CP->getOffset(), OpFlag);
17997 SDLoc DL(CP);
17998 Result = DAG.getNode(getGlobalWrapperKind(), DL, PtrVT, Result);
17999 // With PIC, the address is actually $g + Offset.
18000 if (OpFlag) {
18001 Result =
18002 DAG.getNode(ISD::ADD, DL, PtrVT,
18003 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18004 }
18005
18006 return Result;
18007}
18008
18009SDValue X86TargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
18010 JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
18011
18012 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18013 // global base reg.
18014 unsigned char OpFlag = Subtarget.classifyLocalReference(nullptr);
18015
18016 auto PtrVT = getPointerTy(DAG.getDataLayout());
18017 SDValue Result = DAG.getTargetJumpTable(JT->getIndex(), PtrVT, OpFlag);
18018 SDLoc DL(JT);
18019 Result = DAG.getNode(getGlobalWrapperKind(), DL, PtrVT, Result);
18020
18021 // With PIC, the address is actually $g + Offset.
18022 if (OpFlag)
18023 Result =
18024 DAG.getNode(ISD::ADD, DL, PtrVT,
18025 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), Result);
18026
18027 return Result;
18028}
18029
18030SDValue X86TargetLowering::LowerExternalSymbol(SDValue Op,
18031 SelectionDAG &DAG) const {
18032 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18033}
18034
18035SDValue
18036X86TargetLowering::LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const {
18037 // Create the TargetBlockAddressAddress node.
18038 unsigned char OpFlags =
18039 Subtarget.classifyBlockAddressReference();
18040 const BlockAddress *BA = cast<BlockAddressSDNode>(Op)->getBlockAddress();
18041 int64_t Offset = cast<BlockAddressSDNode>(Op)->getOffset();
18042 SDLoc dl(Op);
18043 auto PtrVT = getPointerTy(DAG.getDataLayout());
18044 SDValue Result = DAG.getTargetBlockAddress(BA, PtrVT, Offset, OpFlags);
18045 Result = DAG.getNode(getGlobalWrapperKind(), dl, PtrVT, Result);
18046
18047 // With PIC, the address is actually $g + Offset.
18048 if (isGlobalRelativeToPICBase(OpFlags)) {
18049 Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18050 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18051 }
18052
18053 return Result;
18054}
18055
18056/// Creates target global address or external symbol nodes for calls or
18057/// other uses.
18058SDValue X86TargetLowering::LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
18059 bool ForCall) const {
18060 // Unpack the global address or external symbol.
18061 const SDLoc &dl = SDLoc(Op);
18062 const GlobalValue *GV = nullptr;
18063 int64_t Offset = 0;
18064 const char *ExternalSym = nullptr;
18065 if (const auto *G = dyn_cast<GlobalAddressSDNode>(Op)) {
18066 GV = G->getGlobal();
18067 Offset = G->getOffset();
18068 } else {
18069 const auto *ES = cast<ExternalSymbolSDNode>(Op);
18070 ExternalSym = ES->getSymbol();
18071 }
18072
18073 // Calculate some flags for address lowering.
18074 const Module &Mod = *DAG.getMachineFunction().getFunction().getParent();
18075 unsigned char OpFlags;
18076 if (ForCall)
18077 OpFlags = Subtarget.classifyGlobalFunctionReference(GV, Mod);
18078 else
18079 OpFlags = Subtarget.classifyGlobalReference(GV, Mod);
18080 bool HasPICReg = isGlobalRelativeToPICBase(OpFlags);
18081 bool NeedsLoad = isGlobalStubReference(OpFlags);
18082
18083 CodeModel::Model M = DAG.getTarget().getCodeModel();
18084 auto PtrVT = getPointerTy(DAG.getDataLayout());
18085 SDValue Result;
18086
18087 if (GV) {
18088 // Create a target global address if this is a global. If possible, fold the
18089 // offset into the global address reference. Otherwise, ADD it on later.
18090 int64_t GlobalOffset = 0;
18091 if (OpFlags == X86II::MO_NO_FLAG &&
18092 X86::isOffsetSuitableForCodeModel(Offset, M)) {
18093 std::swap(GlobalOffset, Offset);
18094 }
18095 Result = DAG.getTargetGlobalAddress(GV, dl, PtrVT, GlobalOffset, OpFlags);
18096 } else {
18097 // If this is not a global address, this must be an external symbol.
18098 Result = DAG.getTargetExternalSymbol(ExternalSym, PtrVT, OpFlags);
18099 }
18100
18101 // If this is a direct call, avoid the wrapper if we don't need to do any
18102 // loads or adds. This allows SDAG ISel to match direct calls.
18103 if (ForCall && !NeedsLoad && !HasPICReg && Offset == 0)
18104 return Result;
18105
18106 Result = DAG.getNode(getGlobalWrapperKind(GV, OpFlags), dl, PtrVT, Result);
18107
18108 // With PIC, the address is actually $g + Offset.
18109 if (HasPICReg) {
18110 Result = DAG.getNode(ISD::ADD, dl, PtrVT,
18111 DAG.getNode(X86ISD::GlobalBaseReg, dl, PtrVT), Result);
18112 }
18113
18114 // For globals that require a load from a stub to get the address, emit the
18115 // load.
18116 if (NeedsLoad)
18117 Result = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Result,
18118 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18119
18120 // If there was a non-zero offset that we didn't fold, create an explicit
18121 // addition for it.
18122 if (Offset != 0)
18123 Result = DAG.getNode(ISD::ADD, dl, PtrVT, Result,
18124 DAG.getConstant(Offset, dl, PtrVT));
18125
18126 return Result;
18127}
18128
18129SDValue
18130X86TargetLowering::LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const {
18131 return LowerGlobalOrExternal(Op, DAG, /*ForCall=*/false);
18132}
18133
18134static SDValue
18135GetTLSADDR(SelectionDAG &DAG, SDValue Chain, GlobalAddressSDNode *GA,
18136 SDValue *InFlag, const EVT PtrVT, unsigned ReturnReg,
18137 unsigned char OperandFlags, bool LocalDynamic = false) {
18138 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18139 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18140 SDLoc dl(GA);
18141 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18142 GA->getValueType(0),
18143 GA->getOffset(),
18144 OperandFlags);
18145
18146 X86ISD::NodeType CallType = LocalDynamic ? X86ISD::TLSBASEADDR
18147 : X86ISD::TLSADDR;
18148
18149 if (InFlag) {
18150 SDValue Ops[] = { Chain, TGA, *InFlag };
18151 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18152 } else {
18153 SDValue Ops[] = { Chain, TGA };
18154 Chain = DAG.getNode(CallType, dl, NodeTys, Ops);
18155 }
18156
18157 // TLSADDR will be codegen'ed as call. Inform MFI that function has calls.
18158 MFI.setAdjustsStack(true);
18159 MFI.setHasCalls(true);
18160
18161 SDValue Flag = Chain.getValue(1);
18162 return DAG.getCopyFromReg(Chain, dl, ReturnReg, PtrVT, Flag);
18163}
18164
18165// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 32 bit
18166static SDValue
18167LowerToTLSGeneralDynamicModel32(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18168 const EVT PtrVT) {
18169 SDValue InFlag;
18170 SDLoc dl(GA); // ? function entry point might be better
18171 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18172 DAG.getNode(X86ISD::GlobalBaseReg,
18173 SDLoc(), PtrVT), InFlag);
18174 InFlag = Chain.getValue(1);
18175
18176 return GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX, X86II::MO_TLSGD);
18177}
18178
18179// Lower ISD::GlobalTLSAddress using the "general dynamic" model, 64 bit
18180static SDValue
18181LowerToTLSGeneralDynamicModel64(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18182 const EVT PtrVT) {
18183 return GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT,
18184 X86::RAX, X86II::MO_TLSGD);
18185}
18186
18187static SDValue LowerToTLSLocalDynamicModel(GlobalAddressSDNode *GA,
18188 SelectionDAG &DAG,
18189 const EVT PtrVT,
18190 bool is64Bit) {
18191 SDLoc dl(GA);
18192
18193 // Get the start address of the TLS block for this module.
18194 X86MachineFunctionInfo *MFI = DAG.getMachineFunction()
18195 .getInfo<X86MachineFunctionInfo>();
18196 MFI->incNumLocalDynamicTLSAccesses();
18197
18198 SDValue Base;
18199 if (is64Bit) {
18200 Base = GetTLSADDR(DAG, DAG.getEntryNode(), GA, nullptr, PtrVT, X86::RAX,
18201 X86II::MO_TLSLD, /*LocalDynamic=*/true);
18202 } else {
18203 SDValue InFlag;
18204 SDValue Chain = DAG.getCopyToReg(DAG.getEntryNode(), dl, X86::EBX,
18205 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT), InFlag);
18206 InFlag = Chain.getValue(1);
18207 Base = GetTLSADDR(DAG, Chain, GA, &InFlag, PtrVT, X86::EAX,
18208 X86II::MO_TLSLDM, /*LocalDynamic=*/true);
18209 }
18210
18211 // Note: the CleanupLocalDynamicTLSPass will remove redundant computations
18212 // of Base.
18213
18214 // Build x@dtpoff.
18215 unsigned char OperandFlags = X86II::MO_DTPOFF;
18216 unsigned WrapperKind = X86ISD::Wrapper;
18217 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18218 GA->getValueType(0),
18219 GA->getOffset(), OperandFlags);
18220 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18221
18222 // Add x@dtpoff with the base.
18223 return DAG.getNode(ISD::ADD, dl, PtrVT, Offset, Base);
18224}
18225
18226// Lower ISD::GlobalTLSAddress using the "initial exec" or "local exec" model.
18227static SDValue LowerToTLSExecModel(GlobalAddressSDNode *GA, SelectionDAG &DAG,
18228 const EVT PtrVT, TLSModel::Model model,
18229 bool is64Bit, bool isPIC) {
18230 SDLoc dl(GA);
18231
18232 // Get the Thread Pointer, which is %gs:0 (32-bit) or %fs:0 (64-bit).
18233 Value *Ptr = Constant::getNullValue(Type::getInt8PtrTy(*DAG.getContext(),
18234 is64Bit ? 257 : 256));
18235
18236 SDValue ThreadPointer =
18237 DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), DAG.getIntPtrConstant(0, dl),
18238 MachinePointerInfo(Ptr));
18239
18240 unsigned char OperandFlags = 0;
18241 // Most TLS accesses are not RIP relative, even on x86-64. One exception is
18242 // initialexec.
18243 unsigned WrapperKind = X86ISD::Wrapper;
18244 if (model == TLSModel::LocalExec) {
18245 OperandFlags = is64Bit ? X86II::MO_TPOFF : X86II::MO_NTPOFF;
18246 } else if (model == TLSModel::InitialExec) {
18247 if (is64Bit) {
18248 OperandFlags = X86II::MO_GOTTPOFF;
18249 WrapperKind = X86ISD::WrapperRIP;
18250 } else {
18251 OperandFlags = isPIC ? X86II::MO_GOTNTPOFF : X86II::MO_INDNTPOFF;
18252 }
18253 } else {
18254 llvm_unreachable("Unexpected model")::llvm::llvm_unreachable_internal("Unexpected model", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18254);
18255 }
18256
18257 // emit "addl x@ntpoff,%eax" (local exec)
18258 // or "addl x@indntpoff,%eax" (initial exec)
18259 // or "addl x@gotntpoff(%ebx) ,%eax" (initial exec, 32-bit pic)
18260 SDValue TGA =
18261 DAG.getTargetGlobalAddress(GA->getGlobal(), dl, GA->getValueType(0),
18262 GA->getOffset(), OperandFlags);
18263 SDValue Offset = DAG.getNode(WrapperKind, dl, PtrVT, TGA);
18264
18265 if (model == TLSModel::InitialExec) {
18266 if (isPIC && !is64Bit) {
18267 Offset = DAG.getNode(ISD::ADD, dl, PtrVT,
18268 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18269 Offset);
18270 }
18271
18272 Offset = DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), Offset,
18273 MachinePointerInfo::getGOT(DAG.getMachineFunction()));
18274 }
18275
18276 // The address of the thread local variable is the add of the thread
18277 // pointer with the offset of the variable.
18278 return DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, Offset);
18279}
18280
18281SDValue
18282X86TargetLowering::LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const {
18283
18284 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Op);
18285
18286 if (DAG.getTarget().useEmulatedTLS())
18287 return LowerToTLSEmulatedModel(GA, DAG);
18288
18289 const GlobalValue *GV = GA->getGlobal();
18290 auto PtrVT = getPointerTy(DAG.getDataLayout());
18291 bool PositionIndependent = isPositionIndependent();
18292
18293 if (Subtarget.isTargetELF()) {
18294 TLSModel::Model model = DAG.getTarget().getTLSModel(GV);
18295 switch (model) {
18296 case TLSModel::GeneralDynamic:
18297 if (Subtarget.is64Bit())
18298 return LowerToTLSGeneralDynamicModel64(GA, DAG, PtrVT);
18299 return LowerToTLSGeneralDynamicModel32(GA, DAG, PtrVT);
18300 case TLSModel::LocalDynamic:
18301 return LowerToTLSLocalDynamicModel(GA, DAG, PtrVT,
18302 Subtarget.is64Bit());
18303 case TLSModel::InitialExec:
18304 case TLSModel::LocalExec:
18305 return LowerToTLSExecModel(GA, DAG, PtrVT, model, Subtarget.is64Bit(),
18306 PositionIndependent);
18307 }
18308 llvm_unreachable("Unknown TLS model.")::llvm::llvm_unreachable_internal("Unknown TLS model.", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18308);
18309 }
18310
18311 if (Subtarget.isTargetDarwin()) {
18312 // Darwin only has one model of TLS. Lower to that.
18313 unsigned char OpFlag = 0;
18314 unsigned WrapperKind = Subtarget.isPICStyleRIPRel() ?
18315 X86ISD::WrapperRIP : X86ISD::Wrapper;
18316
18317 // In PIC mode (unless we're in RIPRel PIC mode) we add an offset to the
18318 // global base reg.
18319 bool PIC32 = PositionIndependent && !Subtarget.is64Bit();
18320 if (PIC32)
18321 OpFlag = X86II::MO_TLVP_PIC_BASE;
18322 else
18323 OpFlag = X86II::MO_TLVP;
18324 SDLoc DL(Op);
18325 SDValue Result = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
18326 GA->getValueType(0),
18327 GA->getOffset(), OpFlag);
18328 SDValue Offset = DAG.getNode(WrapperKind, DL, PtrVT, Result);
18329
18330 // With PIC32, the address is actually $g + Offset.
18331 if (PIC32)
18332 Offset = DAG.getNode(ISD::ADD, DL, PtrVT,
18333 DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(), PtrVT),
18334 Offset);
18335
18336 // Lowering the machine isd will make sure everything is in the right
18337 // location.
18338 SDValue Chain = DAG.getEntryNode();
18339 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
18340 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, DL);
18341 SDValue Args[] = { Chain, Offset };
18342 Chain = DAG.getNode(X86ISD::TLSCALL, DL, NodeTys, Args);
18343 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, DL, true),
18344 DAG.getIntPtrConstant(0, DL, true),
18345 Chain.getValue(1), DL);
18346
18347 // TLSCALL will be codegen'ed as call. Inform MFI that function has calls.
18348 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
18349 MFI.setAdjustsStack(true);
18350
18351 // And our return value (tls address) is in the standard call return value
18352 // location.
18353 unsigned Reg = Subtarget.is64Bit() ? X86::RAX : X86::EAX;
18354 return DAG.getCopyFromReg(Chain, DL, Reg, PtrVT, Chain.getValue(1));
18355 }
18356
18357 if (Subtarget.isOSWindows()) {
18358 // Just use the implicit TLS architecture
18359 // Need to generate something similar to:
18360 // mov rdx, qword [gs:abs 58H]; Load pointer to ThreadLocalStorage
18361 // ; from TEB
18362 // mov ecx, dword [rel _tls_index]: Load index (from C runtime)
18363 // mov rcx, qword [rdx+rcx*8]
18364 // mov eax, .tls$:tlsvar
18365 // [rax+rcx] contains the address
18366 // Windows 64bit: gs:0x58
18367 // Windows 32bit: fs:__tls_array
18368
18369 SDLoc dl(GA);
18370 SDValue Chain = DAG.getEntryNode();
18371
18372 // Get the Thread Pointer, which is %fs:__tls_array (32-bit) or
18373 // %gs:0x58 (64-bit). On MinGW, __tls_array is not available, so directly
18374 // use its literal value of 0x2C.
18375 Value *Ptr = Constant::getNullValue(Subtarget.is64Bit()
18376 ? Type::getInt8PtrTy(*DAG.getContext(),
18377 256)
18378 : Type::getInt32PtrTy(*DAG.getContext(),
18379 257));
18380
18381 SDValue TlsArray = Subtarget.is64Bit()
18382 ? DAG.getIntPtrConstant(0x58, dl)
18383 : (Subtarget.isTargetWindowsGNU()
18384 ? DAG.getIntPtrConstant(0x2C, dl)
18385 : DAG.getExternalSymbol("_tls_array", PtrVT));
18386
18387 SDValue ThreadPointer =
18388 DAG.getLoad(PtrVT, dl, Chain, TlsArray, MachinePointerInfo(Ptr));
18389
18390 SDValue res;
18391 if (GV->getThreadLocalMode() == GlobalVariable::LocalExecTLSModel) {
18392 res = ThreadPointer;
18393 } else {
18394 // Load the _tls_index variable
18395 SDValue IDX = DAG.getExternalSymbol("_tls_index", PtrVT);
18396 if (Subtarget.is64Bit())
18397 IDX = DAG.getExtLoad(ISD::ZEXTLOAD, dl, PtrVT, Chain, IDX,
18398 MachinePointerInfo(), MVT::i32);
18399 else
18400 IDX = DAG.getLoad(PtrVT, dl, Chain, IDX, MachinePointerInfo());
18401
18402 auto &DL = DAG.getDataLayout();
18403 SDValue Scale =
18404 DAG.getConstant(Log2_64_Ceil(DL.getPointerSize()), dl, MVT::i8);
18405 IDX = DAG.getNode(ISD::SHL, dl, PtrVT, IDX, Scale);
18406
18407 res = DAG.getNode(ISD::ADD, dl, PtrVT, ThreadPointer, IDX);
18408 }
18409
18410 res = DAG.getLoad(PtrVT, dl, Chain, res, MachinePointerInfo());
18411
18412 // Get the offset of start of .tls section
18413 SDValue TGA = DAG.getTargetGlobalAddress(GA->getGlobal(), dl,
18414 GA->getValueType(0),
18415 GA->getOffset(), X86II::MO_SECREL);
18416 SDValue Offset = DAG.getNode(X86ISD::Wrapper, dl, PtrVT, TGA);
18417
18418 // The address of the thread local variable is the add of the thread
18419 // pointer with the offset of the variable.
18420 return DAG.getNode(ISD::ADD, dl, PtrVT, res, Offset);
18421 }
18422
18423 llvm_unreachable("TLS not implemented for this target.")::llvm::llvm_unreachable_internal("TLS not implemented for this target."
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18423);
18424}
18425
18426/// Lower SRA_PARTS and friends, which return two i32 values
18427/// and take a 2 x i32 value to shift plus a shift amount.
18428/// TODO: Can this be moved to general expansion code?
18429static SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) {
18430 assert(Op.getNumOperands() == 3 && "Not a double-shift!")((Op.getNumOperands() == 3 && "Not a double-shift!") ?
static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == 3 && \"Not a double-shift!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18430, __PRETTY_FUNCTION__));
18431 MVT VT = Op.getSimpleValueType();
18432 unsigned VTBits = VT.getSizeInBits();
18433 SDLoc dl(Op);
18434 bool isSRA = Op.getOpcode() == ISD::SRA_PARTS;
18435 SDValue ShOpLo = Op.getOperand(0);
18436 SDValue ShOpHi = Op.getOperand(1);
18437 SDValue ShAmt = Op.getOperand(2);
18438 // ISD::FSHL and ISD::FSHR have defined overflow behavior but ISD::SHL and
18439 // ISD::SRA/L nodes haven't. Insert an AND to be safe, it's optimized away
18440 // during isel.
18441 SDValue SafeShAmt = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
18442 DAG.getConstant(VTBits - 1, dl, MVT::i8));
18443 SDValue Tmp1 = isSRA ? DAG.getNode(ISD::SRA, dl, VT, ShOpHi,
18444 DAG.getConstant(VTBits - 1, dl, MVT::i8))
18445 : DAG.getConstant(0, dl, VT);
18446
18447 SDValue Tmp2, Tmp3;
18448 if (Op.getOpcode() == ISD::SHL_PARTS) {
18449 Tmp2 = DAG.getNode(ISD::FSHL, dl, VT, ShOpHi, ShOpLo, ShAmt);
18450 Tmp3 = DAG.getNode(ISD::SHL, dl, VT, ShOpLo, SafeShAmt);
18451 } else {
18452 Tmp2 = DAG.getNode(ISD::FSHR, dl, VT, ShOpHi, ShOpLo, ShAmt);
18453 Tmp3 = DAG.getNode(isSRA ? ISD::SRA : ISD::SRL, dl, VT, ShOpHi, SafeShAmt);
18454 }
18455
18456 // If the shift amount is larger or equal than the width of a part we can't
18457 // rely on the results of shld/shrd. Insert a test and select the appropriate
18458 // values for large shift amounts.
18459 SDValue AndNode = DAG.getNode(ISD::AND, dl, MVT::i8, ShAmt,
18460 DAG.getConstant(VTBits, dl, MVT::i8));
18461 SDValue Cond = DAG.getSetCC(dl, MVT::i8, AndNode,
18462 DAG.getConstant(0, dl, MVT::i8), ISD::SETNE);
18463
18464 SDValue Hi, Lo;
18465 if (Op.getOpcode() == ISD::SHL_PARTS) {
18466 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
18467 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
18468 } else {
18469 Lo = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp3, Tmp2);
18470 Hi = DAG.getNode(ISD::SELECT, dl, VT, Cond, Tmp1, Tmp3);
18471 }
18472
18473 return DAG.getMergeValues({ Lo, Hi }, dl);
18474}
18475
18476static SDValue LowerFunnelShift(SDValue Op, const X86Subtarget &Subtarget,
18477 SelectionDAG &DAG) {
18478 MVT VT = Op.getSimpleValueType();
18479 assert((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) &&(((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR
) && "Unexpected funnel shift opcode!") ? static_cast
<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) && \"Unexpected funnel shift opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18480, __PRETTY_FUNCTION__))
18480 "Unexpected funnel shift opcode!")(((Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR
) && "Unexpected funnel shift opcode!") ? static_cast
<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::FSHL || Op.getOpcode() == ISD::FSHR) && \"Unexpected funnel shift opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18480, __PRETTY_FUNCTION__));
18481
18482 SDLoc DL(Op);
18483 SDValue Op0 = Op.getOperand(0);
18484 SDValue Op1 = Op.getOperand(1);
18485 SDValue Amt = Op.getOperand(2);
18486
18487 bool IsFSHR = Op.getOpcode() == ISD::FSHR;
18488
18489 if (VT.isVector()) {
18490 assert(Subtarget.hasVBMI2() && "Expected VBMI2")((Subtarget.hasVBMI2() && "Expected VBMI2") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasVBMI2() && \"Expected VBMI2\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18490, __PRETTY_FUNCTION__));
18491
18492 if (IsFSHR)
18493 std::swap(Op0, Op1);
18494
18495 APInt APIntShiftAmt;
18496 if (X86::isConstantSplat(Amt, APIntShiftAmt)) {
18497 uint64_t ShiftAmt = APIntShiftAmt.urem(VT.getScalarSizeInBits());
18498 return DAG.getNode(IsFSHR ? X86ISD::VSHRD : X86ISD::VSHLD, DL, VT, Op0,
18499 Op1, DAG.getTargetConstant(ShiftAmt, DL, MVT::i8));
18500 }
18501
18502 return DAG.getNode(IsFSHR ? X86ISD::VSHRDV : X86ISD::VSHLDV, DL, VT,
18503 Op0, Op1, Amt);
18504 }
18505
18506 assert((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&(((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
"Unexpected funnel shift type!") ? static_cast<void> (
0) : __assert_fail ("(VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && \"Unexpected funnel shift type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18507, __PRETTY_FUNCTION__))
18507 "Unexpected funnel shift type!")(((VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) &&
"Unexpected funnel shift type!") ? static_cast<void> (
0) : __assert_fail ("(VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) && \"Unexpected funnel shift type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18507, __PRETTY_FUNCTION__));
18508
18509 // Expand slow SHLD/SHRD cases if we are not optimizing for size.
18510 bool OptForSize = DAG.shouldOptForSize();
18511 if (!OptForSize && Subtarget.isSHLDSlow())
18512 return SDValue();
18513
18514 if (IsFSHR)
18515 std::swap(Op0, Op1);
18516
18517 // i16 needs to modulo the shift amount, but i32/i64 have implicit modulo.
18518 if (VT == MVT::i16)
18519 Amt = DAG.getNode(ISD::AND, DL, Amt.getValueType(), Amt,
18520 DAG.getConstant(15, DL, Amt.getValueType()));
18521
18522 unsigned SHDOp = (IsFSHR ? X86ISD::SHRD : X86ISD::SHLD);
18523 return DAG.getNode(SHDOp, DL, VT, Op0, Op1, Amt);
18524}
18525
18526// Try to use a packed vector operation to handle i64 on 32-bit targets when
18527// AVX512DQ is enabled.
18528static SDValue LowerI64IntToFP_AVX512DQ(SDValue Op, SelectionDAG &DAG,
18529 const X86Subtarget &Subtarget) {
18530 assert((Op.getOpcode() == ISD::SINT_TO_FP ||(((Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD
::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP
|| Op.getOpcode() == ISD::UINT_TO_FP) && "Unexpected opcode!"
) ? static_cast<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP || Op.getOpcode() == ISD::UINT_TO_FP) && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18534, __PRETTY_FUNCTION__))
18531 Op.getOpcode() == ISD::STRICT_SINT_TO_FP ||(((Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD
::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP
|| Op.getOpcode() == ISD::UINT_TO_FP) && "Unexpected opcode!"
) ? static_cast<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP || Op.getOpcode() == ISD::UINT_TO_FP) && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18534, __PRETTY_FUNCTION__))
18532 Op.getOpcode() == ISD::STRICT_UINT_TO_FP ||(((Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD
::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP
|| Op.getOpcode() == ISD::UINT_TO_FP) && "Unexpected opcode!"
) ? static_cast<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP || Op.getOpcode() == ISD::UINT_TO_FP) && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18534, __PRETTY_FUNCTION__))
18533 Op.getOpcode() == ISD::UINT_TO_FP) &&(((Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD
::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP
|| Op.getOpcode() == ISD::UINT_TO_FP) && "Unexpected opcode!"
) ? static_cast<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP || Op.getOpcode() == ISD::UINT_TO_FP) && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18534, __PRETTY_FUNCTION__))
18534 "Unexpected opcode!")(((Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD
::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP
|| Op.getOpcode() == ISD::UINT_TO_FP) && "Unexpected opcode!"
) ? static_cast<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::SINT_TO_FP || Op.getOpcode() == ISD::STRICT_SINT_TO_FP || Op.getOpcode() == ISD::STRICT_UINT_TO_FP || Op.getOpcode() == ISD::UINT_TO_FP) && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18534, __PRETTY_FUNCTION__));
18535 bool IsStrict = Op->isStrictFPOpcode();
18536 unsigned OpNo = IsStrict ? 1 : 0;
18537 SDValue Src = Op.getOperand(OpNo);
18538 MVT SrcVT = Src.getSimpleValueType();
18539 MVT VT = Op.getSimpleValueType();
18540
18541 if (!Subtarget.hasDQI() || SrcVT != MVT::i64 || Subtarget.is64Bit() ||
18542 (VT != MVT::f32 && VT != MVT::f64))
18543 return SDValue();
18544
18545 // Pack the i64 into a vector, do the operation and extract.
18546
18547 // Using 256-bit to ensure result is 128-bits for f32 case.
18548 unsigned NumElts = Subtarget.hasVLX() ? 4 : 8;
18549 MVT VecInVT = MVT::getVectorVT(MVT::i64, NumElts);
18550 MVT VecVT = MVT::getVectorVT(VT, NumElts);
18551
18552 SDLoc dl(Op);
18553 SDValue InVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecInVT, Src);
18554 if (IsStrict) {
18555 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, {VecVT, MVT::Other},
18556 {Op.getOperand(0), InVec});
18557 SDValue Chain = CvtVec.getValue(1);
18558 SDValue Value = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18559 DAG.getIntPtrConstant(0, dl));
18560 return DAG.getMergeValues({Value, Chain}, dl);
18561 }
18562
18563 SDValue CvtVec = DAG.getNode(Op.getOpcode(), dl, VecVT, InVec);
18564
18565 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, CvtVec,
18566 DAG.getIntPtrConstant(0, dl));
18567}
18568
18569static bool useVectorCast(unsigned Opcode, MVT FromVT, MVT ToVT,
18570 const X86Subtarget &Subtarget) {
18571 switch (Opcode) {
18572 case ISD::SINT_TO_FP:
18573 // TODO: Handle wider types with AVX/AVX512.
18574 if (!Subtarget.hasSSE2() || FromVT != MVT::v4i32)
18575 return false;
18576 // CVTDQ2PS or (V)CVTDQ2PD
18577 return ToVT == MVT::v4f32 || (Subtarget.hasAVX() && ToVT == MVT::v4f64);
18578
18579 case ISD::UINT_TO_FP:
18580 // TODO: Handle wider types and i64 elements.
18581 if (!Subtarget.hasAVX512() || FromVT != MVT::v4i32)
18582 return false;
18583 // VCVTUDQ2PS or VCVTUDQ2PD
18584 return ToVT == MVT::v4f32 || ToVT == MVT::v4f64;
18585
18586 default:
18587 return false;
18588 }
18589}
18590
18591/// Given a scalar cast operation that is extracted from a vector, try to
18592/// vectorize the cast op followed by extraction. This will avoid an expensive
18593/// round-trip between XMM and GPR.
18594static SDValue vectorizeExtractedCast(SDValue Cast, SelectionDAG &DAG,
18595 const X86Subtarget &Subtarget) {
18596 // TODO: This could be enhanced to handle smaller integer types by peeking
18597 // through an extend.
18598 SDValue Extract = Cast.getOperand(0);
18599 MVT DestVT = Cast.getSimpleValueType();
18600 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
18601 !isa<ConstantSDNode>(Extract.getOperand(1)))
18602 return SDValue();
18603
18604 // See if we have a 128-bit vector cast op for this type of cast.
18605 SDValue VecOp = Extract.getOperand(0);
18606 MVT FromVT = VecOp.getSimpleValueType();
18607 unsigned NumEltsInXMM = 128 / FromVT.getScalarSizeInBits();
18608 MVT Vec128VT = MVT::getVectorVT(FromVT.getScalarType(), NumEltsInXMM);
18609 MVT ToVT = MVT::getVectorVT(DestVT, NumEltsInXMM);
18610 if (!useVectorCast(Cast.getOpcode(), Vec128VT, ToVT, Subtarget))
18611 return SDValue();
18612
18613 // If we are extracting from a non-zero element, first shuffle the source
18614 // vector to allow extracting from element zero.
18615 SDLoc DL(Cast);
18616 if (!isNullConstant(Extract.getOperand(1))) {
18617 SmallVector<int, 16> Mask(FromVT.getVectorNumElements(), -1);
18618 Mask[0] = Extract.getConstantOperandVal(1);
18619 VecOp = DAG.getVectorShuffle(FromVT, DL, VecOp, DAG.getUNDEF(FromVT), Mask);
18620 }
18621 // If the source vector is wider than 128-bits, extract the low part. Do not
18622 // create an unnecessarily wide vector cast op.
18623 if (FromVT != Vec128VT)
18624 VecOp = extract128BitVector(VecOp, 0, DAG, DL);
18625
18626 // cast (extelt V, 0) --> extelt (cast (extract_subv V)), 0
18627 // cast (extelt V, C) --> extelt (cast (extract_subv (shuffle V, [C...]))), 0
18628 SDValue VCast = DAG.getNode(Cast.getOpcode(), DL, ToVT, VecOp);
18629 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, DestVT, VCast,
18630 DAG.getIntPtrConstant(0, DL));
18631}
18632
18633static SDValue lowerINT_TO_FP_vXi64(SDValue Op, SelectionDAG &DAG,
18634 const X86Subtarget &Subtarget) {
18635 SDLoc DL(Op);
18636 bool IsStrict = Op->isStrictFPOpcode();
18637 MVT VT = Op->getSimpleValueType(0);
18638 SDValue Src = Op->getOperand(IsStrict ? 1 : 0);
18639
18640 if (Subtarget.hasDQI()) {
18641 assert(!Subtarget.hasVLX() && "Unexpected features")((!Subtarget.hasVLX() && "Unexpected features") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.hasVLX() && \"Unexpected features\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18641, __PRETTY_FUNCTION__));
18642
18643 assert((Src.getSimpleValueType() == MVT::v2i64 ||(((Src.getSimpleValueType() == MVT::v2i64 || Src.getSimpleValueType
() == MVT::v4i64) && "Unsupported custom type") ? static_cast
<void> (0) : __assert_fail ("(Src.getSimpleValueType() == MVT::v2i64 || Src.getSimpleValueType() == MVT::v4i64) && \"Unsupported custom type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18645, __PRETTY_FUNCTION__))
18644 Src.getSimpleValueType() == MVT::v4i64) &&(((Src.getSimpleValueType() == MVT::v2i64 || Src.getSimpleValueType
() == MVT::v4i64) && "Unsupported custom type") ? static_cast
<void> (0) : __assert_fail ("(Src.getSimpleValueType() == MVT::v2i64 || Src.getSimpleValueType() == MVT::v4i64) && \"Unsupported custom type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18645, __PRETTY_FUNCTION__))
18645 "Unsupported custom type")(((Src.getSimpleValueType() == MVT::v2i64 || Src.getSimpleValueType
() == MVT::v4i64) && "Unsupported custom type") ? static_cast
<void> (0) : __assert_fail ("(Src.getSimpleValueType() == MVT::v2i64 || Src.getSimpleValueType() == MVT::v4i64) && \"Unsupported custom type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18645, __PRETTY_FUNCTION__));
18646
18647 // With AVX512DQ, but not VLX we need to widen to get a 512-bit result type.
18648 assert((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&(((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&
"Unexpected VT!") ? static_cast<void> (0) : __assert_fail
("(VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18649, __PRETTY_FUNCTION__))
18649 "Unexpected VT!")(((VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) &&
"Unexpected VT!") ? static_cast<void> (0) : __assert_fail
("(VT == MVT::v4f32 || VT == MVT::v2f64 || VT == MVT::v4f64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18649, __PRETTY_FUNCTION__));
18650 MVT WideVT = VT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
18651
18652 // Need to concat with zero vector for strict fp to avoid spurious
18653 // exceptions.
18654 SDValue Tmp = IsStrict ? DAG.getConstant(0, DL, MVT::v8i64)
18655 : DAG.getUNDEF(MVT::v8i64);
18656 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v8i64, Tmp, Src,
18657 DAG.getIntPtrConstant(0, DL));
18658 SDValue Res, Chain;
18659 if (IsStrict) {
18660 Res = DAG.getNode(Op.getOpcode(), DL, {WideVT, MVT::Other},
18661 {Op->getOperand(0), Src});
18662 Chain = Res.getValue(1);
18663 } else {
18664 Res = DAG.getNode(Op.getOpcode(), DL, WideVT, Src);
18665 }
18666
18667 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
18668 DAG.getIntPtrConstant(0, DL));
18669
18670 if (IsStrict)
18671 return DAG.getMergeValues({Res, Chain}, DL);
18672 return Res;
18673 }
18674
18675 bool IsSigned = Op->getOpcode() == ISD::SINT_TO_FP ||
18676 Op->getOpcode() == ISD::STRICT_SINT_TO_FP;
18677 if (VT != MVT::v4f32 || IsSigned)
18678 return SDValue();
18679
18680 SDValue Zero = DAG.getConstant(0, DL, MVT::v4i64);
18681 SDValue One = DAG.getConstant(1, DL, MVT::v4i64);
18682 SDValue Sign = DAG.getNode(ISD::OR, DL, MVT::v4i64,
18683 DAG.getNode(ISD::SRL, DL, MVT::v4i64, Src, One),
18684 DAG.getNode(ISD::AND, DL, MVT::v4i64, Src, One));
18685 SDValue IsNeg = DAG.getSetCC(DL, MVT::v4i64, Src, Zero, ISD::SETLT);
18686 SDValue SignSrc = DAG.getSelect(DL, MVT::v4i64, IsNeg, Sign, Src);
18687 SmallVector<SDValue, 4> SignCvts(4);
18688 SmallVector<SDValue, 4> Chains(4);
18689 for (int i = 0; i != 4; ++i) {
18690 SDValue Src = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, SignSrc,
18691 DAG.getIntPtrConstant(i, DL));
18692 if (IsStrict) {
18693 SignCvts[i] =
18694 DAG.getNode(ISD::STRICT_SINT_TO_FP, DL, {MVT::f32, MVT::Other},
18695 {Op.getOperand(0), Src});
18696 Chains[i] = SignCvts[i].getValue(1);
18697 } else {
18698 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, DL, MVT::f32, Src);
18699 }
18700 }
18701 SDValue SignCvt = DAG.getBuildVector(VT, DL, SignCvts);
18702
18703 SDValue Slow, Chain;
18704 if (IsStrict) {
18705 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
18706 Slow = DAG.getNode(ISD::STRICT_FADD, DL, {MVT::v4f32, MVT::Other},
18707 {Chain, SignCvt, SignCvt});
18708 Chain = Slow.getValue(1);
18709 } else {
18710 Slow = DAG.getNode(ISD::FADD, DL, MVT::v4f32, SignCvt, SignCvt);
18711 }
18712
18713 IsNeg = DAG.getNode(ISD::TRUNCATE, DL, MVT::v4i32, IsNeg);
18714 SDValue Cvt = DAG.getSelect(DL, MVT::v4f32, IsNeg, Slow, SignCvt);
18715
18716 if (IsStrict)
18717 return DAG.getMergeValues({Cvt, Chain}, DL);
18718
18719 return Cvt;
18720}
18721
18722SDValue X86TargetLowering::LowerSINT_TO_FP(SDValue Op,
18723 SelectionDAG &DAG) const {
18724 bool IsStrict = Op->isStrictFPOpcode();
18725 unsigned OpNo = IsStrict ? 1 : 0;
18726 SDValue Src = Op.getOperand(OpNo);
18727 SDValue Chain = IsStrict ? Op->getOperand(0) : DAG.getEntryNode();
18728 MVT SrcVT = Src.getSimpleValueType();
18729 MVT VT = Op.getSimpleValueType();
18730 SDLoc dl(Op);
18731
18732 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
18733 return Extract;
18734
18735 if (SrcVT.isVector()) {
18736 if (SrcVT == MVT::v2i32 && VT == MVT::v2f64) {
18737 // Note: Since v2f64 is a legal type. We don't need to zero extend the
18738 // source for strict FP.
18739 if (IsStrict)
18740 return DAG.getNode(
18741 X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
18742 {Chain, DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
18743 DAG.getUNDEF(SrcVT))});
18744 return DAG.getNode(X86ISD::CVTSI2P, dl, VT,
18745 DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
18746 DAG.getUNDEF(SrcVT)));
18747 }
18748 if (SrcVT == MVT::v2i64 || SrcVT == MVT::v4i64)
18749 return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);
18750
18751 return SDValue();
18752 }
18753
18754 assert(SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&((SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&
"Unknown SINT_TO_FP to lower!") ? static_cast<void> (0
) : __assert_fail ("SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && \"Unknown SINT_TO_FP to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18755, __PRETTY_FUNCTION__))
18755 "Unknown SINT_TO_FP to lower!")((SrcVT <= MVT::i64 && SrcVT >= MVT::i16 &&
"Unknown SINT_TO_FP to lower!") ? static_cast<void> (0
) : __assert_fail ("SrcVT <= MVT::i64 && SrcVT >= MVT::i16 && \"Unknown SINT_TO_FP to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 18755, __PRETTY_FUNCTION__));
18756
18757 bool UseSSEReg = isScalarFPTypeInSSEReg(VT);
18758
18759 // These are really Legal; return the operand so the caller accepts it as
18760 // Legal.
18761 if (SrcVT == MVT::i32 && UseSSEReg)
18762 return Op;
18763 if (SrcVT == MVT::i64 && UseSSEReg && Subtarget.is64Bit())
18764 return Op;
18765
18766 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
18767 return V;
18768
18769 // SSE doesn't have an i16 conversion so we need to promote.
18770 if (SrcVT == MVT::i16 && (UseSSEReg || VT == MVT::f128)) {
18771 SDValue Ext = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::i32, Src);
18772 if (IsStrict)
18773 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
18774 {Chain, Ext});
18775
18776 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Ext);
18777 }
18778
18779 if (VT == MVT::f128)
18780 return LowerF128Call(Op, DAG, RTLIB::getSINTTOFP(SrcVT, VT));
18781
18782 SDValue ValueToStore = Src;
18783 if (SrcVT == MVT::i64 && UseSSEReg && !Subtarget.is64Bit())
18784 // Bitcasting to f64 here allows us to do a single 64-bit store from
18785 // an SSE register, avoiding the store forwarding penalty that would come
18786 // with two 32-bit stores.
18787 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
18788
18789 unsigned Size = SrcVT.getSizeInBits()/8;
18790 MachineFunction &MF = DAG.getMachineFunction();
18791 auto PtrVT = getPointerTy(MF.getDataLayout());
18792 int SSFI = MF.getFrameInfo().CreateStackObject(Size, Size, false);
18793 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
18794 Chain = DAG.getStore(
18795 Chain, dl, ValueToStore, StackSlot,
18796 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI));
18797 std::pair<SDValue, SDValue> Tmp = BuildFILD(Op, SrcVT, Chain, StackSlot, DAG);
18798
18799 if (IsStrict)
18800 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
18801
18802 return Tmp.first;
18803}
18804
18805std::pair<SDValue, SDValue> X86TargetLowering::BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain,
18806 SDValue StackSlot,
18807 SelectionDAG &DAG) const {
18808 // Build the FILD
18809 SDLoc DL(Op);
18810 SDVTList Tys;
18811 bool useSSE = isScalarFPTypeInSSEReg(Op.getValueType());
18812 if (useSSE)
18813 Tys = DAG.getVTList(MVT::f64, MVT::Other, MVT::Glue);
18814 else
18815 Tys = DAG.getVTList(Op.getValueType(), MVT::Other);
18816
18817 unsigned ByteSize = SrcVT.getSizeInBits() / 8;
18818
18819 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(StackSlot);
18820 MachineMemOperand *LoadMMO;
18821 if (FI) {
18822 int SSFI = FI->getIndex();
18823 LoadMMO = DAG.getMachineFunction().getMachineMemOperand(
18824 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
18825 MachineMemOperand::MOLoad, ByteSize, ByteSize);
18826 } else {
18827 LoadMMO = cast<LoadSDNode>(StackSlot)->getMemOperand();
18828 StackSlot = StackSlot.getOperand(1);
18829 }
18830 SDValue FILDOps[] = {Chain, StackSlot};
18831 SDValue Result =
18832 DAG.getMemIntrinsicNode(useSSE ? X86ISD::FILD_FLAG : X86ISD::FILD, DL,
18833 Tys, FILDOps, SrcVT, LoadMMO);
18834 Chain = Result.getValue(1);
18835
18836 if (useSSE) {
18837 SDValue InFlag = Result.getValue(2);
18838
18839 // FIXME: Currently the FST is glued to the FILD_FLAG. This
18840 // shouldn't be necessary except that RFP cannot be live across
18841 // multiple blocks. When stackifier is fixed, they can be uncoupled.
18842 MachineFunction &MF = DAG.getMachineFunction();
18843 unsigned SSFISize = Op.getValueSizeInBits() / 8;
18844 int SSFI = MF.getFrameInfo().CreateStackObject(SSFISize, SSFISize, false);
18845 auto PtrVT = getPointerTy(MF.getDataLayout());
18846 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
18847 Tys = DAG.getVTList(MVT::Other);
18848 SDValue FSTOps[] = {Chain, Result, StackSlot, InFlag};
18849 MachineMemOperand *StoreMMO = DAG.getMachineFunction().getMachineMemOperand(
18850 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
18851 MachineMemOperand::MOStore, SSFISize, SSFISize);
18852
18853 Chain = DAG.getMemIntrinsicNode(X86ISD::FST, DL, Tys, FSTOps,
18854 Op.getValueType(), StoreMMO);
18855 Result = DAG.getLoad(
18856 Op.getValueType(), DL, Chain, StackSlot,
18857 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI));
18858 Chain = Result.getValue(1);
18859 }
18860
18861 return { Result, Chain };
18862}
18863
18864/// Horizontal vector math instructions may be slower than normal math with
18865/// shuffles. Limit horizontal op codegen based on size/speed trade-offs, uarch
18866/// implementation, and likely shuffle complexity of the alternate sequence.
18867static bool shouldUseHorizontalOp(bool IsSingleSource, SelectionDAG &DAG,
18868 const X86Subtarget &Subtarget) {
18869 bool IsOptimizingSize = DAG.shouldOptForSize();
18870 bool HasFastHOps = Subtarget.hasFastHorizontalOps();
18871 return !IsSingleSource || IsOptimizingSize || HasFastHOps;
18872}
18873
18874/// 64-bit unsigned integer to double expansion.
18875static SDValue LowerUINT_TO_FP_i64(SDValue Op, SelectionDAG &DAG,
18876 const X86Subtarget &Subtarget) {
18877 // This algorithm is not obvious. Here it is what we're trying to output:
18878 /*
18879 movq %rax, %xmm0
18880 punpckldq (c0), %xmm0 // c0: (uint4){ 0x43300000U, 0x45300000U, 0U, 0U }
18881 subpd (c1), %xmm0 // c1: (double2){ 0x1.0p52, 0x1.0p52 * 0x1.0p32 }
18882 #ifdef __SSE3__
18883 haddpd %xmm0, %xmm0
18884 #else
18885 pshufd $0x4e, %xmm0, %xmm1
18886 addpd %xmm1, %xmm0
18887 #endif
18888 */
18889
18890 bool IsStrict = Op->isStrictFPOpcode();
18891 unsigned OpNo = IsStrict ? 1 : 0;
18892 SDLoc dl(Op);
18893 LLVMContext *Context = DAG.getContext();
18894
18895 // Build some magic constants.
18896 static const uint32_t CV0[] = { 0x43300000, 0x45300000, 0, 0 };
18897 Constant *C0 = ConstantDataVector::get(*Context, CV0);
18898 auto PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DAG.getDataLayout());
18899 SDValue CPIdx0 = DAG.getConstantPool(C0, PtrVT, 16);
18900
18901 SmallVector<Constant*,2> CV1;
18902 CV1.push_back(
18903 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
18904 APInt(64, 0x4330000000000000ULL))));
18905 CV1.push_back(
18906 ConstantFP::get(*Context, APFloat(APFloat::IEEEdouble(),
18907 APInt(64, 0x4530000000000000ULL))));
18908 Constant *C1 = ConstantVector::get(CV1);
18909 SDValue CPIdx1 = DAG.getConstantPool(C1, PtrVT, 16);
18910
18911 // Load the 64-bit value into an XMM register.
18912 SDValue XR1 =
18913 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Op.getOperand(OpNo));
18914 SDValue CLod0 =
18915 DAG.getLoad(MVT::v4i32, dl, DAG.getEntryNode(), CPIdx0,
18916 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
18917 /* Alignment = */ 16);
18918 SDValue Unpck1 =
18919 getUnpackl(DAG, dl, MVT::v4i32, DAG.getBitcast(MVT::v4i32, XR1), CLod0);
18920
18921 SDValue CLod1 =
18922 DAG.getLoad(MVT::v2f64, dl, CLod0.getValue(1), CPIdx1,
18923 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()),
18924 /* Alignment = */ 16);
18925 SDValue XR2F = DAG.getBitcast(MVT::v2f64, Unpck1);
18926 SDValue Sub;
18927 SDValue Chain;
18928 // TODO: Are there any fast-math-flags to propagate here?
18929 if (IsStrict) {
18930 Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
18931 {Op.getOperand(0), XR2F, CLod1});
18932 Chain = Sub.getValue(1);
18933 } else
18934 Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, XR2F, CLod1);
18935 SDValue Result;
18936
18937 if (!IsStrict && Subtarget.hasSSE3() &&
18938 shouldUseHorizontalOp(true, DAG, Subtarget)) {
18939 // FIXME: Do we need a STRICT version of FHADD?
18940 Result = DAG.getNode(X86ISD::FHADD, dl, MVT::v2f64, Sub, Sub);
18941 } else {
18942 SDValue Shuffle = DAG.getVectorShuffle(MVT::v2f64, dl, Sub, Sub, {1,-1});
18943 if (IsStrict) {
18944 Result = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v2f64, MVT::Other},
18945 {Chain, Shuffle, Sub});
18946 Chain = Result.getValue(1);
18947 } else
18948 Result = DAG.getNode(ISD::FADD, dl, MVT::v2f64, Shuffle, Sub);
18949 }
18950 Result = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64, Result,
18951 DAG.getIntPtrConstant(0, dl));
18952 if (IsStrict)
18953 return DAG.getMergeValues({Result, Chain}, dl);
18954
18955 return Result;
18956}
18957
18958/// 32-bit unsigned integer to float expansion.
18959static SDValue LowerUINT_TO_FP_i32(SDValue Op, SelectionDAG &DAG,
18960 const X86Subtarget &Subtarget) {
18961 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
18962 SDLoc dl(Op);
18963 // FP constant to bias correct the final result.
18964 SDValue Bias = DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl,
18965 MVT::f64);
18966
18967 // Load the 32-bit value into an XMM register.
18968 SDValue Load =
18969 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v4i32, Op.getOperand(OpNo));
18970
18971 // Zero out the upper parts of the register.
18972 Load = getShuffleVectorZeroOrUndef(Load, 0, true, Subtarget, DAG);
18973
18974 // Or the load with the bias.
18975 SDValue Or = DAG.getNode(
18976 ISD::OR, dl, MVT::v2i64,
18977 DAG.getBitcast(MVT::v2i64, Load),
18978 DAG.getBitcast(MVT::v2i64,
18979 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2f64, Bias)));
18980 Or =
18981 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
18982 DAG.getBitcast(MVT::v2f64, Or), DAG.getIntPtrConstant(0, dl));
18983
18984 if (Op.getNode()->isStrictFPOpcode()) {
18985 // Subtract the bias.
18986 // TODO: Are there any fast-math-flags to propagate here?
18987 SDValue Chain = Op.getOperand(0);
18988 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::f64, MVT::Other},
18989 {Chain, Or, Bias});
18990
18991 if (Op.getValueType() == Sub.getValueType())
18992 return Sub;
18993
18994 // Handle final rounding.
18995 std::pair<SDValue, SDValue> ResultPair = DAG.getStrictFPExtendOrRound(
18996 Sub, Sub.getValue(1), dl, Op.getSimpleValueType());
18997
18998 return DAG.getMergeValues({ResultPair.first, ResultPair.second}, dl);
18999 }
19000
19001 // Subtract the bias.
19002 // TODO: Are there any fast-math-flags to propagate here?
19003 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::f64, Or, Bias);
19004
19005 // Handle final rounding.
19006 return DAG.getFPExtendOrRound(Sub, dl, Op.getSimpleValueType());
19007}
19008
19009static SDValue lowerUINT_TO_FP_v2i32(SDValue Op, SelectionDAG &DAG,
19010 const X86Subtarget &Subtarget,
19011 const SDLoc &DL) {
19012 if (Op.getSimpleValueType() != MVT::v2f64)
19013 return SDValue();
19014
19015 bool IsStrict = Op->isStrictFPOpcode();
19016
19017 SDValue N0 = Op.getOperand(IsStrict ? 1 : 0);
19018 assert(N0.getSimpleValueType() == MVT::v2i32 && "Unexpected input type")((N0.getSimpleValueType() == MVT::v2i32 && "Unexpected input type"
) ? static_cast<void> (0) : __assert_fail ("N0.getSimpleValueType() == MVT::v2i32 && \"Unexpected input type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19018, __PRETTY_FUNCTION__));
19019
19020 if (Subtarget.hasAVX512()) {
19021 if (!Subtarget.hasVLX()) {
19022 // Let generic type legalization widen this.
19023 if (!IsStrict)
19024 return SDValue();
19025 // Otherwise pad the integer input with 0s and widen the operation.
19026 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19027 DAG.getConstant(0, DL, MVT::v2i32));
19028 SDValue Res = DAG.getNode(Op->getOpcode(), DL, {MVT::v4f64, MVT::Other},
19029 {Op.getOperand(0), N0});
19030 SDValue Chain = Res.getValue(1);
19031 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2f64, Res,
19032 DAG.getIntPtrConstant(0, DL));
19033 return DAG.getMergeValues({Res, Chain}, DL);
19034 }
19035
19036 // Legalize to v4i32 type.
19037 N0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
19038 DAG.getUNDEF(MVT::v2i32));
19039 if (IsStrict)
19040 return DAG.getNode(X86ISD::STRICT_CVTUI2P, DL, {MVT::v2f64, MVT::Other},
19041 {Op.getOperand(0), N0});
19042 return DAG.getNode(X86ISD::CVTUI2P, DL, MVT::v2f64, N0);
19043 }
19044
19045 // Zero extend to 2i64, OR with the floating point representation of 2^52.
19046 // This gives us the floating point equivalent of 2^52 + the i32 integer
19047 // since double has 52-bits of mantissa. Then subtract 2^52 in floating
19048 // point leaving just our i32 integers in double format.
19049 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v2i64, N0);
19050 SDValue VBias =
19051 DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), DL, MVT::v2f64);
19052 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v2i64, ZExtIn,
19053 DAG.getBitcast(MVT::v2i64, VBias));
19054 Or = DAG.getBitcast(MVT::v2f64, Or);
19055
19056 if (IsStrict)
19057 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v2f64, MVT::Other},
19058 {Op.getOperand(0), Or, VBias});
19059 return DAG.getNode(ISD::FSUB, DL, MVT::v2f64, Or, VBias);
19060}
19061
19062static SDValue lowerUINT_TO_FP_vXi32(SDValue Op, SelectionDAG &DAG,
19063 const X86Subtarget &Subtarget) {
19064 SDLoc DL(Op);
19065 bool IsStrict = Op->isStrictFPOpcode();
19066 SDValue V = Op->getOperand(IsStrict ? 1 : 0);
19067 MVT VecIntVT = V.getSimpleValueType();
19068 assert((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&(((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
"Unsupported custom type") ? static_cast<void> (0) : __assert_fail
("(VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) && \"Unsupported custom type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19069, __PRETTY_FUNCTION__))
19069 "Unsupported custom type")(((VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) &&
"Unsupported custom type") ? static_cast<void> (0) : __assert_fail
("(VecIntVT == MVT::v4i32 || VecIntVT == MVT::v8i32) && \"Unsupported custom type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19069, __PRETTY_FUNCTION__));
19070
19071 if (Subtarget.hasAVX512()) {
19072 // With AVX512, but not VLX we need to widen to get a 512-bit result type.
19073 assert(!Subtarget.hasVLX() && "Unexpected features")((!Subtarget.hasVLX() && "Unexpected features") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.hasVLX() && \"Unexpected features\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19073, __PRETTY_FUNCTION__));
19074 MVT VT = Op->getSimpleValueType(0);
19075
19076 // v8i32->v8f64 is legal with AVX512 so just return it.
19077 if (VT == MVT::v8f64)
19078 return Op;
19079
19080 assert((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&(((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&
"Unexpected VT!") ? static_cast<void> (0) : __assert_fail
("(VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19081, __PRETTY_FUNCTION__))
19081 "Unexpected VT!")(((VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) &&
"Unexpected VT!") ? static_cast<void> (0) : __assert_fail
("(VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v4f64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19081, __PRETTY_FUNCTION__));
19082 MVT WideVT = VT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
19083 MVT WideIntVT = VT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
19084 // Need to concat with zero vector for strict fp to avoid spurious
19085 // exceptions.
19086 SDValue Tmp =
19087 IsStrict ? DAG.getConstant(0, DL, WideIntVT) : DAG.getUNDEF(WideIntVT);
19088 V = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideIntVT, Tmp, V,
19089 DAG.getIntPtrConstant(0, DL));
19090 SDValue Res, Chain;
19091 if (IsStrict) {
19092 Res = DAG.getNode(ISD::STRICT_UINT_TO_FP, DL, {WideVT, MVT::Other},
19093 {Op->getOperand(0), V});
19094 Chain = Res.getValue(1);
19095 } else {
19096 Res = DAG.getNode(ISD::UINT_TO_FP, DL, WideVT, V);
19097 }
19098
19099 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
19100 DAG.getIntPtrConstant(0, DL));
19101
19102 if (IsStrict)
19103 return DAG.getMergeValues({Res, Chain}, DL);
19104 return Res;
19105 }
19106
19107 if (Subtarget.hasAVX() && VecIntVT == MVT::v4i32 &&
19108 Op->getSimpleValueType(0) == MVT::v4f64) {
19109 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::v4i64, V);
19110 SDValue VBias =
19111 DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), DL, MVT::v4f64);
19112 SDValue Or = DAG.getNode(ISD::OR, DL, MVT::v4i64, ZExtIn,
19113 DAG.getBitcast(MVT::v4i64, VBias));
19114 Or = DAG.getBitcast(MVT::v4f64, Or);
19115
19116 if (IsStrict)
19117 return DAG.getNode(ISD::STRICT_FSUB, DL, {MVT::v4f64, MVT::Other},
19118 {Op.getOperand(0), Or, VBias});
19119 return DAG.getNode(ISD::FSUB, DL, MVT::v4f64, Or, VBias);
19120 }
19121
19122 // The algorithm is the following:
19123 // #ifdef __SSE4_1__
19124 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19125 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19126 // (uint4) 0x53000000, 0xaa);
19127 // #else
19128 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19129 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
19130 // #endif
19131 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19132 // return (float4) lo + fhi;
19133
19134 bool Is128 = VecIntVT == MVT::v4i32;
19135 MVT VecFloatVT = Is128 ? MVT::v4f32 : MVT::v8f32;
19136 // If we convert to something else than the supported type, e.g., to v4f64,
19137 // abort early.
19138 if (VecFloatVT != Op->getSimpleValueType(0))
19139 return SDValue();
19140
19141 // In the #idef/#else code, we have in common:
19142 // - The vector of constants:
19143 // -- 0x4b000000
19144 // -- 0x53000000
19145 // - A shift:
19146 // -- v >> 16
19147
19148 // Create the splat vector for 0x4b000000.
19149 SDValue VecCstLow = DAG.getConstant(0x4b000000, DL, VecIntVT);
19150 // Create the splat vector for 0x53000000.
19151 SDValue VecCstHigh = DAG.getConstant(0x53000000, DL, VecIntVT);
19152
19153 // Create the right shift.
19154 SDValue VecCstShift = DAG.getConstant(16, DL, VecIntVT);
19155 SDValue HighShift = DAG.getNode(ISD::SRL, DL, VecIntVT, V, VecCstShift);
19156
19157 SDValue Low, High;
19158 if (Subtarget.hasSSE41()) {
19159 MVT VecI16VT = Is128 ? MVT::v8i16 : MVT::v16i16;
19160 // uint4 lo = _mm_blend_epi16( v, (uint4) 0x4b000000, 0xaa);
19161 SDValue VecCstLowBitcast = DAG.getBitcast(VecI16VT, VecCstLow);
19162 SDValue VecBitcast = DAG.getBitcast(VecI16VT, V);
19163 // Low will be bitcasted right away, so do not bother bitcasting back to its
19164 // original type.
19165 Low = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecBitcast,
19166 VecCstLowBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19167 // uint4 hi = _mm_blend_epi16( _mm_srli_epi32(v,16),
19168 // (uint4) 0x53000000, 0xaa);
19169 SDValue VecCstHighBitcast = DAG.getBitcast(VecI16VT, VecCstHigh);
19170 SDValue VecShiftBitcast = DAG.getBitcast(VecI16VT, HighShift);
19171 // High will be bitcasted right away, so do not bother bitcasting back to
19172 // its original type.
19173 High = DAG.getNode(X86ISD::BLENDI, DL, VecI16VT, VecShiftBitcast,
19174 VecCstHighBitcast, DAG.getTargetConstant(0xaa, DL, MVT::i8));
19175 } else {
19176 SDValue VecCstMask = DAG.getConstant(0xffff, DL, VecIntVT);
19177 // uint4 lo = (v & (uint4) 0xffff) | (uint4) 0x4b000000;
19178 SDValue LowAnd = DAG.getNode(ISD::AND, DL, VecIntVT, V, VecCstMask);
19179 Low = DAG.getNode(ISD::OR, DL, VecIntVT, LowAnd, VecCstLow);
19180
19181 // uint4 hi = (v >> 16) | (uint4) 0x53000000;
19182 High = DAG.getNode(ISD::OR, DL, VecIntVT, HighShift, VecCstHigh);
19183 }
19184
19185 // Create the vector constant for (0x1.0p39f + 0x1.0p23f).
19186 SDValue VecCstFSub = DAG.getConstantFP(
19187 APFloat(APFloat::IEEEsingle(), APInt(32, 0x53000080)), DL, VecFloatVT);
19188
19189 // float4 fhi = (float4) hi - (0x1.0p39f + 0x1.0p23f);
19190 // NOTE: By using fsub of a positive constant instead of fadd of a negative
19191 // constant, we avoid reassociation in MachineCombiner when unsafe-fp-math is
19192 // enabled. See PR24512.
19193 SDValue HighBitcast = DAG.getBitcast(VecFloatVT, High);
19194 // TODO: Are there any fast-math-flags to propagate here?
19195 // (float4) lo;
19196 SDValue LowBitcast = DAG.getBitcast(VecFloatVT, Low);
19197 // return (float4) lo + fhi;
19198 if (IsStrict) {
19199 SDValue FHigh = DAG.getNode(ISD::STRICT_FSUB, DL, {VecFloatVT, MVT::Other},
19200 {Op.getOperand(0), HighBitcast, VecCstFSub});
19201 return DAG.getNode(ISD::STRICT_FADD, DL, {VecFloatVT, MVT::Other},
19202 {FHigh.getValue(1), LowBitcast, FHigh});
19203 }
19204
19205 SDValue FHigh =
19206 DAG.getNode(ISD::FSUB, DL, VecFloatVT, HighBitcast, VecCstFSub);
19207 return DAG.getNode(ISD::FADD, DL, VecFloatVT, LowBitcast, FHigh);
19208}
19209
19210static SDValue lowerUINT_TO_FP_vec(SDValue Op, SelectionDAG &DAG,
19211 const X86Subtarget &Subtarget) {
19212 unsigned OpNo = Op.getNode()->isStrictFPOpcode() ? 1 : 0;
19213 SDValue N0 = Op.getOperand(OpNo);
19214 MVT SrcVT = N0.getSimpleValueType();
19215 SDLoc dl(Op);
19216
19217 switch (SrcVT.SimpleTy) {
19218 default:
19219 llvm_unreachable("Custom UINT_TO_FP is not supported!")::llvm::llvm_unreachable_internal("Custom UINT_TO_FP is not supported!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19219);
19220 case MVT::v2i32:
19221 return lowerUINT_TO_FP_v2i32(Op, DAG, Subtarget, dl);
19222 case MVT::v4i32:
19223 case MVT::v8i32:
19224 return lowerUINT_TO_FP_vXi32(Op, DAG, Subtarget);
19225 case MVT::v2i64:
19226 case MVT::v4i64:
19227 return lowerINT_TO_FP_vXi64(Op, DAG, Subtarget);
19228 }
19229}
19230
19231SDValue X86TargetLowering::LowerUINT_TO_FP(SDValue Op,
19232 SelectionDAG &DAG) const {
19233 bool IsStrict = Op->isStrictFPOpcode();
19234 unsigned OpNo = IsStrict ? 1 : 0;
19235 SDValue Src = Op.getOperand(OpNo);
19236 SDLoc dl(Op);
19237 auto PtrVT = getPointerTy(DAG.getDataLayout());
19238 MVT SrcVT = Src.getSimpleValueType();
19239 MVT DstVT = Op->getSimpleValueType(0);
19240 SDValue Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19241
19242 if (DstVT == MVT::f128)
19243 return LowerF128Call(Op, DAG, RTLIB::getUINTTOFP(SrcVT, DstVT));
19244
19245 if (DstVT.isVector())
19246 return lowerUINT_TO_FP_vec(Op, DAG, Subtarget);
19247
19248 if (SDValue Extract = vectorizeExtractedCast(Op, DAG, Subtarget))
19249 return Extract;
19250
19251 if (Subtarget.hasAVX512() && isScalarFPTypeInSSEReg(DstVT) &&
19252 (SrcVT == MVT::i32 || (SrcVT == MVT::i64 && Subtarget.is64Bit()))) {
19253 // Conversions from unsigned i32 to f32/f64 are legal,
19254 // using VCVTUSI2SS/SD. Same for i64 in 64-bit mode.
19255 return Op;
19256 }
19257
19258 // Promote i32 to i64 and use a signed conversion on 64-bit targets.
19259 if (SrcVT == MVT::i32 && Subtarget.is64Bit()) {
19260 Src = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Src);
19261 if (IsStrict)
19262 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {DstVT, MVT::Other},
19263 {Chain, Src});
19264 return DAG.getNode(ISD::SINT_TO_FP, dl, DstVT, Src);
19265 }
19266
19267 if (SDValue V = LowerI64IntToFP_AVX512DQ(Op, DAG, Subtarget))
19268 return V;
19269
19270 if (SrcVT == MVT::i64 && DstVT == MVT::f64 && X86ScalarSSEf64)
19271 return LowerUINT_TO_FP_i64(Op, DAG, Subtarget);
19272 if (SrcVT == MVT::i32 && X86ScalarSSEf64)
19273 return LowerUINT_TO_FP_i32(Op, DAG, Subtarget);
19274 if (Subtarget.is64Bit() && SrcVT == MVT::i64 && DstVT == MVT::f32)
19275 return SDValue();
19276
19277 // Make a 64-bit buffer, and use it to build an FILD.
19278 SDValue StackSlot = DAG.CreateStackTemporary(MVT::i64);
19279 if (SrcVT == MVT::i32) {
19280 SDValue OffsetSlot = DAG.getMemBasePlusOffset(StackSlot, 4, dl);
19281 SDValue Store1 =
19282 DAG.getStore(Chain, dl, Src, StackSlot, MachinePointerInfo());
19283 SDValue Store2 = DAG.getStore(Store1, dl, DAG.getConstant(0, dl, MVT::i32),
19284 OffsetSlot, MachinePointerInfo());
19285 std::pair<SDValue, SDValue> Tmp =
19286 BuildFILD(Op, MVT::i64, Store2, StackSlot, DAG);
19287 if (IsStrict)
19288 return DAG.getMergeValues({Tmp.first, Tmp.second}, dl);
19289
19290 return Tmp.first;
19291 }
19292
19293 assert(SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP")((SrcVT == MVT::i64 && "Unexpected type in UINT_TO_FP"
) ? static_cast<void> (0) : __assert_fail ("SrcVT == MVT::i64 && \"Unexpected type in UINT_TO_FP\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19293, __PRETTY_FUNCTION__));
19294 SDValue ValueToStore = Src;
19295 if (isScalarFPTypeInSSEReg(Op.getValueType()) && !Subtarget.is64Bit()) {
19296 // Bitcasting to f64 here allows us to do a single 64-bit store from
19297 // an SSE register, avoiding the store forwarding penalty that would come
19298 // with two 32-bit stores.
19299 ValueToStore = DAG.getBitcast(MVT::f64, ValueToStore);
19300 }
19301 SDValue Store =
19302 DAG.getStore(Chain, dl, ValueToStore, StackSlot, MachinePointerInfo());
19303 // For i64 source, we need to add the appropriate power of 2 if the input
19304 // was negative. This is the same as the optimization in
19305 // DAGTypeLegalizer::ExpandIntOp_UNIT_TO_FP, and for it to be safe here,
19306 // we must be careful to do the computation in x87 extended precision, not
19307 // in SSE. (The generic code can't know it's OK to do this, or how to.)
19308 int SSFI = cast<FrameIndexSDNode>(StackSlot)->getIndex();
19309 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
19310 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SSFI),
19311 MachineMemOperand::MOLoad, 8, 8);
19312
19313 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other);
19314 SDValue Ops[] = { Store, StackSlot };
19315 SDValue Fild = DAG.getMemIntrinsicNode(X86ISD::FILD, dl, Tys, Ops,
19316 MVT::i64, MMO);
19317 Chain = Fild.getValue(1);
19318
19319 APInt FF(32, 0x5F800000ULL);
19320
19321 // Check whether the sign bit is set.
19322 SDValue SignSet = DAG.getSetCC(
19323 dl, getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), MVT::i64),
19324 Op.getOperand(OpNo), DAG.getConstant(0, dl, MVT::i64), ISD::SETLT);
19325
19326 // Build a 64 bit pair (0, FF) in the constant pool, with FF in the lo bits.
19327 SDValue FudgePtr = DAG.getConstantPool(
19328 ConstantInt::get(*DAG.getContext(), FF.zext(64)), PtrVT);
19329
19330 // Get a pointer to FF if the sign bit was set, or to 0 otherwise.
19331 SDValue Zero = DAG.getIntPtrConstant(0, dl);
19332 SDValue Four = DAG.getIntPtrConstant(4, dl);
19333 SDValue Offset = DAG.getSelect(dl, Zero.getValueType(), SignSet, Zero, Four);
19334 FudgePtr = DAG.getNode(ISD::ADD, dl, PtrVT, FudgePtr, Offset);
19335
19336 // Load the value out, extending it from f32 to f80.
19337 // FIXME: Avoid the extend by constructing the right constant pool?
19338 SDValue Fudge = DAG.getExtLoad(
19339 ISD::EXTLOAD, dl, MVT::f80, Chain, FudgePtr,
19340 MachinePointerInfo::getConstantPool(DAG.getMachineFunction()), MVT::f32,
19341 /* Alignment = */ 4);
19342 Chain = Fudge.getValue(1);
19343 // Extend everything to 80 bits to force it to be done on x87.
19344 // TODO: Are there any fast-math-flags to propagate here?
19345 if (IsStrict) {
19346 SDValue Add = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::f80, MVT::Other},
19347 {Chain, Fild, Fudge});
19348 // STRICT_FP_ROUND can't handle equal types.
19349 if (DstVT == MVT::f80)
19350 return Add;
19351 return DAG.getNode(ISD::STRICT_FP_ROUND, dl, {DstVT, MVT::Other},
19352 {Add.getValue(1), Add, DAG.getIntPtrConstant(0, dl)});
19353 }
19354 SDValue Add = DAG.getNode(ISD::FADD, dl, MVT::f80, Fild, Fudge);
19355 return DAG.getNode(ISD::FP_ROUND, dl, DstVT, Add,
19356 DAG.getIntPtrConstant(0, dl));
19357}
19358
19359// If the given FP_TO_SINT (IsSigned) or FP_TO_UINT (!IsSigned) operation
19360// is legal, or has an fp128 or f16 source (which needs to be promoted to f32),
19361// just return an SDValue().
19362// Otherwise it is assumed to be a conversion from one of f32, f64 or f80
19363// to i16, i32 or i64, and we lower it to a legal sequence and return the
19364// result.
19365SDValue
19366X86TargetLowering::FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
19367 bool IsSigned, SDValue &Chain) const {
19368 bool IsStrict = Op->isStrictFPOpcode();
19369 SDLoc DL(Op);
19370
19371 EVT DstTy = Op.getValueType();
19372 SDValue Value = Op.getOperand(IsStrict ? 1 : 0);
19373 EVT TheVT = Value.getValueType();
19374 auto PtrVT = getPointerTy(DAG.getDataLayout());
19375
19376 if (TheVT != MVT::f32 && TheVT != MVT::f64 && TheVT != MVT::f80) {
19377 // f16 must be promoted before using the lowering in this routine.
19378 // fp128 does not use this lowering.
19379 return SDValue();
19380 }
19381
19382 // If using FIST to compute an unsigned i64, we'll need some fixup
19383 // to handle values above the maximum signed i64. A FIST is always
19384 // used for the 32-bit subtarget, but also for f80 on a 64-bit target.
19385 bool UnsignedFixup = !IsSigned && DstTy == MVT::i64;
19386
19387 // FIXME: This does not generate an invalid exception if the input does not
19388 // fit in i32. PR44019
19389 if (!IsSigned && DstTy != MVT::i64) {
19390 // Replace the fp-to-uint32 operation with an fp-to-sint64 FIST.
19391 // The low 32 bits of the fist result will have the correct uint32 result.
19392 assert(DstTy == MVT::i32 && "Unexpected FP_TO_UINT")((DstTy == MVT::i32 && "Unexpected FP_TO_UINT") ? static_cast
<void> (0) : __assert_fail ("DstTy == MVT::i32 && \"Unexpected FP_TO_UINT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19392, __PRETTY_FUNCTION__));
19393 DstTy = MVT::i64;
19394 }
19395
19396 assert(DstTy.getSimpleVT() <= MVT::i64 &&((DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT
() >= MVT::i16 && "Unknown FP_TO_INT to lower!") ?
static_cast<void> (0) : __assert_fail ("DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT() >= MVT::i16 && \"Unknown FP_TO_INT to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19398, __PRETTY_FUNCTION__))
19397 DstTy.getSimpleVT() >= MVT::i16 &&((DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT
() >= MVT::i16 && "Unknown FP_TO_INT to lower!") ?
static_cast<void> (0) : __assert_fail ("DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT() >= MVT::i16 && \"Unknown FP_TO_INT to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19398, __PRETTY_FUNCTION__))
19398 "Unknown FP_TO_INT to lower!")((DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT
() >= MVT::i16 && "Unknown FP_TO_INT to lower!") ?
static_cast<void> (0) : __assert_fail ("DstTy.getSimpleVT() <= MVT::i64 && DstTy.getSimpleVT() >= MVT::i16 && \"Unknown FP_TO_INT to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19398, __PRETTY_FUNCTION__));
19399
19400 // We lower FP->int64 into FISTP64 followed by a load from a temporary
19401 // stack slot.
19402 MachineFunction &MF = DAG.getMachineFunction();
19403 unsigned MemSize = DstTy.getStoreSize();
19404 int SSFI = MF.getFrameInfo().CreateStackObject(MemSize, MemSize, false);
19405 SDValue StackSlot = DAG.getFrameIndex(SSFI, PtrVT);
19406
19407 Chain = IsStrict ? Op.getOperand(0) : DAG.getEntryNode();
19408
19409 SDValue Adjust; // 0x0 or 0x80000000, for result sign bit adjustment.
19410
19411 if (UnsignedFixup) {
19412 //
19413 // Conversion to unsigned i64 is implemented with a select,
19414 // depending on whether the source value fits in the range
19415 // of a signed i64. Let Thresh be the FP equivalent of
19416 // 0x8000000000000000ULL.
19417 //
19418 // Adjust = (Value < Thresh) ? 0 : 0x80000000;
19419 // FltOfs = (Value < Thresh) ? 0 : 0x80000000;
19420 // FistSrc = (Value - FltOfs);
19421 // Fist-to-mem64 FistSrc
19422 // Add 0 or 0x800...0ULL to the 64-bit result, which is equivalent
19423 // to XOR'ing the high 32 bits with Adjust.
19424 //
19425 // Being a power of 2, Thresh is exactly representable in all FP formats.
19426 // For X87 we'd like to use the smallest FP type for this constant, but
19427 // for DAG type consistency we have to match the FP operand type.
19428
19429 APFloat Thresh(APFloat::IEEEsingle(), APInt(32, 0x5f000000));
19430 LLVM_ATTRIBUTE_UNUSED__attribute__((__unused__)) APFloat::opStatus Status = APFloat::opOK;
19431 bool LosesInfo = false;
19432 if (TheVT == MVT::f64)
19433 // The rounding mode is irrelevant as the conversion should be exact.
19434 Status = Thresh.convert(APFloat::IEEEdouble(), APFloat::rmNearestTiesToEven,
19435 &LosesInfo);
19436 else if (TheVT == MVT::f80)
19437 Status = Thresh.convert(APFloat::x87DoubleExtended(),
19438 APFloat::rmNearestTiesToEven, &LosesInfo);
19439
19440 assert(Status == APFloat::opOK && !LosesInfo &&((Status == APFloat::opOK && !LosesInfo && "FP conversion should have been exact"
) ? static_cast<void> (0) : __assert_fail ("Status == APFloat::opOK && !LosesInfo && \"FP conversion should have been exact\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19441, __PRETTY_FUNCTION__))
19441 "FP conversion should have been exact")((Status == APFloat::opOK && !LosesInfo && "FP conversion should have been exact"
) ? static_cast<void> (0) : __assert_fail ("Status == APFloat::opOK && !LosesInfo && \"FP conversion should have been exact\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19441, __PRETTY_FUNCTION__));
19442
19443 SDValue ThreshVal = DAG.getConstantFP(Thresh, DL, TheVT);
19444
19445 EVT ResVT = getSetCCResultType(DAG.getDataLayout(),
19446 *DAG.getContext(), TheVT);
19447 SDValue Cmp;
19448 if (IsStrict)
19449 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT,
19450 Chain, /*IsSignaling*/ true);
19451 else
19452 Cmp = DAG.getSetCC(DL, ResVT, Value, ThreshVal, ISD::SETLT);
19453
19454 Adjust = DAG.getSelect(DL, MVT::i64, Cmp,
19455 DAG.getConstant(0, DL, MVT::i64),
19456 DAG.getConstant(APInt::getSignMask(64),
19457 DL, MVT::i64));
19458 SDValue FltOfs = DAG.getSelect(DL, TheVT, Cmp,
19459 DAG.getConstantFP(0.0, DL, TheVT),
19460 ThreshVal);
19461
19462 if (IsStrict) {
19463 Value = DAG.getNode(ISD::STRICT_FSUB, DL, { TheVT, MVT::Other},
19464 { Chain, Value, FltOfs });
19465 Chain = Value.getValue(1);
19466 } else
19467 Value = DAG.getNode(ISD::FSUB, DL, TheVT, Value, FltOfs);
19468 }
19469
19470 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, SSFI);
19471
19472 // FIXME This causes a redundant load/store if the SSE-class value is already
19473 // in memory, such as if it is on the callstack.
19474 if (isScalarFPTypeInSSEReg(TheVT)) {
19475 assert(DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!")((DstTy == MVT::i64 && "Invalid FP_TO_SINT to lower!"
) ? static_cast<void> (0) : __assert_fail ("DstTy == MVT::i64 && \"Invalid FP_TO_SINT to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19475, __PRETTY_FUNCTION__));
19476 Chain = DAG.getStore(Chain, DL, Value, StackSlot, MPI);
19477 SDVTList Tys = DAG.getVTList(TheVT, MVT::Other);
19478 SDValue Ops[] = { Chain, StackSlot };
19479
19480 unsigned FLDSize = TheVT.getStoreSize();
19481 assert(FLDSize <= MemSize && "Stack slot not big enough")((FLDSize <= MemSize && "Stack slot not big enough"
) ? static_cast<void> (0) : __assert_fail ("FLDSize <= MemSize && \"Stack slot not big enough\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19481, __PRETTY_FUNCTION__));
19482 MachineMemOperand *MMO = MF.getMachineMemOperand(
19483 MPI, MachineMemOperand::MOLoad, FLDSize, FLDSize);
19484 Value = DAG.getMemIntrinsicNode(X86ISD::FLD, DL, Tys, Ops, TheVT, MMO);
19485 Chain = Value.getValue(1);
19486 }
19487
19488 // Build the FP_TO_INT*_IN_MEM
19489 MachineMemOperand *MMO = MF.getMachineMemOperand(
19490 MPI, MachineMemOperand::MOStore, MemSize, MemSize);
19491 SDValue Ops[] = { Chain, Value, StackSlot };
19492 SDValue FIST = DAG.getMemIntrinsicNode(X86ISD::FP_TO_INT_IN_MEM, DL,
19493 DAG.getVTList(MVT::Other),
19494 Ops, DstTy, MMO);
19495
19496 SDValue Res = DAG.getLoad(Op.getValueType(), SDLoc(Op), FIST, StackSlot, MPI);
19497 Chain = Res.getValue(1);
19498
19499 // If we need an unsigned fixup, XOR the result with adjust.
19500 if (UnsignedFixup)
19501 Res = DAG.getNode(ISD::XOR, DL, MVT::i64, Res, Adjust);
19502
19503 return Res;
19504}
19505
19506static SDValue LowerAVXExtend(SDValue Op, SelectionDAG &DAG,
19507 const X86Subtarget &Subtarget) {
19508 MVT VT = Op.getSimpleValueType();
19509 SDValue In = Op.getOperand(0);
19510 MVT InVT = In.getSimpleValueType();
19511 SDLoc dl(Op);
19512 unsigned Opc = Op.getOpcode();
19513
19514 assert(VT.isVector() && InVT.isVector() && "Expected vector type")((VT.isVector() && InVT.isVector() && "Expected vector type"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && InVT.isVector() && \"Expected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19514, __PRETTY_FUNCTION__));
19515 assert((Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) &&(((Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) &&
"Unexpected extension opcode") ? static_cast<void> (0)
: __assert_fail ("(Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) && \"Unexpected extension opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19516, __PRETTY_FUNCTION__))
19516 "Unexpected extension opcode")(((Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) &&
"Unexpected extension opcode") ? static_cast<void> (0)
: __assert_fail ("(Opc == ISD::ANY_EXTEND || Opc == ISD::ZERO_EXTEND) && \"Unexpected extension opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19516, __PRETTY_FUNCTION__));
19517 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&((VT.getVectorNumElements() == InVT.getVectorNumElements() &&
"Expected same number of elements") ? static_cast<void>
(0) : __assert_fail ("VT.getVectorNumElements() == InVT.getVectorNumElements() && \"Expected same number of elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19518, __PRETTY_FUNCTION__))
19518 "Expected same number of elements")((VT.getVectorNumElements() == InVT.getVectorNumElements() &&
"Expected same number of elements") ? static_cast<void>
(0) : __assert_fail ("VT.getVectorNumElements() == InVT.getVectorNumElements() && \"Expected same number of elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19518, __PRETTY_FUNCTION__));
19519 assert((VT.getVectorElementType() == MVT::i16 ||(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19522, __PRETTY_FUNCTION__))
19520 VT.getVectorElementType() == MVT::i32 ||(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19522, __PRETTY_FUNCTION__))
19521 VT.getVectorElementType() == MVT::i64) &&(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19522, __PRETTY_FUNCTION__))
19522 "Unexpected element type")(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19522, __PRETTY_FUNCTION__));
19523 assert((InVT.getVectorElementType() == MVT::i8 ||(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19526, __PRETTY_FUNCTION__))
19524 InVT.getVectorElementType() == MVT::i16 ||(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19526, __PRETTY_FUNCTION__))
19525 InVT.getVectorElementType() == MVT::i32) &&(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19526, __PRETTY_FUNCTION__))
19526 "Unexpected element type")(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19526, __PRETTY_FUNCTION__));
19527
19528 unsigned ExtendInVecOpc = getOpcode_EXTEND_VECTOR_INREG(Opc);
19529
19530 // Custom legalize v8i8->v8i64 on CPUs without avx512bw.
19531 if (InVT == MVT::v8i8) {
19532 if (VT != MVT::v8i64)
19533 return SDValue();
19534
19535 In = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op),
19536 MVT::v16i8, In, DAG.getUNDEF(MVT::v8i8));
19537 return DAG.getNode(ExtendInVecOpc, dl, VT, In);
19538 }
19539
19540 if (Subtarget.hasInt256())
19541 return Op;
19542
19543 // Optimize vectors in AVX mode:
19544 //
19545 // v8i16 -> v8i32
19546 // Use vpmovzwd for 4 lower elements v8i16 -> v4i32.
19547 // Use vpunpckhwd for 4 upper elements v8i16 -> v4i32.
19548 // Concat upper and lower parts.
19549 //
19550 // v4i32 -> v4i64
19551 // Use vpmovzdq for 4 lower elements v4i32 -> v2i64.
19552 // Use vpunpckhdq for 4 upper elements v4i32 -> v2i64.
19553 // Concat upper and lower parts.
19554 //
19555 MVT HalfVT = VT.getHalfNumVectorElementsVT();
19556 SDValue OpLo = DAG.getNode(ExtendInVecOpc, dl, HalfVT, In);
19557
19558 // Short-circuit if we can determine that each 128-bit half is the same value.
19559 // Otherwise, this is difficult to match and optimize.
19560 if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(In))
19561 if (hasIdenticalHalvesShuffleMask(Shuf->getMask()))
19562 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpLo);
19563
19564 SDValue ZeroVec = DAG.getConstant(0, dl, InVT);
19565 SDValue Undef = DAG.getUNDEF(InVT);
19566 bool NeedZero = Opc == ISD::ZERO_EXTEND;
19567 SDValue OpHi = getUnpackh(DAG, dl, InVT, In, NeedZero ? ZeroVec : Undef);
19568 OpHi = DAG.getBitcast(HalfVT, OpHi);
19569
19570 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
19571}
19572
19573// Helper to split and extend a v16i1 mask to v16i8 or v16i16.
19574static SDValue SplitAndExtendv16i1(unsigned ExtOpc, MVT VT, SDValue In,
19575 const SDLoc &dl, SelectionDAG &DAG) {
19576 assert((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT.")(((VT == MVT::v16i8 || VT == MVT::v16i16) && "Unexpected VT."
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::v16i8 || VT == MVT::v16i16) && \"Unexpected VT.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19576, __PRETTY_FUNCTION__));
19577 SDValue Lo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
19578 DAG.getIntPtrConstant(0, dl));
19579 SDValue Hi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v8i1, In,
19580 DAG.getIntPtrConstant(8, dl));
19581 Lo = DAG.getNode(ExtOpc, dl, MVT::v8i16, Lo);
19582 Hi = DAG.getNode(ExtOpc, dl, MVT::v8i16, Hi);
19583 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i16, Lo, Hi);
19584 return DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
19585}
19586
19587static SDValue LowerZERO_EXTEND_Mask(SDValue Op,
19588 const X86Subtarget &Subtarget,
19589 SelectionDAG &DAG) {
19590 MVT VT = Op->getSimpleValueType(0);
19591 SDValue In = Op->getOperand(0);
19592 MVT InVT = In.getSimpleValueType();
19593 assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!")((InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!"
) ? static_cast<void> (0) : __assert_fail ("InVT.getVectorElementType() == MVT::i1 && \"Unexpected input type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19593, __PRETTY_FUNCTION__));
19594 SDLoc DL(Op);
19595 unsigned NumElts = VT.getVectorNumElements();
19596
19597 // For all vectors, but vXi8 we can just emit a sign_extend and a shift. This
19598 // avoids a constant pool load.
19599 if (VT.getVectorElementType() != MVT::i8) {
19600 SDValue Extend = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, In);
19601 return DAG.getNode(ISD::SRL, DL, VT, Extend,
19602 DAG.getConstant(VT.getScalarSizeInBits() - 1, DL, VT));
19603 }
19604
19605 // Extend VT if BWI is not supported.
19606 MVT ExtVT = VT;
19607 if (!Subtarget.hasBWI()) {
19608 // If v16i32 is to be avoided, we'll need to split and concatenate.
19609 if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
19610 return SplitAndExtendv16i1(ISD::ZERO_EXTEND, VT, In, DL, DAG);
19611
19612 ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
19613 }
19614
19615 // Widen to 512-bits if VLX is not supported.
19616 MVT WideVT = ExtVT;
19617 if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
19618 NumElts *= 512 / ExtVT.getSizeInBits();
19619 InVT = MVT::getVectorVT(MVT::i1, NumElts);
19620 In = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, InVT, DAG.getUNDEF(InVT),
19621 In, DAG.getIntPtrConstant(0, DL));
19622 WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(),
19623 NumElts);
19624 }
19625
19626 SDValue One = DAG.getConstant(1, DL, WideVT);
19627 SDValue Zero = DAG.getConstant(0, DL, WideVT);
19628
19629 SDValue SelectedVal = DAG.getSelect(DL, WideVT, In, One, Zero);
19630
19631 // Truncate if we had to extend above.
19632 if (VT != ExtVT) {
19633 WideVT = MVT::getVectorVT(MVT::i8, NumElts);
19634 SelectedVal = DAG.getNode(ISD::TRUNCATE, DL, WideVT, SelectedVal);
19635 }
19636
19637 // Extract back to 128/256-bit if we widened.
19638 if (WideVT != VT)
19639 SelectedVal = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, SelectedVal,
19640 DAG.getIntPtrConstant(0, DL));
19641
19642 return SelectedVal;
19643}
19644
19645static SDValue LowerZERO_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
19646 SelectionDAG &DAG) {
19647 SDValue In = Op.getOperand(0);
19648 MVT SVT = In.getSimpleValueType();
19649
19650 if (SVT.getVectorElementType() == MVT::i1)
19651 return LowerZERO_EXTEND_Mask(Op, Subtarget, DAG);
19652
19653 assert(Subtarget.hasAVX() && "Expected AVX support")((Subtarget.hasAVX() && "Expected AVX support") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasAVX() && \"Expected AVX support\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19653, __PRETTY_FUNCTION__));
19654 return LowerAVXExtend(Op, DAG, Subtarget);
19655}
19656
19657/// Helper to recursively truncate vector elements in half with PACKSS/PACKUS.
19658/// It makes use of the fact that vectors with enough leading sign/zero bits
19659/// prevent the PACKSS/PACKUS from saturating the results.
19660/// AVX2 (Int256) sub-targets require extra shuffling as the PACK*S operates
19661/// within each 128-bit lane.
19662static SDValue truncateVectorWithPACK(unsigned Opcode, EVT DstVT, SDValue In,
19663 const SDLoc &DL, SelectionDAG &DAG,
19664 const X86Subtarget &Subtarget) {
19665 assert((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&(((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&
"Unexpected PACK opcode") ? static_cast<void> (0) : __assert_fail
("(Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) && \"Unexpected PACK opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19666, __PRETTY_FUNCTION__))
19666 "Unexpected PACK opcode")(((Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) &&
"Unexpected PACK opcode") ? static_cast<void> (0) : __assert_fail
("(Opcode == X86ISD::PACKSS || Opcode == X86ISD::PACKUS) && \"Unexpected PACK opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19666, __PRETTY_FUNCTION__));
19667 assert(DstVT.isVector() && "VT not a vector?")((DstVT.isVector() && "VT not a vector?") ? static_cast
<void> (0) : __assert_fail ("DstVT.isVector() && \"VT not a vector?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19667, __PRETTY_FUNCTION__));
19668
19669 // Requires SSE2 but AVX512 has fast vector truncate.
19670 if (!Subtarget.hasSSE2())
19671 return SDValue();
19672
19673 EVT SrcVT = In.getValueType();
19674
19675 // No truncation required, we might get here due to recursive calls.
19676 if (SrcVT == DstVT)
19677 return In;
19678
19679 // We only support vector truncation to 64bits or greater from a
19680 // 128bits or greater source.
19681 unsigned DstSizeInBits = DstVT.getSizeInBits();
19682 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
19683 if ((DstSizeInBits % 64) != 0 || (SrcSizeInBits % 128) != 0)
19684 return SDValue();
19685
19686 unsigned NumElems = SrcVT.getVectorNumElements();
19687 if (!isPowerOf2_32(NumElems))
19688 return SDValue();
19689
19690 LLVMContext &Ctx = *DAG.getContext();
19691 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation")((DstVT.getVectorNumElements() == NumElems && "Illegal truncation"
) ? static_cast<void> (0) : __assert_fail ("DstVT.getVectorNumElements() == NumElems && \"Illegal truncation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19691, __PRETTY_FUNCTION__));
19692 assert(SrcSizeInBits > DstSizeInBits && "Illegal truncation")((SrcSizeInBits > DstSizeInBits && "Illegal truncation"
) ? static_cast<void> (0) : __assert_fail ("SrcSizeInBits > DstSizeInBits && \"Illegal truncation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19692, __PRETTY_FUNCTION__));
19693
19694 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
19695
19696 // Pack to the largest type possible:
19697 // vXi64/vXi32 -> PACK*SDW and vXi16 -> PACK*SWB.
19698 EVT InVT = MVT::i16, OutVT = MVT::i8;
19699 if (SrcVT.getScalarSizeInBits() > 16 &&
19700 (Opcode == X86ISD::PACKSS || Subtarget.hasSSE41())) {
19701 InVT = MVT::i32;
19702 OutVT = MVT::i16;
19703 }
19704
19705 // 128bit -> 64bit truncate - PACK 128-bit src in the lower subvector.
19706 if (SrcVT.is128BitVector()) {
19707 InVT = EVT::getVectorVT(Ctx, InVT, 128 / InVT.getSizeInBits());
19708 OutVT = EVT::getVectorVT(Ctx, OutVT, 128 / OutVT.getSizeInBits());
19709 In = DAG.getBitcast(InVT, In);
19710 SDValue Res = DAG.getNode(Opcode, DL, OutVT, In, In);
19711 Res = extractSubVector(Res, 0, DAG, DL, 64);
19712 return DAG.getBitcast(DstVT, Res);
19713 }
19714
19715 // Extract lower/upper subvectors.
19716 unsigned NumSubElts = NumElems / 2;
19717 SDValue Lo = extractSubVector(In, 0 * NumSubElts, DAG, DL, SrcSizeInBits / 2);
19718 SDValue Hi = extractSubVector(In, 1 * NumSubElts, DAG, DL, SrcSizeInBits / 2);
19719
19720 unsigned SubSizeInBits = SrcSizeInBits / 2;
19721 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
19722 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
19723
19724 // 256bit -> 128bit truncate - PACK lower/upper 128-bit subvectors.
19725 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
19726 Lo = DAG.getBitcast(InVT, Lo);
19727 Hi = DAG.getBitcast(InVT, Hi);
19728 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
19729 return DAG.getBitcast(DstVT, Res);
19730 }
19731
19732 // AVX2: 512bit -> 256bit truncate - PACK lower/upper 256-bit subvectors.
19733 // AVX2: 512bit -> 128bit truncate - PACK(PACK, PACK).
19734 if (SrcVT.is512BitVector() && Subtarget.hasInt256()) {
19735 Lo = DAG.getBitcast(InVT, Lo);
19736 Hi = DAG.getBitcast(InVT, Hi);
19737 SDValue Res = DAG.getNode(Opcode, DL, OutVT, Lo, Hi);
19738
19739 // 256-bit PACK(ARG0, ARG1) leaves us with ((LO0,LO1),(HI0,HI1)),
19740 // so we need to shuffle to get ((LO0,HI0),(LO1,HI1)).
19741 // Scale shuffle mask to avoid bitcasts and help ComputeNumSignBits.
19742 SmallVector<int, 64> Mask;
19743 int Scale = 64 / OutVT.getScalarSizeInBits();
19744 scaleShuffleMask<int>(Scale, ArrayRef<int>({ 0, 2, 1, 3 }), Mask);
19745 Res = DAG.getVectorShuffle(OutVT, DL, Res, Res, Mask);
19746
19747 if (DstVT.is256BitVector())
19748 return DAG.getBitcast(DstVT, Res);
19749
19750 // If 512bit -> 128bit truncate another stage.
19751 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
19752 Res = DAG.getBitcast(PackedVT, Res);
19753 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
19754 }
19755
19756 // Recursively pack lower/upper subvectors, concat result and pack again.
19757 assert(SrcSizeInBits >= 256 && "Expected 256-bit vector or greater")((SrcSizeInBits >= 256 && "Expected 256-bit vector or greater"
) ? static_cast<void> (0) : __assert_fail ("SrcSizeInBits >= 256 && \"Expected 256-bit vector or greater\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19757, __PRETTY_FUNCTION__));
19758 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumSubElts);
19759 Lo = truncateVectorWithPACK(Opcode, PackedVT, Lo, DL, DAG, Subtarget);
19760 Hi = truncateVectorWithPACK(Opcode, PackedVT, Hi, DL, DAG, Subtarget);
19761
19762 PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
19763 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
19764 return truncateVectorWithPACK(Opcode, DstVT, Res, DL, DAG, Subtarget);
19765}
19766
19767static SDValue LowerTruncateVecI1(SDValue Op, SelectionDAG &DAG,
19768 const X86Subtarget &Subtarget) {
19769
19770 SDLoc DL(Op);
19771 MVT VT = Op.getSimpleValueType();
19772 SDValue In = Op.getOperand(0);
19773 MVT InVT = In.getSimpleValueType();
19774
19775 assert(VT.getVectorElementType() == MVT::i1 && "Unexpected vector type.")((VT.getVectorElementType() == MVT::i1 && "Unexpected vector type."
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == MVT::i1 && \"Unexpected vector type.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19775, __PRETTY_FUNCTION__));
19776
19777 // Shift LSB to MSB and use VPMOVB/W2M or TESTD/Q.
19778 unsigned ShiftInx = InVT.getScalarSizeInBits() - 1;
19779 if (InVT.getScalarSizeInBits() <= 16) {
19780 if (Subtarget.hasBWI()) {
19781 // legal, will go to VPMOVB2M, VPMOVW2M
19782 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
19783 // We need to shift to get the lsb into sign position.
19784 // Shift packed bytes not supported natively, bitcast to word
19785 MVT ExtVT = MVT::getVectorVT(MVT::i16, InVT.getSizeInBits()/16);
19786 In = DAG.getNode(ISD::SHL, DL, ExtVT,
19787 DAG.getBitcast(ExtVT, In),
19788 DAG.getConstant(ShiftInx, DL, ExtVT));
19789 In = DAG.getBitcast(InVT, In);
19790 }
19791 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT),
19792 In, ISD::SETGT);
19793 }
19794 // Use TESTD/Q, extended vector to packed dword/qword.
19795 assert((InVT.is256BitVector() || InVT.is128BitVector()) &&(((InVT.is256BitVector() || InVT.is128BitVector()) &&
"Unexpected vector type.") ? static_cast<void> (0) : __assert_fail
("(InVT.is256BitVector() || InVT.is128BitVector()) && \"Unexpected vector type.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19796, __PRETTY_FUNCTION__))
19796 "Unexpected vector type.")(((InVT.is256BitVector() || InVT.is128BitVector()) &&
"Unexpected vector type.") ? static_cast<void> (0) : __assert_fail
("(InVT.is256BitVector() || InVT.is128BitVector()) && \"Unexpected vector type.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19796, __PRETTY_FUNCTION__));
19797 unsigned NumElts = InVT.getVectorNumElements();
19798 assert((NumElts == 8 || NumElts == 16) && "Unexpected number of elements")(((NumElts == 8 || NumElts == 16) && "Unexpected number of elements"
) ? static_cast<void> (0) : __assert_fail ("(NumElts == 8 || NumElts == 16) && \"Unexpected number of elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19798, __PRETTY_FUNCTION__));
19799 // We need to change to a wider element type that we have support for.
19800 // For 8 element vectors this is easy, we either extend to v8i32 or v8i64.
19801 // For 16 element vectors we extend to v16i32 unless we are explicitly
19802 // trying to avoid 512-bit vectors. If we are avoiding 512-bit vectors
19803 // we need to split into two 8 element vectors which we can extend to v8i32,
19804 // truncate and concat the results. There's an additional complication if
19805 // the original type is v16i8. In that case we can't split the v16i8 so
19806 // first we pre-extend it to v16i16 which we can split to v8i16, then extend
19807 // to v8i32, truncate that to v8i1 and concat the two halves.
19808 if (NumElts == 16 && !Subtarget.canExtendTo512DQ()) {
19809 if (InVT == MVT::v16i8) {
19810 // First we need to sign extend up to 256-bits so we can split that.
19811 InVT = MVT::v16i16;
19812 In = DAG.getNode(ISD::SIGN_EXTEND, DL, InVT, In);
19813 }
19814 SDValue Lo = extract128BitVector(In, 0, DAG, DL);
19815 SDValue Hi = extract128BitVector(In, 8, DAG, DL);
19816 // We're split now, just emit two truncates and a concat. The two
19817 // truncates will trigger legalization to come back to this function.
19818 Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Lo);
19819 Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::v8i1, Hi);
19820 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
19821 }
19822 // We either have 8 elements or we're allowed to use 512-bit vectors.
19823 // If we have VLX, we want to use the narrowest vector that can get the
19824 // job done so we use vXi32.
19825 MVT EltVT = Subtarget.hasVLX() ? MVT::i32 : MVT::getIntegerVT(512/NumElts);
19826 MVT ExtVT = MVT::getVectorVT(EltVT, NumElts);
19827 In = DAG.getNode(ISD::SIGN_EXTEND, DL, ExtVT, In);
19828 InVT = ExtVT;
19829 ShiftInx = InVT.getScalarSizeInBits() - 1;
19830 }
19831
19832 if (DAG.ComputeNumSignBits(In) < InVT.getScalarSizeInBits()) {
19833 // We need to shift to get the lsb into sign position.
19834 In = DAG.getNode(ISD::SHL, DL, InVT, In,
19835 DAG.getConstant(ShiftInx, DL, InVT));
19836 }
19837 // If we have DQI, emit a pattern that will be iseled as vpmovq2m/vpmovd2m.
19838 if (Subtarget.hasDQI())
19839 return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, InVT), In, ISD::SETGT);
19840 return DAG.getSetCC(DL, VT, In, DAG.getConstant(0, DL, InVT), ISD::SETNE);
19841}
19842
19843SDValue X86TargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
19844 SDLoc DL(Op);
19845 MVT VT = Op.getSimpleValueType();
19846 SDValue In = Op.getOperand(0);
19847 MVT InVT = In.getSimpleValueType();
19848 unsigned InNumEltBits = InVT.getScalarSizeInBits();
19849
19850 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&((VT.getVectorNumElements() == InVT.getVectorNumElements() &&
"Invalid TRUNCATE operation") ? static_cast<void> (0) :
__assert_fail ("VT.getVectorNumElements() == InVT.getVectorNumElements() && \"Invalid TRUNCATE operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19851, __PRETTY_FUNCTION__))
19851 "Invalid TRUNCATE operation")((VT.getVectorNumElements() == InVT.getVectorNumElements() &&
"Invalid TRUNCATE operation") ? static_cast<void> (0) :
__assert_fail ("VT.getVectorNumElements() == InVT.getVectorNumElements() && \"Invalid TRUNCATE operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19851, __PRETTY_FUNCTION__));
19852
19853 // If we're called by the type legalizer, handle a few cases.
19854 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19855 if (!TLI.isTypeLegal(InVT)) {
19856 if ((InVT == MVT::v8i64 || InVT == MVT::v16i32 || InVT == MVT::v16i64) &&
19857 VT.is128BitVector()) {
19858 assert(Subtarget.hasVLX() && "Unexpected subtarget!")((Subtarget.hasVLX() && "Unexpected subtarget!") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasVLX() && \"Unexpected subtarget!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19858, __PRETTY_FUNCTION__));
19859 // The default behavior is to truncate one step, concatenate, and then
19860 // truncate the remainder. We'd rather produce two 64-bit results and
19861 // concatenate those.
19862 SDValue Lo, Hi;
19863 std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
19864
19865 EVT LoVT, HiVT;
19866 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(VT);
19867
19868 Lo = DAG.getNode(ISD::TRUNCATE, DL, LoVT, Lo);
19869 Hi = DAG.getNode(ISD::TRUNCATE, DL, HiVT, Hi);
19870 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
19871 }
19872
19873 // Otherwise let default legalization handle it.
19874 return SDValue();
19875 }
19876
19877 if (VT.getVectorElementType() == MVT::i1)
19878 return LowerTruncateVecI1(Op, DAG, Subtarget);
19879
19880 // vpmovqb/w/d, vpmovdb/w, vpmovwb
19881 if (Subtarget.hasAVX512()) {
19882 // word to byte only under BWI. Otherwise we have to promoted to v16i32
19883 // and then truncate that. But we should only do that if we haven't been
19884 // asked to avoid 512-bit vectors. The actual promotion to v16i32 will be
19885 // handled by isel patterns.
19886 if (InVT != MVT::v16i16 || Subtarget.hasBWI() ||
19887 Subtarget.canExtendTo512DQ())
19888 return Op;
19889 }
19890
19891 unsigned NumPackedSignBits = std::min<unsigned>(VT.getScalarSizeInBits(), 16);
19892 unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8;
19893
19894 // Truncate with PACKUS if we are truncating a vector with leading zero bits
19895 // that extend all the way to the packed/truncated value.
19896 // Pre-SSE41 we can only use PACKUSWB.
19897 KnownBits Known = DAG.computeKnownBits(In);
19898 if ((InNumEltBits - NumPackedZeroBits) <= Known.countMinLeadingZeros())
19899 if (SDValue V =
19900 truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget))
19901 return V;
19902
19903 // Truncate with PACKSS if we are truncating a vector with sign-bits that
19904 // extend all the way to the packed/truncated value.
19905 if ((InNumEltBits - NumPackedSignBits) < DAG.ComputeNumSignBits(In))
19906 if (SDValue V =
19907 truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget))
19908 return V;
19909
19910 // Handle truncation of V256 to V128 using shuffles.
19911 assert(VT.is128BitVector() && InVT.is256BitVector() && "Unexpected types!")((VT.is128BitVector() && InVT.is256BitVector() &&
"Unexpected types!") ? static_cast<void> (0) : __assert_fail
("VT.is128BitVector() && InVT.is256BitVector() && \"Unexpected types!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19911, __PRETTY_FUNCTION__));
19912
19913 if ((VT == MVT::v4i32) && (InVT == MVT::v4i64)) {
19914 // On AVX2, v4i64 -> v4i32 becomes VPERMD.
19915 if (Subtarget.hasInt256()) {
19916 static const int ShufMask[] = {0, 2, 4, 6, -1, -1, -1, -1};
19917 In = DAG.getBitcast(MVT::v8i32, In);
19918 In = DAG.getVectorShuffle(MVT::v8i32, DL, In, In, ShufMask);
19919 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, In,
19920 DAG.getIntPtrConstant(0, DL));
19921 }
19922
19923 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
19924 DAG.getIntPtrConstant(0, DL));
19925 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
19926 DAG.getIntPtrConstant(2, DL));
19927 OpLo = DAG.getBitcast(MVT::v4i32, OpLo);
19928 OpHi = DAG.getBitcast(MVT::v4i32, OpHi);
19929 static const int ShufMask[] = {0, 2, 4, 6};
19930 return DAG.getVectorShuffle(VT, DL, OpLo, OpHi, ShufMask);
19931 }
19932
19933 if ((VT == MVT::v8i16) && (InVT == MVT::v8i32)) {
19934 // On AVX2, v8i32 -> v8i16 becomes PSHUFB.
19935 if (Subtarget.hasInt256()) {
19936 In = DAG.getBitcast(MVT::v32i8, In);
19937
19938 // The PSHUFB mask:
19939 static const int ShufMask1[] = { 0, 1, 4, 5, 8, 9, 12, 13,
19940 -1, -1, -1, -1, -1, -1, -1, -1,
19941 16, 17, 20, 21, 24, 25, 28, 29,
19942 -1, -1, -1, -1, -1, -1, -1, -1 };
19943 In = DAG.getVectorShuffle(MVT::v32i8, DL, In, In, ShufMask1);
19944 In = DAG.getBitcast(MVT::v4i64, In);
19945
19946 static const int ShufMask2[] = {0, 2, -1, -1};
19947 In = DAG.getVectorShuffle(MVT::v4i64, DL, In, In, ShufMask2);
19948 In = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v2i64, In,
19949 DAG.getIntPtrConstant(0, DL));
19950 return DAG.getBitcast(VT, In);
19951 }
19952
19953 SDValue OpLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
19954 DAG.getIntPtrConstant(0, DL));
19955
19956 SDValue OpHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v4i32, In,
19957 DAG.getIntPtrConstant(4, DL));
19958
19959 OpLo = DAG.getBitcast(MVT::v16i8, OpLo);
19960 OpHi = DAG.getBitcast(MVT::v16i8, OpHi);
19961
19962 // The PSHUFB mask:
19963 static const int ShufMask1[] = {0, 1, 4, 5, 8, 9, 12, 13,
19964 -1, -1, -1, -1, -1, -1, -1, -1};
19965
19966 OpLo = DAG.getVectorShuffle(MVT::v16i8, DL, OpLo, OpLo, ShufMask1);
19967 OpHi = DAG.getVectorShuffle(MVT::v16i8, DL, OpHi, OpHi, ShufMask1);
19968
19969 OpLo = DAG.getBitcast(MVT::v4i32, OpLo);
19970 OpHi = DAG.getBitcast(MVT::v4i32, OpHi);
19971
19972 // The MOVLHPS Mask:
19973 static const int ShufMask2[] = {0, 1, 4, 5};
19974 SDValue res = DAG.getVectorShuffle(MVT::v4i32, DL, OpLo, OpHi, ShufMask2);
19975 return DAG.getBitcast(MVT::v8i16, res);
19976 }
19977
19978 if (VT == MVT::v16i8 && InVT == MVT::v16i16) {
19979 // Use an AND to zero uppper bits for PACKUS.
19980 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(255, DL, InVT));
19981
19982 SDValue InLo = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
19983 DAG.getIntPtrConstant(0, DL));
19984 SDValue InHi = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, MVT::v8i16, In,
19985 DAG.getIntPtrConstant(8, DL));
19986 return DAG.getNode(X86ISD::PACKUS, DL, VT, InLo, InHi);
19987 }
19988
19989 llvm_unreachable("All 256->128 cases should have been handled above!")::llvm::llvm_unreachable_internal("All 256->128 cases should have been handled above!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 19989);
19990}
19991
19992SDValue X86TargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const {
19993 bool IsStrict = Op->isStrictFPOpcode();
19994 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
19995 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
19996 MVT VT = Op->getSimpleValueType(0);
19997 SDValue Src = Op.getOperand(IsStrict ? 1 : 0);
19998 MVT SrcVT = Src.getSimpleValueType();
19999 SDLoc dl(Op);
20000
20001 if (VT.isVector()) {
20002 if (VT == MVT::v2i1 && SrcVT == MVT::v2f64) {
20003 MVT ResVT = MVT::v4i32;
20004 MVT TruncVT = MVT::v4i1;
20005 unsigned Opc;
20006 if (IsStrict)
20007 Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
20008 else
20009 Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
20010
20011 if (!IsSigned && !Subtarget.hasVLX()) {
20012 assert(Subtarget.useAVX512Regs() && "Unexpected features!")((Subtarget.useAVX512Regs() && "Unexpected features!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.useAVX512Regs() && \"Unexpected features!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20012, __PRETTY_FUNCTION__));
20013 // Widen to 512-bits.
20014 ResVT = MVT::v8i32;
20015 TruncVT = MVT::v8i1;
20016 Opc = Op.getOpcode();
20017 // Need to concat with zero vector for strict fp to avoid spurious
20018 // exceptions.
20019 // TODO: Should we just do this for non-strict as well?
20020 SDValue Tmp = IsStrict ? DAG.getConstantFP(0.0, dl, MVT::v8f64)
20021 : DAG.getUNDEF(MVT::v8f64);
20022 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8f64, Tmp, Src,
20023 DAG.getIntPtrConstant(0, dl));
20024 }
20025 SDValue Res, Chain;
20026 if (IsStrict) {
20027 Res =
20028 DAG.getNode(Opc, dl, {ResVT, MVT::Other}, {Op->getOperand(0), Src});
20029 Chain = Res.getValue(1);
20030 } else {
20031 Res = DAG.getNode(Opc, dl, ResVT, Src);
20032 }
20033
20034 Res = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Res);
20035 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v2i1, Res,
20036 DAG.getIntPtrConstant(0, dl));
20037 if (IsStrict)
20038 return DAG.getMergeValues({Res, Chain}, dl);
20039 return Res;
20040 }
20041
20042 // v8f64->v8i32 is legal, but we need v8i32 to be custom for v8f32.
20043 if (VT == MVT::v8i32 && SrcVT == MVT::v8f64) {
20044 assert(!IsSigned && "Expected unsigned conversion!")((!IsSigned && "Expected unsigned conversion!") ? static_cast
<void> (0) : __assert_fail ("!IsSigned && \"Expected unsigned conversion!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20044, __PRETTY_FUNCTION__));
20045 assert(Subtarget.useAVX512Regs() && "Requires avx512f")((Subtarget.useAVX512Regs() && "Requires avx512f") ? static_cast
<void> (0) : __assert_fail ("Subtarget.useAVX512Regs() && \"Requires avx512f\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20045, __PRETTY_FUNCTION__));
20046 return Op;
20047 }
20048
20049 // Widen vXi32 fp_to_uint with avx512f to 512-bit source.
20050 if ((VT == MVT::v4i32 || VT == MVT::v8i32) &&
20051 (SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32 || SrcVT == MVT::v8f32)) {
20052 assert(!IsSigned && "Expected unsigned conversion!")((!IsSigned && "Expected unsigned conversion!") ? static_cast
<void> (0) : __assert_fail ("!IsSigned && \"Expected unsigned conversion!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20052, __PRETTY_FUNCTION__));
20053 assert(Subtarget.useAVX512Regs() && !Subtarget.hasVLX() &&((Subtarget.useAVX512Regs() && !Subtarget.hasVLX() &&
"Unexpected features!") ? static_cast<void> (0) : __assert_fail
("Subtarget.useAVX512Regs() && !Subtarget.hasVLX() && \"Unexpected features!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20054, __PRETTY_FUNCTION__))
20054 "Unexpected features!")((Subtarget.useAVX512Regs() && !Subtarget.hasVLX() &&
"Unexpected features!") ? static_cast<void> (0) : __assert_fail
("Subtarget.useAVX512Regs() && !Subtarget.hasVLX() && \"Unexpected features!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20054, __PRETTY_FUNCTION__));
20055 MVT WideVT = SrcVT == MVT::v4f64 ? MVT::v8f64 : MVT::v16f32;
20056 MVT ResVT = SrcVT == MVT::v4f64 ? MVT::v8i32 : MVT::v16i32;
20057 // Need to concat with zero vector for strict fp to avoid spurious
20058 // exceptions.
20059 // TODO: Should we just do this for non-strict as well?
20060 SDValue Tmp =
20061 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
20062 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
20063 DAG.getIntPtrConstant(0, dl));
20064
20065 SDValue Res, Chain;
20066 if (IsStrict) {
20067 Res = DAG.getNode(ISD::STRICT_FP_TO_UINT, dl, {ResVT, MVT::Other},
20068 {Op->getOperand(0), Src});
20069 Chain = Res.getValue(1);
20070 } else {
20071 Res = DAG.getNode(ISD::FP_TO_UINT, dl, ResVT, Src);
20072 }
20073
20074 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20075 DAG.getIntPtrConstant(0, dl));
20076
20077 if (IsStrict)
20078 return DAG.getMergeValues({Res, Chain}, dl);
20079 return Res;
20080 }
20081
20082 // Widen vXi64 fp_to_uint/fp_to_sint with avx512dq to 512-bit source.
20083 if ((VT == MVT::v2i64 || VT == MVT::v4i64) &&
20084 (SrcVT == MVT::v2f64 || SrcVT == MVT::v4f64 || SrcVT == MVT::v4f32)) {
20085 assert(Subtarget.useAVX512Regs() && Subtarget.hasDQI() &&((Subtarget.useAVX512Regs() && Subtarget.hasDQI() &&
!Subtarget.hasVLX() && "Unexpected features!") ? static_cast
<void> (0) : __assert_fail ("Subtarget.useAVX512Regs() && Subtarget.hasDQI() && !Subtarget.hasVLX() && \"Unexpected features!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20086, __PRETTY_FUNCTION__))
20086 !Subtarget.hasVLX() && "Unexpected features!")((Subtarget.useAVX512Regs() && Subtarget.hasDQI() &&
!Subtarget.hasVLX() && "Unexpected features!") ? static_cast
<void> (0) : __assert_fail ("Subtarget.useAVX512Regs() && Subtarget.hasDQI() && !Subtarget.hasVLX() && \"Unexpected features!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20086, __PRETTY_FUNCTION__));
20087 MVT WideVT = SrcVT == MVT::v4f32 ? MVT::v8f32 : MVT::v8f64;
20088 // Need to concat with zero vector for strict fp to avoid spurious
20089 // exceptions.
20090 // TODO: Should we just do this for non-strict as well?
20091 SDValue Tmp =
20092 IsStrict ? DAG.getConstantFP(0.0, dl, WideVT) : DAG.getUNDEF(WideVT);
20093 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, WideVT, Tmp, Src,
20094 DAG.getIntPtrConstant(0, dl));
20095
20096 SDValue Res, Chain;
20097 if (IsStrict) {
20098 Res = DAG.getNode(Op.getOpcode(), dl, {MVT::v8i64, MVT::Other},
20099 {Op->getOperand(0), Src});
20100 Chain = Res.getValue(1);
20101 } else {
20102 Res = DAG.getNode(Op.getOpcode(), dl, MVT::v8i64, Src);
20103 }
20104
20105 Res = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, Res,
20106 DAG.getIntPtrConstant(0, dl));
20107
20108 if (IsStrict)
20109 return DAG.getMergeValues({Res, Chain}, dl);
20110 return Res;
20111 }
20112
20113 if (VT == MVT::v2i64 && SrcVT == MVT::v2f32) {
20114 assert(Subtarget.hasDQI() && Subtarget.hasVLX() && "Requires AVX512DQVL")((Subtarget.hasDQI() && Subtarget.hasVLX() &&
"Requires AVX512DQVL") ? static_cast<void> (0) : __assert_fail
("Subtarget.hasDQI() && Subtarget.hasVLX() && \"Requires AVX512DQVL\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20114, __PRETTY_FUNCTION__));
20115 SDValue Tmp = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
20116 DAG.getUNDEF(MVT::v2f32));
20117 if (IsStrict) {
20118 unsigned Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI
20119 : X86ISD::STRICT_CVTTP2UI;
20120 return DAG.getNode(Opc, dl, {VT, MVT::Other}, {Op->getOperand(0), Tmp});
20121 }
20122 unsigned Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
20123 return DAG.getNode(Opc, dl, VT, Tmp);
20124 }
20125
20126 return SDValue();
20127 }
20128
20129 assert(!VT.isVector())((!VT.isVector()) ? static_cast<void> (0) : __assert_fail
("!VT.isVector()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20129, __PRETTY_FUNCTION__));
20130
20131 bool UseSSEReg = isScalarFPTypeInSSEReg(SrcVT);
20132
20133 if (!IsSigned && UseSSEReg) {
20134 // Conversions from f32/f64 with AVX512 should be legal.
20135 if (Subtarget.hasAVX512())
20136 return Op;
20137
20138 // Use default expansion for i64.
20139 if (VT == MVT::i64)
20140 return SDValue();
20141
20142 assert(VT == MVT::i32 && "Unexpected VT!")((VT == MVT::i32 && "Unexpected VT!") ? static_cast<
void> (0) : __assert_fail ("VT == MVT::i32 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20142, __PRETTY_FUNCTION__));
20143
20144 // Promote i32 to i64 and use a signed operation on 64-bit targets.
20145 // FIXME: This does not generate an invalid exception if the input does not
20146 // fit in i32. PR44019
20147 if (Subtarget.is64Bit()) {
20148 SDValue Res, Chain;
20149 if (IsStrict) {
20150 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { MVT::i64, MVT::Other},
20151 { Op.getOperand(0), Src });
20152 Chain = Res.getValue(1);
20153 } else
20154 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i64, Src);
20155
20156 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20157 if (IsStrict)
20158 return DAG.getMergeValues({ Res, Chain }, dl);
20159 return Res;
20160 }
20161
20162 // Use default expansion for SSE1/2 targets without SSE3. With SSE3 we can
20163 // use fisttp which will be handled later.
20164 if (!Subtarget.hasSSE3())
20165 return SDValue();
20166 }
20167
20168 // Promote i16 to i32 if we can use a SSE operation or the type is f128.
20169 // FIXME: This does not generate an invalid exception if the input does not
20170 // fit in i16. PR44019
20171 if (VT == MVT::i16 && (UseSSEReg || SrcVT == MVT::f128)) {
20172 assert(IsSigned && "Expected i16 FP_TO_UINT to have been promoted!")((IsSigned && "Expected i16 FP_TO_UINT to have been promoted!"
) ? static_cast<void> (0) : __assert_fail ("IsSigned && \"Expected i16 FP_TO_UINT to have been promoted!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20172, __PRETTY_FUNCTION__));
20173 SDValue Res, Chain;
20174 if (IsStrict) {
20175 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, { MVT::i32, MVT::Other},
20176 { Op.getOperand(0), Src });
20177 Chain = Res.getValue(1);
20178 } else
20179 Res = DAG.getNode(ISD::FP_TO_SINT, dl, MVT::i32, Src);
20180
20181 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
20182 if (IsStrict)
20183 return DAG.getMergeValues({ Res, Chain }, dl);
20184 return Res;
20185 }
20186
20187 // If this is a FP_TO_SINT using SSEReg we're done.
20188 if (UseSSEReg && IsSigned)
20189 return Op;
20190
20191 // fp128 needs to use a libcall.
20192 if (SrcVT == MVT::f128) {
20193 RTLIB::Libcall LC;
20194 if (IsSigned)
20195 LC = RTLIB::getFPTOSINT(SrcVT, VT);
20196 else
20197 LC = RTLIB::getFPTOUINT(SrcVT, VT);
20198
20199 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
20200 MakeLibCallOptions CallOptions;
20201 std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, Src, CallOptions,
20202 SDLoc(Op), Chain);
20203
20204 if (IsStrict)
20205 return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
20206
20207 return Tmp.first;
20208 }
20209
20210 // Fall back to X87.
20211 SDValue Chain;
20212 if (SDValue V = FP_TO_INTHelper(Op, DAG, IsSigned, Chain)) {
20213 if (IsStrict)
20214 return DAG.getMergeValues({V, Chain}, dl);
20215 return V;
20216 }
20217
20218 llvm_unreachable("Expected FP_TO_INTHelper to handle all remaining cases.")::llvm::llvm_unreachable_internal("Expected FP_TO_INTHelper to handle all remaining cases."
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20218);
20219}
20220
20221SDValue X86TargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
20222 bool IsStrict = Op->isStrictFPOpcode();
20223
20224 SDLoc DL(Op);
20225 MVT VT = Op.getSimpleValueType();
20226 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
20227 MVT SVT = In.getSimpleValueType();
20228
20229 if (VT == MVT::f128) {
20230 RTLIB::Libcall LC = RTLIB::getFPEXT(SVT, VT);
20231 return LowerF128Call(Op, DAG, LC);
20232 }
20233
20234 assert(SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!")((SVT == MVT::v2f32 && "Only customize MVT::v2f32 type legalization!"
) ? static_cast<void> (0) : __assert_fail ("SVT == MVT::v2f32 && \"Only customize MVT::v2f32 type legalization!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20234, __PRETTY_FUNCTION__));
20235
20236 SDValue Res =
20237 DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4f32, In, DAG.getUNDEF(SVT));
20238 if (IsStrict)
20239 return DAG.getNode(X86ISD::STRICT_VFPEXT, DL, {VT, MVT::Other},
20240 {Op->getOperand(0), Res});
20241 return DAG.getNode(X86ISD::VFPEXT, DL, VT, Res);
20242}
20243
20244SDValue X86TargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
20245 bool IsStrict = Op->isStrictFPOpcode();
20246
20247 MVT VT = Op.getSimpleValueType();
20248 SDValue In = Op.getOperand(IsStrict ? 1 : 0);
20249 MVT SVT = In.getSimpleValueType();
20250
20251 // It's legal except when f128 is involved
20252 if (SVT != MVT::f128)
20253 return Op;
20254
20255 RTLIB::Libcall LC = RTLIB::getFPROUND(SVT, VT);
20256
20257 // FP_ROUND node has a second operand indicating whether it is known to be
20258 // precise. That doesn't take part in the LibCall so we can't directly use
20259 // LowerF128Call.
20260
20261 SDLoc dl(Op);
20262 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
20263 MakeLibCallOptions CallOptions;
20264 std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, LC, VT, In, CallOptions,
20265 dl, Chain);
20266
20267 if (IsStrict)
20268 return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
20269
20270 return Tmp.first;
20271}
20272
20273/// Depending on uarch and/or optimizing for size, we might prefer to use a
20274/// vector operation in place of the typical scalar operation.
20275static SDValue lowerAddSubToHorizontalOp(SDValue Op, SelectionDAG &DAG,
20276 const X86Subtarget &Subtarget) {
20277 // If both operands have other uses, this is probably not profitable.
20278 SDValue LHS = Op.getOperand(0);
20279 SDValue RHS = Op.getOperand(1);
20280 if (!LHS.hasOneUse() && !RHS.hasOneUse())
20281 return Op;
20282
20283 // FP horizontal add/sub were added with SSE3. Integer with SSSE3.
20284 bool IsFP = Op.getSimpleValueType().isFloatingPoint();
20285 if (IsFP && !Subtarget.hasSSE3())
20286 return Op;
20287 if (!IsFP && !Subtarget.hasSSSE3())
20288 return Op;
20289
20290 // Extract from a common vector.
20291 if (LHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
20292 RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
20293 LHS.getOperand(0) != RHS.getOperand(0) ||
20294 !isa<ConstantSDNode>(LHS.getOperand(1)) ||
20295 !isa<ConstantSDNode>(RHS.getOperand(1)) ||
20296 !shouldUseHorizontalOp(true, DAG, Subtarget))
20297 return Op;
20298
20299 // Allow commuted 'hadd' ops.
20300 // TODO: Allow commuted (f)sub by negating the result of (F)HSUB?
20301 unsigned HOpcode;
20302 switch (Op.getOpcode()) {
20303 case ISD::ADD: HOpcode = X86ISD::HADD; break;
20304 case ISD::SUB: HOpcode = X86ISD::HSUB; break;
20305 case ISD::FADD: HOpcode = X86ISD::FHADD; break;
20306 case ISD::FSUB: HOpcode = X86ISD::FHSUB; break;
20307 default:
20308 llvm_unreachable("Trying to lower unsupported opcode to horizontal op")::llvm::llvm_unreachable_internal("Trying to lower unsupported opcode to horizontal op"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20308);
20309 }
20310 unsigned LExtIndex = LHS.getConstantOperandVal(1);
20311 unsigned RExtIndex = RHS.getConstantOperandVal(1);
20312 if ((LExtIndex & 1) == 1 && (RExtIndex & 1) == 0 &&
20313 (HOpcode == X86ISD::HADD || HOpcode == X86ISD::FHADD))
20314 std::swap(LExtIndex, RExtIndex);
20315
20316 if ((LExtIndex & 1) != 0 || RExtIndex != (LExtIndex + 1))
20317 return Op;
20318
20319 SDValue X = LHS.getOperand(0);
20320 EVT VecVT = X.getValueType();
20321 unsigned BitWidth = VecVT.getSizeInBits();
20322 unsigned NumLanes = BitWidth / 128;
20323 unsigned NumEltsPerLane = VecVT.getVectorNumElements() / NumLanes;
20324 assert((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&(((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&
"Not expecting illegal vector widths here") ? static_cast<
void> (0) : __assert_fail ("(BitWidth == 128 || BitWidth == 256 || BitWidth == 512) && \"Not expecting illegal vector widths here\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20325, __PRETTY_FUNCTION__))
20325 "Not expecting illegal vector widths here")(((BitWidth == 128 || BitWidth == 256 || BitWidth == 512) &&
"Not expecting illegal vector widths here") ? static_cast<
void> (0) : __assert_fail ("(BitWidth == 128 || BitWidth == 256 || BitWidth == 512) && \"Not expecting illegal vector widths here\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20325, __PRETTY_FUNCTION__));
20326
20327 // Creating a 256-bit horizontal op would be wasteful, and there is no 512-bit
20328 // equivalent, so extract the 256/512-bit source op to 128-bit if we can.
20329 SDLoc DL(Op);
20330 if (BitWidth == 256 || BitWidth == 512) {
20331 unsigned LaneIdx = LExtIndex / NumEltsPerLane;
20332 X = extract128BitVector(X, LaneIdx * NumEltsPerLane, DAG, DL);
20333 LExtIndex %= NumEltsPerLane;
20334 }
20335
20336 // add (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hadd X, X), 0
20337 // add (extractelt (X, 1), extractelt (X, 0)) --> extractelt (hadd X, X), 0
20338 // add (extractelt (X, 2), extractelt (X, 3)) --> extractelt (hadd X, X), 1
20339 // sub (extractelt (X, 0), extractelt (X, 1)) --> extractelt (hsub X, X), 0
20340 SDValue HOp = DAG.getNode(HOpcode, DL, X.getValueType(), X, X);
20341 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Op.getSimpleValueType(), HOp,
20342 DAG.getIntPtrConstant(LExtIndex / 2, DL));
20343}
20344
20345/// Depending on uarch and/or optimizing for size, we might prefer to use a
20346/// vector operation in place of the typical scalar operation.
20347SDValue X86TargetLowering::lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const {
20348 assert((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) &&(((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::
f64) && "Only expecting float/double") ? static_cast<
void> (0) : __assert_fail ("(Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && \"Only expecting float/double\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20349, __PRETTY_FUNCTION__))
20349 "Only expecting float/double")(((Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::
f64) && "Only expecting float/double") ? static_cast<
void> (0) : __assert_fail ("(Op.getValueType() == MVT::f32 || Op.getValueType() == MVT::f64) && \"Only expecting float/double\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20349, __PRETTY_FUNCTION__));
20350 return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
20351}
20352
20353/// The only differences between FABS and FNEG are the mask and the logic op.
20354/// FNEG also has a folding opportunity for FNEG(FABS(x)).
20355static SDValue LowerFABSorFNEG(SDValue Op, SelectionDAG &DAG) {
20356 assert((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) &&(((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG
) && "Wrong opcode for lowering FABS or FNEG.") ? static_cast
<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) && \"Wrong opcode for lowering FABS or FNEG.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20357, __PRETTY_FUNCTION__))
20357 "Wrong opcode for lowering FABS or FNEG.")(((Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG
) && "Wrong opcode for lowering FABS or FNEG.") ? static_cast
<void> (0) : __assert_fail ("(Op.getOpcode() == ISD::FABS || Op.getOpcode() == ISD::FNEG) && \"Wrong opcode for lowering FABS or FNEG.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20357, __PRETTY_FUNCTION__));
20358
20359 bool IsFABS = (Op.getOpcode() == ISD::FABS);
20360
20361 // If this is a FABS and it has an FNEG user, bail out to fold the combination
20362 // into an FNABS. We'll lower the FABS after that if it is still in use.
20363 if (IsFABS)
20364 for (SDNode *User : Op->uses())
20365 if (User->getOpcode() == ISD::FNEG)
20366 return Op;
20367
20368 SDLoc dl(Op);
20369 MVT VT = Op.getSimpleValueType();
20370
20371 bool IsF128 = (VT == MVT::f128);
20372 assert((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 ||(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFABSorFNEG"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFABSorFNEG\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20375, __PRETTY_FUNCTION__))
20373 VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 ||(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFABSorFNEG"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFABSorFNEG\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20375, __PRETTY_FUNCTION__))
20374 VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) &&(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFABSorFNEG"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFABSorFNEG\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20375, __PRETTY_FUNCTION__))
20375 "Unexpected type in LowerFABSorFNEG")(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFABSorFNEG"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFABSorFNEG\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20375, __PRETTY_FUNCTION__));
20376
20377 // FIXME: Use function attribute "OptimizeForSize" and/or CodeGenOpt::Level to
20378 // decide if we should generate a 16-byte constant mask when we only need 4 or
20379 // 8 bytes for the scalar case.
20380
20381 // There are no scalar bitwise logical SSE/AVX instructions, so we
20382 // generate a 16-byte vector constant and logic op even for the scalar case.
20383 // Using a 16-byte mask allows folding the load of the mask with
20384 // the logic op, so it can save (~4 bytes) on code size.
20385 bool IsFakeVector = !VT.isVector() && !IsF128;
20386 MVT LogicVT = VT;
20387 if (IsFakeVector)
20388 LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32;
20389
20390 unsigned EltBits = VT.getScalarSizeInBits();
20391 // For FABS, mask is 0x7f...; for FNEG, mask is 0x80...
20392 APInt MaskElt = IsFABS ? APInt::getSignedMaxValue(EltBits) :
20393 APInt::getSignMask(EltBits);
20394 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
20395 SDValue Mask = DAG.getConstantFP(APFloat(Sem, MaskElt), dl, LogicVT);
20396
20397 SDValue Op0 = Op.getOperand(0);
20398 bool IsFNABS = !IsFABS && (Op0.getOpcode() == ISD::FABS);
20399 unsigned LogicOp = IsFABS ? X86ISD::FAND :
20400 IsFNABS ? X86ISD::FOR :
20401 X86ISD::FXOR;
20402 SDValue Operand = IsFNABS ? Op0.getOperand(0) : Op0;
20403
20404 if (VT.isVector() || IsF128)
20405 return DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
20406
20407 // For the scalar case extend to a 128-bit vector, perform the logic op,
20408 // and extract the scalar result back out.
20409 Operand = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Operand);
20410 SDValue LogicNode = DAG.getNode(LogicOp, dl, LogicVT, Operand, Mask);
20411 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, LogicNode,
20412 DAG.getIntPtrConstant(0, dl));
20413}
20414
20415static SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) {
20416 SDValue Mag = Op.getOperand(0);
20417 SDValue Sign = Op.getOperand(1);
20418 SDLoc dl(Op);
20419
20420 // If the sign operand is smaller, extend it first.
20421 MVT VT = Op.getSimpleValueType();
20422 if (Sign.getSimpleValueType().bitsLT(VT))
20423 Sign = DAG.getNode(ISD::FP_EXTEND, dl, VT, Sign);
20424
20425 // And if it is bigger, shrink it first.
20426 if (Sign.getSimpleValueType().bitsGT(VT))
20427 Sign = DAG.getNode(ISD::FP_ROUND, dl, VT, Sign, DAG.getIntPtrConstant(1, dl));
20428
20429 // At this point the operands and the result should have the same
20430 // type, and that won't be f80 since that is not custom lowered.
20431 bool IsF128 = (VT == MVT::f128);
20432 assert((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 ||(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFCOPYSIGN"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFCOPYSIGN\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20435, __PRETTY_FUNCTION__))
20433 VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 ||(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFCOPYSIGN"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFCOPYSIGN\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20435, __PRETTY_FUNCTION__))
20434 VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) &&(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFCOPYSIGN"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFCOPYSIGN\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20435, __PRETTY_FUNCTION__))
20435 "Unexpected type in LowerFCOPYSIGN")(((VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT ==
MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT
::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && "Unexpected type in LowerFCOPYSIGN"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::f64 || VT == MVT::f32 || VT == MVT::f128 || VT == MVT::v2f64 || VT == MVT::v4f64 || VT == MVT::v4f32 || VT == MVT::v8f32 || VT == MVT::v8f64 || VT == MVT::v16f32) && \"Unexpected type in LowerFCOPYSIGN\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20435, __PRETTY_FUNCTION__));
20436
20437 const fltSemantics &Sem = SelectionDAG::EVTToAPFloatSemantics(VT);
20438
20439 // Perform all scalar logic operations as 16-byte vectors because there are no
20440 // scalar FP logic instructions in SSE.
20441 // TODO: This isn't necessary. If we used scalar types, we might avoid some
20442 // unnecessary splats, but we might miss load folding opportunities. Should
20443 // this decision be based on OptimizeForSize?
20444 bool IsFakeVector = !VT.isVector() && !IsF128;
20445 MVT LogicVT = VT;
20446 if (IsFakeVector)
20447 LogicVT = (VT == MVT::f64) ? MVT::v2f64 : MVT::v4f32;
20448
20449 // The mask constants are automatically splatted for vector types.
20450 unsigned EltSizeInBits = VT.getScalarSizeInBits();
20451 SDValue SignMask = DAG.getConstantFP(
20452 APFloat(Sem, APInt::getSignMask(EltSizeInBits)), dl, LogicVT);
20453 SDValue MagMask = DAG.getConstantFP(
20454 APFloat(Sem, APInt::getSignedMaxValue(EltSizeInBits)), dl, LogicVT);
20455
20456 // First, clear all bits but the sign bit from the second operand (sign).
20457 if (IsFakeVector)
20458 Sign = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Sign);
20459 SDValue SignBit = DAG.getNode(X86ISD::FAND, dl, LogicVT, Sign, SignMask);
20460
20461 // Next, clear the sign bit from the first operand (magnitude).
20462 // TODO: If we had general constant folding for FP logic ops, this check
20463 // wouldn't be necessary.
20464 SDValue MagBits;
20465 if (ConstantFPSDNode *Op0CN = isConstOrConstSplatFP(Mag)) {
20466 APFloat APF = Op0CN->getValueAPF();
20467 APF.clearSign();
20468 MagBits = DAG.getConstantFP(APF, dl, LogicVT);
20469 } else {
20470 // If the magnitude operand wasn't a constant, we need to AND out the sign.
20471 if (IsFakeVector)
20472 Mag = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, LogicVT, Mag);
20473 MagBits = DAG.getNode(X86ISD::FAND, dl, LogicVT, Mag, MagMask);
20474 }
20475
20476 // OR the magnitude value with the sign bit.
20477 SDValue Or = DAG.getNode(X86ISD::FOR, dl, LogicVT, MagBits, SignBit);
20478 return !IsFakeVector ? Or : DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Or,
20479 DAG.getIntPtrConstant(0, dl));
20480}
20481
20482static SDValue LowerFGETSIGN(SDValue Op, SelectionDAG &DAG) {
20483 SDValue N0 = Op.getOperand(0);
20484 SDLoc dl(Op);
20485 MVT VT = Op.getSimpleValueType();
20486
20487 MVT OpVT = N0.getSimpleValueType();
20488 assert((OpVT == MVT::f32 || OpVT == MVT::f64) &&(((OpVT == MVT::f32 || OpVT == MVT::f64) && "Unexpected type for FGETSIGN"
) ? static_cast<void> (0) : __assert_fail ("(OpVT == MVT::f32 || OpVT == MVT::f64) && \"Unexpected type for FGETSIGN\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20489, __PRETTY_FUNCTION__))
20489 "Unexpected type for FGETSIGN")(((OpVT == MVT::f32 || OpVT == MVT::f64) && "Unexpected type for FGETSIGN"
) ? static_cast<void> (0) : __assert_fail ("(OpVT == MVT::f32 || OpVT == MVT::f64) && \"Unexpected type for FGETSIGN\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20489, __PRETTY_FUNCTION__));
20490
20491 // Lower ISD::FGETSIGN to (AND (X86ISD::MOVMSK ...) 1).
20492 MVT VecVT = (OpVT == MVT::f32 ? MVT::v4f32 : MVT::v2f64);
20493 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, N0);
20494 Res = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32, Res);
20495 Res = DAG.getZExtOrTrunc(Res, dl, VT);
20496 Res = DAG.getNode(ISD::AND, dl, VT, Res, DAG.getConstant(1, dl, VT));
20497 return Res;
20498}
20499
20500/// Helper for creating a X86ISD::SETCC node.
20501static SDValue getSETCC(X86::CondCode Cond, SDValue EFLAGS, const SDLoc &dl,
20502 SelectionDAG &DAG) {
20503 return DAG.getNode(X86ISD::SETCC, dl, MVT::i8,
20504 DAG.getTargetConstant(Cond, dl, MVT::i8), EFLAGS);
20505}
20506
20507/// Helper for matching OR(EXTRACTELT(X,0),OR(EXTRACTELT(X,1),...))
20508/// style scalarized (associative) reduction patterns.
20509static bool matchScalarReduction(SDValue Op, ISD::NodeType BinOp,
20510 SmallVectorImpl<SDValue> &SrcOps) {
20511 SmallVector<SDValue, 8> Opnds;
20512 DenseMap<SDValue, APInt> SrcOpMap;
20513 EVT VT = MVT::Other;
20514
20515 // Recognize a special case where a vector is casted into wide integer to
20516 // test all 0s.
20517 assert(Op.getOpcode() == unsigned(BinOp) &&((Op.getOpcode() == unsigned(BinOp) && "Unexpected bit reduction opcode"
) ? static_cast<void> (0) : __assert_fail ("Op.getOpcode() == unsigned(BinOp) && \"Unexpected bit reduction opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20518, __PRETTY_FUNCTION__))
20518 "Unexpected bit reduction opcode")((Op.getOpcode() == unsigned(BinOp) && "Unexpected bit reduction opcode"
) ? static_cast<void> (0) : __assert_fail ("Op.getOpcode() == unsigned(BinOp) && \"Unexpected bit reduction opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20518, __PRETTY_FUNCTION__));
20519 Opnds.push_back(Op.getOperand(0));
20520 Opnds.push_back(Op.getOperand(1));
20521
20522 for (unsigned Slot = 0, e = Opnds.size(); Slot < e; ++Slot) {
20523 SmallVectorImpl<SDValue>::const_iterator I = Opnds.begin() + Slot;
20524 // BFS traverse all BinOp operands.
20525 if (I->getOpcode() == unsigned(BinOp)) {
20526 Opnds.push_back(I->getOperand(0));
20527 Opnds.push_back(I->getOperand(1));
20528 // Re-evaluate the number of nodes to be traversed.
20529 e += 2; // 2 more nodes (LHS and RHS) are pushed.
20530 continue;
20531 }
20532
20533 // Quit if a non-EXTRACT_VECTOR_ELT
20534 if (I->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
20535 return false;
20536
20537 // Quit if without a constant index.
20538 SDValue Idx = I->getOperand(1);
20539 if (!isa<ConstantSDNode>(Idx))
20540 return false;
20541
20542 SDValue Src = I->getOperand(0);
20543 DenseMap<SDValue, APInt>::iterator M = SrcOpMap.find(Src);
20544 if (M == SrcOpMap.end()) {
20545 VT = Src.getValueType();
20546 // Quit if not the same type.
20547 if (SrcOpMap.begin() != SrcOpMap.end() &&
20548 VT != SrcOpMap.begin()->first.getValueType())
20549 return false;
20550 unsigned NumElts = VT.getVectorNumElements();
20551 APInt EltCount = APInt::getNullValue(NumElts);
20552 M = SrcOpMap.insert(std::make_pair(Src, EltCount)).first;
20553 SrcOps.push_back(Src);
20554 }
20555 // Quit if element already used.
20556 unsigned CIdx = cast<ConstantSDNode>(Idx)->getZExtValue();
20557 if (M->second[CIdx])
20558 return false;
20559 M->second.setBit(CIdx);
20560 }
20561
20562 // Quit if not all elements are used.
20563 for (DenseMap<SDValue, APInt>::const_iterator I = SrcOpMap.begin(),
20564 E = SrcOpMap.end();
20565 I != E; ++I) {
20566 if (!I->second.isAllOnesValue())
20567 return false;
20568 }
20569
20570 return true;
20571}
20572
20573// Check whether an OR'd tree is PTEST-able.
20574static SDValue LowerVectorAllZeroTest(SDValue Op, ISD::CondCode CC,
20575 const X86Subtarget &Subtarget,
20576 SelectionDAG &DAG, SDValue &X86CC) {
20577 assert(Op.getOpcode() == ISD::OR && "Only check OR'd tree.")((Op.getOpcode() == ISD::OR && "Only check OR'd tree."
) ? static_cast<void> (0) : __assert_fail ("Op.getOpcode() == ISD::OR && \"Only check OR'd tree.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20577, __PRETTY_FUNCTION__));
20578
20579 if (!Subtarget.hasSSE41() || !Op->hasOneUse())
20580 return SDValue();
20581
20582 SmallVector<SDValue, 8> VecIns;
20583 if (!matchScalarReduction(Op, ISD::OR, VecIns))
20584 return SDValue();
20585
20586 // Quit if not 128/256-bit vector.
20587 EVT VT = VecIns[0].getValueType();
20588 if (!VT.is128BitVector() && !VT.is256BitVector())
20589 return SDValue();
20590
20591 SDLoc DL(Op);
20592 MVT TestVT = VT.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
20593
20594 // Cast all vectors into TestVT for PTEST.
20595 for (unsigned i = 0, e = VecIns.size(); i < e; ++i)
20596 VecIns[i] = DAG.getBitcast(TestVT, VecIns[i]);
20597
20598 // If more than one full vector is evaluated, OR them first before PTEST.
20599 for (unsigned Slot = 0, e = VecIns.size(); e - Slot > 1; Slot += 2, e += 1) {
20600 // Each iteration will OR 2 nodes and append the result until there is only
20601 // 1 node left, i.e. the final OR'd value of all vectors.
20602 SDValue LHS = VecIns[Slot];
20603 SDValue RHS = VecIns[Slot + 1];
20604 VecIns.push_back(DAG.getNode(ISD::OR, DL, TestVT, LHS, RHS));
20605 }
20606
20607 X86CC = DAG.getTargetConstant(CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE,
20608 DL, MVT::i8);
20609 return DAG.getNode(X86ISD::PTEST, DL, MVT::i32, VecIns.back(), VecIns.back());
20610}
20611
20612/// return true if \c Op has a use that doesn't just read flags.
20613static bool hasNonFlagsUse(SDValue Op) {
20614 for (SDNode::use_iterator UI = Op->use_begin(), UE = Op->use_end(); UI != UE;
20615 ++UI) {
20616 SDNode *User = *UI;
20617 unsigned UOpNo = UI.getOperandNo();
20618 if (User->getOpcode() == ISD::TRUNCATE && User->hasOneUse()) {
20619 // Look pass truncate.
20620 UOpNo = User->use_begin().getOperandNo();
20621 User = *User->use_begin();
20622 }
20623
20624 if (User->getOpcode() != ISD::BRCOND && User->getOpcode() != ISD::SETCC &&
20625 !(User->getOpcode() == ISD::SELECT && UOpNo == 0))
20626 return true;
20627 }
20628 return false;
20629}
20630
20631// Transform to an x86-specific ALU node with flags if there is a chance of
20632// using an RMW op or only the flags are used. Otherwise, leave
20633// the node alone and emit a 'cmp' or 'test' instruction.
20634static bool isProfitableToUseFlagOp(SDValue Op) {
20635 for (SDNode *U : Op->uses())
20636 if (U->getOpcode() != ISD::CopyToReg &&
20637 U->getOpcode() != ISD::SETCC &&
20638 U->getOpcode() != ISD::STORE)
20639 return false;
20640
20641 return true;
20642}
20643
20644/// Emit nodes that will be selected as "test Op0,Op0", or something
20645/// equivalent.
20646static SDValue EmitTest(SDValue Op, unsigned X86CC, const SDLoc &dl,
20647 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
20648 // CF and OF aren't always set the way we want. Determine which
20649 // of these we need.
20650 bool NeedCF = false;
20651 bool NeedOF = false;
20652 switch (X86CC) {
20653 default: break;
20654 case X86::COND_A: case X86::COND_AE:
20655 case X86::COND_B: case X86::COND_BE:
20656 NeedCF = true;
20657 break;
20658 case X86::COND_G: case X86::COND_GE:
20659 case X86::COND_L: case X86::COND_LE:
20660 case X86::COND_O: case X86::COND_NO: {
20661 // Check if we really need to set the
20662 // Overflow flag. If NoSignedWrap is present
20663 // that is not actually needed.
20664 switch (Op->getOpcode()) {
20665 case ISD::ADD:
20666 case ISD::SUB:
20667 case ISD::MUL:
20668 case ISD::SHL:
20669 if (Op.getNode()->getFlags().hasNoSignedWrap())
20670 break;
20671 LLVM_FALLTHROUGH[[gnu::fallthrough]];
20672 default:
20673 NeedOF = true;
20674 break;
20675 }
20676 break;
20677 }
20678 }
20679 // See if we can use the EFLAGS value from the operand instead of
20680 // doing a separate TEST. TEST always sets OF and CF to 0, so unless
20681 // we prove that the arithmetic won't overflow, we can't use OF or CF.
20682 if (Op.getResNo() != 0 || NeedOF || NeedCF) {
20683 // Emit a CMP with 0, which is the TEST pattern.
20684 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
20685 DAG.getConstant(0, dl, Op.getValueType()));
20686 }
20687 unsigned Opcode = 0;
20688 unsigned NumOperands = 0;
20689
20690 SDValue ArithOp = Op;
20691
20692 // NOTICE: In the code below we use ArithOp to hold the arithmetic operation
20693 // which may be the result of a CAST. We use the variable 'Op', which is the
20694 // non-casted variable when we check for possible users.
20695 switch (ArithOp.getOpcode()) {
20696 case ISD::AND:
20697 // If the primary 'and' result isn't used, don't bother using X86ISD::AND,
20698 // because a TEST instruction will be better.
20699 if (!hasNonFlagsUse(Op))
20700 break;
20701
20702 LLVM_FALLTHROUGH[[gnu::fallthrough]];
20703 case ISD::ADD:
20704 case ISD::SUB:
20705 case ISD::OR:
20706 case ISD::XOR:
20707 if (!isProfitableToUseFlagOp(Op))
20708 break;
20709
20710 // Otherwise use a regular EFLAGS-setting instruction.
20711 switch (ArithOp.getOpcode()) {
20712 default: llvm_unreachable("unexpected operator!")::llvm::llvm_unreachable_internal("unexpected operator!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20712);
20713 case ISD::ADD: Opcode = X86ISD::ADD; break;
20714 case ISD::SUB: Opcode = X86ISD::SUB; break;
20715 case ISD::XOR: Opcode = X86ISD::XOR; break;
20716 case ISD::AND: Opcode = X86ISD::AND; break;
20717 case ISD::OR: Opcode = X86ISD::OR; break;
20718 }
20719
20720 NumOperands = 2;
20721 break;
20722 case X86ISD::ADD:
20723 case X86ISD::SUB:
20724 case X86ISD::OR:
20725 case X86ISD::XOR:
20726 case X86ISD::AND:
20727 return SDValue(Op.getNode(), 1);
20728 case ISD::SSUBO:
20729 case ISD::USUBO: {
20730 // /USUBO/SSUBO will become a X86ISD::SUB and we can use its Z flag.
20731 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
20732 return DAG.getNode(X86ISD::SUB, dl, VTs, Op->getOperand(0),
20733 Op->getOperand(1)).getValue(1);
20734 }
20735 default:
20736 break;
20737 }
20738
20739 if (Opcode == 0) {
20740 // Emit a CMP with 0, which is the TEST pattern.
20741 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
20742 DAG.getConstant(0, dl, Op.getValueType()));
20743 }
20744 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
20745 SmallVector<SDValue, 4> Ops(Op->op_begin(), Op->op_begin() + NumOperands);
20746
20747 SDValue New = DAG.getNode(Opcode, dl, VTs, Ops);
20748 DAG.ReplaceAllUsesOfValueWith(SDValue(Op.getNode(), 0), New);
20749 return SDValue(New.getNode(), 1);
20750}
20751
20752/// Emit nodes that will be selected as "cmp Op0,Op1", or something
20753/// equivalent.
20754static std::pair<SDValue, SDValue> EmitCmp(SDValue Op0, SDValue Op1,
20755 unsigned X86CC, const SDLoc &dl,
20756 SelectionDAG &DAG,
20757 const X86Subtarget &Subtarget,
20758 SDValue Chain, bool IsSignaling) {
20759 if (isNullConstant(Op1))
20760 return std::make_pair(EmitTest(Op0, X86CC, dl, DAG, Subtarget), Chain);
20761
20762 EVT CmpVT = Op0.getValueType();
20763
20764 if (CmpVT.isFloatingPoint()) {
20765 if (Chain) {
20766 SDValue Res =
20767 DAG.getNode(IsSignaling ? X86ISD::STRICT_FCMPS : X86ISD::STRICT_FCMP,
20768 dl, {MVT::i32, MVT::Other}, {Chain, Op0, Op1});
20769 return std::make_pair(Res, Res.getValue(1));
20770 }
20771 return std::make_pair(DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op0, Op1),
20772 SDValue());
20773 }
20774
20775 assert((CmpVT == MVT::i8 || CmpVT == MVT::i16 ||(((CmpVT == MVT::i8 || CmpVT == MVT::i16 || CmpVT == MVT::i32
|| CmpVT == MVT::i64) && "Unexpected VT!") ? static_cast
<void> (0) : __assert_fail ("(CmpVT == MVT::i8 || CmpVT == MVT::i16 || CmpVT == MVT::i32 || CmpVT == MVT::i64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20776, __PRETTY_FUNCTION__))
20776 CmpVT == MVT::i32 || CmpVT == MVT::i64) && "Unexpected VT!")(((CmpVT == MVT::i8 || CmpVT == MVT::i16 || CmpVT == MVT::i32
|| CmpVT == MVT::i64) && "Unexpected VT!") ? static_cast
<void> (0) : __assert_fail ("(CmpVT == MVT::i8 || CmpVT == MVT::i16 || CmpVT == MVT::i32 || CmpVT == MVT::i64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20776, __PRETTY_FUNCTION__));
20777
20778 // Only promote the compare up to I32 if it is a 16 bit operation
20779 // with an immediate. 16 bit immediates are to be avoided.
20780 if (CmpVT == MVT::i16 && !Subtarget.isAtom() &&
20781 !DAG.getMachineFunction().getFunction().hasMinSize()) {
20782 ConstantSDNode *COp0 = dyn_cast<ConstantSDNode>(Op0);
20783 ConstantSDNode *COp1 = dyn_cast<ConstantSDNode>(Op1);
20784 // Don't do this if the immediate can fit in 8-bits.
20785 if ((COp0 && !COp0->getAPIntValue().isSignedIntN(8)) ||
20786 (COp1 && !COp1->getAPIntValue().isSignedIntN(8))) {
20787 unsigned ExtendOp =
20788 isX86CCSigned(X86CC) ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
20789 if (X86CC == X86::COND_E || X86CC == X86::COND_NE) {
20790 // For equality comparisons try to use SIGN_EXTEND if the input was
20791 // truncate from something with enough sign bits.
20792 if (Op0.getOpcode() == ISD::TRUNCATE) {
20793 SDValue In = Op0.getOperand(0);
20794 unsigned EffBits =
20795 In.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(In) + 1;
20796 if (EffBits <= 16)
20797 ExtendOp = ISD::SIGN_EXTEND;
20798 } else if (Op1.getOpcode() == ISD::TRUNCATE) {
20799 SDValue In = Op1.getOperand(0);
20800 unsigned EffBits =
20801 In.getScalarValueSizeInBits() - DAG.ComputeNumSignBits(In) + 1;
20802 if (EffBits <= 16)
20803 ExtendOp = ISD::SIGN_EXTEND;
20804 }
20805 }
20806
20807 CmpVT = MVT::i32;
20808 Op0 = DAG.getNode(ExtendOp, dl, CmpVT, Op0);
20809 Op1 = DAG.getNode(ExtendOp, dl, CmpVT, Op1);
20810 }
20811 }
20812
20813 // Try to shrink i64 compares if the input has enough zero bits.
20814 // FIXME: Do this for non-constant compares for constant on LHS?
20815 if (CmpVT == MVT::i64 && isa<ConstantSDNode>(Op1) && !isX86CCSigned(X86CC) &&
20816 Op0.hasOneUse() && // Hacky way to not break CSE opportunities with sub.
20817 cast<ConstantSDNode>(Op1)->getAPIntValue().getActiveBits() <= 32 &&
20818 DAG.MaskedValueIsZero(Op0, APInt::getHighBitsSet(64, 32))) {
20819 CmpVT = MVT::i32;
20820 Op0 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op0);
20821 Op1 = DAG.getNode(ISD::TRUNCATE, dl, CmpVT, Op1);
20822 }
20823
20824 // Use SUB instead of CMP to enable CSE between SUB and CMP.
20825 SDVTList VTs = DAG.getVTList(CmpVT, MVT::i32);
20826 SDValue Sub = DAG.getNode(X86ISD::SUB, dl, VTs, Op0, Op1);
20827 return std::make_pair(Sub.getValue(1), SDValue());
20828}
20829
20830/// Convert a comparison if required by the subtarget.
20831SDValue X86TargetLowering::ConvertCmpIfNecessary(SDValue Cmp,
20832 SelectionDAG &DAG) const {
20833 // If the subtarget does not support the FUCOMI instruction, floating-point
20834 // comparisons have to be converted.
20835 bool IsCmp = Cmp.getOpcode() == X86ISD::CMP;
20836 bool IsStrictCmp = Cmp.getOpcode() == X86ISD::STRICT_FCMP ||
20837 Cmp.getOpcode() == X86ISD::STRICT_FCMPS;
20838
20839 if (Subtarget.hasCMov() || (!IsCmp && !IsStrictCmp) ||
20840 !Cmp.getOperand(IsStrictCmp ? 1 : 0).getValueType().isFloatingPoint() ||
20841 !Cmp.getOperand(IsStrictCmp ? 2 : 1).getValueType().isFloatingPoint())
20842 return Cmp;
20843
20844 // The instruction selector will select an FUCOM instruction instead of
20845 // FUCOMI, which writes the comparison result to FPSW instead of EFLAGS. Hence
20846 // build an SDNode sequence that transfers the result from FPSW into EFLAGS:
20847 // (X86sahf (trunc (srl (X86fp_stsw (trunc (X86any_fcmp ...)), 8))))
20848 SDLoc dl(Cmp);
20849 SDValue TruncFPSW = DAG.getNode(ISD::TRUNCATE, dl, MVT::i16, Cmp);
20850 SDValue FNStSW = DAG.getNode(X86ISD::FNSTSW16r, dl, MVT::i16, TruncFPSW);
20851 SDValue Srl = DAG.getNode(ISD::SRL, dl, MVT::i16, FNStSW,
20852 DAG.getConstant(8, dl, MVT::i8));
20853 SDValue TruncSrl = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Srl);
20854
20855 // Some 64-bit targets lack SAHF support, but they do support FCOMI.
20856 assert(Subtarget.hasLAHFSAHF() && "Target doesn't support SAHF or FCOMI?")((Subtarget.hasLAHFSAHF() && "Target doesn't support SAHF or FCOMI?"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasLAHFSAHF() && \"Target doesn't support SAHF or FCOMI?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20856, __PRETTY_FUNCTION__));
20857 return DAG.getNode(X86ISD::SAHF, dl, MVT::i32, TruncSrl);
20858}
20859
20860/// Check if replacement of SQRT with RSQRT should be disabled.
20861bool X86TargetLowering::isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const {
20862 EVT VT = Op.getValueType();
20863
20864 // We never want to use both SQRT and RSQRT instructions for the same input.
20865 if (DAG.getNodeIfExists(X86ISD::FRSQRT, DAG.getVTList(VT), Op))
20866 return false;
20867
20868 if (VT.isVector())
20869 return Subtarget.hasFastVectorFSQRT();
20870 return Subtarget.hasFastScalarFSQRT();
20871}
20872
20873/// The minimum architected relative accuracy is 2^-12. We need one
20874/// Newton-Raphson step to have a good float result (24 bits of precision).
20875SDValue X86TargetLowering::getSqrtEstimate(SDValue Op,
20876 SelectionDAG &DAG, int Enabled,
20877 int &RefinementSteps,
20878 bool &UseOneConstNR,
20879 bool Reciprocal) const {
20880 EVT VT = Op.getValueType();
20881
20882 // SSE1 has rsqrtss and rsqrtps. AVX adds a 256-bit variant for rsqrtps.
20883 // It is likely not profitable to do this for f64 because a double-precision
20884 // rsqrt estimate with refinement on x86 prior to FMA requires at least 16
20885 // instructions: convert to single, rsqrtss, convert back to double, refine
20886 // (3 steps = at least 13 insts). If an 'rsqrtsd' variant was added to the ISA
20887 // along with FMA, this could be a throughput win.
20888 // TODO: SQRT requires SSE2 to prevent the introduction of an illegal v4i32
20889 // after legalize types.
20890 if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
20891 (VT == MVT::v4f32 && Subtarget.hasSSE1() && Reciprocal) ||
20892 (VT == MVT::v4f32 && Subtarget.hasSSE2() && !Reciprocal) ||
20893 (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
20894 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
20895 if (RefinementSteps == ReciprocalEstimate::Unspecified)
20896 RefinementSteps = 1;
20897
20898 UseOneConstNR = false;
20899 // There is no FSQRT for 512-bits, but there is RSQRT14.
20900 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RSQRT14 : X86ISD::FRSQRT;
20901 return DAG.getNode(Opcode, SDLoc(Op), VT, Op);
20902 }
20903 return SDValue();
20904}
20905
20906/// The minimum architected relative accuracy is 2^-12. We need one
20907/// Newton-Raphson step to have a good float result (24 bits of precision).
20908SDValue X86TargetLowering::getRecipEstimate(SDValue Op, SelectionDAG &DAG,
20909 int Enabled,
20910 int &RefinementSteps) const {
20911 EVT VT = Op.getValueType();
20912
20913 // SSE1 has rcpss and rcpps. AVX adds a 256-bit variant for rcpps.
20914 // It is likely not profitable to do this for f64 because a double-precision
20915 // reciprocal estimate with refinement on x86 prior to FMA requires
20916 // 15 instructions: convert to single, rcpss, convert back to double, refine
20917 // (3 steps = 12 insts). If an 'rcpsd' variant was added to the ISA
20918 // along with FMA, this could be a throughput win.
20919
20920 if ((VT == MVT::f32 && Subtarget.hasSSE1()) ||
20921 (VT == MVT::v4f32 && Subtarget.hasSSE1()) ||
20922 (VT == MVT::v8f32 && Subtarget.hasAVX()) ||
20923 (VT == MVT::v16f32 && Subtarget.useAVX512Regs())) {
20924 // Enable estimate codegen with 1 refinement step for vector division.
20925 // Scalar division estimates are disabled because they break too much
20926 // real-world code. These defaults are intended to match GCC behavior.
20927 if (VT == MVT::f32 && Enabled == ReciprocalEstimate::Unspecified)
20928 return SDValue();
20929
20930 if (RefinementSteps == ReciprocalEstimate::Unspecified)
20931 RefinementSteps = 1;
20932
20933 // There is no FSQRT for 512-bits, but there is RCP14.
20934 unsigned Opcode = VT == MVT::v16f32 ? X86ISD::RCP14 : X86ISD::FRCP;
20935 return DAG.getNode(Opcode, SDLoc(Op), VT, Op);
20936 }
20937 return SDValue();
20938}
20939
20940/// If we have at least two divisions that use the same divisor, convert to
20941/// multiplication by a reciprocal. This may need to be adjusted for a given
20942/// CPU if a division's cost is not at least twice the cost of a multiplication.
20943/// This is because we still need one division to calculate the reciprocal and
20944/// then we need two multiplies by that reciprocal as replacements for the
20945/// original divisions.
20946unsigned X86TargetLowering::combineRepeatedFPDivisors() const {
20947 return 2;
20948}
20949
20950SDValue
20951X86TargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
20952 SelectionDAG &DAG,
20953 SmallVectorImpl<SDNode *> &Created) const {
20954 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
20955 if (isIntDivCheap(N->getValueType(0), Attr))
20956 return SDValue(N,0); // Lower SDIV as SDIV
20957
20958 assert((Divisor.isPowerOf2() || (-Divisor).isPowerOf2()) &&(((Divisor.isPowerOf2() || (-Divisor).isPowerOf2()) &&
"Unexpected divisor!") ? static_cast<void> (0) : __assert_fail
("(Divisor.isPowerOf2() || (-Divisor).isPowerOf2()) && \"Unexpected divisor!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20959, __PRETTY_FUNCTION__))
20959 "Unexpected divisor!")(((Divisor.isPowerOf2() || (-Divisor).isPowerOf2()) &&
"Unexpected divisor!") ? static_cast<void> (0) : __assert_fail
("(Divisor.isPowerOf2() || (-Divisor).isPowerOf2()) && \"Unexpected divisor!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 20959, __PRETTY_FUNCTION__));
20960
20961 // Only perform this transform if CMOV is supported otherwise the select
20962 // below will become a branch.
20963 if (!Subtarget.hasCMov())
20964 return SDValue();
20965
20966 // fold (sdiv X, pow2)
20967 EVT VT = N->getValueType(0);
20968 // FIXME: Support i8.
20969 if (VT != MVT::i16 && VT != MVT::i32 &&
20970 !(Subtarget.is64Bit() && VT == MVT::i64))
20971 return SDValue();
20972
20973 unsigned Lg2 = Divisor.countTrailingZeros();
20974
20975 // If the divisor is 2 or -2, the default expansion is better.
20976 if (Lg2 == 1)
20977 return SDValue();
20978
20979 SDLoc DL(N);
20980 SDValue N0 = N->getOperand(0);
20981 SDValue Zero = DAG.getConstant(0, DL, VT);
20982 APInt Lg2Mask = APInt::getLowBitsSet(VT.getSizeInBits(), Lg2);
20983 SDValue Pow2MinusOne = DAG.getConstant(Lg2Mask, DL, VT);
20984
20985 // If N0 is negative, we need to add (Pow2 - 1) to it before shifting right.
20986 SDValue Cmp = DAG.getSetCC(DL, MVT::i8, N0, Zero, ISD::SETLT);
20987 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, N0, Pow2MinusOne);
20988 SDValue CMov = DAG.getNode(ISD::SELECT, DL, VT, Cmp, Add, N0);
20989
20990 Created.push_back(Cmp.getNode());
20991 Created.push_back(Add.getNode());
20992 Created.push_back(CMov.getNode());
20993
20994 // Divide by pow2.
20995 SDValue SRA =
20996 DAG.getNode(ISD::SRA, DL, VT, CMov, DAG.getConstant(Lg2, DL, MVT::i64));
20997
20998 // If we're dividing by a positive value, we're done. Otherwise, we must
20999 // negate the result.
21000 if (Divisor.isNonNegative())
21001 return SRA;
21002
21003 Created.push_back(SRA.getNode());
21004 return DAG.getNode(ISD::SUB, DL, VT, Zero, SRA);
21005}
21006
21007/// Result of 'and' is compared against zero. Change to a BT node if possible.
21008/// Returns the BT node and the condition code needed to use it.
21009static SDValue LowerAndToBT(SDValue And, ISD::CondCode CC,
21010 const SDLoc &dl, SelectionDAG &DAG,
21011 SDValue &X86CC) {
21012 assert(And.getOpcode() == ISD::AND && "Expected AND node!")((And.getOpcode() == ISD::AND && "Expected AND node!"
) ? static_cast<void> (0) : __assert_fail ("And.getOpcode() == ISD::AND && \"Expected AND node!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21012, __PRETTY_FUNCTION__));
21013 SDValue Op0 = And.getOperand(0);
21014 SDValue Op1 = And.getOperand(1);
21015 if (Op0.getOpcode() == ISD::TRUNCATE)
21016 Op0 = Op0.getOperand(0);
21017 if (Op1.getOpcode() == ISD::TRUNCATE)
21018 Op1 = Op1.getOperand(0);
21019
21020 SDValue Src, BitNo;
21021 if (Op1.getOpcode() == ISD::SHL)
21022 std::swap(Op0, Op1);
21023 if (Op0.getOpcode() == ISD::SHL) {
21024 if (isOneConstant(Op0.getOperand(0))) {
21025 // If we looked past a truncate, check that it's only truncating away
21026 // known zeros.
21027 unsigned BitWidth = Op0.getValueSizeInBits();
21028 unsigned AndBitWidth = And.getValueSizeInBits();
21029 if (BitWidth > AndBitWidth) {
21030 KnownBits Known = DAG.computeKnownBits(Op0);
21031 if (Known.countMinLeadingZeros() < BitWidth - AndBitWidth)
21032 return SDValue();
21033 }
21034 Src = Op1;
21035 BitNo = Op0.getOperand(1);
21036 }
21037 } else if (Op1.getOpcode() == ISD::Constant) {
21038 ConstantSDNode *AndRHS = cast<ConstantSDNode>(Op1);
21039 uint64_t AndRHSVal = AndRHS->getZExtValue();
21040 SDValue AndLHS = Op0;
21041
21042 if (AndRHSVal == 1 && AndLHS.getOpcode() == ISD::SRL) {
21043 Src = AndLHS.getOperand(0);
21044 BitNo = AndLHS.getOperand(1);
21045 } else {
21046 // Use BT if the immediate can't be encoded in a TEST instruction or we
21047 // are optimizing for size and the immedaite won't fit in a byte.
21048 bool OptForSize = DAG.shouldOptForSize();
21049 if ((!isUInt<32>(AndRHSVal) || (OptForSize && !isUInt<8>(AndRHSVal))) &&
21050 isPowerOf2_64(AndRHSVal)) {
21051 Src = AndLHS;
21052 BitNo = DAG.getConstant(Log2_64_Ceil(AndRHSVal), dl,
21053 Src.getValueType());
21054 }
21055 }
21056 }
21057
21058 // No patterns found, give up.
21059 if (!Src.getNode())
21060 return SDValue();
21061
21062 // If Src is i8, promote it to i32 with any_extend. There is no i8 BT
21063 // instruction. Since the shift amount is in-range-or-undefined, we know
21064 // that doing a bittest on the i32 value is ok. We extend to i32 because
21065 // the encoding for the i16 version is larger than the i32 version.
21066 // Also promote i16 to i32 for performance / code size reason.
21067 if (Src.getValueType() == MVT::i8 || Src.getValueType() == MVT::i16)
21068 Src = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i32, Src);
21069
21070 // See if we can use the 32-bit instruction instead of the 64-bit one for a
21071 // shorter encoding. Since the former takes the modulo 32 of BitNo and the
21072 // latter takes the modulo 64, this is only valid if the 5th bit of BitNo is
21073 // known to be zero.
21074 if (Src.getValueType() == MVT::i64 &&
21075 DAG.MaskedValueIsZero(BitNo, APInt(BitNo.getValueSizeInBits(), 32)))
21076 Src = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, Src);
21077
21078 // If the operand types disagree, extend the shift amount to match. Since
21079 // BT ignores high bits (like shifts) we can use anyextend.
21080 if (Src.getValueType() != BitNo.getValueType())
21081 BitNo = DAG.getNode(ISD::ANY_EXTEND, dl, Src.getValueType(), BitNo);
21082
21083 X86CC = DAG.getTargetConstant(CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B,
21084 dl, MVT::i8);
21085 return DAG.getNode(X86ISD::BT, dl, MVT::i32, Src, BitNo);
21086}
21087
21088/// Turns an ISD::CondCode into a value suitable for SSE floating-point mask
21089/// CMPs.
21090static unsigned translateX86FSETCC(ISD::CondCode SetCCOpcode, SDValue &Op0,
21091 SDValue &Op1, bool &IsAlwaysSignaling) {
21092 unsigned SSECC;
21093 bool Swap = false;
21094
21095 // SSE Condition code mapping:
21096 // 0 - EQ
21097 // 1 - LT
21098 // 2 - LE
21099 // 3 - UNORD
21100 // 4 - NEQ
21101 // 5 - NLT
21102 // 6 - NLE
21103 // 7 - ORD
21104 switch (SetCCOpcode) {
21105 default: llvm_unreachable("Unexpected SETCC condition")::llvm::llvm_unreachable_internal("Unexpected SETCC condition"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21105);
21106 case ISD::SETOEQ:
21107 case ISD::SETEQ: SSECC = 0; break;
21108 case ISD::SETOGT:
21109 case ISD::SETGT: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
21110 case ISD::SETLT:
21111 case ISD::SETOLT: SSECC = 1; break;
21112 case ISD::SETOGE:
21113 case ISD::SETGE: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
21114 case ISD::SETLE:
21115 case ISD::SETOLE: SSECC = 2; break;
21116 case ISD::SETUO: SSECC = 3; break;
21117 case ISD::SETUNE:
21118 case ISD::SETNE: SSECC = 4; break;
21119 case ISD::SETULE: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
21120 case ISD::SETUGE: SSECC = 5; break;
21121 case ISD::SETULT: Swap = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
21122 case ISD::SETUGT: SSECC = 6; break;
21123 case ISD::SETO: SSECC = 7; break;
21124 case ISD::SETUEQ: SSECC = 8; break;
21125 case ISD::SETONE: SSECC = 12; break;
21126 }
21127 if (Swap)
21128 std::swap(Op0, Op1);
21129
21130 switch (SetCCOpcode) {
21131 default:
21132 IsAlwaysSignaling = true;
21133 break;
21134 case ISD::SETEQ:
21135 case ISD::SETOEQ:
21136 case ISD::SETUEQ:
21137 case ISD::SETNE:
21138 case ISD::SETONE:
21139 case ISD::SETUNE:
21140 case ISD::SETO:
21141 case ISD::SETUO:
21142 IsAlwaysSignaling = false;
21143 break;
21144 }
21145
21146 return SSECC;
21147}
21148
21149/// Break a VSETCC 256-bit integer VSETCC into two new 128 ones and then
21150/// concatenate the result back.
21151static SDValue Lower256IntVSETCC(SDValue Op, SelectionDAG &DAG) {
21152 MVT VT = Op.getSimpleValueType();
21153
21154 assert(VT.is256BitVector() && Op.getOpcode() == ISD::SETCC &&((VT.is256BitVector() && Op.getOpcode() == ISD::SETCC
&& "Unsupported value type for operation") ? static_cast
<void> (0) : __assert_fail ("VT.is256BitVector() && Op.getOpcode() == ISD::SETCC && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21155, __PRETTY_FUNCTION__))
21155 "Unsupported value type for operation")((VT.is256BitVector() && Op.getOpcode() == ISD::SETCC
&& "Unsupported value type for operation") ? static_cast
<void> (0) : __assert_fail ("VT.is256BitVector() && Op.getOpcode() == ISD::SETCC && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21155, __PRETTY_FUNCTION__));
21156
21157 unsigned NumElems = VT.getVectorNumElements();
21158 SDLoc dl(Op);
21159 SDValue CC = Op.getOperand(2);
21160
21161 // Extract the LHS vectors
21162 SDValue LHS = Op.getOperand(0);
21163 SDValue LHS1 = extract128BitVector(LHS, 0, DAG, dl);
21164 SDValue LHS2 = extract128BitVector(LHS, NumElems / 2, DAG, dl);
21165
21166 // Extract the RHS vectors
21167 SDValue RHS = Op.getOperand(1);
21168 SDValue RHS1 = extract128BitVector(RHS, 0, DAG, dl);
21169 SDValue RHS2 = extract128BitVector(RHS, NumElems / 2, DAG, dl);
21170
21171 // Issue the operation on the smaller types and concatenate the result back
21172 MVT EltVT = VT.getVectorElementType();
21173 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
21174 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
21175 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1, CC),
21176 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2, CC));
21177}
21178
21179static SDValue LowerIntVSETCC_AVX512(SDValue Op, SelectionDAG &DAG) {
21180
21181 SDValue Op0 = Op.getOperand(0);
21182 SDValue Op1 = Op.getOperand(1);
21183 SDValue CC = Op.getOperand(2);
21184 MVT VT = Op.getSimpleValueType();
21185 SDLoc dl(Op);
21186
21187 assert(VT.getVectorElementType() == MVT::i1 &&((VT.getVectorElementType() == MVT::i1 && "Cannot set masked compare for this operation"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == MVT::i1 && \"Cannot set masked compare for this operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21188, __PRETTY_FUNCTION__))
21188 "Cannot set masked compare for this operation")((VT.getVectorElementType() == MVT::i1 && "Cannot set masked compare for this operation"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == MVT::i1 && \"Cannot set masked compare for this operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21188, __PRETTY_FUNCTION__));
21189
21190 ISD::CondCode SetCCOpcode = cast<CondCodeSDNode>(CC)->get();
21191
21192 // Prefer SETGT over SETLT.
21193 if (SetCCOpcode == ISD::SETLT) {
21194 SetCCOpcode = ISD::getSetCCSwappedOperands(SetCCOpcode);
21195 std::swap(Op0, Op1);
21196 }
21197
21198 return DAG.getSetCC(dl, VT, Op0, Op1, SetCCOpcode);
21199}
21200
21201/// Given a buildvector constant, return a new vector constant with each element
21202/// incremented or decremented. If incrementing or decrementing would result in
21203/// unsigned overflow or underflow or this is not a simple vector constant,
21204/// return an empty value.
21205static SDValue incDecVectorConstant(SDValue V, SelectionDAG &DAG, bool IsInc) {
21206 auto *BV = dyn_cast<BuildVectorSDNode>(V.getNode());
21207 if (!BV)
21208 return SDValue();
21209
21210 MVT VT = V.getSimpleValueType();
21211 MVT EltVT = VT.getVectorElementType();
21212 unsigned NumElts = VT.getVectorNumElements();
21213 SmallVector<SDValue, 8> NewVecC;
21214 SDLoc DL(V);
21215 for (unsigned i = 0; i < NumElts; ++i) {
21216 auto *Elt = dyn_cast<ConstantSDNode>(BV->getOperand(i));
21217 if (!Elt || Elt->isOpaque() || Elt->getSimpleValueType(0) != EltVT)
21218 return SDValue();
21219
21220 // Avoid overflow/underflow.
21221 const APInt &EltC = Elt->getAPIntValue();
21222 if ((IsInc && EltC.isMaxValue()) || (!IsInc && EltC.isNullValue()))
21223 return SDValue();
21224
21225 NewVecC.push_back(DAG.getConstant(EltC + (IsInc ? 1 : -1), DL, EltVT));
21226 }
21227
21228 return DAG.getBuildVector(VT, DL, NewVecC);
21229}
21230
21231/// As another special case, use PSUBUS[BW] when it's profitable. E.g. for
21232/// Op0 u<= Op1:
21233/// t = psubus Op0, Op1
21234/// pcmpeq t, <0..0>
21235static SDValue LowerVSETCCWithSUBUS(SDValue Op0, SDValue Op1, MVT VT,
21236 ISD::CondCode Cond, const SDLoc &dl,
21237 const X86Subtarget &Subtarget,
21238 SelectionDAG &DAG) {
21239 if (!Subtarget.hasSSE2())
21240 return SDValue();
21241
21242 MVT VET = VT.getVectorElementType();
21243 if (VET != MVT::i8 && VET != MVT::i16)
21244 return SDValue();
21245
21246 switch (Cond) {
21247 default:
21248 return SDValue();
21249 case ISD::SETULT: {
21250 // If the comparison is against a constant we can turn this into a
21251 // setule. With psubus, setule does not require a swap. This is
21252 // beneficial because the constant in the register is no longer
21253 // destructed as the destination so it can be hoisted out of a loop.
21254 // Only do this pre-AVX since vpcmp* is no longer destructive.
21255 if (Subtarget.hasAVX())
21256 return SDValue();
21257 SDValue ULEOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/false);
21258 if (!ULEOp1)
21259 return SDValue();
21260 Op1 = ULEOp1;
21261 break;
21262 }
21263 case ISD::SETUGT: {
21264 // If the comparison is against a constant, we can turn this into a setuge.
21265 // This is beneficial because materializing a constant 0 for the PCMPEQ is
21266 // probably cheaper than XOR+PCMPGT using 2 different vector constants:
21267 // cmpgt (xor X, SignMaskC) CmpC --> cmpeq (usubsat (CmpC+1), X), 0
21268 SDValue UGEOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/true);
21269 if (!UGEOp1)
21270 return SDValue();
21271 Op1 = Op0;
21272 Op0 = UGEOp1;
21273 break;
21274 }
21275 // Psubus is better than flip-sign because it requires no inversion.
21276 case ISD::SETUGE:
21277 std::swap(Op0, Op1);
21278 break;
21279 case ISD::SETULE:
21280 break;
21281 }
21282
21283 SDValue Result = DAG.getNode(ISD::USUBSAT, dl, VT, Op0, Op1);
21284 return DAG.getNode(X86ISD::PCMPEQ, dl, VT, Result,
21285 DAG.getConstant(0, dl, VT));
21286}
21287
21288static SDValue LowerVSETCC(SDValue Op, const X86Subtarget &Subtarget,
21289 SelectionDAG &DAG) {
21290 bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
21291 Op.getOpcode() == ISD::STRICT_FSETCCS;
21292 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
21293 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
21294 SDValue CC = Op.getOperand(IsStrict ? 3 : 2);
21295 MVT VT = Op->getSimpleValueType(0);
21296 ISD::CondCode Cond = cast<CondCodeSDNode>(CC)->get();
21297 bool isFP = Op1.getSimpleValueType().isFloatingPoint();
21298 SDLoc dl(Op);
21299
21300 if (isFP) {
21301#ifndef NDEBUG
21302 MVT EltVT = Op0.getSimpleValueType().getVectorElementType();
21303 assert(EltVT == MVT::f32 || EltVT == MVT::f64)((EltVT == MVT::f32 || EltVT == MVT::f64) ? static_cast<void
> (0) : __assert_fail ("EltVT == MVT::f32 || EltVT == MVT::f64"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21303, __PRETTY_FUNCTION__));
21304#endif
21305
21306 bool IsSignaling = Op.getOpcode() == ISD::STRICT_FSETCCS;
21307 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21308
21309 unsigned Opc;
21310 if (Subtarget.hasAVX512() && VT.getVectorElementType() == MVT::i1) {
21311 assert(VT.getVectorNumElements() <= 16)((VT.getVectorNumElements() <= 16) ? static_cast<void>
(0) : __assert_fail ("VT.getVectorNumElements() <= 16", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21311, __PRETTY_FUNCTION__));
21312 Opc = IsStrict ? X86ISD::STRICT_CMPM : X86ISD::CMPM;
21313 } else {
21314 Opc = IsStrict ? X86ISD::STRICT_CMPP : X86ISD::CMPP;
21315 // The SSE/AVX packed FP comparison nodes are defined with a
21316 // floating-point vector result that matches the operand type. This allows
21317 // them to work with an SSE1 target (integer vector types are not legal).
21318 VT = Op0.getSimpleValueType();
21319 }
21320
21321 SDValue Cmp;
21322 bool IsAlwaysSignaling;
21323 unsigned SSECC = translateX86FSETCC(Cond, Op0, Op1, IsAlwaysSignaling);
21324 if (!Subtarget.hasAVX()) {
21325 // TODO: We could use following steps to handle a quiet compare with
21326 // signaling encodings.
21327 // 1. Get ordered masks from a quiet ISD::SETO
21328 // 2. Use the masks to mask potential unordered elements in operand A, B
21329 // 3. Get the compare results of masked A, B
21330 // 4. Calculating final result using the mask and result from 3
21331 // But currently, we just fall back to scalar operations.
21332 if (IsStrict && IsAlwaysSignaling && !IsSignaling)
21333 return SDValue();
21334
21335 // Insert an extra signaling instruction to raise exception.
21336 if (IsStrict && !IsAlwaysSignaling && IsSignaling) {
21337 SDValue SignalCmp = DAG.getNode(
21338 Opc, dl, {VT, MVT::Other},
21339 {Chain, Op0, Op1, DAG.getTargetConstant(1, dl, MVT::i8)}); // LT_OS
21340 // FIXME: It seems we need to update the flags of all new strict nodes.
21341 // Otherwise, mayRaiseFPException in MI will return false due to
21342 // NoFPExcept = false by default. However, I didn't find it in other
21343 // patches.
21344 SignalCmp->setFlags(Op->getFlags());
21345 Chain = SignalCmp.getValue(1);
21346 }
21347
21348 // In the two cases not handled by SSE compare predicates (SETUEQ/SETONE),
21349 // emit two comparisons and a logic op to tie them together.
21350 if (SSECC >= 8) {
21351 // LLVM predicate is SETUEQ or SETONE.
21352 unsigned CC0, CC1;
21353 unsigned CombineOpc;
21354 if (Cond == ISD::SETUEQ) {
21355 CC0 = 3; // UNORD
21356 CC1 = 0; // EQ
21357 CombineOpc = X86ISD::FOR;
21358 } else {
21359 assert(Cond == ISD::SETONE)((Cond == ISD::SETONE) ? static_cast<void> (0) : __assert_fail
("Cond == ISD::SETONE", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21359, __PRETTY_FUNCTION__));
21360 CC0 = 7; // ORD
21361 CC1 = 4; // NEQ
21362 CombineOpc = X86ISD::FAND;
21363 }
21364
21365 SDValue Cmp0, Cmp1;
21366 if (IsStrict) {
21367 Cmp0 = DAG.getNode(
21368 Opc, dl, {VT, MVT::Other},
21369 {Chain, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8)});
21370 Cmp1 = DAG.getNode(
21371 Opc, dl, {VT, MVT::Other},
21372 {Chain, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8)});
21373 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Cmp0.getValue(1),
21374 Cmp1.getValue(1));
21375 } else {
21376 Cmp0 = DAG.getNode(
21377 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC0, dl, MVT::i8));
21378 Cmp1 = DAG.getNode(
21379 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(CC1, dl, MVT::i8));
21380 }
21381 Cmp = DAG.getNode(CombineOpc, dl, VT, Cmp0, Cmp1);
21382 } else {
21383 if (IsStrict) {
21384 Cmp = DAG.getNode(
21385 Opc, dl, {VT, MVT::Other},
21386 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
21387 Chain = Cmp.getValue(1);
21388 } else
21389 Cmp = DAG.getNode(
21390 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
21391 }
21392 } else {
21393 // Handle all other FP comparisons here.
21394 if (IsStrict) {
21395 // Make a flip on already signaling CCs before setting bit 4 of AVX CC.
21396 SSECC |= (IsAlwaysSignaling ^ IsSignaling) << 4;
21397 Cmp = DAG.getNode(
21398 Opc, dl, {VT, MVT::Other},
21399 {Chain, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8)});
21400 Chain = Cmp.getValue(1);
21401 } else
21402 Cmp = DAG.getNode(
21403 Opc, dl, VT, Op0, Op1, DAG.getTargetConstant(SSECC, dl, MVT::i8));
21404 }
21405
21406 // If this is SSE/AVX CMPP, bitcast the result back to integer to match the
21407 // result type of SETCC. The bitcast is expected to be optimized away
21408 // during combining/isel.
21409 Cmp = DAG.getBitcast(Op.getSimpleValueType(), Cmp);
21410
21411 if (IsStrict)
21412 return DAG.getMergeValues({Cmp, Chain}, dl);
21413
21414 return Cmp;
21415 }
21416
21417 assert(!IsStrict && "Strict SETCC only handles FP operands.")((!IsStrict && "Strict SETCC only handles FP operands."
) ? static_cast<void> (0) : __assert_fail ("!IsStrict && \"Strict SETCC only handles FP operands.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21417, __PRETTY_FUNCTION__));
21418
21419 MVT VTOp0 = Op0.getSimpleValueType();
21420 (void)VTOp0;
21421 assert(VTOp0 == Op1.getSimpleValueType() &&((VTOp0 == Op1.getSimpleValueType() && "Expected operands with same type!"
) ? static_cast<void> (0) : __assert_fail ("VTOp0 == Op1.getSimpleValueType() && \"Expected operands with same type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21422, __PRETTY_FUNCTION__))
21422 "Expected operands with same type!")((VTOp0 == Op1.getSimpleValueType() && "Expected operands with same type!"
) ? static_cast<void> (0) : __assert_fail ("VTOp0 == Op1.getSimpleValueType() && \"Expected operands with same type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21422, __PRETTY_FUNCTION__));
21423 assert(VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&((VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&
"Invalid number of packed elements for source and destination!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == VTOp0.getVectorNumElements() && \"Invalid number of packed elements for source and destination!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21424, __PRETTY_FUNCTION__))
21424 "Invalid number of packed elements for source and destination!")((VT.getVectorNumElements() == VTOp0.getVectorNumElements() &&
"Invalid number of packed elements for source and destination!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorNumElements() == VTOp0.getVectorNumElements() && \"Invalid number of packed elements for source and destination!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21424, __PRETTY_FUNCTION__));
21425
21426 // The non-AVX512 code below works under the assumption that source and
21427 // destination types are the same.
21428 assert((Subtarget.hasAVX512() || (VT == VTOp0)) &&(((Subtarget.hasAVX512() || (VT == VTOp0)) && "Value types for source and destination must be the same!"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasAVX512() || (VT == VTOp0)) && \"Value types for source and destination must be the same!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21429, __PRETTY_FUNCTION__))
21429 "Value types for source and destination must be the same!")(((Subtarget.hasAVX512() || (VT == VTOp0)) && "Value types for source and destination must be the same!"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.hasAVX512() || (VT == VTOp0)) && \"Value types for source and destination must be the same!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21429, __PRETTY_FUNCTION__));
21430
21431 // The result is boolean, but operands are int/float
21432 if (VT.getVectorElementType() == MVT::i1) {
21433 // In AVX-512 architecture setcc returns mask with i1 elements,
21434 // But there is no compare instruction for i8 and i16 elements in KNL.
21435 assert((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) &&(((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()
) && "Unexpected operand type") ? static_cast<void
> (0) : __assert_fail ("(VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) && \"Unexpected operand type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21436, __PRETTY_FUNCTION__))
21436 "Unexpected operand type")(((VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()
) && "Unexpected operand type") ? static_cast<void
> (0) : __assert_fail ("(VTOp0.getScalarSizeInBits() >= 32 || Subtarget.hasBWI()) && \"Unexpected operand type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21436, __PRETTY_FUNCTION__));
21437 return LowerIntVSETCC_AVX512(Op, DAG);
21438 }
21439
21440 // Lower using XOP integer comparisons.
21441 if (VT.is128BitVector() && Subtarget.hasXOP()) {
21442 // Translate compare code to XOP PCOM compare mode.
21443 unsigned CmpMode = 0;
21444 switch (Cond) {
21445 default: llvm_unreachable("Unexpected SETCC condition")::llvm::llvm_unreachable_internal("Unexpected SETCC condition"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21445);
21446 case ISD::SETULT:
21447 case ISD::SETLT: CmpMode = 0x00; break;
21448 case ISD::SETULE:
21449 case ISD::SETLE: CmpMode = 0x01; break;
21450 case ISD::SETUGT:
21451 case ISD::SETGT: CmpMode = 0x02; break;
21452 case ISD::SETUGE:
21453 case ISD::SETGE: CmpMode = 0x03; break;
21454 case ISD::SETEQ: CmpMode = 0x04; break;
21455 case ISD::SETNE: CmpMode = 0x05; break;
21456 }
21457
21458 // Are we comparing unsigned or signed integers?
21459 unsigned Opc =
21460 ISD::isUnsignedIntSetCC(Cond) ? X86ISD::VPCOMU : X86ISD::VPCOM;
21461
21462 return DAG.getNode(Opc, dl, VT, Op0, Op1,
21463 DAG.getTargetConstant(CmpMode, dl, MVT::i8));
21464 }
21465
21466 // (X & Y) != 0 --> (X & Y) == Y iff Y is power-of-2.
21467 // Revert part of the simplifySetCCWithAnd combine, to avoid an invert.
21468 if (Cond == ISD::SETNE && ISD::isBuildVectorAllZeros(Op1.getNode())) {
21469 SDValue BC0 = peekThroughBitcasts(Op0);
21470 if (BC0.getOpcode() == ISD::AND) {
21471 APInt UndefElts;
21472 SmallVector<APInt, 64> EltBits;
21473 if (getTargetConstantBitsFromNode(BC0.getOperand(1),
21474 VT.getScalarSizeInBits(), UndefElts,
21475 EltBits, false, false)) {
21476 if (llvm::all_of(EltBits, [](APInt &V) { return V.isPowerOf2(); })) {
21477 Cond = ISD::SETEQ;
21478 Op1 = DAG.getBitcast(VT, BC0.getOperand(1));
21479 }
21480 }
21481 }
21482 }
21483
21484 // ICMP_EQ(AND(X,C),C) -> SRA(SHL(X,LOG2(C)),BW-1) iff C is power-of-2.
21485 if (Cond == ISD::SETEQ && Op0.getOpcode() == ISD::AND &&
21486 Op0.getOperand(1) == Op1 && Op0.hasOneUse()) {
21487 ConstantSDNode *C1 = isConstOrConstSplat(Op1);
21488 if (C1 && C1->getAPIntValue().isPowerOf2()) {
21489 unsigned BitWidth = VT.getScalarSizeInBits();
21490 unsigned ShiftAmt = BitWidth - C1->getAPIntValue().logBase2() - 1;
21491
21492 SDValue Result = Op0.getOperand(0);
21493 Result = DAG.getNode(ISD::SHL, dl, VT, Result,
21494 DAG.getConstant(ShiftAmt, dl, VT));
21495 Result = DAG.getNode(ISD::SRA, dl, VT, Result,
21496 DAG.getConstant(BitWidth - 1, dl, VT));
21497 return Result;
21498 }
21499 }
21500
21501 // Break 256-bit integer vector compare into smaller ones.
21502 if (VT.is256BitVector() && !Subtarget.hasInt256())
21503 return Lower256IntVSETCC(Op, DAG);
21504
21505 // If this is a SETNE against the signed minimum value, change it to SETGT.
21506 // If this is a SETNE against the signed maximum value, change it to SETLT.
21507 // which will be swapped to SETGT.
21508 // Otherwise we use PCMPEQ+invert.
21509 APInt ConstValue;
21510 if (Cond == ISD::SETNE &&
21511 ISD::isConstantSplatVector(Op1.getNode(), ConstValue)) {
21512 if (ConstValue.isMinSignedValue())
21513 Cond = ISD::SETGT;
21514 else if (ConstValue.isMaxSignedValue())
21515 Cond = ISD::SETLT;
21516 }
21517
21518 // If both operands are known non-negative, then an unsigned compare is the
21519 // same as a signed compare and there's no need to flip signbits.
21520 // TODO: We could check for more general simplifications here since we're
21521 // computing known bits.
21522 bool FlipSigns = ISD::isUnsignedIntSetCC(Cond) &&
21523 !(DAG.SignBitIsZero(Op0) && DAG.SignBitIsZero(Op1));
21524
21525 // Special case: Use min/max operations for unsigned compares.
21526 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
21527 if (ISD::isUnsignedIntSetCC(Cond) &&
21528 (FlipSigns || ISD::isTrueWhenEqual(Cond)) &&
21529 TLI.isOperationLegal(ISD::UMIN, VT)) {
21530 // If we have a constant operand, increment/decrement it and change the
21531 // condition to avoid an invert.
21532 if (Cond == ISD::SETUGT) {
21533 // X > C --> X >= (C+1) --> X == umax(X, C+1)
21534 if (SDValue UGTOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/true)) {
21535 Op1 = UGTOp1;
21536 Cond = ISD::SETUGE;
21537 }
21538 }
21539 if (Cond == ISD::SETULT) {
21540 // X < C --> X <= (C-1) --> X == umin(X, C-1)
21541 if (SDValue ULTOp1 = incDecVectorConstant(Op1, DAG, /*IsInc*/false)) {
21542 Op1 = ULTOp1;
21543 Cond = ISD::SETULE;
21544 }
21545 }
21546 bool Invert = false;
21547 unsigned Opc;
21548 switch (Cond) {
21549 default: llvm_unreachable("Unexpected condition code")::llvm::llvm_unreachable_internal("Unexpected condition code"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21549);
21550 case ISD::SETUGT: Invert = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
21551 case ISD::SETULE: Opc = ISD::UMIN; break;
21552 case ISD::SETULT: Invert = true; LLVM_FALLTHROUGH[[gnu::fallthrough]];
21553 case ISD::SETUGE: Opc = ISD::UMAX; break;
21554 }
21555
21556 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
21557 Result = DAG.getNode(X86ISD::PCMPEQ, dl, VT, Op0, Result);
21558
21559 // If the logical-not of the result is required, perform that now.
21560 if (Invert)
21561 Result = DAG.getNOT(dl, Result, VT);
21562
21563 return Result;
21564 }
21565
21566 // Try to use SUBUS and PCMPEQ.
21567 if (SDValue V = LowerVSETCCWithSUBUS(Op0, Op1, VT, Cond, dl, Subtarget, DAG))
21568 return V;
21569
21570 // We are handling one of the integer comparisons here. Since SSE only has
21571 // GT and EQ comparisons for integer, swapping operands and multiple
21572 // operations may be required for some comparisons.
21573 unsigned Opc = (Cond == ISD::SETEQ || Cond == ISD::SETNE) ? X86ISD::PCMPEQ
21574 : X86ISD::PCMPGT;
21575 bool Swap = Cond == ISD::SETLT || Cond == ISD::SETULT ||
21576 Cond == ISD::SETGE || Cond == ISD::SETUGE;
21577 bool Invert = Cond == ISD::SETNE ||
21578 (Cond != ISD::SETEQ && ISD::isTrueWhenEqual(Cond));
21579
21580 if (Swap)
21581 std::swap(Op0, Op1);
21582
21583 // Check that the operation in question is available (most are plain SSE2,
21584 // but PCMPGTQ and PCMPEQQ have different requirements).
21585 if (VT == MVT::v2i64) {
21586 if (Opc == X86ISD::PCMPGT && !Subtarget.hasSSE42()) {
21587 assert(Subtarget.hasSSE2() && "Don't know how to lower!")((Subtarget.hasSSE2() && "Don't know how to lower!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Don't know how to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21587, __PRETTY_FUNCTION__));
21588
21589 // Special case for sign bit test. We can use a v4i32 PCMPGT and shuffle
21590 // the odd elements over the even elements.
21591 if (!FlipSigns && !Invert && ISD::isBuildVectorAllZeros(Op0.getNode())) {
21592 Op0 = DAG.getConstant(0, dl, MVT::v4i32);
21593 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
21594
21595 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
21596 static const int MaskHi[] = { 1, 1, 3, 3 };
21597 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
21598
21599 return DAG.getBitcast(VT, Result);
21600 }
21601
21602 if (!FlipSigns && !Invert && ISD::isBuildVectorAllOnes(Op1.getNode())) {
21603 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
21604 Op1 = DAG.getConstant(-1, dl, MVT::v4i32);
21605
21606 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
21607 static const int MaskHi[] = { 1, 1, 3, 3 };
21608 SDValue Result = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
21609
21610 return DAG.getBitcast(VT, Result);
21611 }
21612
21613 // Since SSE has no unsigned integer comparisons, we need to flip the sign
21614 // bits of the inputs before performing those operations. The lower
21615 // compare is always unsigned.
21616 SDValue SB;
21617 if (FlipSigns) {
21618 SB = DAG.getConstant(0x8000000080000000ULL, dl, MVT::v2i64);
21619 } else {
21620 SB = DAG.getConstant(0x0000000080000000ULL, dl, MVT::v2i64);
21621 }
21622 Op0 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op0, SB);
21623 Op1 = DAG.getNode(ISD::XOR, dl, MVT::v2i64, Op1, SB);
21624
21625 // Cast everything to the right type.
21626 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
21627 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
21628
21629 // Emulate PCMPGTQ with (hi1 > hi2) | ((hi1 == hi2) & (lo1 > lo2))
21630 SDValue GT = DAG.getNode(X86ISD::PCMPGT, dl, MVT::v4i32, Op0, Op1);
21631 SDValue EQ = DAG.getNode(X86ISD::PCMPEQ, dl, MVT::v4i32, Op0, Op1);
21632
21633 // Create masks for only the low parts/high parts of the 64 bit integers.
21634 static const int MaskHi[] = { 1, 1, 3, 3 };
21635 static const int MaskLo[] = { 0, 0, 2, 2 };
21636 SDValue EQHi = DAG.getVectorShuffle(MVT::v4i32, dl, EQ, EQ, MaskHi);
21637 SDValue GTLo = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskLo);
21638 SDValue GTHi = DAG.getVectorShuffle(MVT::v4i32, dl, GT, GT, MaskHi);
21639
21640 SDValue Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, EQHi, GTLo);
21641 Result = DAG.getNode(ISD::OR, dl, MVT::v4i32, Result, GTHi);
21642
21643 if (Invert)
21644 Result = DAG.getNOT(dl, Result, MVT::v4i32);
21645
21646 return DAG.getBitcast(VT, Result);
21647 }
21648
21649 if (Opc == X86ISD::PCMPEQ && !Subtarget.hasSSE41()) {
21650 // If pcmpeqq is missing but pcmpeqd is available synthesize pcmpeqq with
21651 // pcmpeqd + pshufd + pand.
21652 assert(Subtarget.hasSSE2() && !FlipSigns && "Don't know how to lower!")((Subtarget.hasSSE2() && !FlipSigns && "Don't know how to lower!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && !FlipSigns && \"Don't know how to lower!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21652, __PRETTY_FUNCTION__));
21653
21654 // First cast everything to the right type.
21655 Op0 = DAG.getBitcast(MVT::v4i32, Op0);
21656 Op1 = DAG.getBitcast(MVT::v4i32, Op1);
21657
21658 // Do the compare.
21659 SDValue Result = DAG.getNode(Opc, dl, MVT::v4i32, Op0, Op1);
21660
21661 // Make sure the lower and upper halves are both all-ones.
21662 static const int Mask[] = { 1, 0, 3, 2 };
21663 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Result, Result, Mask);
21664 Result = DAG.getNode(ISD::AND, dl, MVT::v4i32, Result, Shuf);
21665
21666 if (Invert)
21667 Result = DAG.getNOT(dl, Result, MVT::v4i32);
21668
21669 return DAG.getBitcast(VT, Result);
21670 }
21671 }
21672
21673 // Since SSE has no unsigned integer comparisons, we need to flip the sign
21674 // bits of the inputs before performing those operations.
21675 if (FlipSigns) {
21676 MVT EltVT = VT.getVectorElementType();
21677 SDValue SM = DAG.getConstant(APInt::getSignMask(EltVT.getSizeInBits()), dl,
21678 VT);
21679 Op0 = DAG.getNode(ISD::XOR, dl, VT, Op0, SM);
21680 Op1 = DAG.getNode(ISD::XOR, dl, VT, Op1, SM);
21681 }
21682
21683 SDValue Result = DAG.getNode(Opc, dl, VT, Op0, Op1);
21684
21685 // If the logical-not of the result is required, perform that now.
21686 if (Invert)
21687 Result = DAG.getNOT(dl, Result, VT);
21688
21689 return Result;
21690}
21691
21692// Try to select this as a KORTEST+SETCC or KTEST+SETCC if possible.
21693static SDValue EmitAVX512Test(SDValue Op0, SDValue Op1, ISD::CondCode CC,
21694 const SDLoc &dl, SelectionDAG &DAG,
21695 const X86Subtarget &Subtarget,
21696 SDValue &X86CC) {
21697 // Only support equality comparisons.
21698 if (CC != ISD::SETEQ && CC != ISD::SETNE)
21699 return SDValue();
21700
21701 // Must be a bitcast from vXi1.
21702 if (Op0.getOpcode() != ISD::BITCAST)
21703 return SDValue();
21704
21705 Op0 = Op0.getOperand(0);
21706 MVT VT = Op0.getSimpleValueType();
21707 if (!(Subtarget.hasAVX512() && VT == MVT::v16i1) &&
21708 !(Subtarget.hasDQI() && VT == MVT::v8i1) &&
21709 !(Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1)))
21710 return SDValue();
21711
21712 X86::CondCode X86Cond;
21713 if (isNullConstant(Op1)) {
21714 X86Cond = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
21715 } else if (isAllOnesConstant(Op1)) {
21716 // C flag is set for all ones.
21717 X86Cond = CC == ISD::SETEQ ? X86::COND_B : X86::COND_AE;
21718 } else
21719 return SDValue();
21720
21721 // If the input is an AND, we can combine it's operands into the KTEST.
21722 bool KTestable = false;
21723 if (Subtarget.hasDQI() && (VT == MVT::v8i1 || VT == MVT::v16i1))
21724 KTestable = true;
21725 if (Subtarget.hasBWI() && (VT == MVT::v32i1 || VT == MVT::v64i1))
21726 KTestable = true;
21727 if (!isNullConstant(Op1))
21728 KTestable = false;
21729 if (KTestable && Op0.getOpcode() == ISD::AND && Op0.hasOneUse()) {
21730 SDValue LHS = Op0.getOperand(0);
21731 SDValue RHS = Op0.getOperand(1);
21732 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
21733 return DAG.getNode(X86ISD::KTEST, dl, MVT::i32, LHS, RHS);
21734 }
21735
21736 // If the input is an OR, we can combine it's operands into the KORTEST.
21737 SDValue LHS = Op0;
21738 SDValue RHS = Op0;
21739 if (Op0.getOpcode() == ISD::OR && Op0.hasOneUse()) {
21740 LHS = Op0.getOperand(0);
21741 RHS = Op0.getOperand(1);
21742 }
21743
21744 X86CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
21745 return DAG.getNode(X86ISD::KORTEST, dl, MVT::i32, LHS, RHS);
21746}
21747
21748/// Emit flags for the given setcc condition and operands. Also returns the
21749/// corresponding X86 condition code constant in X86CC.
21750SDValue X86TargetLowering::emitFlagsForSetcc(SDValue Op0, SDValue Op1,
21751 ISD::CondCode CC, const SDLoc &dl,
21752 SelectionDAG &DAG, SDValue &X86CC,
21753 SDValue &Chain,
21754 bool IsSignaling) const {
21755 // Optimize to BT if possible.
21756 // Lower (X & (1 << N)) == 0 to BT(X, N).
21757 // Lower ((X >>u N) & 1) != 0 to BT(X, N).
21758 // Lower ((X >>s N) & 1) != 0 to BT(X, N).
21759 if (Op0.getOpcode() == ISD::AND && Op0.hasOneUse() && isNullConstant(Op1) &&
21760 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
21761 if (SDValue BT = LowerAndToBT(Op0, CC, dl, DAG, X86CC))
21762 return BT;
21763 }
21764
21765 // Try to use PTEST for a tree ORs equality compared with 0.
21766 // TODO: We could do AND tree with all 1s as well by using the C flag.
21767 if (Op0.getOpcode() == ISD::OR && isNullConstant(Op1) &&
21768 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
21769 if (SDValue PTEST = LowerVectorAllZeroTest(Op0, CC, Subtarget, DAG, X86CC))
21770 return PTEST;
21771 }
21772
21773 // Try to lower using KORTEST or KTEST.
21774 if (SDValue Test = EmitAVX512Test(Op0, Op1, CC, dl, DAG, Subtarget, X86CC))
21775 return Test;
21776
21777 // Look for X == 0, X == 1, X != 0, or X != 1. We can simplify some forms of
21778 // these.
21779 if ((isOneConstant(Op1) || isNullConstant(Op1)) &&
21780 (CC == ISD::SETEQ || CC == ISD::SETNE)) {
21781 // If the input is a setcc, then reuse the input setcc or use a new one with
21782 // the inverted condition.
21783 if (Op0.getOpcode() == X86ISD::SETCC) {
21784 bool Invert = (CC == ISD::SETNE) ^ isNullConstant(Op1);
21785
21786 X86CC = Op0.getOperand(0);
21787 if (Invert) {
21788 X86::CondCode CCode = (X86::CondCode)Op0.getConstantOperandVal(0);
21789 CCode = X86::GetOppositeBranchCondition(CCode);
21790 X86CC = DAG.getTargetConstant(CCode, dl, MVT::i8);
21791 }
21792
21793 return Op0.getOperand(1);
21794 }
21795 }
21796
21797 // Try to use the carry flag from the add in place of an separate CMP for:
21798 // (seteq (add X, -1), -1). Similar for setne.
21799 if (isAllOnesConstant(Op1) && Op0.getOpcode() == ISD::ADD &&
21800 Op0.getOperand(1) == Op1 && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
21801 if (isProfitableToUseFlagOp(Op0)) {
21802 SDVTList VTs = DAG.getVTList(Op0.getValueType(), MVT::i32);
21803
21804 SDValue New = DAG.getNode(X86ISD::ADD, dl, VTs, Op0.getOperand(0),
21805 Op0.getOperand(1));
21806 DAG.ReplaceAllUsesOfValueWith(SDValue(Op0.getNode(), 0), New);
21807 X86::CondCode CCode = CC == ISD::SETEQ ? X86::COND_AE : X86::COND_B;
21808 X86CC = DAG.getTargetConstant(CCode, dl, MVT::i8);
21809 return SDValue(New.getNode(), 1);
21810 }
21811 }
21812
21813 bool IsFP = Op1.getSimpleValueType().isFloatingPoint();
21814 X86::CondCode CondCode = TranslateX86CC(CC, dl, IsFP, Op0, Op1, DAG);
21815 if (CondCode == X86::COND_INVALID)
21816 return SDValue();
21817
21818 std::pair<SDValue, SDValue> Tmp =
21819 EmitCmp(Op0, Op1, CondCode, dl, DAG, Subtarget, Chain, IsSignaling);
21820 SDValue EFLAGS = Tmp.first;
21821 if (Chain)
21822 Chain = Tmp.second;
21823 EFLAGS = ConvertCmpIfNecessary(EFLAGS, DAG);
21824 X86CC = DAG.getTargetConstant(CondCode, dl, MVT::i8);
21825 return EFLAGS;
21826}
21827
21828SDValue X86TargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
21829
21830 bool IsStrict = Op.getOpcode() == ISD::STRICT_FSETCC ||
21831 Op.getOpcode() == ISD::STRICT_FSETCCS;
21832 MVT VT = Op->getSimpleValueType(0);
21833
21834 if (VT.isVector()) return LowerVSETCC(Op, Subtarget, DAG);
21835
21836 assert(VT == MVT::i8 && "SetCC type must be 8-bit integer")((VT == MVT::i8 && "SetCC type must be 8-bit integer"
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::i8 && \"SetCC type must be 8-bit integer\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21836, __PRETTY_FUNCTION__));
21837 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
21838 SDValue Op0 = Op.getOperand(IsStrict ? 1 : 0);
21839 SDValue Op1 = Op.getOperand(IsStrict ? 2 : 1);
21840 SDLoc dl(Op);
21841 ISD::CondCode CC =
21842 cast<CondCodeSDNode>(Op.getOperand(IsStrict ? 3 : 2))->get();
21843
21844 // Handle f128 first, since one possible outcome is a normal integer
21845 // comparison which gets handled by emitFlagsForSetcc.
21846 if (Op0.getValueType() == MVT::f128) {
21847 softenSetCCOperands(DAG, MVT::f128, Op0, Op1, CC, dl, Op0, Op1, Chain,
21848 Op.getOpcode() == ISD::STRICT_FSETCCS);
21849
21850 // If softenSetCCOperands returned a scalar, use it.
21851 if (!Op1.getNode()) {
21852 assert(Op0.getValueType() == Op.getValueType() &&((Op0.getValueType() == Op.getValueType() && "Unexpected setcc expansion!"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType() == Op.getValueType() && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21853, __PRETTY_FUNCTION__))
21853 "Unexpected setcc expansion!")((Op0.getValueType() == Op.getValueType() && "Unexpected setcc expansion!"
) ? static_cast<void> (0) : __assert_fail ("Op0.getValueType() == Op.getValueType() && \"Unexpected setcc expansion!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21853, __PRETTY_FUNCTION__));
21854 if (IsStrict)
21855 return DAG.getMergeValues({Op0, Chain}, dl);
21856 return Op0;
21857 }
21858 }
21859
21860 SDValue X86CC;
21861 SDValue EFLAGS = emitFlagsForSetcc(Op0, Op1, CC, dl, DAG, X86CC, Chain,
21862 Op.getOpcode() == ISD::STRICT_FSETCCS);
21863 if (!EFLAGS)
21864 return SDValue();
21865
21866 SDValue Res = DAG.getNode(X86ISD::SETCC, dl, MVT::i8, X86CC, EFLAGS);
21867
21868 if (IsStrict)
21869 return DAG.getMergeValues({Res, Chain}, dl);
21870
21871 return Res;
21872}
21873
21874SDValue X86TargetLowering::LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const {
21875 SDValue LHS = Op.getOperand(0);
21876 SDValue RHS = Op.getOperand(1);
21877 SDValue Carry = Op.getOperand(2);
21878 SDValue Cond = Op.getOperand(3);
21879 SDLoc DL(Op);
21880
21881 assert(LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only.")((LHS.getSimpleValueType().isInteger() && "SETCCCARRY is integer only."
) ? static_cast<void> (0) : __assert_fail ("LHS.getSimpleValueType().isInteger() && \"SETCCCARRY is integer only.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21881, __PRETTY_FUNCTION__));
21882 X86::CondCode CC = TranslateIntegerX86CC(cast<CondCodeSDNode>(Cond)->get());
21883
21884 // Recreate the carry if needed.
21885 EVT CarryVT = Carry.getValueType();
21886 APInt NegOne = APInt::getAllOnesValue(CarryVT.getScalarSizeInBits());
21887 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
21888 Carry, DAG.getConstant(NegOne, DL, CarryVT));
21889
21890 SDVTList VTs = DAG.getVTList(LHS.getValueType(), MVT::i32);
21891 SDValue Cmp = DAG.getNode(X86ISD::SBB, DL, VTs, LHS, RHS, Carry.getValue(1));
21892 return getSETCC(CC, Cmp.getValue(1), DL, DAG);
21893}
21894
21895// This function returns three things: the arithmetic computation itself
21896// (Value), an EFLAGS result (Overflow), and a condition code (Cond). The
21897// flag and the condition code define the case in which the arithmetic
21898// computation overflows.
21899static std::pair<SDValue, SDValue>
21900getX86XALUOOp(X86::CondCode &Cond, SDValue Op, SelectionDAG &DAG) {
21901 assert(Op.getResNo() == 0 && "Unexpected result number!")((Op.getResNo() == 0 && "Unexpected result number!") ?
static_cast<void> (0) : __assert_fail ("Op.getResNo() == 0 && \"Unexpected result number!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21901, __PRETTY_FUNCTION__));
21902 SDValue Value, Overflow;
21903 SDValue LHS = Op.getOperand(0);
21904 SDValue RHS = Op.getOperand(1);
21905 unsigned BaseOp = 0;
21906 SDLoc DL(Op);
21907 switch (Op.getOpcode()) {
21908 default: llvm_unreachable("Unknown ovf instruction!")::llvm::llvm_unreachable_internal("Unknown ovf instruction!",
"/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21908);
21909 case ISD::SADDO:
21910 BaseOp = X86ISD::ADD;
21911 Cond = X86::COND_O;
21912 break;
21913 case ISD::UADDO:
21914 BaseOp = X86ISD::ADD;
21915 Cond = isOneConstant(RHS) ? X86::COND_E : X86::COND_B;
21916 break;
21917 case ISD::SSUBO:
21918 BaseOp = X86ISD::SUB;
21919 Cond = X86::COND_O;
21920 break;
21921 case ISD::USUBO:
21922 BaseOp = X86ISD::SUB;
21923 Cond = X86::COND_B;
21924 break;
21925 case ISD::SMULO:
21926 BaseOp = X86ISD::SMUL;
21927 Cond = X86::COND_O;
21928 break;
21929 case ISD::UMULO:
21930 BaseOp = X86ISD::UMUL;
21931 Cond = X86::COND_O;
21932 break;
21933 }
21934
21935 if (BaseOp) {
21936 // Also sets EFLAGS.
21937 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
21938 Value = DAG.getNode(BaseOp, DL, VTs, LHS, RHS);
21939 Overflow = Value.getValue(1);
21940 }
21941
21942 return std::make_pair(Value, Overflow);
21943}
21944
21945static SDValue LowerXALUO(SDValue Op, SelectionDAG &DAG) {
21946 // Lower the "add/sub/mul with overflow" instruction into a regular ins plus
21947 // a "setcc" instruction that checks the overflow flag. The "brcond" lowering
21948 // looks for this combo and may remove the "setcc" instruction if the "setcc"
21949 // has only one use.
21950 SDLoc DL(Op);
21951 X86::CondCode Cond;
21952 SDValue Value, Overflow;
21953 std::tie(Value, Overflow) = getX86XALUOOp(Cond, Op, DAG);
21954
21955 SDValue SetCC = getSETCC(Cond, Overflow, DL, DAG);
21956 assert(Op->getValueType(1) == MVT::i8 && "Unexpected VT!")((Op->getValueType(1) == MVT::i8 && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("Op->getValueType(1) == MVT::i8 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 21956, __PRETTY_FUNCTION__));
21957 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(), Value, SetCC);
21958}
21959
21960/// Return true if opcode is a X86 logical comparison.
21961static bool isX86LogicalCmp(SDValue Op) {
21962 unsigned Opc = Op.getOpcode();
21963 if (Opc == X86ISD::CMP || Opc == X86ISD::COMI || Opc == X86ISD::UCOMI ||
21964 Opc == X86ISD::SAHF)
21965 return true;
21966 if (Op.getResNo() == 1 &&
21967 (Opc == X86ISD::ADD || Opc == X86ISD::SUB || Opc == X86ISD::ADC ||
21968 Opc == X86ISD::SBB || Opc == X86ISD::SMUL || Opc == X86ISD::UMUL ||
21969 Opc == X86ISD::OR || Opc == X86ISD::XOR || Opc == X86ISD::AND))
21970 return true;
21971
21972 return false;
21973}
21974
21975static bool isTruncWithZeroHighBitsInput(SDValue V, SelectionDAG &DAG) {
21976 if (V.getOpcode() != ISD::TRUNCATE)
21977 return false;
21978
21979 SDValue VOp0 = V.getOperand(0);
21980 unsigned InBits = VOp0.getValueSizeInBits();
21981 unsigned Bits = V.getValueSizeInBits();
21982 return DAG.MaskedValueIsZero(VOp0, APInt::getHighBitsSet(InBits,InBits-Bits));
21983}
21984
21985SDValue X86TargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const {
21986 bool AddTest = true;
21987 SDValue Cond = Op.getOperand(0);
21988 SDValue Op1 = Op.getOperand(1);
21989 SDValue Op2 = Op.getOperand(2);
21990 SDLoc DL(Op);
21991 MVT VT = Op1.getSimpleValueType();
21992 SDValue CC;
21993
21994 // Lower FP selects into a CMP/AND/ANDN/OR sequence when the necessary SSE ops
21995 // are available or VBLENDV if AVX is available.
21996 // Otherwise FP cmovs get lowered into a less efficient branch sequence later.
21997 if (Cond.getOpcode() == ISD::SETCC &&
21998 ((Subtarget.hasSSE2() && VT == MVT::f64) ||
21999 (Subtarget.hasSSE1() && VT == MVT::f32)) &&
22000 VT == Cond.getOperand(0).getSimpleValueType() && Cond->hasOneUse()) {
22001 SDValue CondOp0 = Cond.getOperand(0), CondOp1 = Cond.getOperand(1);
22002 bool IsAlwaysSignaling;
22003 unsigned SSECC =
22004 translateX86FSETCC(cast<CondCodeSDNode>(Cond.getOperand(2))->get(),
22005 CondOp0, CondOp1, IsAlwaysSignaling);
22006
22007 if (Subtarget.hasAVX512()) {
22008 SDValue Cmp =
22009 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CondOp0, CondOp1,
22010 DAG.getTargetConstant(SSECC, DL, MVT::i8));
22011 assert(!VT.isVector() && "Not a scalar type?")((!VT.isVector() && "Not a scalar type?") ? static_cast
<void> (0) : __assert_fail ("!VT.isVector() && \"Not a scalar type?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22011, __PRETTY_FUNCTION__));
22012 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
22013 }
22014
22015 if (SSECC < 8 || Subtarget.hasAVX()) {
22016 SDValue Cmp = DAG.getNode(X86ISD::FSETCC, DL, VT, CondOp0, CondOp1,
22017 DAG.getTargetConstant(SSECC, DL, MVT::i8));
22018
22019 // If we have AVX, we can use a variable vector select (VBLENDV) instead
22020 // of 3 logic instructions for size savings and potentially speed.
22021 // Unfortunately, there is no scalar form of VBLENDV.
22022
22023 // If either operand is a +0.0 constant, don't try this. We can expect to
22024 // optimize away at least one of the logic instructions later in that
22025 // case, so that sequence would be faster than a variable blend.
22026
22027 // BLENDV was introduced with SSE 4.1, but the 2 register form implicitly
22028 // uses XMM0 as the selection register. That may need just as many
22029 // instructions as the AND/ANDN/OR sequence due to register moves, so
22030 // don't bother.
22031 if (Subtarget.hasAVX() && !isNullFPConstant(Op1) &&
22032 !isNullFPConstant(Op2)) {
22033 // Convert to vectors, do a VSELECT, and convert back to scalar.
22034 // All of the conversions should be optimized away.
22035 MVT VecVT = VT == MVT::f32 ? MVT::v4f32 : MVT::v2f64;
22036 SDValue VOp1 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op1);
22037 SDValue VOp2 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Op2);
22038 SDValue VCmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Cmp);
22039
22040 MVT VCmpVT = VT == MVT::f32 ? MVT::v4i32 : MVT::v2i64;
22041 VCmp = DAG.getBitcast(VCmpVT, VCmp);
22042
22043 SDValue VSel = DAG.getSelect(DL, VecVT, VCmp, VOp1, VOp2);
22044
22045 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
22046 VSel, DAG.getIntPtrConstant(0, DL));
22047 }
22048 SDValue AndN = DAG.getNode(X86ISD::FANDN, DL, VT, Cmp, Op2);
22049 SDValue And = DAG.getNode(X86ISD::FAND, DL, VT, Cmp, Op1);
22050 return DAG.getNode(X86ISD::FOR, DL, VT, AndN, And);
22051 }
22052 }
22053
22054 // AVX512 fallback is to lower selects of scalar floats to masked moves.
22055 if ((VT == MVT::f64 || VT == MVT::f32) && Subtarget.hasAVX512()) {
22056 SDValue Cmp = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1, Cond);
22057 return DAG.getNode(X86ISD::SELECTS, DL, VT, Cmp, Op1, Op2);
22058 }
22059
22060 // For v64i1 without 64-bit support we need to split and rejoin.
22061 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
22062 assert(Subtarget.hasBWI() && "Expected BWI to be legal")((Subtarget.hasBWI() && "Expected BWI to be legal") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI() && \"Expected BWI to be legal\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22062, __PRETTY_FUNCTION__));
22063 SDValue Op1Lo = extractSubVector(Op1, 0, DAG, DL, 32);
22064 SDValue Op2Lo = extractSubVector(Op2, 0, DAG, DL, 32);
22065 SDValue Op1Hi = extractSubVector(Op1, 32, DAG, DL, 32);
22066 SDValue Op2Hi = extractSubVector(Op2, 32, DAG, DL, 32);
22067 SDValue Lo = DAG.getSelect(DL, MVT::v32i1, Cond, Op1Lo, Op2Lo);
22068 SDValue Hi = DAG.getSelect(DL, MVT::v32i1, Cond, Op1Hi, Op2Hi);
22069 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
22070 }
22071
22072 if (VT.isVector() && VT.getVectorElementType() == MVT::i1) {
22073 SDValue Op1Scalar;
22074 if (ISD::isBuildVectorOfConstantSDNodes(Op1.getNode()))
22075 Op1Scalar = ConvertI1VectorToInteger(Op1, DAG);
22076 else if (Op1.getOpcode() == ISD::BITCAST && Op1.getOperand(0))
22077 Op1Scalar = Op1.getOperand(0);
22078 SDValue Op2Scalar;
22079 if (ISD::isBuildVectorOfConstantSDNodes(Op2.getNode()))
22080 Op2Scalar = ConvertI1VectorToInteger(Op2, DAG);
22081 else if (Op2.getOpcode() == ISD::BITCAST && Op2.getOperand(0))
22082 Op2Scalar = Op2.getOperand(0);
22083 if (Op1Scalar.getNode() && Op2Scalar.getNode()) {
22084 SDValue newSelect = DAG.getSelect(DL, Op1Scalar.getValueType(), Cond,
22085 Op1Scalar, Op2Scalar);
22086 if (newSelect.getValueSizeInBits() == VT.getSizeInBits())
22087 return DAG.getBitcast(VT, newSelect);
22088 SDValue ExtVec = DAG.getBitcast(MVT::v8i1, newSelect);
22089 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, ExtVec,
22090 DAG.getIntPtrConstant(0, DL));
22091 }
22092 }
22093
22094 if (Cond.getOpcode() == ISD::SETCC ||
22095 Cond.getOpcode() == ISD::STRICT_FSETCC ||
22096 Cond.getOpcode() == ISD::STRICT_FSETCCS) {
22097 if (SDValue NewCond = LowerSETCC(Cond, DAG)) {
22098 Cond = NewCond;
22099 // If the condition was updated, it's possible that the operands of the
22100 // select were also updated (for example, EmitTest has a RAUW). Refresh
22101 // the local references to the select operands in case they got stale.
22102 Op1 = Op.getOperand(1);
22103 Op2 = Op.getOperand(2);
22104 }
22105 }
22106
22107 // (select (x == 0), -1, y) -> (sign_bit (x - 1)) | y
22108 // (select (x == 0), y, -1) -> ~(sign_bit (x - 1)) | y
22109 // (select (x != 0), y, -1) -> (sign_bit (x - 1)) | y
22110 // (select (x != 0), -1, y) -> ~(sign_bit (x - 1)) | y
22111 // (select (and (x , 0x1) == 0), y, (z ^ y) ) -> (-(and (x , 0x1)) & z ) ^ y
22112 // (select (and (x , 0x1) == 0), y, (z | y) ) -> (-(and (x , 0x1)) & z ) | y
22113 if (Cond.getOpcode() == X86ISD::SETCC &&
22114 Cond.getOperand(1).getOpcode() == X86ISD::CMP &&
22115 isNullConstant(Cond.getOperand(1).getOperand(1))) {
22116 SDValue Cmp = Cond.getOperand(1);
22117 unsigned CondCode = Cond.getConstantOperandVal(0);
22118
22119 if ((isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
22120 (CondCode == X86::COND_E || CondCode == X86::COND_NE)) {
22121 SDValue Y = isAllOnesConstant(Op2) ? Op1 : Op2;
22122 SDValue CmpOp0 = Cmp.getOperand(0);
22123
22124 // Apply further optimizations for special cases
22125 // (select (x != 0), -1, 0) -> neg & sbb
22126 // (select (x == 0), 0, -1) -> neg & sbb
22127 if (isNullConstant(Y) &&
22128 (isAllOnesConstant(Op1) == (CondCode == X86::COND_NE))) {
22129 SDValue Zero = DAG.getConstant(0, DL, CmpOp0.getValueType());
22130 SDValue CmpZero = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Zero, CmpOp0);
22131 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22132 Zero = DAG.getConstant(0, DL, Op.getValueType());
22133 return DAG.getNode(X86ISD::SBB, DL, VTs, Zero, Zero, CmpZero);
22134 }
22135
22136 Cmp = DAG.getNode(X86ISD::CMP, DL, MVT::i32,
22137 CmpOp0, DAG.getConstant(1, DL, CmpOp0.getValueType()));
22138 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
22139
22140 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
22141 SDValue Zero = DAG.getConstant(0, DL, Op.getValueType());
22142 SDValue Res = // Res = 0 or -1.
22143 DAG.getNode(X86ISD::SBB, DL, VTs, Zero, Zero, Cmp);
22144
22145 if (isAllOnesConstant(Op1) != (CondCode == X86::COND_E))
22146 Res = DAG.getNOT(DL, Res, Res.getValueType());
22147
22148 if (!isNullConstant(Op2))
22149 Res = DAG.getNode(ISD::OR, DL, Res.getValueType(), Res, Y);
22150 return Res;
22151 } else if (!Subtarget.hasCMov() && CondCode == X86::COND_E &&
22152 Cmp.getOperand(0).getOpcode() == ISD::AND &&
22153 isOneConstant(Cmp.getOperand(0).getOperand(1))) {
22154 SDValue CmpOp0 = Cmp.getOperand(0);
22155 SDValue Src1, Src2;
22156 // true if Op2 is XOR or OR operator and one of its operands
22157 // is equal to Op1
22158 // ( a , a op b) || ( b , a op b)
22159 auto isOrXorPattern = [&]() {
22160 if ((Op2.getOpcode() == ISD::XOR || Op2.getOpcode() == ISD::OR) &&
22161 (Op2.getOperand(0) == Op1 || Op2.getOperand(1) == Op1)) {
22162 Src1 =
22163 Op2.getOperand(0) == Op1 ? Op2.getOperand(1) : Op2.getOperand(0);
22164 Src2 = Op1;
22165 return true;
22166 }
22167 return false;
22168 };
22169
22170 if (isOrXorPattern()) {
22171 SDValue Neg;
22172 unsigned int CmpSz = CmpOp0.getSimpleValueType().getSizeInBits();
22173 // we need mask of all zeros or ones with same size of the other
22174 // operands.
22175 if (CmpSz > VT.getSizeInBits())
22176 Neg = DAG.getNode(ISD::TRUNCATE, DL, VT, CmpOp0);
22177 else if (CmpSz < VT.getSizeInBits())
22178 Neg = DAG.getNode(ISD::AND, DL, VT,
22179 DAG.getNode(ISD::ANY_EXTEND, DL, VT, CmpOp0.getOperand(0)),
22180 DAG.getConstant(1, DL, VT));
22181 else
22182 Neg = CmpOp0;
22183 SDValue Mask = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
22184 Neg); // -(and (x, 0x1))
22185 SDValue And = DAG.getNode(ISD::AND, DL, VT, Mask, Src1); // Mask & z
22186 return DAG.getNode(Op2.getOpcode(), DL, VT, And, Src2); // And Op y
22187 }
22188 }
22189 }
22190
22191 // Look past (and (setcc_carry (cmp ...)), 1).
22192 if (Cond.getOpcode() == ISD::AND &&
22193 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
22194 isOneConstant(Cond.getOperand(1)))
22195 Cond = Cond.getOperand(0);
22196
22197 // If condition flag is set by a X86ISD::CMP, then use it as the condition
22198 // setting operand in place of the X86ISD::SETCC.
22199 unsigned CondOpcode = Cond.getOpcode();
22200 if (CondOpcode == X86ISD::SETCC ||
22201 CondOpcode == X86ISD::SETCC_CARRY) {
22202 CC = Cond.getOperand(0);
22203
22204 SDValue Cmp = Cond.getOperand(1);
22205 bool IllegalFPCMov = false;
22206 if (VT.isFloatingPoint() && !VT.isVector() &&
22207 !isScalarFPTypeInSSEReg(VT)) // FPStack?
22208 IllegalFPCMov = !hasFPCMov(cast<ConstantSDNode>(CC)->getSExtValue());
22209
22210 if ((isX86LogicalCmp(Cmp) && !IllegalFPCMov) ||
22211 Cmp.getOpcode() == X86ISD::BT) { // FIXME
22212 Cond = Cmp;
22213 AddTest = false;
22214 }
22215 } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
22216 CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
22217 CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
22218 SDValue Value;
22219 X86::CondCode X86Cond;
22220 std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
22221
22222 CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
22223 AddTest = false;
22224 }
22225
22226 if (AddTest) {
22227 // Look past the truncate if the high bits are known zero.
22228 if (isTruncWithZeroHighBitsInput(Cond, DAG))
22229 Cond = Cond.getOperand(0);
22230
22231 // We know the result of AND is compared against zero. Try to match
22232 // it to BT.
22233 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
22234 SDValue BTCC;
22235 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, BTCC)) {
22236 CC = BTCC;
22237 Cond = BT;
22238 AddTest = false;
22239 }
22240 }
22241 }
22242
22243 if (AddTest) {
22244 CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
22245 Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
22246 }
22247
22248 // a < b ? -1 : 0 -> RES = ~setcc_carry
22249 // a < b ? 0 : -1 -> RES = setcc_carry
22250 // a >= b ? -1 : 0 -> RES = setcc_carry
22251 // a >= b ? 0 : -1 -> RES = ~setcc_carry
22252 if (Cond.getOpcode() == X86ISD::SUB) {
22253 Cond = ConvertCmpIfNecessary(Cond, DAG);
22254 unsigned CondCode = cast<ConstantSDNode>(CC)->getZExtValue();
22255
22256 if ((CondCode == X86::COND_AE || CondCode == X86::COND_B) &&
22257 (isAllOnesConstant(Op1) || isAllOnesConstant(Op2)) &&
22258 (isNullConstant(Op1) || isNullConstant(Op2))) {
22259 SDValue Res =
22260 DAG.getNode(X86ISD::SETCC_CARRY, DL, Op.getValueType(),
22261 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cond);
22262 if (isAllOnesConstant(Op1) != (CondCode == X86::COND_B))
22263 return DAG.getNOT(DL, Res, Res.getValueType());
22264 return Res;
22265 }
22266 }
22267
22268 // X86 doesn't have an i8 cmov. If both operands are the result of a truncate
22269 // widen the cmov and push the truncate through. This avoids introducing a new
22270 // branch during isel and doesn't add any extensions.
22271 if (Op.getValueType() == MVT::i8 &&
22272 Op1.getOpcode() == ISD::TRUNCATE && Op2.getOpcode() == ISD::TRUNCATE) {
22273 SDValue T1 = Op1.getOperand(0), T2 = Op2.getOperand(0);
22274 if (T1.getValueType() == T2.getValueType() &&
22275 // Blacklist CopyFromReg to avoid partial register stalls.
22276 T1.getOpcode() != ISD::CopyFromReg && T2.getOpcode()!=ISD::CopyFromReg){
22277 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, T1.getValueType(), T2, T1,
22278 CC, Cond);
22279 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
22280 }
22281 }
22282
22283 // Or finally, promote i8 cmovs if we have CMOV,
22284 // or i16 cmovs if it won't prevent folding a load.
22285 // FIXME: we should not limit promotion of i8 case to only when the CMOV is
22286 // legal, but EmitLoweredSelect() can not deal with these extensions
22287 // being inserted between two CMOV's. (in i16 case too TBN)
22288 // https://bugs.llvm.org/show_bug.cgi?id=40974
22289 if ((Op.getValueType() == MVT::i8 && Subtarget.hasCMov()) ||
22290 (Op.getValueType() == MVT::i16 && !MayFoldLoad(Op1) &&
22291 !MayFoldLoad(Op2))) {
22292 Op1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op1);
22293 Op2 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Op2);
22294 SDValue Ops[] = { Op2, Op1, CC, Cond };
22295 SDValue Cmov = DAG.getNode(X86ISD::CMOV, DL, MVT::i32, Ops);
22296 return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), Cmov);
22297 }
22298
22299 // X86ISD::CMOV means set the result (which is operand 1) to the RHS if
22300 // condition is true.
22301 SDValue Ops[] = { Op2, Op1, CC, Cond };
22302 return DAG.getNode(X86ISD::CMOV, DL, Op.getValueType(), Ops);
22303}
22304
22305static SDValue LowerSIGN_EXTEND_Mask(SDValue Op,
22306 const X86Subtarget &Subtarget,
22307 SelectionDAG &DAG) {
22308 MVT VT = Op->getSimpleValueType(0);
22309 SDValue In = Op->getOperand(0);
22310 MVT InVT = In.getSimpleValueType();
22311 assert(InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!")((InVT.getVectorElementType() == MVT::i1 && "Unexpected input type!"
) ? static_cast<void> (0) : __assert_fail ("InVT.getVectorElementType() == MVT::i1 && \"Unexpected input type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22311, __PRETTY_FUNCTION__));
22312 MVT VTElt = VT.getVectorElementType();
22313 SDLoc dl(Op);
22314
22315 unsigned NumElts = VT.getVectorNumElements();
22316
22317 // Extend VT if the scalar type is i8/i16 and BWI is not supported.
22318 MVT ExtVT = VT;
22319 if (!Subtarget.hasBWI() && VTElt.getSizeInBits() <= 16) {
22320 // If v16i32 is to be avoided, we'll need to split and concatenate.
22321 if (NumElts == 16 && !Subtarget.canExtendTo512DQ())
22322 return SplitAndExtendv16i1(Op.getOpcode(), VT, In, dl, DAG);
22323
22324 ExtVT = MVT::getVectorVT(MVT::i32, NumElts);
22325 }
22326
22327 // Widen to 512-bits if VLX is not supported.
22328 MVT WideVT = ExtVT;
22329 if (!ExtVT.is512BitVector() && !Subtarget.hasVLX()) {
22330 NumElts *= 512 / ExtVT.getSizeInBits();
22331 InVT = MVT::getVectorVT(MVT::i1, NumElts);
22332 In = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, InVT, DAG.getUNDEF(InVT),
22333 In, DAG.getIntPtrConstant(0, dl));
22334 WideVT = MVT::getVectorVT(ExtVT.getVectorElementType(), NumElts);
22335 }
22336
22337 SDValue V;
22338 MVT WideEltVT = WideVT.getVectorElementType();
22339 if ((Subtarget.hasDQI() && WideEltVT.getSizeInBits() >= 32) ||
22340 (Subtarget.hasBWI() && WideEltVT.getSizeInBits() <= 16)) {
22341 V = DAG.getNode(Op.getOpcode(), dl, WideVT, In);
22342 } else {
22343 SDValue NegOne = DAG.getConstant(-1, dl, WideVT);
22344 SDValue Zero = DAG.getConstant(0, dl, WideVT);
22345 V = DAG.getSelect(dl, WideVT, In, NegOne, Zero);
22346 }
22347
22348 // Truncate if we had to extend i16/i8 above.
22349 if (VT != ExtVT) {
22350 WideVT = MVT::getVectorVT(VTElt, NumElts);
22351 V = DAG.getNode(ISD::TRUNCATE, dl, WideVT, V);
22352 }
22353
22354 // Extract back to 128/256-bit if we widened.
22355 if (WideVT != VT)
22356 V = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, V,
22357 DAG.getIntPtrConstant(0, dl));
22358
22359 return V;
22360}
22361
22362static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
22363 SelectionDAG &DAG) {
22364 SDValue In = Op->getOperand(0);
22365 MVT InVT = In.getSimpleValueType();
22366
22367 if (InVT.getVectorElementType() == MVT::i1)
22368 return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG);
22369
22370 assert(Subtarget.hasAVX() && "Expected AVX support")((Subtarget.hasAVX() && "Expected AVX support") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasAVX() && \"Expected AVX support\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22370, __PRETTY_FUNCTION__));
22371 return LowerAVXExtend(Op, DAG, Subtarget);
22372}
22373
22374// Lowering for SIGN_EXTEND_VECTOR_INREG and ZERO_EXTEND_VECTOR_INREG.
22375// For sign extend this needs to handle all vector sizes and SSE4.1 and
22376// non-SSE4.1 targets. For zero extend this should only handle inputs of
22377// MVT::v64i8 when BWI is not supported, but AVX512 is.
22378static SDValue LowerEXTEND_VECTOR_INREG(SDValue Op,
22379 const X86Subtarget &Subtarget,
22380 SelectionDAG &DAG) {
22381 SDValue In = Op->getOperand(0);
22382 MVT VT = Op->getSimpleValueType(0);
22383 MVT InVT = In.getSimpleValueType();
22384
22385 MVT SVT = VT.getVectorElementType();
22386 MVT InSVT = InVT.getVectorElementType();
22387 assert(SVT.getSizeInBits() > InSVT.getSizeInBits())((SVT.getSizeInBits() > InSVT.getSizeInBits()) ? static_cast
<void> (0) : __assert_fail ("SVT.getSizeInBits() > InSVT.getSizeInBits()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22387, __PRETTY_FUNCTION__));
22388
22389 if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16)
22390 return SDValue();
22391 if (InSVT != MVT::i32 && InSVT != MVT::i16 && InSVT != MVT::i8)
22392 return SDValue();
22393 if (!(VT.is128BitVector() && Subtarget.hasSSE2()) &&
22394 !(VT.is256BitVector() && Subtarget.hasAVX()) &&
22395 !(VT.is512BitVector() && Subtarget.hasAVX512()))
22396 return SDValue();
22397
22398 SDLoc dl(Op);
22399 unsigned Opc = Op.getOpcode();
22400 unsigned NumElts = VT.getVectorNumElements();
22401
22402 // For 256-bit vectors, we only need the lower (128-bit) half of the input.
22403 // For 512-bit vectors, we need 128-bits or 256-bits.
22404 if (InVT.getSizeInBits() > 128) {
22405 // Input needs to be at least the same number of elements as output, and
22406 // at least 128-bits.
22407 int InSize = InSVT.getSizeInBits() * NumElts;
22408 In = extractSubVector(In, 0, DAG, dl, std::max(InSize, 128));
22409 InVT = In.getSimpleValueType();
22410 }
22411
22412 // SSE41 targets can use the pmov[sz]x* instructions directly for 128-bit results,
22413 // so are legal and shouldn't occur here. AVX2/AVX512 pmovsx* instructions still
22414 // need to be handled here for 256/512-bit results.
22415 if (Subtarget.hasInt256()) {
22416 assert(VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension")((VT.getSizeInBits() > 128 && "Unexpected 128-bit vector extension"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() > 128 && \"Unexpected 128-bit vector extension\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22416, __PRETTY_FUNCTION__));
22417
22418 if (InVT.getVectorNumElements() != NumElts)
22419 return DAG.getNode(Op.getOpcode(), dl, VT, In);
22420
22421 // FIXME: Apparently we create inreg operations that could be regular
22422 // extends.
22423 unsigned ExtOpc =
22424 Opc == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SIGN_EXTEND
22425 : ISD::ZERO_EXTEND;
22426 return DAG.getNode(ExtOpc, dl, VT, In);
22427 }
22428
22429 // pre-AVX2 256-bit extensions need to be split into 128-bit instructions.
22430 if (Subtarget.hasAVX()) {
22431 assert(VT.is256BitVector() && "256-bit vector expected")((VT.is256BitVector() && "256-bit vector expected") ?
static_cast<void> (0) : __assert_fail ("VT.is256BitVector() && \"256-bit vector expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22431, __PRETTY_FUNCTION__));
22432 MVT HalfVT = VT.getHalfNumVectorElementsVT();
22433 int HalfNumElts = HalfVT.getVectorNumElements();
22434
22435 unsigned NumSrcElts = InVT.getVectorNumElements();
22436 SmallVector<int, 16> HiMask(NumSrcElts, SM_SentinelUndef);
22437 for (int i = 0; i != HalfNumElts; ++i)
22438 HiMask[i] = HalfNumElts + i;
22439
22440 SDValue Lo = DAG.getNode(Opc, dl, HalfVT, In);
22441 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, DAG.getUNDEF(InVT), HiMask);
22442 Hi = DAG.getNode(Opc, dl, HalfVT, Hi);
22443 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
22444 }
22445
22446 // We should only get here for sign extend.
22447 assert(Opc == ISD::SIGN_EXTEND_VECTOR_INREG && "Unexpected opcode!")((Opc == ISD::SIGN_EXTEND_VECTOR_INREG && "Unexpected opcode!"
) ? static_cast<void> (0) : __assert_fail ("Opc == ISD::SIGN_EXTEND_VECTOR_INREG && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22447, __PRETTY_FUNCTION__));
22448 assert(VT.is128BitVector() && InVT.is128BitVector() && "Unexpected VTs")((VT.is128BitVector() && InVT.is128BitVector() &&
"Unexpected VTs") ? static_cast<void> (0) : __assert_fail
("VT.is128BitVector() && InVT.is128BitVector() && \"Unexpected VTs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22448, __PRETTY_FUNCTION__));
22449
22450 // pre-SSE41 targets unpack lower lanes and then sign-extend using SRAI.
22451 SDValue Curr = In;
22452 SDValue SignExt = Curr;
22453
22454 // As SRAI is only available on i16/i32 types, we expand only up to i32
22455 // and handle i64 separately.
22456 if (InVT != MVT::v4i32) {
22457 MVT DestVT = VT == MVT::v2i64 ? MVT::v4i32 : VT;
22458
22459 unsigned DestWidth = DestVT.getScalarSizeInBits();
22460 unsigned Scale = DestWidth / InSVT.getSizeInBits();
22461
22462 unsigned InNumElts = InVT.getVectorNumElements();
22463 unsigned DestElts = DestVT.getVectorNumElements();
22464
22465 // Build a shuffle mask that takes each input element and places it in the
22466 // MSBs of the new element size.
22467 SmallVector<int, 16> Mask(InNumElts, SM_SentinelUndef);
22468 for (unsigned i = 0; i != DestElts; ++i)
22469 Mask[i * Scale + (Scale - 1)] = i;
22470
22471 Curr = DAG.getVectorShuffle(InVT, dl, In, In, Mask);
22472 Curr = DAG.getBitcast(DestVT, Curr);
22473
22474 unsigned SignExtShift = DestWidth - InSVT.getSizeInBits();
22475 SignExt = DAG.getNode(X86ISD::VSRAI, dl, DestVT, Curr,
22476 DAG.getTargetConstant(SignExtShift, dl, MVT::i8));
22477 }
22478
22479 if (VT == MVT::v2i64) {
22480 assert(Curr.getValueType() == MVT::v4i32 && "Unexpected input VT")((Curr.getValueType() == MVT::v4i32 && "Unexpected input VT"
) ? static_cast<void> (0) : __assert_fail ("Curr.getValueType() == MVT::v4i32 && \"Unexpected input VT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22480, __PRETTY_FUNCTION__));
22481 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
22482 SDValue Sign = DAG.getSetCC(dl, MVT::v4i32, Zero, Curr, ISD::SETGT);
22483 SignExt = DAG.getVectorShuffle(MVT::v4i32, dl, SignExt, Sign, {0, 4, 1, 5});
22484 SignExt = DAG.getBitcast(VT, SignExt);
22485 }
22486
22487 return SignExt;
22488}
22489
22490static SDValue LowerSIGN_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
22491 SelectionDAG &DAG) {
22492 MVT VT = Op->getSimpleValueType(0);
22493 SDValue In = Op->getOperand(0);
22494 MVT InVT = In.getSimpleValueType();
22495 SDLoc dl(Op);
22496
22497 if (InVT.getVectorElementType() == MVT::i1)
22498 return LowerSIGN_EXTEND_Mask(Op, Subtarget, DAG);
22499
22500 assert(VT.isVector() && InVT.isVector() && "Expected vector type")((VT.isVector() && InVT.isVector() && "Expected vector type"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && InVT.isVector() && \"Expected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22500, __PRETTY_FUNCTION__));
22501 assert(VT.getVectorNumElements() == InVT.getVectorNumElements() &&((VT.getVectorNumElements() == InVT.getVectorNumElements() &&
"Expected same number of elements") ? static_cast<void>
(0) : __assert_fail ("VT.getVectorNumElements() == InVT.getVectorNumElements() && \"Expected same number of elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22502, __PRETTY_FUNCTION__))
22502 "Expected same number of elements")((VT.getVectorNumElements() == InVT.getVectorNumElements() &&
"Expected same number of elements") ? static_cast<void>
(0) : __assert_fail ("VT.getVectorNumElements() == InVT.getVectorNumElements() && \"Expected same number of elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22502, __PRETTY_FUNCTION__));
22503 assert((VT.getVectorElementType() == MVT::i16 ||(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22506, __PRETTY_FUNCTION__))
22504 VT.getVectorElementType() == MVT::i32 ||(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22506, __PRETTY_FUNCTION__))
22505 VT.getVectorElementType() == MVT::i64) &&(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22506, __PRETTY_FUNCTION__))
22506 "Unexpected element type")(((VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType
() == MVT::i32 || VT.getVectorElementType() == MVT::i64) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(VT.getVectorElementType() == MVT::i16 || VT.getVectorElementType() == MVT::i32 || VT.getVectorElementType() == MVT::i64) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22506, __PRETTY_FUNCTION__));
22507 assert((InVT.getVectorElementType() == MVT::i8 ||(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22510, __PRETTY_FUNCTION__))
22508 InVT.getVectorElementType() == MVT::i16 ||(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22510, __PRETTY_FUNCTION__))
22509 InVT.getVectorElementType() == MVT::i32) &&(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22510, __PRETTY_FUNCTION__))
22510 "Unexpected element type")(((InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType
() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) &&
"Unexpected element type") ? static_cast<void> (0) : __assert_fail
("(InVT.getVectorElementType() == MVT::i8 || InVT.getVectorElementType() == MVT::i16 || InVT.getVectorElementType() == MVT::i32) && \"Unexpected element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22510, __PRETTY_FUNCTION__));
22511
22512 // Custom legalize v8i8->v8i64 on CPUs without avx512bw.
22513 if (InVT == MVT::v8i8) {
22514 if (VT != MVT::v8i64)
22515 return SDValue();
22516
22517 In = DAG.getNode(ISD::CONCAT_VECTORS, SDLoc(Op),
22518 MVT::v16i8, In, DAG.getUNDEF(MVT::v8i8));
22519 return DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, VT, In);
22520 }
22521
22522 if (Subtarget.hasInt256())
22523 return Op;
22524
22525 // Optimize vectors in AVX mode
22526 // Sign extend v8i16 to v8i32 and
22527 // v4i32 to v4i64
22528 //
22529 // Divide input vector into two parts
22530 // for v4i32 the high shuffle mask will be {2, 3, -1, -1}
22531 // use vpmovsx instruction to extend v4i32 -> v2i64; v8i16 -> v4i32
22532 // concat the vectors to original VT
22533 MVT HalfVT = VT.getHalfNumVectorElementsVT();
22534 SDValue OpLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, In);
22535
22536 unsigned NumElems = InVT.getVectorNumElements();
22537 SmallVector<int,8> ShufMask(NumElems, -1);
22538 for (unsigned i = 0; i != NumElems/2; ++i)
22539 ShufMask[i] = i + NumElems/2;
22540
22541 SDValue OpHi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
22542 OpHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, HalfVT, OpHi);
22543
22544 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, OpLo, OpHi);
22545}
22546
22547/// Change a vector store into a pair of half-size vector stores.
22548static SDValue splitVectorStore(StoreSDNode *Store, SelectionDAG &DAG) {
22549 SDValue StoredVal = Store->getValue();
22550 assert((StoredVal.getValueType().is256BitVector() ||(((StoredVal.getValueType().is256BitVector() || StoredVal.getValueType
().is512BitVector()) && "Expecting 256/512-bit op") ?
static_cast<void> (0) : __assert_fail ("(StoredVal.getValueType().is256BitVector() || StoredVal.getValueType().is512BitVector()) && \"Expecting 256/512-bit op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22552, __PRETTY_FUNCTION__))
22551 StoredVal.getValueType().is512BitVector()) &&(((StoredVal.getValueType().is256BitVector() || StoredVal.getValueType
().is512BitVector()) && "Expecting 256/512-bit op") ?
static_cast<void> (0) : __assert_fail ("(StoredVal.getValueType().is256BitVector() || StoredVal.getValueType().is512BitVector()) && \"Expecting 256/512-bit op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22552, __PRETTY_FUNCTION__))
22552 "Expecting 256/512-bit op")(((StoredVal.getValueType().is256BitVector() || StoredVal.getValueType
().is512BitVector()) && "Expecting 256/512-bit op") ?
static_cast<void> (0) : __assert_fail ("(StoredVal.getValueType().is256BitVector() || StoredVal.getValueType().is512BitVector()) && \"Expecting 256/512-bit op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22552, __PRETTY_FUNCTION__));
22553
22554 // Splitting volatile memory ops is not allowed unless the operation was not
22555 // legal to begin with. Assume the input store is legal (this transform is
22556 // only used for targets with AVX). Note: It is possible that we have an
22557 // illegal type like v2i128, and so we could allow splitting a volatile store
22558 // in that case if that is important.
22559 if (!Store->isSimple())
22560 return SDValue();
22561
22562 EVT StoreVT = StoredVal.getValueType();
22563 unsigned NumElems = StoreVT.getVectorNumElements();
22564 unsigned HalfSize = StoredVal.getValueSizeInBits() / 2;
22565 unsigned HalfAlign = (128 == HalfSize ? 16 : 32);
22566
22567 SDLoc DL(Store);
22568 SDValue Value0 = extractSubVector(StoredVal, 0, DAG, DL, HalfSize);
22569 SDValue Value1 = extractSubVector(StoredVal, NumElems / 2, DAG, DL, HalfSize);
22570 SDValue Ptr0 = Store->getBasePtr();
22571 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, HalfAlign, DL);
22572 unsigned Alignment = Store->getAlignment();
22573 SDValue Ch0 =
22574 DAG.getStore(Store->getChain(), DL, Value0, Ptr0, Store->getPointerInfo(),
22575 Alignment, Store->getMemOperand()->getFlags());
22576 SDValue Ch1 = DAG.getStore(Store->getChain(), DL, Value1, Ptr1,
22577 Store->getPointerInfo().getWithOffset(HalfAlign),
22578 MinAlign(Alignment, HalfAlign),
22579 Store->getMemOperand()->getFlags());
22580 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Ch0, Ch1);
22581}
22582
22583/// Scalarize a vector store, bitcasting to TargetVT to determine the scalar
22584/// type.
22585static SDValue scalarizeVectorStore(StoreSDNode *Store, MVT StoreVT,
22586 SelectionDAG &DAG) {
22587 SDValue StoredVal = Store->getValue();
22588 assert(StoreVT.is128BitVector() &&((StoreVT.is128BitVector() && StoredVal.getValueType(
).is128BitVector() && "Expecting 128-bit op") ? static_cast
<void> (0) : __assert_fail ("StoreVT.is128BitVector() && StoredVal.getValueType().is128BitVector() && \"Expecting 128-bit op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22589, __PRETTY_FUNCTION__))
22589 StoredVal.getValueType().is128BitVector() && "Expecting 128-bit op")((StoreVT.is128BitVector() && StoredVal.getValueType(
).is128BitVector() && "Expecting 128-bit op") ? static_cast
<void> (0) : __assert_fail ("StoreVT.is128BitVector() && StoredVal.getValueType().is128BitVector() && \"Expecting 128-bit op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22589, __PRETTY_FUNCTION__));
22590 StoredVal = DAG.getBitcast(StoreVT, StoredVal);
22591
22592 // Splitting volatile memory ops is not allowed unless the operation was not
22593 // legal to begin with. We are assuming the input op is legal (this transform
22594 // is only used for targets with AVX).
22595 if (!Store->isSimple())
22596 return SDValue();
22597
22598 MVT StoreSVT = StoreVT.getScalarType();
22599 unsigned NumElems = StoreVT.getVectorNumElements();
22600 unsigned ScalarSize = StoreSVT.getStoreSize();
22601 unsigned Alignment = Store->getAlignment();
22602
22603 SDLoc DL(Store);
22604 SmallVector<SDValue, 4> Stores;
22605 for (unsigned i = 0; i != NumElems; ++i) {
22606 unsigned Offset = i * ScalarSize;
22607 SDValue Ptr = DAG.getMemBasePlusOffset(Store->getBasePtr(), Offset, DL);
22608 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, StoreSVT, StoredVal,
22609 DAG.getIntPtrConstant(i, DL));
22610 SDValue Ch = DAG.getStore(Store->getChain(), DL, Scl, Ptr,
22611 Store->getPointerInfo().getWithOffset(Offset),
22612 MinAlign(Alignment, Offset),
22613 Store->getMemOperand()->getFlags());
22614 Stores.push_back(Ch);
22615 }
22616 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Stores);
22617}
22618
22619static SDValue LowerStore(SDValue Op, const X86Subtarget &Subtarget,
22620 SelectionDAG &DAG) {
22621 StoreSDNode *St = cast<StoreSDNode>(Op.getNode());
22622 SDLoc dl(St);
22623 SDValue StoredVal = St->getValue();
22624
22625 // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 stores.
22626 if (StoredVal.getValueType().isVector() &&
22627 StoredVal.getValueType().getVectorElementType() == MVT::i1) {
22628 assert(StoredVal.getValueType().getVectorNumElements() <= 8 &&((StoredVal.getValueType().getVectorNumElements() <= 8 &&
"Unexpected VT") ? static_cast<void> (0) : __assert_fail
("StoredVal.getValueType().getVectorNumElements() <= 8 && \"Unexpected VT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22629, __PRETTY_FUNCTION__))
22629 "Unexpected VT")((StoredVal.getValueType().getVectorNumElements() <= 8 &&
"Unexpected VT") ? static_cast<void> (0) : __assert_fail
("StoredVal.getValueType().getVectorNumElements() <= 8 && \"Unexpected VT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22629, __PRETTY_FUNCTION__));
22630 assert(!St->isTruncatingStore() && "Expected non-truncating store")((!St->isTruncatingStore() && "Expected non-truncating store"
) ? static_cast<void> (0) : __assert_fail ("!St->isTruncatingStore() && \"Expected non-truncating store\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22630, __PRETTY_FUNCTION__));
22631 assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&((Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
"Expected AVX512F without AVX512DQI") ? static_cast<void>
(0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasDQI() && \"Expected AVX512F without AVX512DQI\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22632, __PRETTY_FUNCTION__))
22632 "Expected AVX512F without AVX512DQI")((Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
"Expected AVX512F without AVX512DQI") ? static_cast<void>
(0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasDQI() && \"Expected AVX512F without AVX512DQI\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22632, __PRETTY_FUNCTION__));
22633
22634 StoredVal = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
22635 DAG.getUNDEF(MVT::v16i1), StoredVal,
22636 DAG.getIntPtrConstant(0, dl));
22637 StoredVal = DAG.getBitcast(MVT::i16, StoredVal);
22638 StoredVal = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, StoredVal);
22639
22640 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
22641 St->getPointerInfo(), St->getAlignment(),
22642 St->getMemOperand()->getFlags());
22643 }
22644
22645 if (St->isTruncatingStore())
22646 return SDValue();
22647
22648 // If this is a 256-bit store of concatenated ops, we are better off splitting
22649 // that store into two 128-bit stores. This avoids spurious use of 256-bit ops
22650 // and each half can execute independently. Some cores would split the op into
22651 // halves anyway, so the concat (vinsertf128) is purely an extra op.
22652 MVT StoreVT = StoredVal.getSimpleValueType();
22653 if (StoreVT.is256BitVector()) {
22654 SmallVector<SDValue, 4> CatOps;
22655 if (StoredVal.hasOneUse() && collectConcatOps(StoredVal.getNode(), CatOps))
22656 return splitVectorStore(St, DAG);
22657 return SDValue();
22658 }
22659
22660 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
22661 assert(StoreVT.isVector() && StoreVT.getSizeInBits() == 64 &&((StoreVT.isVector() && StoreVT.getSizeInBits() == 64
&& "Unexpected VT") ? static_cast<void> (0) : __assert_fail
("StoreVT.isVector() && StoreVT.getSizeInBits() == 64 && \"Unexpected VT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22662, __PRETTY_FUNCTION__))
22662 "Unexpected VT")((StoreVT.isVector() && StoreVT.getSizeInBits() == 64
&& "Unexpected VT") ? static_cast<void> (0) : __assert_fail
("StoreVT.isVector() && StoreVT.getSizeInBits() == 64 && \"Unexpected VT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22662, __PRETTY_FUNCTION__));
22663 assert(TLI.getTypeAction(*DAG.getContext(), StoreVT) ==((TLI.getTypeAction(*DAG.getContext(), StoreVT) == TargetLowering
::TypeWidenVector && "Unexpected type action!") ? static_cast
<void> (0) : __assert_fail ("TLI.getTypeAction(*DAG.getContext(), StoreVT) == TargetLowering::TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22664, __PRETTY_FUNCTION__))
22664 TargetLowering::TypeWidenVector && "Unexpected type action!")((TLI.getTypeAction(*DAG.getContext(), StoreVT) == TargetLowering
::TypeWidenVector && "Unexpected type action!") ? static_cast
<void> (0) : __assert_fail ("TLI.getTypeAction(*DAG.getContext(), StoreVT) == TargetLowering::TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22664, __PRETTY_FUNCTION__));
22665
22666 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), StoreVT);
22667 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, StoredVal,
22668 DAG.getUNDEF(StoreVT));
22669
22670 if (Subtarget.hasSSE2()) {
22671 // Widen the vector, cast to a v2x64 type, extract the single 64-bit element
22672 // and store it.
22673 MVT StVT = Subtarget.is64Bit() && StoreVT.isInteger() ? MVT::i64 : MVT::f64;
22674 MVT CastVT = MVT::getVectorVT(StVT, 2);
22675 StoredVal = DAG.getBitcast(CastVT, StoredVal);
22676 StoredVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, StVT, StoredVal,
22677 DAG.getIntPtrConstant(0, dl));
22678
22679 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
22680 St->getPointerInfo(), St->getAlignment(),
22681 St->getMemOperand()->getFlags());
22682 }
22683 assert(Subtarget.hasSSE1() && "Expected SSE")((Subtarget.hasSSE1() && "Expected SSE") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasSSE1() && \"Expected SSE\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22683, __PRETTY_FUNCTION__));
22684 SDVTList Tys = DAG.getVTList(MVT::Other);
22685 SDValue Ops[] = {St->getChain(), StoredVal, St->getBasePtr()};
22686 return DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys, Ops, MVT::i64,
22687 St->getMemOperand());
22688}
22689
22690// Lower vector extended loads using a shuffle. If SSSE3 is not available we
22691// may emit an illegal shuffle but the expansion is still better than scalar
22692// code. We generate sext/sext_invec for SEXTLOADs if it's available, otherwise
22693// we'll emit a shuffle and a arithmetic shift.
22694// FIXME: Is the expansion actually better than scalar code? It doesn't seem so.
22695// TODO: It is possible to support ZExt by zeroing the undef values during
22696// the shuffle phase or after the shuffle.
22697static SDValue LowerLoad(SDValue Op, const X86Subtarget &Subtarget,
22698 SelectionDAG &DAG) {
22699 MVT RegVT = Op.getSimpleValueType();
22700 assert(RegVT.isVector() && "We only custom lower vector loads.")((RegVT.isVector() && "We only custom lower vector loads."
) ? static_cast<void> (0) : __assert_fail ("RegVT.isVector() && \"We only custom lower vector loads.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22700, __PRETTY_FUNCTION__));
22701 assert(RegVT.isInteger() &&((RegVT.isInteger() && "We only custom lower integer vector loads."
) ? static_cast<void> (0) : __assert_fail ("RegVT.isInteger() && \"We only custom lower integer vector loads.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22702, __PRETTY_FUNCTION__))
22702 "We only custom lower integer vector loads.")((RegVT.isInteger() && "We only custom lower integer vector loads."
) ? static_cast<void> (0) : __assert_fail ("RegVT.isInteger() && \"We only custom lower integer vector loads.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22702, __PRETTY_FUNCTION__));
22703
22704 LoadSDNode *Ld = cast<LoadSDNode>(Op.getNode());
22705 SDLoc dl(Ld);
22706
22707 // Without AVX512DQ, we need to use a scalar type for v2i1/v4i1/v8i1 loads.
22708 if (RegVT.getVectorElementType() == MVT::i1) {
22709 assert(EVT(RegVT) == Ld->getMemoryVT() && "Expected non-extending load")((EVT(RegVT) == Ld->getMemoryVT() && "Expected non-extending load"
) ? static_cast<void> (0) : __assert_fail ("EVT(RegVT) == Ld->getMemoryVT() && \"Expected non-extending load\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22709, __PRETTY_FUNCTION__));
22710 assert(RegVT.getVectorNumElements() <= 8 && "Unexpected VT")((RegVT.getVectorNumElements() <= 8 && "Unexpected VT"
) ? static_cast<void> (0) : __assert_fail ("RegVT.getVectorNumElements() <= 8 && \"Unexpected VT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22710, __PRETTY_FUNCTION__));
22711 assert(Subtarget.hasAVX512() && !Subtarget.hasDQI() &&((Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
"Expected AVX512F without AVX512DQI") ? static_cast<void>
(0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasDQI() && \"Expected AVX512F without AVX512DQI\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22712, __PRETTY_FUNCTION__))
22712 "Expected AVX512F without AVX512DQI")((Subtarget.hasAVX512() && !Subtarget.hasDQI() &&
"Expected AVX512F without AVX512DQI") ? static_cast<void>
(0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasDQI() && \"Expected AVX512F without AVX512DQI\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22712, __PRETTY_FUNCTION__));
22713
22714 SDValue NewLd = DAG.getLoad(MVT::i8, dl, Ld->getChain(), Ld->getBasePtr(),
22715 Ld->getPointerInfo(), Ld->getAlignment(),
22716 Ld->getMemOperand()->getFlags());
22717
22718 // Replace chain users with the new chain.
22719 assert(NewLd->getNumValues() == 2 && "Loads must carry a chain!")((NewLd->getNumValues() == 2 && "Loads must carry a chain!"
) ? static_cast<void> (0) : __assert_fail ("NewLd->getNumValues() == 2 && \"Loads must carry a chain!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22719, __PRETTY_FUNCTION__));
22720
22721 SDValue Val = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i16, NewLd);
22722 Val = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, RegVT,
22723 DAG.getBitcast(MVT::v16i1, Val),
22724 DAG.getIntPtrConstant(0, dl));
22725 return DAG.getMergeValues({Val, NewLd.getValue(1)}, dl);
22726 }
22727
22728 return SDValue();
22729}
22730
22731/// Return true if node is an ISD::AND or ISD::OR of two X86ISD::SETCC nodes
22732/// each of which has no other use apart from the AND / OR.
22733static bool isAndOrOfSetCCs(SDValue Op, unsigned &Opc) {
22734 Opc = Op.getOpcode();
22735 if (Opc != ISD::OR && Opc != ISD::AND)
22736 return false;
22737 return (Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
22738 Op.getOperand(0).hasOneUse() &&
22739 Op.getOperand(1).getOpcode() == X86ISD::SETCC &&
22740 Op.getOperand(1).hasOneUse());
22741}
22742
22743/// Return true if node is an ISD::XOR of a X86ISD::SETCC and 1 and that the
22744/// SETCC node has a single use.
22745static bool isXor1OfSetCC(SDValue Op) {
22746 if (Op.getOpcode() != ISD::XOR)
22747 return false;
22748 if (isOneConstant(Op.getOperand(1)))
22749 return Op.getOperand(0).getOpcode() == X86ISD::SETCC &&
22750 Op.getOperand(0).hasOneUse();
22751 return false;
22752}
22753
22754SDValue X86TargetLowering::LowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
22755 bool addTest = true;
22756 SDValue Chain = Op.getOperand(0);
22757 SDValue Cond = Op.getOperand(1);
22758 SDValue Dest = Op.getOperand(2);
22759 SDLoc dl(Op);
22760 SDValue CC;
22761 bool Inverted = false;
22762
22763 if (Cond.getOpcode() == ISD::SETCC) {
22764 // Check for setcc([su]{add,sub,mul}o == 0).
22765 if (cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETEQ &&
22766 isNullConstant(Cond.getOperand(1)) &&
22767 Cond.getOperand(0).getResNo() == 1 &&
22768 (Cond.getOperand(0).getOpcode() == ISD::SADDO ||
22769 Cond.getOperand(0).getOpcode() == ISD::UADDO ||
22770 Cond.getOperand(0).getOpcode() == ISD::SSUBO ||
22771 Cond.getOperand(0).getOpcode() == ISD::USUBO ||
22772 Cond.getOperand(0).getOpcode() == ISD::SMULO ||
22773 Cond.getOperand(0).getOpcode() == ISD::UMULO)) {
22774 Inverted = true;
22775 Cond = Cond.getOperand(0);
22776 } else {
22777 if (SDValue NewCond = LowerSETCC(Cond, DAG))
22778 Cond = NewCond;
22779 }
22780 }
22781#if 0
22782 // FIXME: LowerXALUO doesn't handle these!!
22783 else if (Cond.getOpcode() == X86ISD::ADD ||
22784 Cond.getOpcode() == X86ISD::SUB ||
22785 Cond.getOpcode() == X86ISD::SMUL ||
22786 Cond.getOpcode() == X86ISD::UMUL)
22787 Cond = LowerXALUO(Cond, DAG);
22788#endif
22789
22790 // Look pass (and (setcc_carry (cmp ...)), 1).
22791 if (Cond.getOpcode() == ISD::AND &&
22792 Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
22793 isOneConstant(Cond.getOperand(1)))
22794 Cond = Cond.getOperand(0);
22795
22796 // If condition flag is set by a X86ISD::CMP, then use it as the condition
22797 // setting operand in place of the X86ISD::SETCC.
22798 unsigned CondOpcode = Cond.getOpcode();
22799 if (CondOpcode == X86ISD::SETCC ||
22800 CondOpcode == X86ISD::SETCC_CARRY) {
22801 CC = Cond.getOperand(0);
22802
22803 SDValue Cmp = Cond.getOperand(1);
22804 unsigned Opc = Cmp.getOpcode();
22805 // FIXME: WHY THE SPECIAL CASING OF LogicalCmp??
22806 if (isX86LogicalCmp(Cmp) || Opc == X86ISD::BT) {
22807 Cond = Cmp;
22808 addTest = false;
22809 } else {
22810 switch (cast<ConstantSDNode>(CC)->getZExtValue()) {
22811 default: break;
22812 case X86::COND_O:
22813 case X86::COND_B:
22814 // These can only come from an arithmetic instruction with overflow,
22815 // e.g. SADDO, UADDO.
22816 Cond = Cond.getOperand(1);
22817 addTest = false;
22818 break;
22819 }
22820 }
22821 }
22822 CondOpcode = Cond.getOpcode();
22823 if (CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
22824 CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
22825 CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
22826 SDValue Value;
22827 X86::CondCode X86Cond;
22828 std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
22829
22830 if (Inverted)
22831 X86Cond = X86::GetOppositeBranchCondition(X86Cond);
22832
22833 CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
22834 addTest = false;
22835 } else {
22836 unsigned CondOpc;
22837 if (Cond.hasOneUse() && isAndOrOfSetCCs(Cond, CondOpc)) {
22838 SDValue Cmp = Cond.getOperand(0).getOperand(1);
22839 if (CondOpc == ISD::OR) {
22840 // Also, recognize the pattern generated by an FCMP_UNE. We can emit
22841 // two branches instead of an explicit OR instruction with a
22842 // separate test.
22843 if (Cmp == Cond.getOperand(1).getOperand(1) &&
22844 isX86LogicalCmp(Cmp)) {
22845 CC = Cond.getOperand(0).getOperand(0);
22846 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
22847 Chain, Dest, CC, Cmp);
22848 CC = Cond.getOperand(1).getOperand(0);
22849 Cond = Cmp;
22850 addTest = false;
22851 }
22852 } else { // ISD::AND
22853 // Also, recognize the pattern generated by an FCMP_OEQ. We can emit
22854 // two branches instead of an explicit AND instruction with a
22855 // separate test. However, we only do this if this block doesn't
22856 // have a fall-through edge, because this requires an explicit
22857 // jmp when the condition is false.
22858 if (Cmp == Cond.getOperand(1).getOperand(1) &&
22859 isX86LogicalCmp(Cmp) &&
22860 Op.getNode()->hasOneUse()) {
22861 X86::CondCode CCode0 =
22862 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
22863 CCode0 = X86::GetOppositeBranchCondition(CCode0);
22864 CC = DAG.getTargetConstant(CCode0, dl, MVT::i8);
22865 SDNode *User = *Op.getNode()->use_begin();
22866 // Look for an unconditional branch following this conditional branch.
22867 // We need this because we need to reverse the successors in order
22868 // to implement FCMP_OEQ.
22869 if (User->getOpcode() == ISD::BR) {
22870 SDValue FalseBB = User->getOperand(1);
22871 SDNode *NewBR =
22872 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
22873 assert(NewBR == User)((NewBR == User) ? static_cast<void> (0) : __assert_fail
("NewBR == User", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22873, __PRETTY_FUNCTION__));
22874 (void)NewBR;
22875 Dest = FalseBB;
22876
22877 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(), Chain,
22878 Dest, CC, Cmp);
22879 X86::CondCode CCode1 =
22880 (X86::CondCode)Cond.getOperand(1).getConstantOperandVal(0);
22881 CCode1 = X86::GetOppositeBranchCondition(CCode1);
22882 CC = DAG.getTargetConstant(CCode1, dl, MVT::i8);
22883 Cond = Cmp;
22884 addTest = false;
22885 }
22886 }
22887 }
22888 } else if (Cond.hasOneUse() && isXor1OfSetCC(Cond)) {
22889 // Recognize for xorb (setcc), 1 patterns. The xor inverts the condition.
22890 // It should be transformed during dag combiner except when the condition
22891 // is set by a arithmetics with overflow node.
22892 X86::CondCode CCode =
22893 (X86::CondCode)Cond.getOperand(0).getConstantOperandVal(0);
22894 CCode = X86::GetOppositeBranchCondition(CCode);
22895 CC = DAG.getTargetConstant(CCode, dl, MVT::i8);
22896 Cond = Cond.getOperand(0).getOperand(1);
22897 addTest = false;
22898 } else if (Cond.getOpcode() == ISD::SETCC &&
22899 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETOEQ) {
22900 // For FCMP_OEQ, we can emit
22901 // two branches instead of an explicit AND instruction with a
22902 // separate test. However, we only do this if this block doesn't
22903 // have a fall-through edge, because this requires an explicit
22904 // jmp when the condition is false.
22905 if (Op.getNode()->hasOneUse()) {
22906 SDNode *User = *Op.getNode()->use_begin();
22907 // Look for an unconditional branch following this conditional branch.
22908 // We need this because we need to reverse the successors in order
22909 // to implement FCMP_OEQ.
22910 if (User->getOpcode() == ISD::BR) {
22911 SDValue FalseBB = User->getOperand(1);
22912 SDNode *NewBR =
22913 DAG.UpdateNodeOperands(User, User->getOperand(0), Dest);
22914 assert(NewBR == User)((NewBR == User) ? static_cast<void> (0) : __assert_fail
("NewBR == User", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 22914, __PRETTY_FUNCTION__));
22915 (void)NewBR;
22916 Dest = FalseBB;
22917
22918 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
22919 Cond.getOperand(0), Cond.getOperand(1));
22920 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
22921 CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
22922 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
22923 Chain, Dest, CC, Cmp);
22924 CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
22925 Cond = Cmp;
22926 addTest = false;
22927 }
22928 }
22929 } else if (Cond.getOpcode() == ISD::SETCC &&
22930 cast<CondCodeSDNode>(Cond.getOperand(2))->get() == ISD::SETUNE) {
22931 // For FCMP_UNE, we can emit
22932 // two branches instead of an explicit OR instruction with a
22933 // separate test.
22934 SDValue Cmp = DAG.getNode(X86ISD::CMP, dl, MVT::i32,
22935 Cond.getOperand(0), Cond.getOperand(1));
22936 Cmp = ConvertCmpIfNecessary(Cmp, DAG);
22937 CC = DAG.getTargetConstant(X86::COND_NE, dl, MVT::i8);
22938 Chain = DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
22939 Chain, Dest, CC, Cmp);
22940 CC = DAG.getTargetConstant(X86::COND_P, dl, MVT::i8);
22941 Cond = Cmp;
22942 addTest = false;
22943 }
22944 }
22945
22946 if (addTest) {
22947 // Look pass the truncate if the high bits are known zero.
22948 if (isTruncWithZeroHighBitsInput(Cond, DAG))
22949 Cond = Cond.getOperand(0);
22950
22951 // We know the result of AND is compared against zero. Try to match
22952 // it to BT.
22953 if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
22954 SDValue BTCC;
22955 if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, dl, DAG, BTCC)) {
22956 CC = BTCC;
22957 Cond = BT;
22958 addTest = false;
22959 }
22960 }
22961 }
22962
22963 if (addTest) {
22964 X86::CondCode X86Cond = Inverted ? X86::COND_E : X86::COND_NE;
22965 CC = DAG.getTargetConstant(X86Cond, dl, MVT::i8);
22966 Cond = EmitTest(Cond, X86Cond, dl, DAG, Subtarget);
22967 }
22968 Cond = ConvertCmpIfNecessary(Cond, DAG);
22969 return DAG.getNode(X86ISD::BRCOND, dl, Op.getValueType(),
22970 Chain, Dest, CC, Cond);
22971}
22972
22973// Lower dynamic stack allocation to _alloca call for Cygwin/Mingw targets.
22974// Calls to _alloca are needed to probe the stack when allocating more than 4k
22975// bytes in one go. Touching the stack at 4K increments is necessary to ensure
22976// that the guard pages used by the OS virtual memory manager are allocated in
22977// correct sequence.
22978SDValue
22979X86TargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
22980 SelectionDAG &DAG) const {
22981 MachineFunction &MF = DAG.getMachineFunction();
22982 bool SplitStack = MF.shouldSplitStack();
22983 bool EmitStackProbe = !getStackProbeSymbolName(MF).empty();
22984 bool Lower = (Subtarget.isOSWindows() && !Subtarget.isTargetMachO()) ||
22985 SplitStack || EmitStackProbe;
22986 SDLoc dl(Op);
22987
22988 // Get the inputs.
22989 SDNode *Node = Op.getNode();
22990 SDValue Chain = Op.getOperand(0);
22991 SDValue Size = Op.getOperand(1);
22992 MaybeAlign Alignment(Op.getConstantOperandVal(2));
22993 EVT VT = Node->getValueType(0);
22994
22995 // Chain the dynamic stack allocation so that it doesn't modify the stack
22996 // pointer when other instructions are using the stack.
22997 Chain = DAG.getCALLSEQ_START(Chain, 0, 0, dl);
22998
22999 bool Is64Bit = Subtarget.is64Bit();
23000 MVT SPTy = getPointerTy(DAG.getDataLayout());
23001
23002 SDValue Result;
23003 if (!Lower) {
23004 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23005 unsigned SPReg = TLI.getStackPointerRegisterToSaveRestore();
23006 assert(SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23007, __PRETTY_FUNCTION__))
23007 " not tell us which reg is the stack pointer!")((SPReg && "Target cannot require DYNAMIC_STACKALLOC expansion and"
" not tell us which reg is the stack pointer!") ? static_cast
<void> (0) : __assert_fail ("SPReg && \"Target cannot require DYNAMIC_STACKALLOC expansion and\" \" not tell us which reg is the stack pointer!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23007, __PRETTY_FUNCTION__));
23008
23009 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, VT);
23010 Chain = SP.getValue(1);
23011 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
23012 const Align StackAlign(TFI.getStackAlignment());
23013 Result = DAG.getNode(ISD::SUB, dl, VT, SP, Size); // Value
23014 if (Alignment && Alignment > StackAlign)
23015 Result =
23016 DAG.getNode(ISD::AND, dl, VT, Result,
23017 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
23018 Chain = DAG.getCopyToReg(Chain, dl, SPReg, Result); // Output chain
23019 } else if (SplitStack) {
23020 MachineRegisterInfo &MRI = MF.getRegInfo();
23021
23022 if (Is64Bit) {
23023 // The 64 bit implementation of segmented stacks needs to clobber both r10
23024 // r11. This makes it impossible to use it along with nested parameters.
23025 const Function &F = MF.getFunction();
23026 for (const auto &A : F.args()) {
23027 if (A.hasNestAttr())
23028 report_fatal_error("Cannot use segmented stacks with functions that "
23029 "have nested arguments.");
23030 }
23031 }
23032
23033 const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
23034 Register Vreg = MRI.createVirtualRegister(AddrRegClass);
23035 Chain = DAG.getCopyToReg(Chain, dl, Vreg, Size);
23036 Result = DAG.getNode(X86ISD::SEG_ALLOCA, dl, SPTy, Chain,
23037 DAG.getRegister(Vreg, SPTy));
23038 } else {
23039 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
23040 Chain = DAG.getNode(X86ISD::WIN_ALLOCA, dl, NodeTys, Chain, Size);
23041 MF.getInfo<X86MachineFunctionInfo>()->setHasWinAlloca(true);
23042
23043 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
23044 Register SPReg = RegInfo->getStackRegister();
23045 SDValue SP = DAG.getCopyFromReg(Chain, dl, SPReg, SPTy);
23046 Chain = SP.getValue(1);
23047
23048 if (Alignment) {
23049 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
23050 DAG.getConstant(~(Alignment->value() - 1ULL), dl, VT));
23051 Chain = DAG.getCopyToReg(Chain, dl, SPReg, SP);
23052 }
23053
23054 Result = SP;
23055 }
23056
23057 Chain = DAG.getCALLSEQ_END(Chain, DAG.getIntPtrConstant(0, dl, true),
23058 DAG.getIntPtrConstant(0, dl, true), SDValue(), dl);
23059
23060 SDValue Ops[2] = {Result, Chain};
23061 return DAG.getMergeValues(Ops, dl);
23062}
23063
23064SDValue X86TargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
23065 MachineFunction &MF = DAG.getMachineFunction();
23066 auto PtrVT = getPointerTy(MF.getDataLayout());
23067 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
23068
23069 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
23070 SDLoc DL(Op);
23071
23072 if (!Subtarget.is64Bit() ||
23073 Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv())) {
23074 // vastart just stores the address of the VarArgsFrameIndex slot into the
23075 // memory location argument.
23076 SDValue FR = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
23077 return DAG.getStore(Op.getOperand(0), DL, FR, Op.getOperand(1),
23078 MachinePointerInfo(SV));
23079 }
23080
23081 // __va_list_tag:
23082 // gp_offset (0 - 6 * 8)
23083 // fp_offset (48 - 48 + 8 * 16)
23084 // overflow_arg_area (point to parameters coming in memory).
23085 // reg_save_area
23086 SmallVector<SDValue, 8> MemOps;
23087 SDValue FIN = Op.getOperand(1);
23088 // Store gp_offset
23089 SDValue Store = DAG.getStore(
23090 Op.getOperand(0), DL,
23091 DAG.getConstant(FuncInfo->getVarArgsGPOffset(), DL, MVT::i32), FIN,
23092 MachinePointerInfo(SV));
23093 MemOps.push_back(Store);
23094
23095 // Store fp_offset
23096 FIN = DAG.getMemBasePlusOffset(FIN, 4, DL);
23097 Store = DAG.getStore(
23098 Op.getOperand(0), DL,
23099 DAG.getConstant(FuncInfo->getVarArgsFPOffset(), DL, MVT::i32), FIN,
23100 MachinePointerInfo(SV, 4));
23101 MemOps.push_back(Store);
23102
23103 // Store ptr to overflow_arg_area
23104 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(4, DL));
23105 SDValue OVFIN = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(), PtrVT);
23106 Store =
23107 DAG.getStore(Op.getOperand(0), DL, OVFIN, FIN, MachinePointerInfo(SV, 8));
23108 MemOps.push_back(Store);
23109
23110 // Store ptr to reg_save_area.
23111 FIN = DAG.getNode(ISD::ADD, DL, PtrVT, FIN, DAG.getIntPtrConstant(
23112 Subtarget.isTarget64BitLP64() ? 8 : 4, DL));
23113 SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(), PtrVT);
23114 Store = DAG.getStore(
23115 Op.getOperand(0), DL, RSFIN, FIN,
23116 MachinePointerInfo(SV, Subtarget.isTarget64BitLP64() ? 16 : 12));
23117 MemOps.push_back(Store);
23118 return DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOps);
23119}
23120
23121SDValue X86TargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
23122 assert(Subtarget.is64Bit() &&((Subtarget.is64Bit() && "LowerVAARG only handles 64-bit va_arg!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.is64Bit() && \"LowerVAARG only handles 64-bit va_arg!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23123, __PRETTY_FUNCTION__))
23123 "LowerVAARG only handles 64-bit va_arg!")((Subtarget.is64Bit() && "LowerVAARG only handles 64-bit va_arg!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.is64Bit() && \"LowerVAARG only handles 64-bit va_arg!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23123, __PRETTY_FUNCTION__));
23124 assert(Op.getNumOperands() == 4)((Op.getNumOperands() == 4) ? static_cast<void> (0) : __assert_fail
("Op.getNumOperands() == 4", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23124, __PRETTY_FUNCTION__));
23125
23126 MachineFunction &MF = DAG.getMachineFunction();
23127 if (Subtarget.isCallingConvWin64(MF.getFunction().getCallingConv()))
23128 // The Win64 ABI uses char* instead of a structure.
23129 return DAG.expandVAArg(Op.getNode());
23130
23131 SDValue Chain = Op.getOperand(0);
23132 SDValue SrcPtr = Op.getOperand(1);
23133 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
23134 unsigned Align = Op.getConstantOperandVal(3);
23135 SDLoc dl(Op);
23136
23137 EVT ArgVT = Op.getNode()->getValueType(0);
23138 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
23139 uint32_t ArgSize = DAG.getDataLayout().getTypeAllocSize(ArgTy);
23140 uint8_t ArgMode;
23141
23142 // Decide which area this value should be read from.
23143 // TODO: Implement the AMD64 ABI in its entirety. This simple
23144 // selection mechanism works only for the basic types.
23145 if (ArgVT == MVT::f80) {
23146 llvm_unreachable("va_arg for f80 not yet implemented")::llvm::llvm_unreachable_internal("va_arg for f80 not yet implemented"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23146);
23147 } else if (ArgVT.isFloatingPoint() && ArgSize <= 16 /*bytes*/) {
23148 ArgMode = 2; // Argument passed in XMM register. Use fp_offset.
23149 } else if (ArgVT.isInteger() && ArgSize <= 32 /*bytes*/) {
23150 ArgMode = 1; // Argument passed in GPR64 register(s). Use gp_offset.
23151 } else {
23152 llvm_unreachable("Unhandled argument type in LowerVAARG")::llvm::llvm_unreachable_internal("Unhandled argument type in LowerVAARG"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23152);
23153 }
23154
23155 if (ArgMode == 2) {
23156 // Sanity Check: Make sure using fp_offset makes sense.
23157 assert(!Subtarget.useSoftFloat() &&((!Subtarget.useSoftFloat() && !(MF.getFunction().hasFnAttribute
(Attribute::NoImplicitFloat)) && Subtarget.hasSSE1())
? static_cast<void> (0) : __assert_fail ("!Subtarget.useSoftFloat() && !(MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) && Subtarget.hasSSE1()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23159, __PRETTY_FUNCTION__))
23158 !(MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) &&((!Subtarget.useSoftFloat() && !(MF.getFunction().hasFnAttribute
(Attribute::NoImplicitFloat)) && Subtarget.hasSSE1())
? static_cast<void> (0) : __assert_fail ("!Subtarget.useSoftFloat() && !(MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) && Subtarget.hasSSE1()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23159, __PRETTY_FUNCTION__))
23159 Subtarget.hasSSE1())((!Subtarget.useSoftFloat() && !(MF.getFunction().hasFnAttribute
(Attribute::NoImplicitFloat)) && Subtarget.hasSSE1())
? static_cast<void> (0) : __assert_fail ("!Subtarget.useSoftFloat() && !(MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat)) && Subtarget.hasSSE1()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23159, __PRETTY_FUNCTION__));
23160 }
23161
23162 // Insert VAARG_64 node into the DAG
23163 // VAARG_64 returns two values: Variable Argument Address, Chain
23164 SDValue InstOps[] = {Chain, SrcPtr, DAG.getConstant(ArgSize, dl, MVT::i32),
23165 DAG.getConstant(ArgMode, dl, MVT::i8),
23166 DAG.getConstant(Align, dl, MVT::i32)};
23167 SDVTList VTs = DAG.getVTList(getPointerTy(DAG.getDataLayout()), MVT::Other);
23168 SDValue VAARG = DAG.getMemIntrinsicNode(
23169 X86ISD::VAARG_64, dl,
23170 VTs, InstOps, MVT::i64,
23171 MachinePointerInfo(SV),
23172 /*Align=*/0,
23173 MachineMemOperand::MOLoad | MachineMemOperand::MOStore);
23174 Chain = VAARG.getValue(1);
23175
23176 // Load the next argument and return it
23177 return DAG.getLoad(ArgVT, dl, Chain, VAARG, MachinePointerInfo());
23178}
23179
23180static SDValue LowerVACOPY(SDValue Op, const X86Subtarget &Subtarget,
23181 SelectionDAG &DAG) {
23182 // X86-64 va_list is a struct { i32, i32, i8*, i8* }, except on Windows,
23183 // where a va_list is still an i8*.
23184 assert(Subtarget.is64Bit() && "This code only handles 64-bit va_copy!")((Subtarget.is64Bit() && "This code only handles 64-bit va_copy!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.is64Bit() && \"This code only handles 64-bit va_copy!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23184, __PRETTY_FUNCTION__));
23185 if (Subtarget.isCallingConvWin64(
23186 DAG.getMachineFunction().getFunction().getCallingConv()))
23187 // Probably a Win64 va_copy.
23188 return DAG.expandVACopy(Op.getNode());
23189
23190 SDValue Chain = Op.getOperand(0);
23191 SDValue DstPtr = Op.getOperand(1);
23192 SDValue SrcPtr = Op.getOperand(2);
23193 const Value *DstSV = cast<SrcValueSDNode>(Op.getOperand(3))->getValue();
23194 const Value *SrcSV = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
23195 SDLoc DL(Op);
23196
23197 return DAG.getMemcpy(Chain, DL, DstPtr, SrcPtr,
23198 DAG.getIntPtrConstant(24, DL), 8, /*isVolatile*/false,
23199 false, false,
23200 MachinePointerInfo(DstSV), MachinePointerInfo(SrcSV));
23201}
23202
23203// Helper to get immediate/variable SSE shift opcode from other shift opcodes.
23204static unsigned getTargetVShiftUniformOpcode(unsigned Opc, bool IsVariable) {
23205 switch (Opc) {
23206 case ISD::SHL:
23207 case X86ISD::VSHL:
23208 case X86ISD::VSHLI:
23209 return IsVariable ? X86ISD::VSHL : X86ISD::VSHLI;
23210 case ISD::SRL:
23211 case X86ISD::VSRL:
23212 case X86ISD::VSRLI:
23213 return IsVariable ? X86ISD::VSRL : X86ISD::VSRLI;
23214 case ISD::SRA:
23215 case X86ISD::VSRA:
23216 case X86ISD::VSRAI:
23217 return IsVariable ? X86ISD::VSRA : X86ISD::VSRAI;
23218 }
23219 llvm_unreachable("Unknown target vector shift node")::llvm::llvm_unreachable_internal("Unknown target vector shift node"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23219);
23220}
23221
23222/// Handle vector element shifts where the shift amount is a constant.
23223/// Takes immediate version of shift as input.
23224static SDValue getTargetVShiftByConstNode(unsigned Opc, const SDLoc &dl, MVT VT,
23225 SDValue SrcOp, uint64_t ShiftAmt,
23226 SelectionDAG &DAG) {
23227 MVT ElementType = VT.getVectorElementType();
23228
23229 // Bitcast the source vector to the output type, this is mainly necessary for
23230 // vXi8/vXi64 shifts.
23231 if (VT != SrcOp.getSimpleValueType())
23232 SrcOp = DAG.getBitcast(VT, SrcOp);
23233
23234 // Fold this packed shift into its first operand if ShiftAmt is 0.
23235 if (ShiftAmt == 0)
23236 return SrcOp;
23237
23238 // Check for ShiftAmt >= element width
23239 if (ShiftAmt >= ElementType.getSizeInBits()) {
23240 if (Opc == X86ISD::VSRAI)
23241 ShiftAmt = ElementType.getSizeInBits() - 1;
23242 else
23243 return DAG.getConstant(0, dl, VT);
23244 }
23245
23246 assert((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI)(((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD
::VSRAI) && "Unknown target vector shift-by-constant node"
) ? static_cast<void> (0) : __assert_fail ("(Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI) && \"Unknown target vector shift-by-constant node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23247, __PRETTY_FUNCTION__))
23247 && "Unknown target vector shift-by-constant node")(((Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD
::VSRAI) && "Unknown target vector shift-by-constant node"
) ? static_cast<void> (0) : __assert_fail ("(Opc == X86ISD::VSHLI || Opc == X86ISD::VSRLI || Opc == X86ISD::VSRAI) && \"Unknown target vector shift-by-constant node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23247, __PRETTY_FUNCTION__));
23248
23249 // Fold this packed vector shift into a build vector if SrcOp is a
23250 // vector of Constants or UNDEFs.
23251 if (ISD::isBuildVectorOfConstantSDNodes(SrcOp.getNode())) {
23252 SmallVector<SDValue, 8> Elts;
23253 unsigned NumElts = SrcOp->getNumOperands();
23254
23255 switch (Opc) {
23256 default: llvm_unreachable("Unknown opcode!")::llvm::llvm_unreachable_internal("Unknown opcode!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23256);
23257 case X86ISD::VSHLI:
23258 for (unsigned i = 0; i != NumElts; ++i) {
23259 SDValue CurrentOp = SrcOp->getOperand(i);
23260 if (CurrentOp->isUndef()) {
23261 Elts.push_back(CurrentOp);
23262 continue;
23263 }
23264 auto *ND = cast<ConstantSDNode>(CurrentOp);
23265 const APInt &C = ND->getAPIntValue();
23266 Elts.push_back(DAG.getConstant(C.shl(ShiftAmt), dl, ElementType));
23267 }
23268 break;
23269 case X86ISD::VSRLI:
23270 for (unsigned i = 0; i != NumElts; ++i) {
23271 SDValue CurrentOp = SrcOp->getOperand(i);
23272 if (CurrentOp->isUndef()) {
23273 Elts.push_back(CurrentOp);
23274 continue;
23275 }
23276 auto *ND = cast<ConstantSDNode>(CurrentOp);
23277 const APInt &C = ND->getAPIntValue();
23278 Elts.push_back(DAG.getConstant(C.lshr(ShiftAmt), dl, ElementType));
23279 }
23280 break;
23281 case X86ISD::VSRAI:
23282 for (unsigned i = 0; i != NumElts; ++i) {
23283 SDValue CurrentOp = SrcOp->getOperand(i);
23284 if (CurrentOp->isUndef()) {
23285 Elts.push_back(CurrentOp);
23286 continue;
23287 }
23288 auto *ND = cast<ConstantSDNode>(CurrentOp);
23289 const APInt &C = ND->getAPIntValue();
23290 Elts.push_back(DAG.getConstant(C.ashr(ShiftAmt), dl, ElementType));
23291 }
23292 break;
23293 }
23294
23295 return DAG.getBuildVector(VT, dl, Elts);
23296 }
23297
23298 return DAG.getNode(Opc, dl, VT, SrcOp,
23299 DAG.getTargetConstant(ShiftAmt, dl, MVT::i8));
23300}
23301
23302/// Handle vector element shifts where the shift amount may or may not be a
23303/// constant. Takes immediate version of shift as input.
23304static SDValue getTargetVShiftNode(unsigned Opc, const SDLoc &dl, MVT VT,
23305 SDValue SrcOp, SDValue ShAmt,
23306 const X86Subtarget &Subtarget,
23307 SelectionDAG &DAG) {
23308 MVT SVT = ShAmt.getSimpleValueType();
23309 assert((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!")(((SVT == MVT::i32 || SVT == MVT::i64) && "Unexpected value type!"
) ? static_cast<void> (0) : __assert_fail ("(SVT == MVT::i32 || SVT == MVT::i64) && \"Unexpected value type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23309, __PRETTY_FUNCTION__));
23310
23311 // Catch shift-by-constant.
23312 if (ConstantSDNode *CShAmt = dyn_cast<ConstantSDNode>(ShAmt))
23313 return getTargetVShiftByConstNode(Opc, dl, VT, SrcOp,
23314 CShAmt->getZExtValue(), DAG);
23315
23316 // Change opcode to non-immediate version.
23317 Opc = getTargetVShiftUniformOpcode(Opc, true);
23318
23319 // Need to build a vector containing shift amount.
23320 // SSE/AVX packed shifts only use the lower 64-bit of the shift count.
23321 // +====================+============+=======================================+
23322 // | ShAmt is | HasSSE4.1? | Construct ShAmt vector as |
23323 // +====================+============+=======================================+
23324 // | i64 | Yes, No | Use ShAmt as lowest elt |
23325 // | i32 | Yes | zero-extend in-reg |
23326 // | (i32 zext(i16/i8)) | Yes | zero-extend in-reg |
23327 // | (i32 zext(i16/i8)) | No | byte-shift-in-reg |
23328 // | i16/i32 | No | v4i32 build_vector(ShAmt, 0, ud, ud)) |
23329 // +====================+============+=======================================+
23330
23331 if (SVT == MVT::i64)
23332 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(ShAmt), MVT::v2i64, ShAmt);
23333 else if (ShAmt.getOpcode() == ISD::ZERO_EXTEND &&
23334 ShAmt.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
23335 (ShAmt.getOperand(0).getSimpleValueType() == MVT::i16 ||
23336 ShAmt.getOperand(0).getSimpleValueType() == MVT::i8)) {
23337 ShAmt = ShAmt.getOperand(0);
23338 MVT AmtTy = ShAmt.getSimpleValueType() == MVT::i8 ? MVT::v16i8 : MVT::v8i16;
23339 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(ShAmt), AmtTy, ShAmt);
23340 if (Subtarget.hasSSE41())
23341 ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
23342 MVT::v2i64, ShAmt);
23343 else {
23344 SDValue ByteShift = DAG.getTargetConstant(
23345 (128 - AmtTy.getScalarSizeInBits()) / 8, SDLoc(ShAmt), MVT::i8);
23346 ShAmt = DAG.getBitcast(MVT::v16i8, ShAmt);
23347 ShAmt = DAG.getNode(X86ISD::VSHLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
23348 ByteShift);
23349 ShAmt = DAG.getNode(X86ISD::VSRLDQ, SDLoc(ShAmt), MVT::v16i8, ShAmt,
23350 ByteShift);
23351 }
23352 } else if (Subtarget.hasSSE41() &&
23353 ShAmt.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
23354 ShAmt = DAG.getNode(ISD::SCALAR_TO_VECTOR, SDLoc(ShAmt), MVT::v4i32, ShAmt);
23355 ShAmt = DAG.getNode(ISD::ZERO_EXTEND_VECTOR_INREG, SDLoc(ShAmt),
23356 MVT::v2i64, ShAmt);
23357 } else {
23358 SDValue ShOps[4] = {ShAmt, DAG.getConstant(0, dl, SVT), DAG.getUNDEF(SVT),
23359 DAG.getUNDEF(SVT)};
23360 ShAmt = DAG.getBuildVector(MVT::v4i32, dl, ShOps);
23361 }
23362
23363 // The return type has to be a 128-bit type with the same element
23364 // type as the input type.
23365 MVT EltVT = VT.getVectorElementType();
23366 MVT ShVT = MVT::getVectorVT(EltVT, 128 / EltVT.getSizeInBits());
23367
23368 ShAmt = DAG.getBitcast(ShVT, ShAmt);
23369 return DAG.getNode(Opc, dl, VT, SrcOp, ShAmt);
23370}
23371
23372/// Return Mask with the necessary casting or extending
23373/// for \p Mask according to \p MaskVT when lowering masking intrinsics
23374static SDValue getMaskNode(SDValue Mask, MVT MaskVT,
23375 const X86Subtarget &Subtarget, SelectionDAG &DAG,
23376 const SDLoc &dl) {
23377
23378 if (isAllOnesConstant(Mask))
23379 return DAG.getConstant(1, dl, MaskVT);
23380 if (X86::isZeroNode(Mask))
23381 return DAG.getConstant(0, dl, MaskVT);
23382
23383 assert(MaskVT.bitsLE(Mask.getSimpleValueType()) && "Unexpected mask size!")((MaskVT.bitsLE(Mask.getSimpleValueType()) && "Unexpected mask size!"
) ? static_cast<void> (0) : __assert_fail ("MaskVT.bitsLE(Mask.getSimpleValueType()) && \"Unexpected mask size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23383, __PRETTY_FUNCTION__));
23384
23385 if (Mask.getSimpleValueType() == MVT::i64 && Subtarget.is32Bit()) {
23386 assert(MaskVT == MVT::v64i1 && "Expected v64i1 mask!")((MaskVT == MVT::v64i1 && "Expected v64i1 mask!") ? static_cast
<void> (0) : __assert_fail ("MaskVT == MVT::v64i1 && \"Expected v64i1 mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23386, __PRETTY_FUNCTION__));
23387 assert(Subtarget.hasBWI() && "Expected AVX512BW target!")((Subtarget.hasBWI() && "Expected AVX512BW target!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasBWI() && \"Expected AVX512BW target!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23387, __PRETTY_FUNCTION__));
23388 // In case 32bit mode, bitcast i64 is illegal, extend/split it.
23389 SDValue Lo, Hi;
23390 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask,
23391 DAG.getConstant(0, dl, MVT::i32));
23392 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Mask,
23393 DAG.getConstant(1, dl, MVT::i32));
23394
23395 Lo = DAG.getBitcast(MVT::v32i1, Lo);
23396 Hi = DAG.getBitcast(MVT::v32i1, Hi);
23397
23398 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
23399 } else {
23400 MVT BitcastVT = MVT::getVectorVT(MVT::i1,
23401 Mask.getSimpleValueType().getSizeInBits());
23402 // In case when MaskVT equals v2i1 or v4i1, low 2 or 4 elements
23403 // are extracted by EXTRACT_SUBVECTOR.
23404 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MaskVT,
23405 DAG.getBitcast(BitcastVT, Mask),
23406 DAG.getIntPtrConstant(0, dl));
23407 }
23408}
23409
23410/// Return (and \p Op, \p Mask) for compare instructions or
23411/// (vselect \p Mask, \p Op, \p PreservedSrc) for others along with the
23412/// necessary casting or extending for \p Mask when lowering masking intrinsics
23413static SDValue getVectorMaskingNode(SDValue Op, SDValue Mask,
23414 SDValue PreservedSrc,
23415 const X86Subtarget &Subtarget,
23416 SelectionDAG &DAG) {
23417 MVT VT = Op.getSimpleValueType();
23418 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
23419 unsigned OpcodeSelect = ISD::VSELECT;
23420 SDLoc dl(Op);
23421
23422 if (isAllOnesConstant(Mask))
23423 return Op;
23424
23425 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
23426
23427 if (PreservedSrc.isUndef())
23428 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
23429 return DAG.getNode(OpcodeSelect, dl, VT, VMask, Op, PreservedSrc);
23430}
23431
23432/// Creates an SDNode for a predicated scalar operation.
23433/// \returns (X86vselect \p Mask, \p Op, \p PreservedSrc).
23434/// The mask is coming as MVT::i8 and it should be transformed
23435/// to MVT::v1i1 while lowering masking intrinsics.
23436/// The main difference between ScalarMaskingNode and VectorMaskingNode is using
23437/// "X86select" instead of "vselect". We just can't create the "vselect" node
23438/// for a scalar instruction.
23439static SDValue getScalarMaskingNode(SDValue Op, SDValue Mask,
23440 SDValue PreservedSrc,
23441 const X86Subtarget &Subtarget,
23442 SelectionDAG &DAG) {
23443
23444 if (auto *MaskConst = dyn_cast<ConstantSDNode>(Mask))
23445 if (MaskConst->getZExtValue() & 0x1)
23446 return Op;
23447
23448 MVT VT = Op.getSimpleValueType();
23449 SDLoc dl(Op);
23450
23451 assert(Mask.getValueType() == MVT::i8 && "Unexpect type")((Mask.getValueType() == MVT::i8 && "Unexpect type") ?
static_cast<void> (0) : __assert_fail ("Mask.getValueType() == MVT::i8 && \"Unexpect type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23451, __PRETTY_FUNCTION__));
23452 SDValue IMask = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, MVT::v1i1,
23453 DAG.getBitcast(MVT::v8i1, Mask),
23454 DAG.getIntPtrConstant(0, dl));
23455 if (Op.getOpcode() == X86ISD::FSETCCM ||
23456 Op.getOpcode() == X86ISD::FSETCCM_SAE ||
23457 Op.getOpcode() == X86ISD::VFPCLASSS)
23458 return DAG.getNode(ISD::AND, dl, VT, Op, IMask);
23459
23460 if (PreservedSrc.isUndef())
23461 PreservedSrc = getZeroVector(VT, Subtarget, DAG, dl);
23462 return DAG.getNode(X86ISD::SELECTS, dl, VT, IMask, Op, PreservedSrc);
23463}
23464
23465static int getSEHRegistrationNodeSize(const Function *Fn) {
23466 if (!Fn->hasPersonalityFn())
23467 report_fatal_error(
23468 "querying registration node size for function without personality");
23469 // The RegNodeSize is 6 32-bit words for SEH and 4 for C++ EH. See
23470 // WinEHStatePass for the full struct definition.
23471 switch (classifyEHPersonality(Fn->getPersonalityFn())) {
23472 case EHPersonality::MSVC_X86SEH: return 24;
23473 case EHPersonality::MSVC_CXX: return 16;
23474 default: break;
23475 }
23476 report_fatal_error(
23477 "can only recover FP for 32-bit MSVC EH personality functions");
23478}
23479
23480/// When the MSVC runtime transfers control to us, either to an outlined
23481/// function or when returning to a parent frame after catching an exception, we
23482/// recover the parent frame pointer by doing arithmetic on the incoming EBP.
23483/// Here's the math:
23484/// RegNodeBase = EntryEBP - RegNodeSize
23485/// ParentFP = RegNodeBase - ParentFrameOffset
23486/// Subtracting RegNodeSize takes us to the offset of the registration node, and
23487/// subtracting the offset (negative on x86) takes us back to the parent FP.
23488static SDValue recoverFramePointer(SelectionDAG &DAG, const Function *Fn,
23489 SDValue EntryEBP) {
23490 MachineFunction &MF = DAG.getMachineFunction();
23491 SDLoc dl;
23492
23493 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
23494 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
23495
23496 // It's possible that the parent function no longer has a personality function
23497 // if the exceptional code was optimized away, in which case we just return
23498 // the incoming EBP.
23499 if (!Fn->hasPersonalityFn())
23500 return EntryEBP;
23501
23502 // Get an MCSymbol that will ultimately resolve to the frame offset of the EH
23503 // registration, or the .set_setframe offset.
23504 MCSymbol *OffsetSym =
23505 MF.getMMI().getContext().getOrCreateParentFrameOffsetSymbol(
23506 GlobalValue::dropLLVMManglingEscape(Fn->getName()));
23507 SDValue OffsetSymVal = DAG.getMCSymbol(OffsetSym, PtrVT);
23508 SDValue ParentFrameOffset =
23509 DAG.getNode(ISD::LOCAL_RECOVER, dl, PtrVT, OffsetSymVal);
23510
23511 // Return EntryEBP + ParentFrameOffset for x64. This adjusts from RSP after
23512 // prologue to RBP in the parent function.
23513 const X86Subtarget &Subtarget =
23514 static_cast<const X86Subtarget &>(DAG.getSubtarget());
23515 if (Subtarget.is64Bit())
23516 return DAG.getNode(ISD::ADD, dl, PtrVT, EntryEBP, ParentFrameOffset);
23517
23518 int RegNodeSize = getSEHRegistrationNodeSize(Fn);
23519 // RegNodeBase = EntryEBP - RegNodeSize
23520 // ParentFP = RegNodeBase - ParentFrameOffset
23521 SDValue RegNodeBase = DAG.getNode(ISD::SUB, dl, PtrVT, EntryEBP,
23522 DAG.getConstant(RegNodeSize, dl, PtrVT));
23523 return DAG.getNode(ISD::SUB, dl, PtrVT, RegNodeBase, ParentFrameOffset);
23524}
23525
23526SDValue X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
23527 SelectionDAG &DAG) const {
23528 // Helper to detect if the operand is CUR_DIRECTION rounding mode.
23529 auto isRoundModeCurDirection = [](SDValue Rnd) {
23530 if (auto *C = dyn_cast<ConstantSDNode>(Rnd))
23531 return C->getAPIntValue() == X86::STATIC_ROUNDING::CUR_DIRECTION;
23532
23533 return false;
23534 };
23535 auto isRoundModeSAE = [](SDValue Rnd) {
23536 if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
23537 unsigned RC = C->getZExtValue();
23538 if (RC & X86::STATIC_ROUNDING::NO_EXC) {
23539 // Clear the NO_EXC bit and check remaining bits.
23540 RC ^= X86::STATIC_ROUNDING::NO_EXC;
23541 // As a convenience we allow no other bits or explicitly
23542 // current direction.
23543 return RC == 0 || RC == X86::STATIC_ROUNDING::CUR_DIRECTION;
23544 }
23545 }
23546
23547 return false;
23548 };
23549 auto isRoundModeSAEToX = [](SDValue Rnd, unsigned &RC) {
23550 if (auto *C = dyn_cast<ConstantSDNode>(Rnd)) {
23551 RC = C->getZExtValue();
23552 if (RC & X86::STATIC_ROUNDING::NO_EXC) {
23553 // Clear the NO_EXC bit and check remaining bits.
23554 RC ^= X86::STATIC_ROUNDING::NO_EXC;
23555 return RC == X86::STATIC_ROUNDING::TO_NEAREST_INT ||
23556 RC == X86::STATIC_ROUNDING::TO_NEG_INF ||
23557 RC == X86::STATIC_ROUNDING::TO_POS_INF ||
23558 RC == X86::STATIC_ROUNDING::TO_ZERO;
23559 }
23560 }
23561
23562 return false;
23563 };
23564
23565 SDLoc dl(Op);
23566 unsigned IntNo = Op.getConstantOperandVal(0);
23567 MVT VT = Op.getSimpleValueType();
23568 const IntrinsicData* IntrData = getIntrinsicWithoutChain(IntNo);
23569
23570 if (IntrData) {
23571 switch(IntrData->Type) {
23572 case INTR_TYPE_1OP: {
23573 // We specify 2 possible opcodes for intrinsics with rounding modes.
23574 // First, we check if the intrinsic may have non-default rounding mode,
23575 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
23576 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
23577 if (IntrWithRoundingModeOpcode != 0) {
23578 SDValue Rnd = Op.getOperand(2);
23579 unsigned RC = 0;
23580 if (isRoundModeSAEToX(Rnd, RC))
23581 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
23582 Op.getOperand(1),
23583 DAG.getTargetConstant(RC, dl, MVT::i32));
23584 if (!isRoundModeCurDirection(Rnd))
23585 return SDValue();
23586 }
23587 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
23588 Op.getOperand(1));
23589 }
23590 case INTR_TYPE_1OP_SAE: {
23591 SDValue Sae = Op.getOperand(2);
23592
23593 unsigned Opc;
23594 if (isRoundModeCurDirection(Sae))
23595 Opc = IntrData->Opc0;
23596 else if (isRoundModeSAE(Sae))
23597 Opc = IntrData->Opc1;
23598 else
23599 return SDValue();
23600
23601 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1));
23602 }
23603 case INTR_TYPE_2OP: {
23604 SDValue Src2 = Op.getOperand(2);
23605
23606 // We specify 2 possible opcodes for intrinsics with rounding modes.
23607 // First, we check if the intrinsic may have non-default rounding mode,
23608 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
23609 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
23610 if (IntrWithRoundingModeOpcode != 0) {
23611 SDValue Rnd = Op.getOperand(3);
23612 unsigned RC = 0;
23613 if (isRoundModeSAEToX(Rnd, RC))
23614 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
23615 Op.getOperand(1), Src2,
23616 DAG.getTargetConstant(RC, dl, MVT::i32));
23617 if (!isRoundModeCurDirection(Rnd))
23618 return SDValue();
23619 }
23620
23621 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
23622 Op.getOperand(1), Src2);
23623 }
23624 case INTR_TYPE_2OP_SAE: {
23625 SDValue Sae = Op.getOperand(3);
23626
23627 unsigned Opc;
23628 if (isRoundModeCurDirection(Sae))
23629 Opc = IntrData->Opc0;
23630 else if (isRoundModeSAE(Sae))
23631 Opc = IntrData->Opc1;
23632 else
23633 return SDValue();
23634
23635 return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
23636 Op.getOperand(2));
23637 }
23638 case INTR_TYPE_3OP:
23639 case INTR_TYPE_3OP_IMM8: {
23640 SDValue Src1 = Op.getOperand(1);
23641 SDValue Src2 = Op.getOperand(2);
23642 SDValue Src3 = Op.getOperand(3);
23643
23644 // We specify 2 possible opcodes for intrinsics with rounding modes.
23645 // First, we check if the intrinsic may have non-default rounding mode,
23646 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
23647 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
23648 if (IntrWithRoundingModeOpcode != 0) {
23649 SDValue Rnd = Op.getOperand(4);
23650 unsigned RC = 0;
23651 if (isRoundModeSAEToX(Rnd, RC))
23652 return DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
23653 Src1, Src2, Src3,
23654 DAG.getTargetConstant(RC, dl, MVT::i32));
23655 if (!isRoundModeCurDirection(Rnd))
23656 return SDValue();
23657 }
23658
23659 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
23660 {Src1, Src2, Src3});
23661 }
23662 case INTR_TYPE_4OP:
23663 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Op.getOperand(1),
23664 Op.getOperand(2), Op.getOperand(3), Op.getOperand(4));
23665 case INTR_TYPE_1OP_MASK: {
23666 SDValue Src = Op.getOperand(1);
23667 SDValue PassThru = Op.getOperand(2);
23668 SDValue Mask = Op.getOperand(3);
23669 // We add rounding mode to the Node when
23670 // - RC Opcode is specified and
23671 // - RC is not "current direction".
23672 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
23673 if (IntrWithRoundingModeOpcode != 0) {
23674 SDValue Rnd = Op.getOperand(4);
23675 unsigned RC = 0;
23676 if (isRoundModeSAEToX(Rnd, RC))
23677 return getVectorMaskingNode(
23678 DAG.getNode(IntrWithRoundingModeOpcode, dl, Op.getValueType(),
23679 Src, DAG.getTargetConstant(RC, dl, MVT::i32)),
23680 Mask, PassThru, Subtarget, DAG);
23681 if (!isRoundModeCurDirection(Rnd))
23682 return SDValue();
23683 }
23684 return getVectorMaskingNode(
23685 DAG.getNode(IntrData->Opc0, dl, VT, Src), Mask, PassThru,
23686 Subtarget, DAG);
23687 }
23688 case INTR_TYPE_1OP_MASK_SAE: {
23689 SDValue Src = Op.getOperand(1);
23690 SDValue PassThru = Op.getOperand(2);
23691 SDValue Mask = Op.getOperand(3);
23692 SDValue Rnd = Op.getOperand(4);
23693
23694 unsigned Opc;
23695 if (isRoundModeCurDirection(Rnd))
23696 Opc = IntrData->Opc0;
23697 else if (isRoundModeSAE(Rnd))
23698 Opc = IntrData->Opc1;
23699 else
23700 return SDValue();
23701
23702 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src), Mask, PassThru,
23703 Subtarget, DAG);
23704 }
23705 case INTR_TYPE_SCALAR_MASK: {
23706 SDValue Src1 = Op.getOperand(1);
23707 SDValue Src2 = Op.getOperand(2);
23708 SDValue passThru = Op.getOperand(3);
23709 SDValue Mask = Op.getOperand(4);
23710 unsigned IntrWithRoundingModeOpcode = IntrData->Opc1;
23711 // There are 2 kinds of intrinsics in this group:
23712 // (1) With suppress-all-exceptions (sae) or rounding mode- 6 operands
23713 // (2) With rounding mode and sae - 7 operands.
23714 bool HasRounding = IntrWithRoundingModeOpcode != 0;
23715 if (Op.getNumOperands() == (5U + HasRounding)) {
23716 if (HasRounding) {
23717 SDValue Rnd = Op.getOperand(5);
23718 unsigned RC = 0;
23719 if (isRoundModeSAEToX(Rnd, RC))
23720 return getScalarMaskingNode(
23721 DAG.getNode(IntrWithRoundingModeOpcode, dl, VT, Src1, Src2,
23722 DAG.getTargetConstant(RC, dl, MVT::i32)),
23723 Mask, passThru, Subtarget, DAG);
23724 if (!isRoundModeCurDirection(Rnd))
23725 return SDValue();
23726 }
23727 return getScalarMaskingNode(DAG.getNode(IntrData->Opc0, dl, VT, Src1,
23728 Src2),
23729 Mask, passThru, Subtarget, DAG);
23730 }
23731
23732 assert(Op.getNumOperands() == (6U + HasRounding) &&((Op.getNumOperands() == (6U + HasRounding) && "Unexpected intrinsic form"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == (6U + HasRounding) && \"Unexpected intrinsic form\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23733, __PRETTY_FUNCTION__))
23733 "Unexpected intrinsic form")((Op.getNumOperands() == (6U + HasRounding) && "Unexpected intrinsic form"
) ? static_cast<void> (0) : __assert_fail ("Op.getNumOperands() == (6U + HasRounding) && \"Unexpected intrinsic form\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23733, __PRETTY_FUNCTION__));
23734 SDValue RoundingMode = Op.getOperand(5);
23735 unsigned Opc = IntrData->Opc0;
23736 if (HasRounding) {
23737 SDValue Sae = Op.getOperand(6);
23738 if (isRoundModeSAE(Sae))
23739 Opc = IntrWithRoundingModeOpcode;
23740 else if (!isRoundModeCurDirection(Sae))
23741 return SDValue();
23742 }
23743 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1,
23744 Src2, RoundingMode),
23745 Mask, passThru, Subtarget, DAG);
23746 }
23747 case INTR_TYPE_SCALAR_MASK_RND: {
23748 SDValue Src1 = Op.getOperand(1);
23749 SDValue Src2 = Op.getOperand(2);
23750 SDValue passThru = Op.getOperand(3);
23751 SDValue Mask = Op.getOperand(4);
23752 SDValue Rnd = Op.getOperand(5);
23753
23754 SDValue NewOp;
23755 unsigned RC = 0;
23756 if (isRoundModeCurDirection(Rnd))
23757 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
23758 else if (isRoundModeSAEToX(Rnd, RC))
23759 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
23760 DAG.getTargetConstant(RC, dl, MVT::i32));
23761 else
23762 return SDValue();
23763
23764 return getScalarMaskingNode(NewOp, Mask, passThru, Subtarget, DAG);
23765 }
23766 case INTR_TYPE_SCALAR_MASK_SAE: {
23767 SDValue Src1 = Op.getOperand(1);
23768 SDValue Src2 = Op.getOperand(2);
23769 SDValue passThru = Op.getOperand(3);
23770 SDValue Mask = Op.getOperand(4);
23771 SDValue Sae = Op.getOperand(5);
23772 unsigned Opc;
23773 if (isRoundModeCurDirection(Sae))
23774 Opc = IntrData->Opc0;
23775 else if (isRoundModeSAE(Sae))
23776 Opc = IntrData->Opc1;
23777 else
23778 return SDValue();
23779
23780 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
23781 Mask, passThru, Subtarget, DAG);
23782 }
23783 case INTR_TYPE_2OP_MASK: {
23784 SDValue Src1 = Op.getOperand(1);
23785 SDValue Src2 = Op.getOperand(2);
23786 SDValue PassThru = Op.getOperand(3);
23787 SDValue Mask = Op.getOperand(4);
23788 SDValue NewOp;
23789 if (IntrData->Opc1 != 0) {
23790 SDValue Rnd = Op.getOperand(5);
23791 unsigned RC = 0;
23792 if (isRoundModeSAEToX(Rnd, RC))
23793 NewOp = DAG.getNode(IntrData->Opc1, dl, VT, Src1, Src2,
23794 DAG.getTargetConstant(RC, dl, MVT::i32));
23795 else if (!isRoundModeCurDirection(Rnd))
23796 return SDValue();
23797 }
23798 if (!NewOp)
23799 NewOp = DAG.getNode(IntrData->Opc0, dl, VT, Src1, Src2);
23800 return getVectorMaskingNode(NewOp, Mask, PassThru, Subtarget, DAG);
23801 }
23802 case INTR_TYPE_2OP_MASK_SAE: {
23803 SDValue Src1 = Op.getOperand(1);
23804 SDValue Src2 = Op.getOperand(2);
23805 SDValue PassThru = Op.getOperand(3);
23806 SDValue Mask = Op.getOperand(4);
23807
23808 unsigned Opc = IntrData->Opc0;
23809 if (IntrData->Opc1 != 0) {
23810 SDValue Sae = Op.getOperand(5);
23811 if (isRoundModeSAE(Sae))
23812 Opc = IntrData->Opc1;
23813 else if (!isRoundModeCurDirection(Sae))
23814 return SDValue();
23815 }
23816
23817 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2),
23818 Mask, PassThru, Subtarget, DAG);
23819 }
23820 case INTR_TYPE_3OP_SCALAR_MASK_SAE: {
23821 SDValue Src1 = Op.getOperand(1);
23822 SDValue Src2 = Op.getOperand(2);
23823 SDValue Src3 = Op.getOperand(3);
23824 SDValue PassThru = Op.getOperand(4);
23825 SDValue Mask = Op.getOperand(5);
23826 SDValue Sae = Op.getOperand(6);
23827 unsigned Opc;
23828 if (isRoundModeCurDirection(Sae))
23829 Opc = IntrData->Opc0;
23830 else if (isRoundModeSAE(Sae))
23831 Opc = IntrData->Opc1;
23832 else
23833 return SDValue();
23834
23835 return getScalarMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
23836 Mask, PassThru, Subtarget, DAG);
23837 }
23838 case INTR_TYPE_3OP_MASK_SAE: {
23839 SDValue Src1 = Op.getOperand(1);
23840 SDValue Src2 = Op.getOperand(2);
23841 SDValue Src3 = Op.getOperand(3);
23842 SDValue PassThru = Op.getOperand(4);
23843 SDValue Mask = Op.getOperand(5);
23844
23845 unsigned Opc = IntrData->Opc0;
23846 if (IntrData->Opc1 != 0) {
23847 SDValue Sae = Op.getOperand(6);
23848 if (isRoundModeSAE(Sae))
23849 Opc = IntrData->Opc1;
23850 else if (!isRoundModeCurDirection(Sae))
23851 return SDValue();
23852 }
23853 return getVectorMaskingNode(DAG.getNode(Opc, dl, VT, Src1, Src2, Src3),
23854 Mask, PassThru, Subtarget, DAG);
23855 }
23856 case BLENDV: {
23857 SDValue Src1 = Op.getOperand(1);
23858 SDValue Src2 = Op.getOperand(2);
23859 SDValue Src3 = Op.getOperand(3);
23860
23861 EVT MaskVT = Src3.getValueType().changeVectorElementTypeToInteger();
23862 Src3 = DAG.getBitcast(MaskVT, Src3);
23863
23864 // Reverse the operands to match VSELECT order.
23865 return DAG.getNode(IntrData->Opc0, dl, VT, Src3, Src2, Src1);
23866 }
23867 case VPERM_2OP : {
23868 SDValue Src1 = Op.getOperand(1);
23869 SDValue Src2 = Op.getOperand(2);
23870
23871 // Swap Src1 and Src2 in the node creation
23872 return DAG.getNode(IntrData->Opc0, dl, VT,Src2, Src1);
23873 }
23874 case IFMA_OP:
23875 // NOTE: We need to swizzle the operands to pass the multiply operands
23876 // first.
23877 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
23878 Op.getOperand(2), Op.getOperand(3), Op.getOperand(1));
23879 case FPCLASSS: {
23880 SDValue Src1 = Op.getOperand(1);
23881 SDValue Imm = Op.getOperand(2);
23882 SDValue Mask = Op.getOperand(3);
23883 SDValue FPclass = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Imm);
23884 SDValue FPclassMask = getScalarMaskingNode(FPclass, Mask, SDValue(),
23885 Subtarget, DAG);
23886 // Need to fill with zeros to ensure the bitcast will produce zeroes
23887 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
23888 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
23889 DAG.getConstant(0, dl, MVT::v8i1),
23890 FPclassMask, DAG.getIntPtrConstant(0, dl));
23891 return DAG.getBitcast(MVT::i8, Ins);
23892 }
23893
23894 case CMP_MASK_CC: {
23895 MVT MaskVT = Op.getSimpleValueType();
23896 SDValue CC = Op.getOperand(3);
23897 // We specify 2 possible opcodes for intrinsics with rounding modes.
23898 // First, we check if the intrinsic may have non-default rounding mode,
23899 // (IntrData->Opc1 != 0), then we check the rounding mode operand.
23900 if (IntrData->Opc1 != 0) {
23901 SDValue Sae = Op.getOperand(4);
23902 if (isRoundModeSAE(Sae))
23903 return DAG.getNode(IntrData->Opc1, dl, MaskVT, Op.getOperand(1),
23904 Op.getOperand(2), CC, Sae);
23905 if (!isRoundModeCurDirection(Sae))
23906 return SDValue();
23907 }
23908 //default rounding mode
23909 return DAG.getNode(IntrData->Opc0, dl, MaskVT,
23910 {Op.getOperand(1), Op.getOperand(2), CC});
23911 }
23912 case CMP_MASK_SCALAR_CC: {
23913 SDValue Src1 = Op.getOperand(1);
23914 SDValue Src2 = Op.getOperand(2);
23915 SDValue CC = Op.getOperand(3);
23916 SDValue Mask = Op.getOperand(4);
23917
23918 SDValue Cmp;
23919 if (IntrData->Opc1 != 0) {
23920 SDValue Sae = Op.getOperand(5);
23921 if (isRoundModeSAE(Sae))
23922 Cmp = DAG.getNode(IntrData->Opc1, dl, MVT::v1i1, Src1, Src2, CC, Sae);
23923 else if (!isRoundModeCurDirection(Sae))
23924 return SDValue();
23925 }
23926 //default rounding mode
23927 if (!Cmp.getNode())
23928 Cmp = DAG.getNode(IntrData->Opc0, dl, MVT::v1i1, Src1, Src2, CC);
23929
23930 SDValue CmpMask = getScalarMaskingNode(Cmp, Mask, SDValue(),
23931 Subtarget, DAG);
23932 // Need to fill with zeros to ensure the bitcast will produce zeroes
23933 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
23934 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v8i1,
23935 DAG.getConstant(0, dl, MVT::v8i1),
23936 CmpMask, DAG.getIntPtrConstant(0, dl));
23937 return DAG.getBitcast(MVT::i8, Ins);
23938 }
23939 case COMI: { // Comparison intrinsics
23940 ISD::CondCode CC = (ISD::CondCode)IntrData->Opc1;
23941 SDValue LHS = Op.getOperand(1);
23942 SDValue RHS = Op.getOperand(2);
23943 SDValue Comi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, LHS, RHS);
23944 SDValue InvComi = DAG.getNode(IntrData->Opc0, dl, MVT::i32, RHS, LHS);
23945 SDValue SetCC;
23946 switch (CC) {
23947 case ISD::SETEQ: { // (ZF = 0 and PF = 0)
23948 SetCC = getSETCC(X86::COND_E, Comi, dl, DAG);
23949 SDValue SetNP = getSETCC(X86::COND_NP, Comi, dl, DAG);
23950 SetCC = DAG.getNode(ISD::AND, dl, MVT::i8, SetCC, SetNP);
23951 break;
23952 }
23953 case ISD::SETNE: { // (ZF = 1 or PF = 1)
23954 SetCC = getSETCC(X86::COND_NE, Comi, dl, DAG);
23955 SDValue SetP = getSETCC(X86::COND_P, Comi, dl, DAG);
23956 SetCC = DAG.getNode(ISD::OR, dl, MVT::i8, SetCC, SetP);
23957 break;
23958 }
23959 case ISD::SETGT: // (CF = 0 and ZF = 0)
23960 SetCC = getSETCC(X86::COND_A, Comi, dl, DAG);
23961 break;
23962 case ISD::SETLT: { // The condition is opposite to GT. Swap the operands.
23963 SetCC = getSETCC(X86::COND_A, InvComi, dl, DAG);
23964 break;
23965 }
23966 case ISD::SETGE: // CF = 0
23967 SetCC = getSETCC(X86::COND_AE, Comi, dl, DAG);
23968 break;
23969 case ISD::SETLE: // The condition is opposite to GE. Swap the operands.
23970 SetCC = getSETCC(X86::COND_AE, InvComi, dl, DAG);
23971 break;
23972 default:
23973 llvm_unreachable("Unexpected illegal condition!")::llvm::llvm_unreachable_internal("Unexpected illegal condition!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 23973);
23974 }
23975 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
23976 }
23977 case COMI_RM: { // Comparison intrinsics with Sae
23978 SDValue LHS = Op.getOperand(1);
23979 SDValue RHS = Op.getOperand(2);
23980 unsigned CondVal = Op.getConstantOperandVal(3);
23981 SDValue Sae = Op.getOperand(4);
23982
23983 SDValue FCmp;
23984 if (isRoundModeCurDirection(Sae))
23985 FCmp = DAG.getNode(X86ISD::FSETCCM, dl, MVT::v1i1, LHS, RHS,
23986 DAG.getTargetConstant(CondVal, dl, MVT::i8));
23987 else if (isRoundModeSAE(Sae))
23988 FCmp = DAG.getNode(X86ISD::FSETCCM_SAE, dl, MVT::v1i1, LHS, RHS,
23989 DAG.getTargetConstant(CondVal, dl, MVT::i8), Sae);
23990 else
23991 return SDValue();
23992 // Need to fill with zeros to ensure the bitcast will produce zeroes
23993 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
23994 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, dl, MVT::v16i1,
23995 DAG.getConstant(0, dl, MVT::v16i1),
23996 FCmp, DAG.getIntPtrConstant(0, dl));
23997 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32,
23998 DAG.getBitcast(MVT::i16, Ins));
23999 }
24000 case VSHIFT:
24001 return getTargetVShiftNode(IntrData->Opc0, dl, Op.getSimpleValueType(),
24002 Op.getOperand(1), Op.getOperand(2), Subtarget,
24003 DAG);
24004 case COMPRESS_EXPAND_IN_REG: {
24005 SDValue Mask = Op.getOperand(3);
24006 SDValue DataToCompress = Op.getOperand(1);
24007 SDValue PassThru = Op.getOperand(2);
24008 if (ISD::isBuildVectorAllOnes(Mask.getNode())) // return data as is
24009 return Op.getOperand(1);
24010
24011 // Avoid false dependency.
24012 if (PassThru.isUndef())
24013 PassThru = DAG.getConstant(0, dl, VT);
24014
24015 return DAG.getNode(IntrData->Opc0, dl, VT, DataToCompress, PassThru,
24016 Mask);
24017 }
24018 case FIXUPIMM:
24019 case FIXUPIMM_MASKZ: {
24020 SDValue Src1 = Op.getOperand(1);
24021 SDValue Src2 = Op.getOperand(2);
24022 SDValue Src3 = Op.getOperand(3);
24023 SDValue Imm = Op.getOperand(4);
24024 SDValue Mask = Op.getOperand(5);
24025 SDValue Passthru = (IntrData->Type == FIXUPIMM)
24026 ? Src1
24027 : getZeroVector(VT, Subtarget, DAG, dl);
24028
24029 unsigned Opc = IntrData->Opc0;
24030 if (IntrData->Opc1 != 0) {
24031 SDValue Sae = Op.getOperand(6);
24032 if (isRoundModeSAE(Sae))
24033 Opc = IntrData->Opc1;
24034 else if (!isRoundModeCurDirection(Sae))
24035 return SDValue();
24036 }
24037
24038 SDValue FixupImm = DAG.getNode(Opc, dl, VT, Src1, Src2, Src3, Imm);
24039
24040 if (Opc == X86ISD::VFIXUPIMM || Opc == X86ISD::VFIXUPIMM_SAE)
24041 return getVectorMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
24042
24043 return getScalarMaskingNode(FixupImm, Mask, Passthru, Subtarget, DAG);
24044 }
24045 case ROUNDP: {
24046 assert(IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode")((IntrData->Opc0 == X86ISD::VRNDSCALE && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("IntrData->Opc0 == X86ISD::VRNDSCALE && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24046, __PRETTY_FUNCTION__));
24047 // Clear the upper bits of the rounding immediate so that the legacy
24048 // intrinsic can't trigger the scaling behavior of VRNDSCALE.
24049 auto Round = cast<ConstantSDNode>(Op.getOperand(2));
24050 SDValue RoundingMode =
24051 DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
24052 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
24053 Op.getOperand(1), RoundingMode);
24054 }
24055 case ROUNDS: {
24056 assert(IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode")((IntrData->Opc0 == X86ISD::VRNDSCALES && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("IntrData->Opc0 == X86ISD::VRNDSCALES && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24056, __PRETTY_FUNCTION__));
24057 // Clear the upper bits of the rounding immediate so that the legacy
24058 // intrinsic can't trigger the scaling behavior of VRNDSCALE.
24059 auto Round = cast<ConstantSDNode>(Op.getOperand(3));
24060 SDValue RoundingMode =
24061 DAG.getTargetConstant(Round->getZExtValue() & 0xf, dl, MVT::i32);
24062 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
24063 Op.getOperand(1), Op.getOperand(2), RoundingMode);
24064 }
24065 case BEXTRI: {
24066 assert(IntrData->Opc0 == X86ISD::BEXTR && "Unexpected opcode")((IntrData->Opc0 == X86ISD::BEXTR && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("IntrData->Opc0 == X86ISD::BEXTR && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24066, __PRETTY_FUNCTION__));
24067
24068 // The control is a TargetConstant, but we need to convert it to a
24069 // ConstantSDNode.
24070 uint64_t Imm = Op.getConstantOperandVal(2);
24071 SDValue Control = DAG.getConstant(Imm, dl, Op.getValueType());
24072 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(),
24073 Op.getOperand(1), Control);
24074 }
24075 // ADC/ADCX/SBB
24076 case ADX: {
24077 SDVTList CFVTs = DAG.getVTList(Op->getValueType(0), MVT::i32);
24078 SDVTList VTs = DAG.getVTList(Op.getOperand(2).getValueType(), MVT::i32);
24079
24080 SDValue Res;
24081 // If the carry in is zero, then we should just use ADD/SUB instead of
24082 // ADC/SBB.
24083 if (isNullConstant(Op.getOperand(1))) {
24084 Res = DAG.getNode(IntrData->Opc1, dl, VTs, Op.getOperand(2),
24085 Op.getOperand(3));
24086 } else {
24087 SDValue GenCF = DAG.getNode(X86ISD::ADD, dl, CFVTs, Op.getOperand(1),
24088 DAG.getConstant(-1, dl, MVT::i8));
24089 Res = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(2),
24090 Op.getOperand(3), GenCF.getValue(1));
24091 }
24092 SDValue SetCC = getSETCC(X86::COND_B, Res.getValue(1), dl, DAG);
24093 SDValue Results[] = { SetCC, Res };
24094 return DAG.getMergeValues(Results, dl);
24095 }
24096 case CVTPD2PS_MASK:
24097 case CVTPD2DQ_MASK:
24098 case CVTQQ2PS_MASK:
24099 case TRUNCATE_TO_REG: {
24100 SDValue Src = Op.getOperand(1);
24101 SDValue PassThru = Op.getOperand(2);
24102 SDValue Mask = Op.getOperand(3);
24103
24104 if (isAllOnesConstant(Mask))
24105 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
24106
24107 MVT SrcVT = Src.getSimpleValueType();
24108 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
24109 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24110 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(),
24111 {Src, PassThru, Mask});
24112 }
24113 case CVTPS2PH_MASK: {
24114 SDValue Src = Op.getOperand(1);
24115 SDValue Rnd = Op.getOperand(2);
24116 SDValue PassThru = Op.getOperand(3);
24117 SDValue Mask = Op.getOperand(4);
24118
24119 if (isAllOnesConstant(Mask))
24120 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src, Rnd);
24121
24122 MVT SrcVT = Src.getSimpleValueType();
24123 MVT MaskVT = MVT::getVectorVT(MVT::i1, SrcVT.getVectorNumElements());
24124 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24125 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, Rnd,
24126 PassThru, Mask);
24127
24128 }
24129 case CVTNEPS2BF16_MASK: {
24130 SDValue Src = Op.getOperand(1);
24131 SDValue PassThru = Op.getOperand(2);
24132 SDValue Mask = Op.getOperand(3);
24133
24134 if (ISD::isBuildVectorAllOnes(Mask.getNode()))
24135 return DAG.getNode(IntrData->Opc0, dl, Op.getValueType(), Src);
24136
24137 // Break false dependency.
24138 if (PassThru.isUndef())
24139 PassThru = DAG.getConstant(0, dl, PassThru.getValueType());
24140
24141 return DAG.getNode(IntrData->Opc1, dl, Op.getValueType(), Src, PassThru,
24142 Mask);
24143 }
24144 default:
24145 break;
24146 }
24147 }
24148
24149 switch (IntNo) {
24150 default: return SDValue(); // Don't custom lower most intrinsics.
24151
24152 // ptest and testp intrinsics. The intrinsic these come from are designed to
24153 // return an integer value, not just an instruction so lower it to the ptest
24154 // or testp pattern and a setcc for the result.
24155 case Intrinsic::x86_avx512_ktestc_b:
24156 case Intrinsic::x86_avx512_ktestc_w:
24157 case Intrinsic::x86_avx512_ktestc_d:
24158 case Intrinsic::x86_avx512_ktestc_q:
24159 case Intrinsic::x86_avx512_ktestz_b:
24160 case Intrinsic::x86_avx512_ktestz_w:
24161 case Intrinsic::x86_avx512_ktestz_d:
24162 case Intrinsic::x86_avx512_ktestz_q:
24163 case Intrinsic::x86_sse41_ptestz:
24164 case Intrinsic::x86_sse41_ptestc:
24165 case Intrinsic::x86_sse41_ptestnzc:
24166 case Intrinsic::x86_avx_ptestz_256:
24167 case Intrinsic::x86_avx_ptestc_256:
24168 case Intrinsic::x86_avx_ptestnzc_256:
24169 case Intrinsic::x86_avx_vtestz_ps:
24170 case Intrinsic::x86_avx_vtestc_ps:
24171 case Intrinsic::x86_avx_vtestnzc_ps:
24172 case Intrinsic::x86_avx_vtestz_pd:
24173 case Intrinsic::x86_avx_vtestc_pd:
24174 case Intrinsic::x86_avx_vtestnzc_pd:
24175 case Intrinsic::x86_avx_vtestz_ps_256:
24176 case Intrinsic::x86_avx_vtestc_ps_256:
24177 case Intrinsic::x86_avx_vtestnzc_ps_256:
24178 case Intrinsic::x86_avx_vtestz_pd_256:
24179 case Intrinsic::x86_avx_vtestc_pd_256:
24180 case Intrinsic::x86_avx_vtestnzc_pd_256: {
24181 unsigned TestOpc = X86ISD::PTEST;
24182 X86::CondCode X86CC;
24183 switch (IntNo) {
24184 default: llvm_unreachable("Bad fallthrough in Intrinsic lowering.")::llvm::llvm_unreachable_internal("Bad fallthrough in Intrinsic lowering."
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24184);
24185 case Intrinsic::x86_avx512_ktestc_b:
24186 case Intrinsic::x86_avx512_ktestc_w:
24187 case Intrinsic::x86_avx512_ktestc_d:
24188 case Intrinsic::x86_avx512_ktestc_q:
24189 // CF = 1
24190 TestOpc = X86ISD::KTEST;
24191 X86CC = X86::COND_B;
24192 break;
24193 case Intrinsic::x86_avx512_ktestz_b:
24194 case Intrinsic::x86_avx512_ktestz_w:
24195 case Intrinsic::x86_avx512_ktestz_d:
24196 case Intrinsic::x86_avx512_ktestz_q:
24197 TestOpc = X86ISD::KTEST;
24198 X86CC = X86::COND_E;
24199 break;
24200 case Intrinsic::x86_avx_vtestz_ps:
24201 case Intrinsic::x86_avx_vtestz_pd:
24202 case Intrinsic::x86_avx_vtestz_ps_256:
24203 case Intrinsic::x86_avx_vtestz_pd_256:
24204 TestOpc = X86ISD::TESTP;
24205 LLVM_FALLTHROUGH[[gnu::fallthrough]];
24206 case Intrinsic::x86_sse41_ptestz:
24207 case Intrinsic::x86_avx_ptestz_256:
24208 // ZF = 1
24209 X86CC = X86::COND_E;
24210 break;
24211 case Intrinsic::x86_avx_vtestc_ps:
24212 case Intrinsic::x86_avx_vtestc_pd:
24213 case Intrinsic::x86_avx_vtestc_ps_256:
24214 case Intrinsic::x86_avx_vtestc_pd_256:
24215 TestOpc = X86ISD::TESTP;
24216 LLVM_FALLTHROUGH[[gnu::fallthrough]];
24217 case Intrinsic::x86_sse41_ptestc:
24218 case Intrinsic::x86_avx_ptestc_256:
24219 // CF = 1
24220 X86CC = X86::COND_B;
24221 break;
24222 case Intrinsic::x86_avx_vtestnzc_ps:
24223 case Intrinsic::x86_avx_vtestnzc_pd:
24224 case Intrinsic::x86_avx_vtestnzc_ps_256:
24225 case Intrinsic::x86_avx_vtestnzc_pd_256:
24226 TestOpc = X86ISD::TESTP;
24227 LLVM_FALLTHROUGH[[gnu::fallthrough]];
24228 case Intrinsic::x86_sse41_ptestnzc:
24229 case Intrinsic::x86_avx_ptestnzc_256:
24230 // ZF and CF = 0
24231 X86CC = X86::COND_A;
24232 break;
24233 }
24234
24235 SDValue LHS = Op.getOperand(1);
24236 SDValue RHS = Op.getOperand(2);
24237 SDValue Test = DAG.getNode(TestOpc, dl, MVT::i32, LHS, RHS);
24238 SDValue SetCC = getSETCC(X86CC, Test, dl, DAG);
24239 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
24240 }
24241
24242 case Intrinsic::x86_sse42_pcmpistria128:
24243 case Intrinsic::x86_sse42_pcmpestria128:
24244 case Intrinsic::x86_sse42_pcmpistric128:
24245 case Intrinsic::x86_sse42_pcmpestric128:
24246 case Intrinsic::x86_sse42_pcmpistrio128:
24247 case Intrinsic::x86_sse42_pcmpestrio128:
24248 case Intrinsic::x86_sse42_pcmpistris128:
24249 case Intrinsic::x86_sse42_pcmpestris128:
24250 case Intrinsic::x86_sse42_pcmpistriz128:
24251 case Intrinsic::x86_sse42_pcmpestriz128: {
24252 unsigned Opcode;
24253 X86::CondCode X86CC;
24254 switch (IntNo) {
24255 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24255); // Can't reach here.
24256 case Intrinsic::x86_sse42_pcmpistria128:
24257 Opcode = X86ISD::PCMPISTR;
24258 X86CC = X86::COND_A;
24259 break;
24260 case Intrinsic::x86_sse42_pcmpestria128:
24261 Opcode = X86ISD::PCMPESTR;
24262 X86CC = X86::COND_A;
24263 break;
24264 case Intrinsic::x86_sse42_pcmpistric128:
24265 Opcode = X86ISD::PCMPISTR;
24266 X86CC = X86::COND_B;
24267 break;
24268 case Intrinsic::x86_sse42_pcmpestric128:
24269 Opcode = X86ISD::PCMPESTR;
24270 X86CC = X86::COND_B;
24271 break;
24272 case Intrinsic::x86_sse42_pcmpistrio128:
24273 Opcode = X86ISD::PCMPISTR;
24274 X86CC = X86::COND_O;
24275 break;
24276 case Intrinsic::x86_sse42_pcmpestrio128:
24277 Opcode = X86ISD::PCMPESTR;
24278 X86CC = X86::COND_O;
24279 break;
24280 case Intrinsic::x86_sse42_pcmpistris128:
24281 Opcode = X86ISD::PCMPISTR;
24282 X86CC = X86::COND_S;
24283 break;
24284 case Intrinsic::x86_sse42_pcmpestris128:
24285 Opcode = X86ISD::PCMPESTR;
24286 X86CC = X86::COND_S;
24287 break;
24288 case Intrinsic::x86_sse42_pcmpistriz128:
24289 Opcode = X86ISD::PCMPISTR;
24290 X86CC = X86::COND_E;
24291 break;
24292 case Intrinsic::x86_sse42_pcmpestriz128:
24293 Opcode = X86ISD::PCMPESTR;
24294 X86CC = X86::COND_E;
24295 break;
24296 }
24297 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
24298 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
24299 SDValue PCMP = DAG.getNode(Opcode, dl, VTs, NewOps).getValue(2);
24300 SDValue SetCC = getSETCC(X86CC, PCMP, dl, DAG);
24301 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, SetCC);
24302 }
24303
24304 case Intrinsic::x86_sse42_pcmpistri128:
24305 case Intrinsic::x86_sse42_pcmpestri128: {
24306 unsigned Opcode;
24307 if (IntNo == Intrinsic::x86_sse42_pcmpistri128)
24308 Opcode = X86ISD::PCMPISTR;
24309 else
24310 Opcode = X86ISD::PCMPESTR;
24311
24312 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
24313 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
24314 return DAG.getNode(Opcode, dl, VTs, NewOps);
24315 }
24316
24317 case Intrinsic::x86_sse42_pcmpistrm128:
24318 case Intrinsic::x86_sse42_pcmpestrm128: {
24319 unsigned Opcode;
24320 if (IntNo == Intrinsic::x86_sse42_pcmpistrm128)
24321 Opcode = X86ISD::PCMPISTR;
24322 else
24323 Opcode = X86ISD::PCMPESTR;
24324
24325 SmallVector<SDValue, 5> NewOps(Op->op_begin()+1, Op->op_end());
24326 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::v16i8, MVT::i32);
24327 return DAG.getNode(Opcode, dl, VTs, NewOps).getValue(1);
24328 }
24329
24330 case Intrinsic::eh_sjlj_lsda: {
24331 MachineFunction &MF = DAG.getMachineFunction();
24332 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24333 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
24334 auto &Context = MF.getMMI().getContext();
24335 MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
24336 Twine(MF.getFunctionNumber()));
24337 return DAG.getNode(getGlobalWrapperKind(), dl, VT,
24338 DAG.getMCSymbol(S, PtrVT));
24339 }
24340
24341 case Intrinsic::x86_seh_lsda: {
24342 // Compute the symbol for the LSDA. We know it'll get emitted later.
24343 MachineFunction &MF = DAG.getMachineFunction();
24344 SDValue Op1 = Op.getOperand(1);
24345 auto *Fn = cast<Function>(cast<GlobalAddressSDNode>(Op1)->getGlobal());
24346 MCSymbol *LSDASym = MF.getMMI().getContext().getOrCreateLSDASymbol(
24347 GlobalValue::dropLLVMManglingEscape(Fn->getName()));
24348
24349 // Generate a simple absolute symbol reference. This intrinsic is only
24350 // supported on 32-bit Windows, which isn't PIC.
24351 SDValue Result = DAG.getMCSymbol(LSDASym, VT);
24352 return DAG.getNode(X86ISD::Wrapper, dl, VT, Result);
24353 }
24354
24355 case Intrinsic::eh_recoverfp: {
24356 SDValue FnOp = Op.getOperand(1);
24357 SDValue IncomingFPOp = Op.getOperand(2);
24358 GlobalAddressSDNode *GSD = dyn_cast<GlobalAddressSDNode>(FnOp);
24359 auto *Fn = dyn_cast_or_null<Function>(GSD ? GSD->getGlobal() : nullptr);
24360 if (!Fn)
24361 report_fatal_error(
24362 "llvm.eh.recoverfp must take a function as the first argument");
24363 return recoverFramePointer(DAG, Fn, IncomingFPOp);
24364 }
24365
24366 case Intrinsic::localaddress: {
24367 // Returns one of the stack, base, or frame pointer registers, depending on
24368 // which is used to reference local variables.
24369 MachineFunction &MF = DAG.getMachineFunction();
24370 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
24371 unsigned Reg;
24372 if (RegInfo->hasBasePointer(MF))
24373 Reg = RegInfo->getBaseRegister();
24374 else { // Handles the SP or FP case.
24375 bool CantUseFP = RegInfo->needsStackRealignment(MF);
24376 if (CantUseFP)
24377 Reg = RegInfo->getPtrSizedStackRegister(MF);
24378 else
24379 Reg = RegInfo->getPtrSizedFrameRegister(MF);
24380 }
24381 return DAG.getCopyFromReg(DAG.getEntryNode(), dl, Reg, VT);
24382 }
24383
24384 case Intrinsic::x86_avx512_vp2intersect_q_512:
24385 case Intrinsic::x86_avx512_vp2intersect_q_256:
24386 case Intrinsic::x86_avx512_vp2intersect_q_128:
24387 case Intrinsic::x86_avx512_vp2intersect_d_512:
24388 case Intrinsic::x86_avx512_vp2intersect_d_256:
24389 case Intrinsic::x86_avx512_vp2intersect_d_128: {
24390 MVT MaskVT = Op.getSimpleValueType();
24391
24392 SDVTList VTs = DAG.getVTList(MVT::Untyped, MVT::Other);
24393 SDLoc DL(Op);
24394
24395 SDValue Operation =
24396 DAG.getNode(X86ISD::VP2INTERSECT, DL, VTs,
24397 Op->getOperand(1), Op->getOperand(2));
24398
24399 SDValue Result0 = DAG.getTargetExtractSubreg(X86::sub_mask_0, DL,
24400 MaskVT, Operation);
24401 SDValue Result1 = DAG.getTargetExtractSubreg(X86::sub_mask_1, DL,
24402 MaskVT, Operation);
24403 return DAG.getMergeValues({Result0, Result1}, DL);
24404 }
24405 case Intrinsic::x86_mmx_pslli_w:
24406 case Intrinsic::x86_mmx_pslli_d:
24407 case Intrinsic::x86_mmx_pslli_q:
24408 case Intrinsic::x86_mmx_psrli_w:
24409 case Intrinsic::x86_mmx_psrli_d:
24410 case Intrinsic::x86_mmx_psrli_q:
24411 case Intrinsic::x86_mmx_psrai_w:
24412 case Intrinsic::x86_mmx_psrai_d: {
24413 SDLoc DL(Op);
24414 SDValue ShAmt = Op.getOperand(2);
24415 // If the argument is a constant, convert it to a target constant.
24416 if (auto *C = dyn_cast<ConstantSDNode>(ShAmt)) {
24417 // Clamp out of bounds shift amounts since they will otherwise be masked
24418 // to 8-bits which may make it no longer out of bounds.
24419 unsigned ShiftAmount = C->getAPIntValue().getLimitedValue(255);
24420 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
24421 Op.getOperand(0), Op.getOperand(1),
24422 DAG.getTargetConstant(ShiftAmount, DL, MVT::i32));
24423 }
24424
24425 unsigned NewIntrinsic;
24426 switch (IntNo) {
24427 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24427); // Can't reach here.
24428 case Intrinsic::x86_mmx_pslli_w:
24429 NewIntrinsic = Intrinsic::x86_mmx_psll_w;
24430 break;
24431 case Intrinsic::x86_mmx_pslli_d:
24432 NewIntrinsic = Intrinsic::x86_mmx_psll_d;
24433 break;
24434 case Intrinsic::x86_mmx_pslli_q:
24435 NewIntrinsic = Intrinsic::x86_mmx_psll_q;
24436 break;
24437 case Intrinsic::x86_mmx_psrli_w:
24438 NewIntrinsic = Intrinsic::x86_mmx_psrl_w;
24439 break;
24440 case Intrinsic::x86_mmx_psrli_d:
24441 NewIntrinsic = Intrinsic::x86_mmx_psrl_d;
24442 break;
24443 case Intrinsic::x86_mmx_psrli_q:
24444 NewIntrinsic = Intrinsic::x86_mmx_psrl_q;
24445 break;
24446 case Intrinsic::x86_mmx_psrai_w:
24447 NewIntrinsic = Intrinsic::x86_mmx_psra_w;
24448 break;
24449 case Intrinsic::x86_mmx_psrai_d:
24450 NewIntrinsic = Intrinsic::x86_mmx_psra_d;
24451 break;
24452 }
24453
24454 // The vector shift intrinsics with scalars uses 32b shift amounts but
24455 // the sse2/mmx shift instructions reads 64 bits. Copy the 32 bits to an
24456 // MMX register.
24457 ShAmt = DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, ShAmt);
24458 return DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, Op.getValueType(),
24459 DAG.getConstant(NewIntrinsic, DL, MVT::i32),
24460 Op.getOperand(1), ShAmt);
24461
24462 }
24463 }
24464}
24465
24466static SDValue getAVX2GatherNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
24467 SDValue Src, SDValue Mask, SDValue Base,
24468 SDValue Index, SDValue ScaleOp, SDValue Chain,
24469 const X86Subtarget &Subtarget) {
24470 SDLoc dl(Op);
24471 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
24472 // Scale must be constant.
24473 if (!C)
24474 return SDValue();
24475 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24476 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
24477 TLI.getPointerTy(DAG.getDataLayout()));
24478 EVT MaskVT = Mask.getValueType().changeVectorElementTypeToInteger();
24479 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
24480 // If source is undef or we know it won't be used, use a zero vector
24481 // to break register dependency.
24482 // TODO: use undef instead and let BreakFalseDeps deal with it?
24483 if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
24484 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
24485
24486 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
24487
24488 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
24489 SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
24490 VTs, Ops, dl, MemIntr->getMemoryVT(), MemIntr->getMemOperand());
24491 return DAG.getMergeValues({ Res, Res.getValue(2) }, dl);
24492}
24493
24494static SDValue getGatherNode(SDValue Op, SelectionDAG &DAG,
24495 SDValue Src, SDValue Mask, SDValue Base,
24496 SDValue Index, SDValue ScaleOp, SDValue Chain,
24497 const X86Subtarget &Subtarget) {
24498 MVT VT = Op.getSimpleValueType();
24499 SDLoc dl(Op);
24500 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
24501 // Scale must be constant.
24502 if (!C)
24503 return SDValue();
24504 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24505 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
24506 TLI.getPointerTy(DAG.getDataLayout()));
24507 unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
24508 VT.getVectorNumElements());
24509 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
24510
24511 // We support two versions of the gather intrinsics. One with scalar mask and
24512 // one with vXi1 mask. Convert scalar to vXi1 if necessary.
24513 if (Mask.getValueType() != MaskVT)
24514 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24515
24516 SDVTList VTs = DAG.getVTList(Op.getValueType(), MaskVT, MVT::Other);
24517 // If source is undef or we know it won't be used, use a zero vector
24518 // to break register dependency.
24519 // TODO: use undef instead and let BreakFalseDeps deal with it?
24520 if (Src.isUndef() || ISD::isBuildVectorAllOnes(Mask.getNode()))
24521 Src = getZeroVector(Op.getSimpleValueType(), Subtarget, DAG, dl);
24522
24523 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
24524
24525 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale };
24526 SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
24527 VTs, Ops, dl, MemIntr->getMemoryVT(), MemIntr->getMemOperand());
24528 return DAG.getMergeValues({ Res, Res.getValue(2) }, dl);
24529}
24530
24531static SDValue getScatterNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
24532 SDValue Src, SDValue Mask, SDValue Base,
24533 SDValue Index, SDValue ScaleOp, SDValue Chain,
24534 const X86Subtarget &Subtarget) {
24535 SDLoc dl(Op);
24536 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
24537 // Scale must be constant.
24538 if (!C)
24539 return SDValue();
24540 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24541 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
24542 TLI.getPointerTy(DAG.getDataLayout()));
24543 unsigned MinElts = std::min(Index.getSimpleValueType().getVectorNumElements(),
24544 Src.getSimpleValueType().getVectorNumElements());
24545 MVT MaskVT = MVT::getVectorVT(MVT::i1, MinElts);
24546
24547 // We support two versions of the scatter intrinsics. One with scalar mask and
24548 // one with vXi1 mask. Convert scalar to vXi1 if necessary.
24549 if (Mask.getValueType() != MaskVT)
24550 Mask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24551
24552 MemIntrinsicSDNode *MemIntr = cast<MemIntrinsicSDNode>(Op);
24553
24554 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other);
24555 SDValue Ops[] = {Chain, Src, Mask, Base, Index, Scale};
24556 SDValue Res = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
24557 VTs, Ops, dl, MemIntr->getMemoryVT(), MemIntr->getMemOperand());
24558 return Res.getValue(1);
24559}
24560
24561static SDValue getPrefetchNode(unsigned Opc, SDValue Op, SelectionDAG &DAG,
24562 SDValue Mask, SDValue Base, SDValue Index,
24563 SDValue ScaleOp, SDValue Chain,
24564 const X86Subtarget &Subtarget) {
24565 SDLoc dl(Op);
24566 auto *C = dyn_cast<ConstantSDNode>(ScaleOp);
24567 // Scale must be constant.
24568 if (!C)
24569 return SDValue();
24570 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
24571 SDValue Scale = DAG.getTargetConstant(C->getZExtValue(), dl,
24572 TLI.getPointerTy(DAG.getDataLayout()));
24573 SDValue Disp = DAG.getTargetConstant(0, dl, MVT::i32);
24574 SDValue Segment = DAG.getRegister(0, MVT::i32);
24575 MVT MaskVT =
24576 MVT::getVectorVT(MVT::i1, Index.getSimpleValueType().getVectorNumElements());
24577 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24578 SDValue Ops[] = {VMask, Base, Scale, Index, Disp, Segment, Chain};
24579 SDNode *Res = DAG.getMachineNode(Opc, dl, MVT::Other, Ops);
24580 return SDValue(Res, 0);
24581}
24582
24583/// Handles the lowering of builtin intrinsics with chain that return their
24584/// value into registers EDX:EAX.
24585/// If operand ScrReg is a valid register identifier, then operand 2 of N is
24586/// copied to SrcReg. The assumption is that SrcReg is an implicit input to
24587/// TargetOpcode.
24588/// Returns a Glue value which can be used to add extra copy-from-reg if the
24589/// expanded intrinsics implicitly defines extra registers (i.e. not just
24590/// EDX:EAX).
24591static SDValue expandIntrinsicWChainHelper(SDNode *N, const SDLoc &DL,
24592 SelectionDAG &DAG,
24593 unsigned TargetOpcode,
24594 unsigned SrcReg,
24595 const X86Subtarget &Subtarget,
24596 SmallVectorImpl<SDValue> &Results) {
24597 SDValue Chain = N->getOperand(0);
24598 SDValue Glue;
24599
24600 if (SrcReg) {
24601 assert(N->getNumOperands() == 3 && "Unexpected number of operands!")((N->getNumOperands() == 3 && "Unexpected number of operands!"
) ? static_cast<void> (0) : __assert_fail ("N->getNumOperands() == 3 && \"Unexpected number of operands!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24601, __PRETTY_FUNCTION__));
24602 Chain = DAG.getCopyToReg(Chain, DL, SrcReg, N->getOperand(2), Glue);
24603 Glue = Chain.getValue(1);
24604 }
24605
24606 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
24607 SDValue N1Ops[] = {Chain, Glue};
24608 SDNode *N1 = DAG.getMachineNode(
24609 TargetOpcode, DL, Tys, ArrayRef<SDValue>(N1Ops, Glue.getNode() ? 2 : 1));
24610 Chain = SDValue(N1, 0);
24611
24612 // Reads the content of XCR and returns it in registers EDX:EAX.
24613 SDValue LO, HI;
24614 if (Subtarget.is64Bit()) {
24615 LO = DAG.getCopyFromReg(Chain, DL, X86::RAX, MVT::i64, SDValue(N1, 1));
24616 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::RDX, MVT::i64,
24617 LO.getValue(2));
24618 } else {
24619 LO = DAG.getCopyFromReg(Chain, DL, X86::EAX, MVT::i32, SDValue(N1, 1));
24620 HI = DAG.getCopyFromReg(LO.getValue(1), DL, X86::EDX, MVT::i32,
24621 LO.getValue(2));
24622 }
24623 Chain = HI.getValue(1);
24624 Glue = HI.getValue(2);
24625
24626 if (Subtarget.is64Bit()) {
24627 // Merge the two 32-bit values into a 64-bit one.
24628 SDValue Tmp = DAG.getNode(ISD::SHL, DL, MVT::i64, HI,
24629 DAG.getConstant(32, DL, MVT::i8));
24630 Results.push_back(DAG.getNode(ISD::OR, DL, MVT::i64, LO, Tmp));
24631 Results.push_back(Chain);
24632 return Glue;
24633 }
24634
24635 // Use a buildpair to merge the two 32-bit values into a 64-bit one.
24636 SDValue Ops[] = { LO, HI };
24637 SDValue Pair = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64, Ops);
24638 Results.push_back(Pair);
24639 Results.push_back(Chain);
24640 return Glue;
24641}
24642
24643/// Handles the lowering of builtin intrinsics that read the time stamp counter
24644/// (x86_rdtsc and x86_rdtscp). This function is also used to custom lower
24645/// READCYCLECOUNTER nodes.
24646static void getReadTimeStampCounter(SDNode *N, const SDLoc &DL, unsigned Opcode,
24647 SelectionDAG &DAG,
24648 const X86Subtarget &Subtarget,
24649 SmallVectorImpl<SDValue> &Results) {
24650 // The processor's time-stamp counter (a 64-bit MSR) is stored into the
24651 // EDX:EAX registers. EDX is loaded with the high-order 32 bits of the MSR
24652 // and the EAX register is loaded with the low-order 32 bits.
24653 SDValue Glue = expandIntrinsicWChainHelper(N, DL, DAG, Opcode,
24654 /* NoRegister */0, Subtarget,
24655 Results);
24656 if (Opcode != X86::RDTSCP)
24657 return;
24658
24659 SDValue Chain = Results[1];
24660 // Instruction RDTSCP loads the IA32:TSC_AUX_MSR (address C000_0103H) into
24661 // the ECX register. Add 'ecx' explicitly to the chain.
24662 SDValue ecx = DAG.getCopyFromReg(Chain, DL, X86::ECX, MVT::i32, Glue);
24663 Results[1] = ecx;
24664 Results.push_back(ecx.getValue(1));
24665}
24666
24667static SDValue LowerREADCYCLECOUNTER(SDValue Op, const X86Subtarget &Subtarget,
24668 SelectionDAG &DAG) {
24669 SmallVector<SDValue, 3> Results;
24670 SDLoc DL(Op);
24671 getReadTimeStampCounter(Op.getNode(), DL, X86::RDTSC, DAG, Subtarget,
24672 Results);
24673 return DAG.getMergeValues(Results, DL);
24674}
24675
24676static SDValue MarkEHRegistrationNode(SDValue Op, SelectionDAG &DAG) {
24677 MachineFunction &MF = DAG.getMachineFunction();
24678 SDValue Chain = Op.getOperand(0);
24679 SDValue RegNode = Op.getOperand(2);
24680 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
24681 if (!EHInfo)
24682 report_fatal_error("EH registrations only live in functions using WinEH");
24683
24684 // Cast the operand to an alloca, and remember the frame index.
24685 auto *FINode = dyn_cast<FrameIndexSDNode>(RegNode);
24686 if (!FINode)
24687 report_fatal_error("llvm.x86.seh.ehregnode expects a static alloca");
24688 EHInfo->EHRegNodeFrameIndex = FINode->getIndex();
24689
24690 // Return the chain operand without making any DAG nodes.
24691 return Chain;
24692}
24693
24694static SDValue MarkEHGuard(SDValue Op, SelectionDAG &DAG) {
24695 MachineFunction &MF = DAG.getMachineFunction();
24696 SDValue Chain = Op.getOperand(0);
24697 SDValue EHGuard = Op.getOperand(2);
24698 WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo();
24699 if (!EHInfo)
24700 report_fatal_error("EHGuard only live in functions using WinEH");
24701
24702 // Cast the operand to an alloca, and remember the frame index.
24703 auto *FINode = dyn_cast<FrameIndexSDNode>(EHGuard);
24704 if (!FINode)
24705 report_fatal_error("llvm.x86.seh.ehguard expects a static alloca");
24706 EHInfo->EHGuardFrameIndex = FINode->getIndex();
24707
24708 // Return the chain operand without making any DAG nodes.
24709 return Chain;
24710}
24711
24712/// Emit Truncating Store with signed or unsigned saturation.
24713static SDValue
24714EmitTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &Dl, SDValue Val,
24715 SDValue Ptr, EVT MemVT, MachineMemOperand *MMO,
24716 SelectionDAG &DAG) {
24717
24718 SDVTList VTs = DAG.getVTList(MVT::Other);
24719 SDValue Undef = DAG.getUNDEF(Ptr.getValueType());
24720 SDValue Ops[] = { Chain, Val, Ptr, Undef };
24721 return SignedSat ?
24722 DAG.getTargetMemSDNode<TruncSStoreSDNode>(VTs, Ops, Dl, MemVT, MMO) :
24723 DAG.getTargetMemSDNode<TruncUSStoreSDNode>(VTs, Ops, Dl, MemVT, MMO);
24724}
24725
24726/// Emit Masked Truncating Store with signed or unsigned saturation.
24727static SDValue
24728EmitMaskedTruncSStore(bool SignedSat, SDValue Chain, const SDLoc &Dl,
24729 SDValue Val, SDValue Ptr, SDValue Mask, EVT MemVT,
24730 MachineMemOperand *MMO, SelectionDAG &DAG) {
24731
24732 SDVTList VTs = DAG.getVTList(MVT::Other);
24733 SDValue Ops[] = { Chain, Val, Ptr, Mask };
24734 return SignedSat ?
24735 DAG.getTargetMemSDNode<MaskedTruncSStoreSDNode>(VTs, Ops, Dl, MemVT, MMO) :
24736 DAG.getTargetMemSDNode<MaskedTruncUSStoreSDNode>(VTs, Ops, Dl, MemVT, MMO);
24737}
24738
24739static SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, const X86Subtarget &Subtarget,
24740 SelectionDAG &DAG) {
24741 unsigned IntNo = Op.getConstantOperandVal(1);
24742 const IntrinsicData *IntrData = getIntrinsicWithChain(IntNo);
24743 if (!IntrData) {
24744 switch (IntNo) {
24745 case llvm::Intrinsic::x86_seh_ehregnode:
24746 return MarkEHRegistrationNode(Op, DAG);
24747 case llvm::Intrinsic::x86_seh_ehguard:
24748 return MarkEHGuard(Op, DAG);
24749 case llvm::Intrinsic::x86_rdpkru: {
24750 SDLoc dl(Op);
24751 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
24752 // Create a RDPKRU node and pass 0 to the ECX parameter.
24753 return DAG.getNode(X86ISD::RDPKRU, dl, VTs, Op.getOperand(0),
24754 DAG.getConstant(0, dl, MVT::i32));
24755 }
24756 case llvm::Intrinsic::x86_wrpkru: {
24757 SDLoc dl(Op);
24758 // Create a WRPKRU node, pass the input to the EAX parameter, and pass 0
24759 // to the EDX and ECX parameters.
24760 return DAG.getNode(X86ISD::WRPKRU, dl, MVT::Other,
24761 Op.getOperand(0), Op.getOperand(2),
24762 DAG.getConstant(0, dl, MVT::i32),
24763 DAG.getConstant(0, dl, MVT::i32));
24764 }
24765 case llvm::Intrinsic::x86_flags_read_u32:
24766 case llvm::Intrinsic::x86_flags_read_u64:
24767 case llvm::Intrinsic::x86_flags_write_u32:
24768 case llvm::Intrinsic::x86_flags_write_u64: {
24769 // We need a frame pointer because this will get lowered to a PUSH/POP
24770 // sequence.
24771 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
24772 MFI.setHasCopyImplyingStackAdjustment(true);
24773 // Don't do anything here, we will expand these intrinsics out later
24774 // during FinalizeISel in EmitInstrWithCustomInserter.
24775 return Op;
24776 }
24777 case Intrinsic::x86_lwpins32:
24778 case Intrinsic::x86_lwpins64:
24779 case Intrinsic::x86_umwait:
24780 case Intrinsic::x86_tpause: {
24781 SDLoc dl(Op);
24782 SDValue Chain = Op->getOperand(0);
24783 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
24784 unsigned Opcode;
24785
24786 switch (IntNo) {
24787 default: llvm_unreachable("Impossible intrinsic")::llvm::llvm_unreachable_internal("Impossible intrinsic", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24787);
24788 case Intrinsic::x86_umwait:
24789 Opcode = X86ISD::UMWAIT;
24790 break;
24791 case Intrinsic::x86_tpause:
24792 Opcode = X86ISD::TPAUSE;
24793 break;
24794 case Intrinsic::x86_lwpins32:
24795 case Intrinsic::x86_lwpins64:
24796 Opcode = X86ISD::LWPINS;
24797 break;
24798 }
24799
24800 SDValue Operation =
24801 DAG.getNode(Opcode, dl, VTs, Chain, Op->getOperand(2),
24802 Op->getOperand(3), Op->getOperand(4));
24803 SDValue SetCC = getSETCC(X86::COND_B, Operation.getValue(0), dl, DAG);
24804 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
24805 Operation.getValue(1));
24806 }
24807 case Intrinsic::x86_enqcmd:
24808 case Intrinsic::x86_enqcmds: {
24809 SDLoc dl(Op);
24810 SDValue Chain = Op.getOperand(0);
24811 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::Other);
24812 unsigned Opcode;
24813 switch (IntNo) {
24814 default: llvm_unreachable("Impossible intrinsic!")::llvm::llvm_unreachable_internal("Impossible intrinsic!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24814);
24815 case Intrinsic::x86_enqcmd:
24816 Opcode = X86ISD::ENQCMD;
24817 break;
24818 case Intrinsic::x86_enqcmds:
24819 Opcode = X86ISD::ENQCMDS;
24820 break;
24821 }
24822 SDValue Operation = DAG.getNode(Opcode, dl, VTs, Chain, Op.getOperand(2),
24823 Op.getOperand(3));
24824 SDValue SetCC = getSETCC(X86::COND_E, Operation.getValue(0), dl, DAG);
24825 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), SetCC,
24826 Operation.getValue(1));
24827 }
24828 }
24829 return SDValue();
24830 }
24831
24832 SDLoc dl(Op);
24833 switch(IntrData->Type) {
24834 default: llvm_unreachable("Unknown Intrinsic Type")::llvm::llvm_unreachable_internal("Unknown Intrinsic Type", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24834);
24835 case RDSEED:
24836 case RDRAND: {
24837 // Emit the node with the right value type.
24838 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::i32, MVT::Other);
24839 SDValue Result = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
24840
24841 // If the value returned by RDRAND/RDSEED was valid (CF=1), return 1.
24842 // Otherwise return the value from Rand, which is always 0, casted to i32.
24843 SDValue Ops[] = {DAG.getZExtOrTrunc(Result, dl, Op->getValueType(1)),
24844 DAG.getConstant(1, dl, Op->getValueType(1)),
24845 DAG.getTargetConstant(X86::COND_B, dl, MVT::i8),
24846 SDValue(Result.getNode(), 1)};
24847 SDValue isValid = DAG.getNode(X86ISD::CMOV, dl, Op->getValueType(1), Ops);
24848
24849 // Return { result, isValid, chain }.
24850 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(), Result, isValid,
24851 SDValue(Result.getNode(), 2));
24852 }
24853 case GATHER_AVX2: {
24854 SDValue Chain = Op.getOperand(0);
24855 SDValue Src = Op.getOperand(2);
24856 SDValue Base = Op.getOperand(3);
24857 SDValue Index = Op.getOperand(4);
24858 SDValue Mask = Op.getOperand(5);
24859 SDValue Scale = Op.getOperand(6);
24860 return getAVX2GatherNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
24861 Scale, Chain, Subtarget);
24862 }
24863 case GATHER: {
24864 //gather(v1, mask, index, base, scale);
24865 SDValue Chain = Op.getOperand(0);
24866 SDValue Src = Op.getOperand(2);
24867 SDValue Base = Op.getOperand(3);
24868 SDValue Index = Op.getOperand(4);
24869 SDValue Mask = Op.getOperand(5);
24870 SDValue Scale = Op.getOperand(6);
24871 return getGatherNode(Op, DAG, Src, Mask, Base, Index, Scale,
24872 Chain, Subtarget);
24873 }
24874 case SCATTER: {
24875 //scatter(base, mask, index, v1, scale);
24876 SDValue Chain = Op.getOperand(0);
24877 SDValue Base = Op.getOperand(2);
24878 SDValue Mask = Op.getOperand(3);
24879 SDValue Index = Op.getOperand(4);
24880 SDValue Src = Op.getOperand(5);
24881 SDValue Scale = Op.getOperand(6);
24882 return getScatterNode(IntrData->Opc0, Op, DAG, Src, Mask, Base, Index,
24883 Scale, Chain, Subtarget);
24884 }
24885 case PREFETCH: {
24886 const APInt &HintVal = Op.getConstantOperandAPInt(6);
24887 assert((HintVal == 2 || HintVal == 3) &&(((HintVal == 2 || HintVal == 3) && "Wrong prefetch hint in intrinsic: should be 2 or 3"
) ? static_cast<void> (0) : __assert_fail ("(HintVal == 2 || HintVal == 3) && \"Wrong prefetch hint in intrinsic: should be 2 or 3\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24888, __PRETTY_FUNCTION__))
24888 "Wrong prefetch hint in intrinsic: should be 2 or 3")(((HintVal == 2 || HintVal == 3) && "Wrong prefetch hint in intrinsic: should be 2 or 3"
) ? static_cast<void> (0) : __assert_fail ("(HintVal == 2 || HintVal == 3) && \"Wrong prefetch hint in intrinsic: should be 2 or 3\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24888, __PRETTY_FUNCTION__));
24889 unsigned Opcode = (HintVal == 2 ? IntrData->Opc1 : IntrData->Opc0);
24890 SDValue Chain = Op.getOperand(0);
24891 SDValue Mask = Op.getOperand(2);
24892 SDValue Index = Op.getOperand(3);
24893 SDValue Base = Op.getOperand(4);
24894 SDValue Scale = Op.getOperand(5);
24895 return getPrefetchNode(Opcode, Op, DAG, Mask, Base, Index, Scale, Chain,
24896 Subtarget);
24897 }
24898 // Read Time Stamp Counter (RDTSC) and Processor ID (RDTSCP).
24899 case RDTSC: {
24900 SmallVector<SDValue, 2> Results;
24901 getReadTimeStampCounter(Op.getNode(), dl, IntrData->Opc0, DAG, Subtarget,
24902 Results);
24903 return DAG.getMergeValues(Results, dl);
24904 }
24905 // Read Performance Monitoring Counters.
24906 case RDPMC:
24907 // GetExtended Control Register.
24908 case XGETBV: {
24909 SmallVector<SDValue, 2> Results;
24910
24911 // RDPMC uses ECX to select the index of the performance counter to read.
24912 // XGETBV uses ECX to select the index of the XCR register to return.
24913 // The result is stored into registers EDX:EAX.
24914 expandIntrinsicWChainHelper(Op.getNode(), dl, DAG, IntrData->Opc0, X86::ECX,
24915 Subtarget, Results);
24916 return DAG.getMergeValues(Results, dl);
24917 }
24918 // XTEST intrinsics.
24919 case XTEST: {
24920 SDVTList VTs = DAG.getVTList(Op->getValueType(0), MVT::Other);
24921 SDValue InTrans = DAG.getNode(IntrData->Opc0, dl, VTs, Op.getOperand(0));
24922
24923 SDValue SetCC = getSETCC(X86::COND_NE, InTrans, dl, DAG);
24924 SDValue Ret = DAG.getNode(ISD::ZERO_EXTEND, dl, Op->getValueType(0), SetCC);
24925 return DAG.getNode(ISD::MERGE_VALUES, dl, Op->getVTList(),
24926 Ret, SDValue(InTrans.getNode(), 1));
24927 }
24928 case TRUNCATE_TO_MEM_VI8:
24929 case TRUNCATE_TO_MEM_VI16:
24930 case TRUNCATE_TO_MEM_VI32: {
24931 SDValue Mask = Op.getOperand(4);
24932 SDValue DataToTruncate = Op.getOperand(3);
24933 SDValue Addr = Op.getOperand(2);
24934 SDValue Chain = Op.getOperand(0);
24935
24936 MemIntrinsicSDNode *MemIntr = dyn_cast<MemIntrinsicSDNode>(Op);
24937 assert(MemIntr && "Expected MemIntrinsicSDNode!")((MemIntr && "Expected MemIntrinsicSDNode!") ? static_cast
<void> (0) : __assert_fail ("MemIntr && \"Expected MemIntrinsicSDNode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24937, __PRETTY_FUNCTION__));
24938
24939 EVT MemVT = MemIntr->getMemoryVT();
24940
24941 uint16_t TruncationOp = IntrData->Opc0;
24942 switch (TruncationOp) {
24943 case X86ISD::VTRUNC: {
24944 if (isAllOnesConstant(Mask)) // return just a truncate store
24945 return DAG.getTruncStore(Chain, dl, DataToTruncate, Addr, MemVT,
24946 MemIntr->getMemOperand());
24947
24948 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
24949 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24950 SDValue Offset = DAG.getUNDEF(VMask.getValueType());
24951
24952 return DAG.getMaskedStore(Chain, dl, DataToTruncate, Addr, Offset, VMask,
24953 MemVT, MemIntr->getMemOperand(), ISD::UNINDEXED,
24954 true /* truncating */);
24955 }
24956 case X86ISD::VTRUNCUS:
24957 case X86ISD::VTRUNCS: {
24958 bool IsSigned = (TruncationOp == X86ISD::VTRUNCS);
24959 if (isAllOnesConstant(Mask))
24960 return EmitTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr, MemVT,
24961 MemIntr->getMemOperand(), DAG);
24962
24963 MVT MaskVT = MVT::getVectorVT(MVT::i1, MemVT.getVectorNumElements());
24964 SDValue VMask = getMaskNode(Mask, MaskVT, Subtarget, DAG, dl);
24965
24966 return EmitMaskedTruncSStore(IsSigned, Chain, dl, DataToTruncate, Addr,
24967 VMask, MemVT, MemIntr->getMemOperand(), DAG);
24968 }
24969 default:
24970 llvm_unreachable("Unsupported truncstore intrinsic")::llvm::llvm_unreachable_internal("Unsupported truncstore intrinsic"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 24970);
24971 }
24972 }
24973 }
24974}
24975
24976SDValue X86TargetLowering::LowerRETURNADDR(SDValue Op,
24977 SelectionDAG &DAG) const {
24978 MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
24979 MFI.setReturnAddressIsTaken(true);
24980
24981 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
24982 return SDValue();
24983
24984 unsigned Depth = Op.getConstantOperandVal(0);
24985 SDLoc dl(Op);
24986 EVT PtrVT = getPointerTy(DAG.getDataLayout());
24987
24988 if (Depth > 0) {
24989 SDValue FrameAddr = LowerFRAMEADDR(Op, DAG);
24990 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
24991 SDValue Offset = DAG.getConstant(RegInfo->getSlotSize(), dl, PtrVT);
24992 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(),
24993 DAG.getNode(ISD::ADD, dl, PtrVT, FrameAddr, Offset),
24994 MachinePointerInfo());
24995 }
24996
24997 // Just load the return address.
24998 SDValue RetAddrFI = getReturnAddressFrameIndex(DAG);
24999 return DAG.getLoad(PtrVT, dl, DAG.getEntryNode(), RetAddrFI,
25000 MachinePointerInfo());
25001}
25002
25003SDValue X86TargetLowering::LowerADDROFRETURNADDR(SDValue Op,
25004 SelectionDAG &DAG) const {
25005 DAG.getMachineFunction().getFrameInfo().setReturnAddressIsTaken(true);
25006 return getReturnAddressFrameIndex(DAG);
25007}
25008
25009SDValue X86TargetLowering::LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const {
25010 MachineFunction &MF = DAG.getMachineFunction();
25011 MachineFrameInfo &MFI = MF.getFrameInfo();
25012 X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
25013 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
25014 EVT VT = Op.getValueType();
25015
25016 MFI.setFrameAddressIsTaken(true);
25017
25018 if (MF.getTarget().getMCAsmInfo()->usesWindowsCFI()) {
25019 // Depth > 0 makes no sense on targets which use Windows unwind codes. It
25020 // is not possible to crawl up the stack without looking at the unwind codes
25021 // simultaneously.
25022 int FrameAddrIndex = FuncInfo->getFAIndex();
25023 if (!FrameAddrIndex) {
25024 // Set up a frame object for the return address.
25025 unsigned SlotSize = RegInfo->getSlotSize();
25026 FrameAddrIndex = MF.getFrameInfo().CreateFixedObject(
25027 SlotSize, /*SPOffset=*/0, /*IsImmutable=*/false);
25028 FuncInfo->setFAIndex(FrameAddrIndex);
25029 }
25030 return DAG.getFrameIndex(FrameAddrIndex, VT);
25031 }
25032
25033 unsigned FrameReg =
25034 RegInfo->getPtrSizedFrameRegister(DAG.getMachineFunction());
25035 SDLoc dl(Op); // FIXME probably not meaningful
25036 unsigned Depth = Op.getConstantOperandVal(0);
25037 assert(((FrameReg == X86::RBP && VT == MVT::i64) ||((((FrameReg == X86::RBP && VT == MVT::i64) || (FrameReg
== X86::EBP && VT == MVT::i32)) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("((FrameReg == X86::RBP && VT == MVT::i64) || (FrameReg == X86::EBP && VT == MVT::i32)) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25039, __PRETTY_FUNCTION__))
25038 (FrameReg == X86::EBP && VT == MVT::i32)) &&((((FrameReg == X86::RBP && VT == MVT::i64) || (FrameReg
== X86::EBP && VT == MVT::i32)) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("((FrameReg == X86::RBP && VT == MVT::i64) || (FrameReg == X86::EBP && VT == MVT::i32)) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25039, __PRETTY_FUNCTION__))
25039 "Invalid Frame Register!")((((FrameReg == X86::RBP && VT == MVT::i64) || (FrameReg
== X86::EBP && VT == MVT::i32)) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("((FrameReg == X86::RBP && VT == MVT::i64) || (FrameReg == X86::EBP && VT == MVT::i32)) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25039, __PRETTY_FUNCTION__));
25040 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, VT);
25041 while (Depth--)
25042 FrameAddr = DAG.getLoad(VT, dl, DAG.getEntryNode(), FrameAddr,
25043 MachinePointerInfo());
25044 return FrameAddr;
25045}
25046
25047// FIXME? Maybe this could be a TableGen attribute on some registers and
25048// this table could be generated automatically from RegInfo.
25049Register X86TargetLowering::getRegisterByName(const char* RegName, LLT VT,
25050 const MachineFunction &MF) const {
25051 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
25052
25053 Register Reg = StringSwitch<unsigned>(RegName)
25054 .Case("esp", X86::ESP)
25055 .Case("rsp", X86::RSP)
25056 .Case("ebp", X86::EBP)
25057 .Case("rbp", X86::RBP)
25058 .Default(0);
25059
25060 if (Reg == X86::EBP || Reg == X86::RBP) {
25061 if (!TFI.hasFP(MF))
25062 report_fatal_error("register " + StringRef(RegName) +
25063 " is allocatable: function has no frame pointer");
25064#ifndef NDEBUG
25065 else {
25066 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
25067 Register FrameReg = RegInfo->getPtrSizedFrameRegister(MF);
25068 assert((FrameReg == X86::EBP || FrameReg == X86::RBP) &&(((FrameReg == X86::EBP || FrameReg == X86::RBP) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("(FrameReg == X86::EBP || FrameReg == X86::RBP) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25069, __PRETTY_FUNCTION__))
25069 "Invalid Frame Register!")(((FrameReg == X86::EBP || FrameReg == X86::RBP) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("(FrameReg == X86::EBP || FrameReg == X86::RBP) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25069, __PRETTY_FUNCTION__));
25070 }
25071#endif
25072 }
25073
25074 if (Reg)
25075 return Reg;
25076
25077 report_fatal_error("Invalid register name global variable");
25078}
25079
25080SDValue X86TargetLowering::LowerFRAME_TO_ARGS_OFFSET(SDValue Op,
25081 SelectionDAG &DAG) const {
25082 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
25083 return DAG.getIntPtrConstant(2 * RegInfo->getSlotSize(), SDLoc(Op));
25084}
25085
25086unsigned X86TargetLowering::getExceptionPointerRegister(
25087 const Constant *PersonalityFn) const {
25088 if (classifyEHPersonality(PersonalityFn) == EHPersonality::CoreCLR)
25089 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
25090
25091 return Subtarget.isTarget64BitLP64() ? X86::RAX : X86::EAX;
25092}
25093
25094unsigned X86TargetLowering::getExceptionSelectorRegister(
25095 const Constant *PersonalityFn) const {
25096 // Funclet personalities don't use selectors (the runtime does the selection).
25097 assert(!isFuncletEHPersonality(classifyEHPersonality(PersonalityFn)))((!isFuncletEHPersonality(classifyEHPersonality(PersonalityFn
))) ? static_cast<void> (0) : __assert_fail ("!isFuncletEHPersonality(classifyEHPersonality(PersonalityFn))"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25097, __PRETTY_FUNCTION__));
25098 return Subtarget.isTarget64BitLP64() ? X86::RDX : X86::EDX;
25099}
25100
25101bool X86TargetLowering::needsFixedCatchObjects() const {
25102 return Subtarget.isTargetWin64();
25103}
25104
25105SDValue X86TargetLowering::LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const {
25106 SDValue Chain = Op.getOperand(0);
25107 SDValue Offset = Op.getOperand(1);
25108 SDValue Handler = Op.getOperand(2);
25109 SDLoc dl (Op);
25110
25111 EVT PtrVT = getPointerTy(DAG.getDataLayout());
25112 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
25113 Register FrameReg = RegInfo->getFrameRegister(DAG.getMachineFunction());
25114 assert(((FrameReg == X86::RBP && PtrVT == MVT::i64) ||((((FrameReg == X86::RBP && PtrVT == MVT::i64) || (FrameReg
== X86::EBP && PtrVT == MVT::i32)) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("((FrameReg == X86::RBP && PtrVT == MVT::i64) || (FrameReg == X86::EBP && PtrVT == MVT::i32)) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25116, __PRETTY_FUNCTION__))
25115 (FrameReg == X86::EBP && PtrVT == MVT::i32)) &&((((FrameReg == X86::RBP && PtrVT == MVT::i64) || (FrameReg
== X86::EBP && PtrVT == MVT::i32)) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("((FrameReg == X86::RBP && PtrVT == MVT::i64) || (FrameReg == X86::EBP && PtrVT == MVT::i32)) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25116, __PRETTY_FUNCTION__))
25116 "Invalid Frame Register!")((((FrameReg == X86::RBP && PtrVT == MVT::i64) || (FrameReg
== X86::EBP && PtrVT == MVT::i32)) && "Invalid Frame Register!"
) ? static_cast<void> (0) : __assert_fail ("((FrameReg == X86::RBP && PtrVT == MVT::i64) || (FrameReg == X86::EBP && PtrVT == MVT::i32)) && \"Invalid Frame Register!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25116, __PRETTY_FUNCTION__));
25117 SDValue Frame = DAG.getCopyFromReg(DAG.getEntryNode(), dl, FrameReg, PtrVT);
25118 unsigned StoreAddrReg = (PtrVT == MVT::i64) ? X86::RCX : X86::ECX;
25119
25120 SDValue StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, Frame,
25121 DAG.getIntPtrConstant(RegInfo->getSlotSize(),
25122 dl));
25123 StoreAddr = DAG.getNode(ISD::ADD, dl, PtrVT, StoreAddr, Offset);
25124 Chain = DAG.getStore(Chain, dl, Handler, StoreAddr, MachinePointerInfo());
25125 Chain = DAG.getCopyToReg(Chain, dl, StoreAddrReg, StoreAddr);
25126
25127 return DAG.getNode(X86ISD::EH_RETURN, dl, MVT::Other, Chain,
25128 DAG.getRegister(StoreAddrReg, PtrVT));
25129}
25130
25131SDValue X86TargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
25132 SelectionDAG &DAG) const {
25133 SDLoc DL(Op);
25134 // If the subtarget is not 64bit, we may need the global base reg
25135 // after isel expand pseudo, i.e., after CGBR pass ran.
25136 // Therefore, ask for the GlobalBaseReg now, so that the pass
25137 // inserts the code for us in case we need it.
25138 // Otherwise, we will end up in a situation where we will
25139 // reference a virtual register that is not defined!
25140 if (!Subtarget.is64Bit()) {
25141 const X86InstrInfo *TII = Subtarget.getInstrInfo();
25142 (void)TII->getGlobalBaseReg(&DAG.getMachineFunction());
25143 }
25144 return DAG.getNode(X86ISD::EH_SJLJ_SETJMP, DL,
25145 DAG.getVTList(MVT::i32, MVT::Other),
25146 Op.getOperand(0), Op.getOperand(1));
25147}
25148
25149SDValue X86TargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
25150 SelectionDAG &DAG) const {
25151 SDLoc DL(Op);
25152 return DAG.getNode(X86ISD::EH_SJLJ_LONGJMP, DL, MVT::Other,
25153 Op.getOperand(0), Op.getOperand(1));
25154}
25155
25156SDValue X86TargetLowering::lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op,
25157 SelectionDAG &DAG) const {
25158 SDLoc DL(Op);
25159 return DAG.getNode(X86ISD::EH_SJLJ_SETUP_DISPATCH, DL, MVT::Other,
25160 Op.getOperand(0));
25161}
25162
25163static SDValue LowerADJUST_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) {
25164 return Op.getOperand(0);
25165}
25166
25167SDValue X86TargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
25168 SelectionDAG &DAG) const {
25169 SDValue Root = Op.getOperand(0);
25170 SDValue Trmp = Op.getOperand(1); // trampoline
25171 SDValue FPtr = Op.getOperand(2); // nested function
25172 SDValue Nest = Op.getOperand(3); // 'nest' parameter value
25173 SDLoc dl (Op);
25174
25175 const Value *TrmpAddr = cast<SrcValueSDNode>(Op.getOperand(4))->getValue();
25176 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
25177
25178 if (Subtarget.is64Bit()) {
25179 SDValue OutChains[6];
25180
25181 // Large code-model.
25182 const unsigned char JMP64r = 0xFF; // 64-bit jmp through register opcode.
25183 const unsigned char MOV64ri = 0xB8; // X86::MOV64ri opcode.
25184
25185 const unsigned char N86R10 = TRI->getEncodingValue(X86::R10) & 0x7;
25186 const unsigned char N86R11 = TRI->getEncodingValue(X86::R11) & 0x7;
25187
25188 const unsigned char REX_WB = 0x40 | 0x08 | 0x01; // REX prefix
25189
25190 // Load the pointer to the nested function into R11.
25191 unsigned OpCode = ((MOV64ri | N86R11) << 8) | REX_WB; // movabsq r11
25192 SDValue Addr = Trmp;
25193 OutChains[0] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
25194 Addr, MachinePointerInfo(TrmpAddr));
25195
25196 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
25197 DAG.getConstant(2, dl, MVT::i64));
25198 OutChains[1] =
25199 DAG.getStore(Root, dl, FPtr, Addr, MachinePointerInfo(TrmpAddr, 2),
25200 /* Alignment = */ 2);
25201
25202 // Load the 'nest' parameter value into R10.
25203 // R10 is specified in X86CallingConv.td
25204 OpCode = ((MOV64ri | N86R10) << 8) | REX_WB; // movabsq r10
25205 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
25206 DAG.getConstant(10, dl, MVT::i64));
25207 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
25208 Addr, MachinePointerInfo(TrmpAddr, 10));
25209
25210 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
25211 DAG.getConstant(12, dl, MVT::i64));
25212 OutChains[3] =
25213 DAG.getStore(Root, dl, Nest, Addr, MachinePointerInfo(TrmpAddr, 12),
25214 /* Alignment = */ 2);
25215
25216 // Jump to the nested function.
25217 OpCode = (JMP64r << 8) | REX_WB; // jmpq *...
25218 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
25219 DAG.getConstant(20, dl, MVT::i64));
25220 OutChains[4] = DAG.getStore(Root, dl, DAG.getConstant(OpCode, dl, MVT::i16),
25221 Addr, MachinePointerInfo(TrmpAddr, 20));
25222
25223 unsigned char ModRM = N86R11 | (4 << 3) | (3 << 6); // ...r11
25224 Addr = DAG.getNode(ISD::ADD, dl, MVT::i64, Trmp,
25225 DAG.getConstant(22, dl, MVT::i64));
25226 OutChains[5] = DAG.getStore(Root, dl, DAG.getConstant(ModRM, dl, MVT::i8),
25227 Addr, MachinePointerInfo(TrmpAddr, 22));
25228
25229 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
25230 } else {
25231 const Function *Func =
25232 cast<Function>(cast<SrcValueSDNode>(Op.getOperand(5))->getValue());
25233 CallingConv::ID CC = Func->getCallingConv();
25234 unsigned NestReg;
25235
25236 switch (CC) {
25237 default:
25238 llvm_unreachable("Unsupported calling convention")::llvm::llvm_unreachable_internal("Unsupported calling convention"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25238);
25239 case CallingConv::C:
25240 case CallingConv::X86_StdCall: {
25241 // Pass 'nest' parameter in ECX.
25242 // Must be kept in sync with X86CallingConv.td
25243 NestReg = X86::ECX;
25244
25245 // Check that ECX wasn't needed by an 'inreg' parameter.
25246 FunctionType *FTy = Func->getFunctionType();
25247 const AttributeList &Attrs = Func->getAttributes();
25248
25249 if (!Attrs.isEmpty() && !Func->isVarArg()) {
25250 unsigned InRegCount = 0;
25251 unsigned Idx = 1;
25252
25253 for (FunctionType::param_iterator I = FTy->param_begin(),
25254 E = FTy->param_end(); I != E; ++I, ++Idx)
25255 if (Attrs.hasAttribute(Idx, Attribute::InReg)) {
25256 auto &DL = DAG.getDataLayout();
25257 // FIXME: should only count parameters that are lowered to integers.
25258 InRegCount += (DL.getTypeSizeInBits(*I) + 31) / 32;
25259 }
25260
25261 if (InRegCount > 2) {
25262 report_fatal_error("Nest register in use - reduce number of inreg"
25263 " parameters!");
25264 }
25265 }
25266 break;
25267 }
25268 case CallingConv::X86_FastCall:
25269 case CallingConv::X86_ThisCall:
25270 case CallingConv::Fast:
25271 case CallingConv::Tail:
25272 // Pass 'nest' parameter in EAX.
25273 // Must be kept in sync with X86CallingConv.td
25274 NestReg = X86::EAX;
25275 break;
25276 }
25277
25278 SDValue OutChains[4];
25279 SDValue Addr, Disp;
25280
25281 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
25282 DAG.getConstant(10, dl, MVT::i32));
25283 Disp = DAG.getNode(ISD::SUB, dl, MVT::i32, FPtr, Addr);
25284
25285 // This is storing the opcode for MOV32ri.
25286 const unsigned char MOV32ri = 0xB8; // X86::MOV32ri's opcode byte.
25287 const unsigned char N86Reg = TRI->getEncodingValue(NestReg) & 0x7;
25288 OutChains[0] =
25289 DAG.getStore(Root, dl, DAG.getConstant(MOV32ri | N86Reg, dl, MVT::i8),
25290 Trmp, MachinePointerInfo(TrmpAddr));
25291
25292 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
25293 DAG.getConstant(1, dl, MVT::i32));
25294 OutChains[1] =
25295 DAG.getStore(Root, dl, Nest, Addr, MachinePointerInfo(TrmpAddr, 1),
25296 /* Alignment = */ 1);
25297
25298 const unsigned char JMP = 0xE9; // jmp <32bit dst> opcode.
25299 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
25300 DAG.getConstant(5, dl, MVT::i32));
25301 OutChains[2] = DAG.getStore(Root, dl, DAG.getConstant(JMP, dl, MVT::i8),
25302 Addr, MachinePointerInfo(TrmpAddr, 5),
25303 /* Alignment = */ 1);
25304
25305 Addr = DAG.getNode(ISD::ADD, dl, MVT::i32, Trmp,
25306 DAG.getConstant(6, dl, MVT::i32));
25307 OutChains[3] =
25308 DAG.getStore(Root, dl, Disp, Addr, MachinePointerInfo(TrmpAddr, 6),
25309 /* Alignment = */ 1);
25310
25311 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, OutChains);
25312 }
25313}
25314
25315SDValue X86TargetLowering::LowerFLT_ROUNDS_(SDValue Op,
25316 SelectionDAG &DAG) const {
25317 /*
25318 The rounding mode is in bits 11:10 of FPSR, and has the following
25319 settings:
25320 00 Round to nearest
25321 01 Round to -inf
25322 10 Round to +inf
25323 11 Round to 0
25324
25325 FLT_ROUNDS, on the other hand, expects the following:
25326 -1 Undefined
25327 0 Round to 0
25328 1 Round to nearest
25329 2 Round to +inf
25330 3 Round to -inf
25331
25332 To perform the conversion, we do:
25333 (((((FPSR & 0x800) >> 11) | ((FPSR & 0x400) >> 9)) + 1) & 3)
25334 */
25335
25336 MachineFunction &MF = DAG.getMachineFunction();
25337 const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
25338 const Align StackAlignment(TFI.getStackAlignment());
25339 MVT VT = Op.getSimpleValueType();
25340 SDLoc DL(Op);
25341
25342 // Save FP Control Word to stack slot
25343 int SSFI =
25344 MF.getFrameInfo().CreateStackObject(2, StackAlignment.value(), false);
25345 SDValue StackSlot =
25346 DAG.getFrameIndex(SSFI, getPointerTy(DAG.getDataLayout()));
25347
25348 MachineMemOperand *MMO =
25349 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, SSFI),
25350 MachineMemOperand::MOStore, 2, 2);
25351
25352 SDValue Ops[] = { DAG.getEntryNode(), StackSlot };
25353 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::FNSTCW16m, DL,
25354 DAG.getVTList(MVT::Other),
25355 Ops, MVT::i16, MMO);
25356
25357 // Load FP Control Word from stack slot
25358 SDValue CWD =
25359 DAG.getLoad(MVT::i16, DL, Chain, StackSlot, MachinePointerInfo());
25360
25361 // Transform as necessary
25362 SDValue CWD1 =
25363 DAG.getNode(ISD::SRL, DL, MVT::i16,
25364 DAG.getNode(ISD::AND, DL, MVT::i16,
25365 CWD, DAG.getConstant(0x800, DL, MVT::i16)),
25366 DAG.getConstant(11, DL, MVT::i8));
25367 SDValue CWD2 =
25368 DAG.getNode(ISD::SRL, DL, MVT::i16,
25369 DAG.getNode(ISD::AND, DL, MVT::i16,
25370 CWD, DAG.getConstant(0x400, DL, MVT::i16)),
25371 DAG.getConstant(9, DL, MVT::i8));
25372
25373 SDValue RetVal =
25374 DAG.getNode(ISD::AND, DL, MVT::i16,
25375 DAG.getNode(ISD::ADD, DL, MVT::i16,
25376 DAG.getNode(ISD::OR, DL, MVT::i16, CWD1, CWD2),
25377 DAG.getConstant(1, DL, MVT::i16)),
25378 DAG.getConstant(3, DL, MVT::i16));
25379
25380 return DAG.getNode((VT.getSizeInBits() < 16 ?
25381 ISD::TRUNCATE : ISD::ZERO_EXTEND), DL, VT, RetVal);
25382}
25383
25384// Split an unary integer op into 2 half sized ops.
25385static SDValue LowerVectorIntUnary(SDValue Op, SelectionDAG &DAG) {
25386 MVT VT = Op.getSimpleValueType();
25387 unsigned NumElems = VT.getVectorNumElements();
25388 unsigned SizeInBits = VT.getSizeInBits();
25389 MVT EltVT = VT.getVectorElementType();
25390 SDValue Src = Op.getOperand(0);
25391 assert(EltVT == Src.getSimpleValueType().getVectorElementType() &&((EltVT == Src.getSimpleValueType().getVectorElementType() &&
"Src and Op should have the same element type!") ? static_cast
<void> (0) : __assert_fail ("EltVT == Src.getSimpleValueType().getVectorElementType() && \"Src and Op should have the same element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25392, __PRETTY_FUNCTION__))
25392 "Src and Op should have the same element type!")((EltVT == Src.getSimpleValueType().getVectorElementType() &&
"Src and Op should have the same element type!") ? static_cast
<void> (0) : __assert_fail ("EltVT == Src.getSimpleValueType().getVectorElementType() && \"Src and Op should have the same element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25392, __PRETTY_FUNCTION__));
25393
25394 // Extract the Lo/Hi vectors
25395 SDLoc dl(Op);
25396 SDValue Lo = extractSubVector(Src, 0, DAG, dl, SizeInBits / 2);
25397 SDValue Hi = extractSubVector(Src, NumElems / 2, DAG, dl, SizeInBits / 2);
25398
25399 MVT NewVT = MVT::getVectorVT(EltVT, NumElems / 2);
25400 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
25401 DAG.getNode(Op.getOpcode(), dl, NewVT, Lo),
25402 DAG.getNode(Op.getOpcode(), dl, NewVT, Hi));
25403}
25404
25405// Decompose 256-bit ops into smaller 128-bit ops.
25406static SDValue Lower256IntUnary(SDValue Op, SelectionDAG &DAG) {
25407 assert(Op.getSimpleValueType().is256BitVector() &&((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25409, __PRETTY_FUNCTION__))
25408 Op.getSimpleValueType().isInteger() &&((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25409, __PRETTY_FUNCTION__))
25409 "Only handle AVX 256-bit vector integer operation")((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25409, __PRETTY_FUNCTION__));
25410 return LowerVectorIntUnary(Op, DAG);
25411}
25412
25413// Decompose 512-bit ops into smaller 256-bit ops.
25414static SDValue Lower512IntUnary(SDValue Op, SelectionDAG &DAG) {
25415 assert(Op.getSimpleValueType().is512BitVector() &&((Op.getSimpleValueType().is512BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 512-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is512BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 512-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25417, __PRETTY_FUNCTION__))
25416 Op.getSimpleValueType().isInteger() &&((Op.getSimpleValueType().is512BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 512-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is512BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 512-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25417, __PRETTY_FUNCTION__))
25417 "Only handle AVX 512-bit vector integer operation")((Op.getSimpleValueType().is512BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 512-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is512BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 512-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25417, __PRETTY_FUNCTION__));
25418 return LowerVectorIntUnary(Op, DAG);
25419}
25420
25421/// Lower a vector CTLZ using native supported vector CTLZ instruction.
25422//
25423// i8/i16 vector implemented using dword LZCNT vector instruction
25424// ( sub(trunc(lzcnt(zext32(x)))) ). In case zext32(x) is illegal,
25425// split the vector, perform operation on it's Lo a Hi part and
25426// concatenate the results.
25427static SDValue LowerVectorCTLZ_AVX512CDI(SDValue Op, SelectionDAG &DAG,
25428 const X86Subtarget &Subtarget) {
25429 assert(Op.getOpcode() == ISD::CTLZ)((Op.getOpcode() == ISD::CTLZ) ? static_cast<void> (0) :
__assert_fail ("Op.getOpcode() == ISD::CTLZ", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25429, __PRETTY_FUNCTION__));
25430 SDLoc dl(Op);
25431 MVT VT = Op.getSimpleValueType();
25432 MVT EltVT = VT.getVectorElementType();
25433 unsigned NumElems = VT.getVectorNumElements();
25434
25435 assert((EltVT == MVT::i8 || EltVT == MVT::i16) &&(((EltVT == MVT::i8 || EltVT == MVT::i16) && "Unsupported element type"
) ? static_cast<void> (0) : __assert_fail ("(EltVT == MVT::i8 || EltVT == MVT::i16) && \"Unsupported element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25436, __PRETTY_FUNCTION__))
25436 "Unsupported element type")(((EltVT == MVT::i8 || EltVT == MVT::i16) && "Unsupported element type"
) ? static_cast<void> (0) : __assert_fail ("(EltVT == MVT::i8 || EltVT == MVT::i16) && \"Unsupported element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25436, __PRETTY_FUNCTION__));
25437
25438 // Split vector, it's Lo and Hi parts will be handled in next iteration.
25439 if (NumElems > 16 ||
25440 (NumElems == 16 && !Subtarget.canExtendTo512DQ()))
25441 return LowerVectorIntUnary(Op, DAG);
25442
25443 MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
25444 assert((NewVT.is256BitVector() || NewVT.is512BitVector()) &&(((NewVT.is256BitVector() || NewVT.is512BitVector()) &&
"Unsupported value type for operation") ? static_cast<void
> (0) : __assert_fail ("(NewVT.is256BitVector() || NewVT.is512BitVector()) && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25445, __PRETTY_FUNCTION__))
25445 "Unsupported value type for operation")(((NewVT.is256BitVector() || NewVT.is512BitVector()) &&
"Unsupported value type for operation") ? static_cast<void
> (0) : __assert_fail ("(NewVT.is256BitVector() || NewVT.is512BitVector()) && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25445, __PRETTY_FUNCTION__));
25446
25447 // Use native supported vector instruction vplzcntd.
25448 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, NewVT, Op.getOperand(0));
25449 SDValue CtlzNode = DAG.getNode(ISD::CTLZ, dl, NewVT, Op);
25450 SDValue TruncNode = DAG.getNode(ISD::TRUNCATE, dl, VT, CtlzNode);
25451 SDValue Delta = DAG.getConstant(32 - EltVT.getSizeInBits(), dl, VT);
25452
25453 return DAG.getNode(ISD::SUB, dl, VT, TruncNode, Delta);
25454}
25455
25456// Lower CTLZ using a PSHUFB lookup table implementation.
25457static SDValue LowerVectorCTLZInRegLUT(SDValue Op, const SDLoc &DL,
25458 const X86Subtarget &Subtarget,
25459 SelectionDAG &DAG) {
25460 MVT VT = Op.getSimpleValueType();
25461 int NumElts = VT.getVectorNumElements();
25462 int NumBytes = NumElts * (VT.getScalarSizeInBits() / 8);
25463 MVT CurrVT = MVT::getVectorVT(MVT::i8, NumBytes);
25464
25465 // Per-nibble leading zero PSHUFB lookup table.
25466 const int LUT[16] = {/* 0 */ 4, /* 1 */ 3, /* 2 */ 2, /* 3 */ 2,
25467 /* 4 */ 1, /* 5 */ 1, /* 6 */ 1, /* 7 */ 1,
25468 /* 8 */ 0, /* 9 */ 0, /* a */ 0, /* b */ 0,
25469 /* c */ 0, /* d */ 0, /* e */ 0, /* f */ 0};
25470
25471 SmallVector<SDValue, 64> LUTVec;
25472 for (int i = 0; i < NumBytes; ++i)
25473 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
25474 SDValue InRegLUT = DAG.getBuildVector(CurrVT, DL, LUTVec);
25475
25476 // Begin by bitcasting the input to byte vector, then split those bytes
25477 // into lo/hi nibbles and use the PSHUFB LUT to perform CLTZ on each of them.
25478 // If the hi input nibble is zero then we add both results together, otherwise
25479 // we just take the hi result (by masking the lo result to zero before the
25480 // add).
25481 SDValue Op0 = DAG.getBitcast(CurrVT, Op.getOperand(0));
25482 SDValue Zero = DAG.getConstant(0, DL, CurrVT);
25483
25484 SDValue NibbleShift = DAG.getConstant(0x4, DL, CurrVT);
25485 SDValue Lo = Op0;
25486 SDValue Hi = DAG.getNode(ISD::SRL, DL, CurrVT, Op0, NibbleShift);
25487 SDValue HiZ;
25488 if (CurrVT.is512BitVector()) {
25489 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
25490 HiZ = DAG.getSetCC(DL, MaskVT, Hi, Zero, ISD::SETEQ);
25491 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
25492 } else {
25493 HiZ = DAG.getSetCC(DL, CurrVT, Hi, Zero, ISD::SETEQ);
25494 }
25495
25496 Lo = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Lo);
25497 Hi = DAG.getNode(X86ISD::PSHUFB, DL, CurrVT, InRegLUT, Hi);
25498 Lo = DAG.getNode(ISD::AND, DL, CurrVT, Lo, HiZ);
25499 SDValue Res = DAG.getNode(ISD::ADD, DL, CurrVT, Lo, Hi);
25500
25501 // Merge result back from vXi8 back to VT, working on the lo/hi halves
25502 // of the current vector width in the same way we did for the nibbles.
25503 // If the upper half of the input element is zero then add the halves'
25504 // leading zero counts together, otherwise just use the upper half's.
25505 // Double the width of the result until we are at target width.
25506 while (CurrVT != VT) {
25507 int CurrScalarSizeInBits = CurrVT.getScalarSizeInBits();
25508 int CurrNumElts = CurrVT.getVectorNumElements();
25509 MVT NextSVT = MVT::getIntegerVT(CurrScalarSizeInBits * 2);
25510 MVT NextVT = MVT::getVectorVT(NextSVT, CurrNumElts / 2);
25511 SDValue Shift = DAG.getConstant(CurrScalarSizeInBits, DL, NextVT);
25512
25513 // Check if the upper half of the input element is zero.
25514 if (CurrVT.is512BitVector()) {
25515 MVT MaskVT = MVT::getVectorVT(MVT::i1, CurrVT.getVectorNumElements());
25516 HiZ = DAG.getSetCC(DL, MaskVT, DAG.getBitcast(CurrVT, Op0),
25517 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
25518 HiZ = DAG.getNode(ISD::SIGN_EXTEND, DL, CurrVT, HiZ);
25519 } else {
25520 HiZ = DAG.getSetCC(DL, CurrVT, DAG.getBitcast(CurrVT, Op0),
25521 DAG.getBitcast(CurrVT, Zero), ISD::SETEQ);
25522 }
25523 HiZ = DAG.getBitcast(NextVT, HiZ);
25524
25525 // Move the upper/lower halves to the lower bits as we'll be extending to
25526 // NextVT. Mask the lower result to zero if HiZ is true and add the results
25527 // together.
25528 SDValue ResNext = Res = DAG.getBitcast(NextVT, Res);
25529 SDValue R0 = DAG.getNode(ISD::SRL, DL, NextVT, ResNext, Shift);
25530 SDValue R1 = DAG.getNode(ISD::SRL, DL, NextVT, HiZ, Shift);
25531 R1 = DAG.getNode(ISD::AND, DL, NextVT, ResNext, R1);
25532 Res = DAG.getNode(ISD::ADD, DL, NextVT, R0, R1);
25533 CurrVT = NextVT;
25534 }
25535
25536 return Res;
25537}
25538
25539static SDValue LowerVectorCTLZ(SDValue Op, const SDLoc &DL,
25540 const X86Subtarget &Subtarget,
25541 SelectionDAG &DAG) {
25542 MVT VT = Op.getSimpleValueType();
25543
25544 if (Subtarget.hasCDI() &&
25545 // vXi8 vectors need to be promoted to 512-bits for vXi32.
25546 (Subtarget.canExtendTo512DQ() || VT.getVectorElementType() != MVT::i8))
25547 return LowerVectorCTLZ_AVX512CDI(Op, DAG, Subtarget);
25548
25549 // Decompose 256-bit ops into smaller 128-bit ops.
25550 if (VT.is256BitVector() && !Subtarget.hasInt256())
25551 return Lower256IntUnary(Op, DAG);
25552
25553 // Decompose 512-bit ops into smaller 256-bit ops.
25554 if (VT.is512BitVector() && !Subtarget.hasBWI())
25555 return Lower512IntUnary(Op, DAG);
25556
25557 assert(Subtarget.hasSSSE3() && "Expected SSSE3 support for PSHUFB")((Subtarget.hasSSSE3() && "Expected SSSE3 support for PSHUFB"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSSE3() && \"Expected SSSE3 support for PSHUFB\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25557, __PRETTY_FUNCTION__));
25558 return LowerVectorCTLZInRegLUT(Op, DL, Subtarget, DAG);
25559}
25560
25561static SDValue LowerCTLZ(SDValue Op, const X86Subtarget &Subtarget,
25562 SelectionDAG &DAG) {
25563 MVT VT = Op.getSimpleValueType();
25564 MVT OpVT = VT;
25565 unsigned NumBits = VT.getSizeInBits();
25566 SDLoc dl(Op);
25567 unsigned Opc = Op.getOpcode();
25568
25569 if (VT.isVector())
25570 return LowerVectorCTLZ(Op, dl, Subtarget, DAG);
25571
25572 Op = Op.getOperand(0);
25573 if (VT == MVT::i8) {
25574 // Zero extend to i32 since there is not an i8 bsr.
25575 OpVT = MVT::i32;
25576 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, OpVT, Op);
25577 }
25578
25579 // Issue a bsr (scan bits in reverse) which also sets EFLAGS.
25580 SDVTList VTs = DAG.getVTList(OpVT, MVT::i32);
25581 Op = DAG.getNode(X86ISD::BSR, dl, VTs, Op);
25582
25583 if (Opc == ISD::CTLZ) {
25584 // If src is zero (i.e. bsr sets ZF), returns NumBits.
25585 SDValue Ops[] = {Op, DAG.getConstant(NumBits + NumBits - 1, dl, OpVT),
25586 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
25587 Op.getValue(1)};
25588 Op = DAG.getNode(X86ISD::CMOV, dl, OpVT, Ops);
25589 }
25590
25591 // Finally xor with NumBits-1.
25592 Op = DAG.getNode(ISD::XOR, dl, OpVT, Op,
25593 DAG.getConstant(NumBits - 1, dl, OpVT));
25594
25595 if (VT == MVT::i8)
25596 Op = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Op);
25597 return Op;
25598}
25599
25600static SDValue LowerCTTZ(SDValue Op, const X86Subtarget &Subtarget,
25601 SelectionDAG &DAG) {
25602 MVT VT = Op.getSimpleValueType();
25603 unsigned NumBits = VT.getScalarSizeInBits();
25604 SDValue N0 = Op.getOperand(0);
25605 SDLoc dl(Op);
25606
25607 assert(!VT.isVector() && Op.getOpcode() == ISD::CTTZ &&((!VT.isVector() && Op.getOpcode() == ISD::CTTZ &&
"Only scalar CTTZ requires custom lowering") ? static_cast<
void> (0) : __assert_fail ("!VT.isVector() && Op.getOpcode() == ISD::CTTZ && \"Only scalar CTTZ requires custom lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25608, __PRETTY_FUNCTION__))
25608 "Only scalar CTTZ requires custom lowering")((!VT.isVector() && Op.getOpcode() == ISD::CTTZ &&
"Only scalar CTTZ requires custom lowering") ? static_cast<
void> (0) : __assert_fail ("!VT.isVector() && Op.getOpcode() == ISD::CTTZ && \"Only scalar CTTZ requires custom lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25608, __PRETTY_FUNCTION__));
25609
25610 // Issue a bsf (scan bits forward) which also sets EFLAGS.
25611 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
25612 Op = DAG.getNode(X86ISD::BSF, dl, VTs, N0);
25613
25614 // If src is zero (i.e. bsf sets ZF), returns NumBits.
25615 SDValue Ops[] = {Op, DAG.getConstant(NumBits, dl, VT),
25616 DAG.getTargetConstant(X86::COND_E, dl, MVT::i8),
25617 Op.getValue(1)};
25618 return DAG.getNode(X86ISD::CMOV, dl, VT, Ops);
25619}
25620
25621/// Break a 256-bit integer operation into two new 128-bit ones and then
25622/// concatenate the result back.
25623static SDValue split256IntArith(SDValue Op, SelectionDAG &DAG) {
25624 MVT VT = Op.getSimpleValueType();
25625
25626 assert(VT.is256BitVector() && VT.isInteger() &&((VT.is256BitVector() && VT.isInteger() && "Unsupported value type for operation"
) ? static_cast<void> (0) : __assert_fail ("VT.is256BitVector() && VT.isInteger() && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25627, __PRETTY_FUNCTION__))
25627 "Unsupported value type for operation")((VT.is256BitVector() && VT.isInteger() && "Unsupported value type for operation"
) ? static_cast<void> (0) : __assert_fail ("VT.is256BitVector() && VT.isInteger() && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25627, __PRETTY_FUNCTION__));
25628
25629 unsigned NumElems = VT.getVectorNumElements();
25630 SDLoc dl(Op);
25631
25632 // Extract the LHS vectors
25633 SDValue LHS = Op.getOperand(0);
25634 SDValue LHS1 = extract128BitVector(LHS, 0, DAG, dl);
25635 SDValue LHS2 = extract128BitVector(LHS, NumElems / 2, DAG, dl);
25636
25637 // Extract the RHS vectors
25638 SDValue RHS = Op.getOperand(1);
25639 SDValue RHS1 = extract128BitVector(RHS, 0, DAG, dl);
25640 SDValue RHS2 = extract128BitVector(RHS, NumElems / 2, DAG, dl);
25641
25642 MVT EltVT = VT.getVectorElementType();
25643 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
25644
25645 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
25646 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
25647 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
25648}
25649
25650/// Break a 512-bit integer operation into two new 256-bit ones and then
25651/// concatenate the result back.
25652static SDValue split512IntArith(SDValue Op, SelectionDAG &DAG) {
25653 MVT VT = Op.getSimpleValueType();
25654
25655 assert(VT.is512BitVector() && VT.isInteger() &&((VT.is512BitVector() && VT.isInteger() && "Unsupported value type for operation"
) ? static_cast<void> (0) : __assert_fail ("VT.is512BitVector() && VT.isInteger() && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25656, __PRETTY_FUNCTION__))
25656 "Unsupported value type for operation")((VT.is512BitVector() && VT.isInteger() && "Unsupported value type for operation"
) ? static_cast<void> (0) : __assert_fail ("VT.is512BitVector() && VT.isInteger() && \"Unsupported value type for operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25656, __PRETTY_FUNCTION__));
25657
25658 unsigned NumElems = VT.getVectorNumElements();
25659 SDLoc dl(Op);
25660
25661 // Extract the LHS vectors
25662 SDValue LHS = Op.getOperand(0);
25663 SDValue LHS1 = extract256BitVector(LHS, 0, DAG, dl);
25664 SDValue LHS2 = extract256BitVector(LHS, NumElems / 2, DAG, dl);
25665
25666 // Extract the RHS vectors
25667 SDValue RHS = Op.getOperand(1);
25668 SDValue RHS1 = extract256BitVector(RHS, 0, DAG, dl);
25669 SDValue RHS2 = extract256BitVector(RHS, NumElems / 2, DAG, dl);
25670
25671 MVT EltVT = VT.getVectorElementType();
25672 MVT NewVT = MVT::getVectorVT(EltVT, NumElems/2);
25673
25674 return DAG.getNode(ISD::CONCAT_VECTORS, dl, VT,
25675 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS1, RHS1),
25676 DAG.getNode(Op.getOpcode(), dl, NewVT, LHS2, RHS2));
25677}
25678
25679static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
25680 const X86Subtarget &Subtarget) {
25681 MVT VT = Op.getSimpleValueType();
25682 if (VT == MVT::i16 || VT == MVT::i32)
25683 return lowerAddSubToHorizontalOp(Op, DAG, Subtarget);
25684
25685 if (VT.getScalarType() == MVT::i1)
25686 return DAG.getNode(ISD::XOR, SDLoc(Op), VT,
25687 Op.getOperand(0), Op.getOperand(1));
25688
25689 assert(Op.getSimpleValueType().is256BitVector() &&((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25691, __PRETTY_FUNCTION__))
25690 Op.getSimpleValueType().isInteger() &&((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25691, __PRETTY_FUNCTION__))
25691 "Only handle AVX 256-bit vector integer operation")((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25691, __PRETTY_FUNCTION__));
25692 return split256IntArith(Op, DAG);
25693}
25694
25695static SDValue LowerADDSAT_SUBSAT(SDValue Op, SelectionDAG &DAG,
25696 const X86Subtarget &Subtarget) {
25697 MVT VT = Op.getSimpleValueType();
25698 SDValue X = Op.getOperand(0), Y = Op.getOperand(1);
25699 unsigned Opcode = Op.getOpcode();
25700 if (VT.getScalarType() == MVT::i1) {
25701 SDLoc dl(Op);
25702 switch (Opcode) {
25703 default: llvm_unreachable("Expected saturated arithmetic opcode")::llvm::llvm_unreachable_internal("Expected saturated arithmetic opcode"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25703);
25704 case ISD::UADDSAT:
25705 case ISD::SADDSAT:
25706 // *addsat i1 X, Y --> X | Y
25707 return DAG.getNode(ISD::OR, dl, VT, X, Y);
25708 case ISD::USUBSAT:
25709 case ISD::SSUBSAT:
25710 // *subsat i1 X, Y --> X & ~Y
25711 return DAG.getNode(ISD::AND, dl, VT, X, DAG.getNOT(dl, Y, VT));
25712 }
25713 }
25714
25715 if (VT.is128BitVector()) {
25716 // Avoid the generic expansion with min/max if we don't have pminu*/pmaxu*.
25717 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
25718 EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
25719 *DAG.getContext(), VT);
25720 SDLoc DL(Op);
25721 if (Opcode == ISD::UADDSAT && !TLI.isOperationLegal(ISD::UMIN, VT)) {
25722 // uaddsat X, Y --> (X >u (X + Y)) ? -1 : X + Y
25723 SDValue Add = DAG.getNode(ISD::ADD, DL, VT, X, Y);
25724 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Add, ISD::SETUGT);
25725 return DAG.getSelect(DL, VT, Cmp, DAG.getAllOnesConstant(DL, VT), Add);
25726 }
25727 if (Opcode == ISD::USUBSAT && !TLI.isOperationLegal(ISD::UMAX, VT)) {
25728 // usubsat X, Y --> (X >u Y) ? X - Y : 0
25729 SDValue Sub = DAG.getNode(ISD::SUB, DL, VT, X, Y);
25730 SDValue Cmp = DAG.getSetCC(DL, SetCCResultType, X, Y, ISD::SETUGT);
25731 return DAG.getSelect(DL, VT, Cmp, Sub, DAG.getConstant(0, DL, VT));
25732 }
25733 // Use default expansion.
25734 return SDValue();
25735 }
25736
25737 assert(Op.getSimpleValueType().is256BitVector() &&((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25739, __PRETTY_FUNCTION__))
25738 Op.getSimpleValueType().isInteger() &&((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25739, __PRETTY_FUNCTION__))
25739 "Only handle AVX 256-bit vector integer operation")((Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType
().isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().is256BitVector() && Op.getSimpleValueType().isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25739, __PRETTY_FUNCTION__));
25740 return split256IntArith(Op, DAG);
25741}
25742
25743static SDValue LowerABS(SDValue Op, const X86Subtarget &Subtarget,
25744 SelectionDAG &DAG) {
25745 MVT VT = Op.getSimpleValueType();
25746 if (VT == MVT::i16 || VT == MVT::i32 || VT == MVT::i64) {
25747 // Since X86 does not have CMOV for 8-bit integer, we don't convert
25748 // 8-bit integer abs to NEG and CMOV.
25749 SDLoc DL(Op);
25750 SDValue N0 = Op.getOperand(0);
25751 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, DAG.getVTList(VT, MVT::i32),
25752 DAG.getConstant(0, DL, VT), N0);
25753 SDValue Ops[] = {N0, Neg, DAG.getTargetConstant(X86::COND_GE, DL, MVT::i8),
25754 SDValue(Neg.getNode(), 1)};
25755 return DAG.getNode(X86ISD::CMOV, DL, VT, Ops);
25756 }
25757
25758 // ABS(vXi64 X) --> VPBLENDVPD(X, 0-X, X).
25759 if ((VT == MVT::v2i64 || VT == MVT::v4i64) && Subtarget.hasSSE41()) {
25760 SDLoc DL(Op);
25761 SDValue Src = Op.getOperand(0);
25762 SDValue Sub =
25763 DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Src);
25764 return DAG.getNode(X86ISD::BLENDV, DL, VT, Src, Sub, Src);
25765 }
25766
25767 if (VT.is256BitVector() && !Subtarget.hasInt256()) {
25768 assert(VT.isInteger() &&((VT.isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25769, __PRETTY_FUNCTION__))
25769 "Only handle AVX 256-bit vector integer operation")((VT.isInteger() && "Only handle AVX 256-bit vector integer operation"
) ? static_cast<void> (0) : __assert_fail ("VT.isInteger() && \"Only handle AVX 256-bit vector integer operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25769, __PRETTY_FUNCTION__));
25770 return Lower256IntUnary(Op, DAG);
25771 }
25772
25773 // Default to expand.
25774 return SDValue();
25775}
25776
25777static SDValue LowerMINMAX(SDValue Op, SelectionDAG &DAG) {
25778 MVT VT = Op.getSimpleValueType();
25779
25780 // For AVX1 cases, split to use legal ops (everything but v4i64).
25781 if (VT.getScalarType() != MVT::i64 && VT.is256BitVector())
25782 return split256IntArith(Op, DAG);
25783
25784 SDLoc DL(Op);
25785 unsigned Opcode = Op.getOpcode();
25786 SDValue N0 = Op.getOperand(0);
25787 SDValue N1 = Op.getOperand(1);
25788
25789 // For pre-SSE41, we can perform UMIN/UMAX v8i16 by flipping the signbit,
25790 // using the SMIN/SMAX instructions and flipping the signbit back.
25791 if (VT == MVT::v8i16) {
25792 assert((Opcode == ISD::UMIN || Opcode == ISD::UMAX) &&(((Opcode == ISD::UMIN || Opcode == ISD::UMAX) && "Unexpected MIN/MAX opcode"
) ? static_cast<void> (0) : __assert_fail ("(Opcode == ISD::UMIN || Opcode == ISD::UMAX) && \"Unexpected MIN/MAX opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25793, __PRETTY_FUNCTION__))
25793 "Unexpected MIN/MAX opcode")(((Opcode == ISD::UMIN || Opcode == ISD::UMAX) && "Unexpected MIN/MAX opcode"
) ? static_cast<void> (0) : __assert_fail ("(Opcode == ISD::UMIN || Opcode == ISD::UMAX) && \"Unexpected MIN/MAX opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25793, __PRETTY_FUNCTION__));
25794 SDValue Sign = DAG.getConstant(APInt::getSignedMinValue(16), DL, VT);
25795 N0 = DAG.getNode(ISD::XOR, DL, VT, N0, Sign);
25796 N1 = DAG.getNode(ISD::XOR, DL, VT, N1, Sign);
25797 Opcode = (Opcode == ISD::UMIN ? ISD::SMIN : ISD::SMAX);
25798 SDValue Result = DAG.getNode(Opcode, DL, VT, N0, N1);
25799 return DAG.getNode(ISD::XOR, DL, VT, Result, Sign);
25800 }
25801
25802 // Else, expand to a compare/select.
25803 ISD::CondCode CC;
25804 switch (Opcode) {
25805 case ISD::SMIN: CC = ISD::CondCode::SETLT; break;
25806 case ISD::SMAX: CC = ISD::CondCode::SETGT; break;
25807 case ISD::UMIN: CC = ISD::CondCode::SETULT; break;
25808 case ISD::UMAX: CC = ISD::CondCode::SETUGT; break;
25809 default: llvm_unreachable("Unknown MINMAX opcode")::llvm::llvm_unreachable_internal("Unknown MINMAX opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25809);
25810 }
25811
25812 SDValue Cond = DAG.getSetCC(DL, VT, N0, N1, CC);
25813 return DAG.getSelect(DL, VT, Cond, N0, N1);
25814}
25815
25816static SDValue LowerMUL(SDValue Op, const X86Subtarget &Subtarget,
25817 SelectionDAG &DAG) {
25818 SDLoc dl(Op);
25819 MVT VT = Op.getSimpleValueType();
25820
25821 if (VT.getScalarType() == MVT::i1)
25822 return DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0), Op.getOperand(1));
25823
25824 // Decompose 256-bit ops into 128-bit ops.
25825 if (VT.is256BitVector() && !Subtarget.hasInt256())
25826 return split256IntArith(Op, DAG);
25827
25828 SDValue A = Op.getOperand(0);
25829 SDValue B = Op.getOperand(1);
25830
25831 // Lower v16i8/v32i8/v64i8 mul as sign-extension to v8i16/v16i16/v32i16
25832 // vector pairs, multiply and truncate.
25833 if (VT == MVT::v16i8 || VT == MVT::v32i8 || VT == MVT::v64i8) {
25834 unsigned NumElts = VT.getVectorNumElements();
25835
25836 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
25837 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
25838 MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
25839 return DAG.getNode(
25840 ISD::TRUNCATE, dl, VT,
25841 DAG.getNode(ISD::MUL, dl, ExVT,
25842 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, A),
25843 DAG.getNode(ISD::ANY_EXTEND, dl, ExVT, B)));
25844 }
25845
25846 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
25847
25848 // Extract the lo/hi parts to any extend to i16.
25849 // We're going to mask off the low byte of each result element of the
25850 // pmullw, so it doesn't matter what's in the high byte of each 16-bit
25851 // element.
25852 SDValue Undef = DAG.getUNDEF(VT);
25853 SDValue ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Undef));
25854 SDValue AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Undef));
25855
25856 SDValue BLo, BHi;
25857 if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
25858 // If the LHS is a constant, manually unpackl/unpackh.
25859 SmallVector<SDValue, 16> LoOps, HiOps;
25860 for (unsigned i = 0; i != NumElts; i += 16) {
25861 for (unsigned j = 0; j != 8; ++j) {
25862 LoOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j), dl,
25863 MVT::i16));
25864 HiOps.push_back(DAG.getAnyExtOrTrunc(B.getOperand(i + j + 8), dl,
25865 MVT::i16));
25866 }
25867 }
25868
25869 BLo = DAG.getBuildVector(ExVT, dl, LoOps);
25870 BHi = DAG.getBuildVector(ExVT, dl, HiOps);
25871 } else {
25872 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Undef));
25873 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Undef));
25874 }
25875
25876 // Multiply, mask the lower 8bits of the lo/hi results and pack.
25877 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
25878 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
25879 RLo = DAG.getNode(ISD::AND, dl, ExVT, RLo, DAG.getConstant(255, dl, ExVT));
25880 RHi = DAG.getNode(ISD::AND, dl, ExVT, RHi, DAG.getConstant(255, dl, ExVT));
25881 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
25882 }
25883
25884 // Lower v4i32 mul as 2x shuffle, 2x pmuludq, 2x shuffle.
25885 if (VT == MVT::v4i32) {
25886 assert(Subtarget.hasSSE2() && !Subtarget.hasSSE41() &&((Subtarget.hasSSE2() && !Subtarget.hasSSE41() &&
"Should not custom lower when pmulld is available!") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasSSE2() && !Subtarget.hasSSE41() && \"Should not custom lower when pmulld is available!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25887, __PRETTY_FUNCTION__))
25887 "Should not custom lower when pmulld is available!")((Subtarget.hasSSE2() && !Subtarget.hasSSE41() &&
"Should not custom lower when pmulld is available!") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasSSE2() && !Subtarget.hasSSE41() && \"Should not custom lower when pmulld is available!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25887, __PRETTY_FUNCTION__));
25888
25889 // Extract the odd parts.
25890 static const int UnpackMask[] = { 1, -1, 3, -1 };
25891 SDValue Aodds = DAG.getVectorShuffle(VT, dl, A, A, UnpackMask);
25892 SDValue Bodds = DAG.getVectorShuffle(VT, dl, B, B, UnpackMask);
25893
25894 // Multiply the even parts.
25895 SDValue Evens = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
25896 DAG.getBitcast(MVT::v2i64, A),
25897 DAG.getBitcast(MVT::v2i64, B));
25898 // Now multiply odd parts.
25899 SDValue Odds = DAG.getNode(X86ISD::PMULUDQ, dl, MVT::v2i64,
25900 DAG.getBitcast(MVT::v2i64, Aodds),
25901 DAG.getBitcast(MVT::v2i64, Bodds));
25902
25903 Evens = DAG.getBitcast(VT, Evens);
25904 Odds = DAG.getBitcast(VT, Odds);
25905
25906 // Merge the two vectors back together with a shuffle. This expands into 2
25907 // shuffles.
25908 static const int ShufMask[] = { 0, 4, 2, 6 };
25909 return DAG.getVectorShuffle(VT, dl, Evens, Odds, ShufMask);
25910 }
25911
25912 assert((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&(((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
"Only know how to lower V2I64/V4I64/V8I64 multiply") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && \"Only know how to lower V2I64/V4I64/V8I64 multiply\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25913, __PRETTY_FUNCTION__))
25913 "Only know how to lower V2I64/V4I64/V8I64 multiply")(((VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) &&
"Only know how to lower V2I64/V4I64/V8I64 multiply") ? static_cast
<void> (0) : __assert_fail ("(VT == MVT::v2i64 || VT == MVT::v4i64 || VT == MVT::v8i64) && \"Only know how to lower V2I64/V4I64/V8I64 multiply\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25913, __PRETTY_FUNCTION__));
25914 assert(!Subtarget.hasDQI() && "DQI should use MULLQ")((!Subtarget.hasDQI() && "DQI should use MULLQ") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.hasDQI() && \"DQI should use MULLQ\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25914, __PRETTY_FUNCTION__));
25915
25916 // Ahi = psrlqi(a, 32);
25917 // Bhi = psrlqi(b, 32);
25918 //
25919 // AloBlo = pmuludq(a, b);
25920 // AloBhi = pmuludq(a, Bhi);
25921 // AhiBlo = pmuludq(Ahi, b);
25922 //
25923 // Hi = psllqi(AloBhi + AhiBlo, 32);
25924 // return AloBlo + Hi;
25925 KnownBits AKnown = DAG.computeKnownBits(A);
25926 KnownBits BKnown = DAG.computeKnownBits(B);
25927
25928 APInt LowerBitsMask = APInt::getLowBitsSet(64, 32);
25929 bool ALoIsZero = LowerBitsMask.isSubsetOf(AKnown.Zero);
25930 bool BLoIsZero = LowerBitsMask.isSubsetOf(BKnown.Zero);
25931
25932 APInt UpperBitsMask = APInt::getHighBitsSet(64, 32);
25933 bool AHiIsZero = UpperBitsMask.isSubsetOf(AKnown.Zero);
25934 bool BHiIsZero = UpperBitsMask.isSubsetOf(BKnown.Zero);
25935
25936 SDValue Zero = DAG.getConstant(0, dl, VT);
25937
25938 // Only multiply lo/hi halves that aren't known to be zero.
25939 SDValue AloBlo = Zero;
25940 if (!ALoIsZero && !BLoIsZero)
25941 AloBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, B);
25942
25943 SDValue AloBhi = Zero;
25944 if (!ALoIsZero && !BHiIsZero) {
25945 SDValue Bhi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, B, 32, DAG);
25946 AloBhi = DAG.getNode(X86ISD::PMULUDQ, dl, VT, A, Bhi);
25947 }
25948
25949 SDValue AhiBlo = Zero;
25950 if (!AHiIsZero && !BLoIsZero) {
25951 SDValue Ahi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, A, 32, DAG);
25952 AhiBlo = DAG.getNode(X86ISD::PMULUDQ, dl, VT, Ahi, B);
25953 }
25954
25955 SDValue Hi = DAG.getNode(ISD::ADD, dl, VT, AloBhi, AhiBlo);
25956 Hi = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Hi, 32, DAG);
25957
25958 return DAG.getNode(ISD::ADD, dl, VT, AloBlo, Hi);
25959}
25960
25961static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
25962 SelectionDAG &DAG) {
25963 SDLoc dl(Op);
25964 MVT VT = Op.getSimpleValueType();
25965 bool IsSigned = Op->getOpcode() == ISD::MULHS;
25966 unsigned NumElts = VT.getVectorNumElements();
25967 SDValue A = Op.getOperand(0);
25968 SDValue B = Op.getOperand(1);
25969
25970 // Decompose 256-bit ops into 128-bit ops.
25971 if (VT.is256BitVector() && !Subtarget.hasInt256())
25972 return split256IntArith(Op, DAG);
25973
25974 if (VT == MVT::v4i32 || VT == MVT::v8i32 || VT == MVT::v16i32) {
25975 assert((VT == MVT::v4i32 && Subtarget.hasSSE2()) ||(((VT == MVT::v4i32 && Subtarget.hasSSE2()) || (VT ==
MVT::v8i32 && Subtarget.hasInt256()) || (VT == MVT::
v16i32 && Subtarget.hasAVX512())) ? static_cast<void
> (0) : __assert_fail ("(VT == MVT::v4i32 && Subtarget.hasSSE2()) || (VT == MVT::v8i32 && Subtarget.hasInt256()) || (VT == MVT::v16i32 && Subtarget.hasAVX512())"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25977, __PRETTY_FUNCTION__))
25976 (VT == MVT::v8i32 && Subtarget.hasInt256()) ||(((VT == MVT::v4i32 && Subtarget.hasSSE2()) || (VT ==
MVT::v8i32 && Subtarget.hasInt256()) || (VT == MVT::
v16i32 && Subtarget.hasAVX512())) ? static_cast<void
> (0) : __assert_fail ("(VT == MVT::v4i32 && Subtarget.hasSSE2()) || (VT == MVT::v8i32 && Subtarget.hasInt256()) || (VT == MVT::v16i32 && Subtarget.hasAVX512())"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25977, __PRETTY_FUNCTION__))
25977 (VT == MVT::v16i32 && Subtarget.hasAVX512()))(((VT == MVT::v4i32 && Subtarget.hasSSE2()) || (VT ==
MVT::v8i32 && Subtarget.hasInt256()) || (VT == MVT::
v16i32 && Subtarget.hasAVX512())) ? static_cast<void
> (0) : __assert_fail ("(VT == MVT::v4i32 && Subtarget.hasSSE2()) || (VT == MVT::v8i32 && Subtarget.hasInt256()) || (VT == MVT::v16i32 && Subtarget.hasAVX512())"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 25977, __PRETTY_FUNCTION__));
25978
25979 // PMULxD operations multiply each even value (starting at 0) of LHS with
25980 // the related value of RHS and produce a widen result.
25981 // E.g., PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
25982 // => <2 x i64> <ae|cg>
25983 //
25984 // In other word, to have all the results, we need to perform two PMULxD:
25985 // 1. one with the even values.
25986 // 2. one with the odd values.
25987 // To achieve #2, with need to place the odd values at an even position.
25988 //
25989 // Place the odd value at an even position (basically, shift all values 1
25990 // step to the left):
25991 const int Mask[] = {1, -1, 3, -1, 5, -1, 7, -1,
25992 9, -1, 11, -1, 13, -1, 15, -1};
25993 // <a|b|c|d> => <b|undef|d|undef>
25994 SDValue Odd0 = DAG.getVectorShuffle(VT, dl, A, A,
25995 makeArrayRef(&Mask[0], NumElts));
25996 // <e|f|g|h> => <f|undef|h|undef>
25997 SDValue Odd1 = DAG.getVectorShuffle(VT, dl, B, B,
25998 makeArrayRef(&Mask[0], NumElts));
25999
26000 // Emit two multiplies, one for the lower 2 ints and one for the higher 2
26001 // ints.
26002 MVT MulVT = MVT::getVectorVT(MVT::i64, NumElts / 2);
26003 unsigned Opcode =
26004 (IsSigned && Subtarget.hasSSE41()) ? X86ISD::PMULDQ : X86ISD::PMULUDQ;
26005 // PMULUDQ <4 x i32> <a|b|c|d>, <4 x i32> <e|f|g|h>
26006 // => <2 x i64> <ae|cg>
26007 SDValue Mul1 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
26008 DAG.getBitcast(MulVT, A),
26009 DAG.getBitcast(MulVT, B)));
26010 // PMULUDQ <4 x i32> <b|undef|d|undef>, <4 x i32> <f|undef|h|undef>
26011 // => <2 x i64> <bf|dh>
26012 SDValue Mul2 = DAG.getBitcast(VT, DAG.getNode(Opcode, dl, MulVT,
26013 DAG.getBitcast(MulVT, Odd0),
26014 DAG.getBitcast(MulVT, Odd1)));
26015
26016 // Shuffle it back into the right order.
26017 SmallVector<int, 16> ShufMask(NumElts);
26018 for (int i = 0; i != (int)NumElts; ++i)
26019 ShufMask[i] = (i / 2) * 2 + ((i % 2) * NumElts) + 1;
26020
26021 SDValue Res = DAG.getVectorShuffle(VT, dl, Mul1, Mul2, ShufMask);
26022
26023 // If we have a signed multiply but no PMULDQ fix up the result of an
26024 // unsigned multiply.
26025 if (IsSigned && !Subtarget.hasSSE41()) {
26026 SDValue Zero = DAG.getConstant(0, dl, VT);
26027 SDValue T1 = DAG.getNode(ISD::AND, dl, VT,
26028 DAG.getSetCC(dl, VT, Zero, A, ISD::SETGT), B);
26029 SDValue T2 = DAG.getNode(ISD::AND, dl, VT,
26030 DAG.getSetCC(dl, VT, Zero, B, ISD::SETGT), A);
26031
26032 SDValue Fixup = DAG.getNode(ISD::ADD, dl, VT, T1, T2);
26033 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Fixup);
26034 }
26035
26036 return Res;
26037 }
26038
26039 // Only i8 vectors should need custom lowering after this.
26040 assert((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) ||(((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget
.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI
())) && "Unsupported vector type") ? static_cast<void
> (0) : __assert_fail ("(VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI())) && \"Unsupported vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26042, __PRETTY_FUNCTION__))
26041 (VT == MVT::v64i8 && Subtarget.hasBWI())) &&(((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget
.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI
())) && "Unsupported vector type") ? static_cast<void
> (0) : __assert_fail ("(VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI())) && \"Unsupported vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26042, __PRETTY_FUNCTION__))
26042 "Unsupported vector type")(((VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget
.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI
())) && "Unsupported vector type") ? static_cast<void
> (0) : __assert_fail ("(VT == MVT::v16i8 || (VT == MVT::v32i8 && Subtarget.hasInt256()) || (VT == MVT::v64i8 && Subtarget.hasBWI())) && \"Unsupported vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26042, __PRETTY_FUNCTION__));
26043
26044 // Lower v16i8/v32i8 as extension to v8i16/v16i16 vector pairs, multiply,
26045 // logical shift down the upper half and pack back to i8.
26046
26047 // With SSE41 we can use sign/zero extend, but for pre-SSE41 we unpack
26048 // and then ashr/lshr the upper bits down to the lower bits before multiply.
26049 unsigned ExAVX = IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
26050
26051 if ((VT == MVT::v16i8 && Subtarget.hasInt256()) ||
26052 (VT == MVT::v32i8 && Subtarget.canExtendTo512BW())) {
26053 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
26054 SDValue ExA = DAG.getNode(ExAVX, dl, ExVT, A);
26055 SDValue ExB = DAG.getNode(ExAVX, dl, ExVT, B);
26056 SDValue Mul = DAG.getNode(ISD::MUL, dl, ExVT, ExA, ExB);
26057 Mul = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Mul, 8, DAG);
26058 return DAG.getNode(ISD::TRUNCATE, dl, VT, Mul);
26059 }
26060
26061 // For signed 512-bit vectors, split into 256-bit vectors to allow the
26062 // sign-extension to occur.
26063 if (VT == MVT::v64i8 && IsSigned)
26064 return split512IntArith(Op, DAG);
26065
26066 // Signed AVX2 implementation - extend xmm subvectors to ymm.
26067 if (VT == MVT::v32i8 && IsSigned) {
26068 MVT ExVT = MVT::v16i16;
26069 SDValue ALo = extract128BitVector(A, 0, DAG, dl);
26070 SDValue BLo = extract128BitVector(B, 0, DAG, dl);
26071 SDValue AHi = extract128BitVector(A, NumElts / 2, DAG, dl);
26072 SDValue BHi = extract128BitVector(B, NumElts / 2, DAG, dl);
26073 ALo = DAG.getNode(ExAVX, dl, ExVT, ALo);
26074 BLo = DAG.getNode(ExAVX, dl, ExVT, BLo);
26075 AHi = DAG.getNode(ExAVX, dl, ExVT, AHi);
26076 BHi = DAG.getNode(ExAVX, dl, ExVT, BHi);
26077 SDValue Lo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
26078 SDValue Hi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
26079 Lo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Lo, 8, DAG);
26080 Hi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, Hi, 8, DAG);
26081
26082 // Bitcast back to VT and then pack all the even elements from Lo and Hi.
26083 // Shuffle lowering should turn this into PACKUS+PERMQ
26084 Lo = DAG.getBitcast(VT, Lo);
26085 Hi = DAG.getBitcast(VT, Hi);
26086 return DAG.getVectorShuffle(VT, dl, Lo, Hi,
26087 { 0, 2, 4, 6, 8, 10, 12, 14,
26088 16, 18, 20, 22, 24, 26, 28, 30,
26089 32, 34, 36, 38, 40, 42, 44, 46,
26090 48, 50, 52, 54, 56, 58, 60, 62});
26091 }
26092
26093 // For signed v16i8 and all unsigned vXi8 we will unpack the low and high
26094 // half of each 128 bit lane to widen to a vXi16 type. Do the multiplies,
26095 // shift the results and pack the half lane results back together.
26096
26097 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
26098
26099 static const int PSHUFDMask[] = { 8, 9, 10, 11, 12, 13, 14, 15,
26100 -1, -1, -1, -1, -1, -1, -1, -1};
26101
26102 // Extract the lo parts and zero/sign extend to i16.
26103 // Only use SSE4.1 instructions for signed v16i8 where using unpack requires
26104 // shifts to sign extend. Using unpack for unsigned only requires an xor to
26105 // create zeros and a copy due to tied registers contraints pre-avx. But using
26106 // zero_extend_vector_inreg would require an additional pshufd for the high
26107 // part.
26108
26109 SDValue ALo, AHi;
26110 if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) {
26111 ALo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, A);
26112
26113 AHi = DAG.getVectorShuffle(VT, dl, A, A, PSHUFDMask);
26114 AHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, AHi);
26115 } else if (IsSigned) {
26116 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), A));
26117 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), A));
26118
26119 ALo = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, ALo, 8, DAG);
26120 AHi = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, AHi, 8, DAG);
26121 } else {
26122 ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A,
26123 DAG.getConstant(0, dl, VT)));
26124 AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A,
26125 DAG.getConstant(0, dl, VT)));
26126 }
26127
26128 SDValue BLo, BHi;
26129 if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
26130 // If the LHS is a constant, manually unpackl/unpackh and extend.
26131 SmallVector<SDValue, 16> LoOps, HiOps;
26132 for (unsigned i = 0; i != NumElts; i += 16) {
26133 for (unsigned j = 0; j != 8; ++j) {
26134 SDValue LoOp = B.getOperand(i + j);
26135 SDValue HiOp = B.getOperand(i + j + 8);
26136
26137 if (IsSigned) {
26138 LoOp = DAG.getSExtOrTrunc(LoOp, dl, MVT::i16);
26139 HiOp = DAG.getSExtOrTrunc(HiOp, dl, MVT::i16);
26140 } else {
26141 LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
26142 HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
26143 }
26144
26145 LoOps.push_back(LoOp);
26146 HiOps.push_back(HiOp);
26147 }
26148 }
26149
26150 BLo = DAG.getBuildVector(ExVT, dl, LoOps);
26151 BHi = DAG.getBuildVector(ExVT, dl, HiOps);
26152 } else if (IsSigned && VT == MVT::v16i8 && Subtarget.hasSSE41()) {
26153 BLo = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, B);
26154
26155 BHi = DAG.getVectorShuffle(VT, dl, B, B, PSHUFDMask);
26156 BHi = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, dl, ExVT, BHi);
26157 } else if (IsSigned) {
26158 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), B));
26159 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), B));
26160
26161 BLo = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, BLo, 8, DAG);
26162 BHi = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, BHi, 8, DAG);
26163 } else {
26164 BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B,
26165 DAG.getConstant(0, dl, VT)));
26166 BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B,
26167 DAG.getConstant(0, dl, VT)));
26168 }
26169
26170 // Multiply, lshr the upper 8bits to the lower 8bits of the lo/hi results and
26171 // pack back to vXi8.
26172 SDValue RLo = DAG.getNode(ISD::MUL, dl, ExVT, ALo, BLo);
26173 SDValue RHi = DAG.getNode(ISD::MUL, dl, ExVT, AHi, BHi);
26174 RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, RLo, 8, DAG);
26175 RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExVT, RHi, 8, DAG);
26176
26177 // Bitcast back to VT and then pack all the even elements from Lo and Hi.
26178 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
26179}
26180
26181SDValue X86TargetLowering::LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const {
26182 assert(Subtarget.isTargetWin64() && "Unexpected target")((Subtarget.isTargetWin64() && "Unexpected target") ?
static_cast<void> (0) : __assert_fail ("Subtarget.isTargetWin64() && \"Unexpected target\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26182, __PRETTY_FUNCTION__));
26183 EVT VT = Op.getValueType();
26184 assert(VT.isInteger() && VT.getSizeInBits() == 128 &&((VT.isInteger() && VT.getSizeInBits() == 128 &&
"Unexpected return type for lowering") ? static_cast<void
> (0) : __assert_fail ("VT.isInteger() && VT.getSizeInBits() == 128 && \"Unexpected return type for lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26185, __PRETTY_FUNCTION__))
26185 "Unexpected return type for lowering")((VT.isInteger() && VT.getSizeInBits() == 128 &&
"Unexpected return type for lowering") ? static_cast<void
> (0) : __assert_fail ("VT.isInteger() && VT.getSizeInBits() == 128 && \"Unexpected return type for lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26185, __PRETTY_FUNCTION__));
26186
26187 RTLIB::Libcall LC;
26188 bool isSigned;
26189 switch (Op->getOpcode()) {
26190 default: llvm_unreachable("Unexpected request for libcall!")::llvm::llvm_unreachable_internal("Unexpected request for libcall!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26190);
26191 case ISD::SDIV: isSigned = true; LC = RTLIB::SDIV_I128; break;
26192 case ISD::UDIV: isSigned = false; LC = RTLIB::UDIV_I128; break;
26193 case ISD::SREM: isSigned = true; LC = RTLIB::SREM_I128; break;
26194 case ISD::UREM: isSigned = false; LC = RTLIB::UREM_I128; break;
26195 case ISD::SDIVREM: isSigned = true; LC = RTLIB::SDIVREM_I128; break;
26196 case ISD::UDIVREM: isSigned = false; LC = RTLIB::UDIVREM_I128; break;
26197 }
26198
26199 SDLoc dl(Op);
26200 SDValue InChain = DAG.getEntryNode();
26201
26202 TargetLowering::ArgListTy Args;
26203 TargetLowering::ArgListEntry Entry;
26204 for (unsigned i = 0, e = Op->getNumOperands(); i != e; ++i) {
26205 EVT ArgVT = Op->getOperand(i).getValueType();
26206 assert(ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&((ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
"Unexpected argument type for lowering") ? static_cast<void
> (0) : __assert_fail ("ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 && \"Unexpected argument type for lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26207, __PRETTY_FUNCTION__))
26207 "Unexpected argument type for lowering")((ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 &&
"Unexpected argument type for lowering") ? static_cast<void
> (0) : __assert_fail ("ArgVT.isInteger() && ArgVT.getSizeInBits() == 128 && \"Unexpected argument type for lowering\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26207, __PRETTY_FUNCTION__));
26208 SDValue StackPtr = DAG.CreateStackTemporary(ArgVT, 16);
26209 Entry.Node = StackPtr;
26210 InChain = DAG.getStore(InChain, dl, Op->getOperand(i), StackPtr,
26211 MachinePointerInfo(), /* Alignment = */ 16);
26212 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
26213 Entry.Ty = PointerType::get(ArgTy,0);
26214 Entry.IsSExt = false;
26215 Entry.IsZExt = false;
26216 Args.push_back(Entry);
26217 }
26218
26219 SDValue Callee = DAG.getExternalSymbol(getLibcallName(LC),
26220 getPointerTy(DAG.getDataLayout()));
26221
26222 TargetLowering::CallLoweringInfo CLI(DAG);
26223 CLI.setDebugLoc(dl)
26224 .setChain(InChain)
26225 .setLibCallee(
26226 getLibcallCallingConv(LC),
26227 static_cast<EVT>(MVT::v2i64).getTypeForEVT(*DAG.getContext()), Callee,
26228 std::move(Args))
26229 .setInRegister()
26230 .setSExtResult(isSigned)
26231 .setZExtResult(!isSigned);
26232
26233 std::pair<SDValue, SDValue> CallInfo = LowerCallTo(CLI);
26234 return DAG.getBitcast(VT, CallInfo.first);
26235}
26236
26237// Return true if the required (according to Opcode) shift-imm form is natively
26238// supported by the Subtarget
26239static bool SupportedVectorShiftWithImm(MVT VT, const X86Subtarget &Subtarget,
26240 unsigned Opcode) {
26241 if (VT.getScalarSizeInBits() < 16)
26242 return false;
26243
26244 if (VT.is512BitVector() && Subtarget.hasAVX512() &&
26245 (VT.getScalarSizeInBits() > 16 || Subtarget.hasBWI()))
26246 return true;
26247
26248 bool LShift = (VT.is128BitVector() && Subtarget.hasSSE2()) ||
26249 (VT.is256BitVector() && Subtarget.hasInt256());
26250
26251 bool AShift = LShift && (Subtarget.hasAVX512() ||
26252 (VT != MVT::v2i64 && VT != MVT::v4i64));
26253 return (Opcode == ISD::SRA) ? AShift : LShift;
26254}
26255
26256// The shift amount is a variable, but it is the same for all vector lanes.
26257// These instructions are defined together with shift-immediate.
26258static
26259bool SupportedVectorShiftWithBaseAmnt(MVT VT, const X86Subtarget &Subtarget,
26260 unsigned Opcode) {
26261 return SupportedVectorShiftWithImm(VT, Subtarget, Opcode);
26262}
26263
26264// Return true if the required (according to Opcode) variable-shift form is
26265// natively supported by the Subtarget
26266static bool SupportedVectorVarShift(MVT VT, const X86Subtarget &Subtarget,
26267 unsigned Opcode) {
26268
26269 if (!Subtarget.hasInt256() || VT.getScalarSizeInBits() < 16)
26270 return false;
26271
26272 // vXi16 supported only on AVX-512, BWI
26273 if (VT.getScalarSizeInBits() == 16 && !Subtarget.hasBWI())
26274 return false;
26275
26276 if (Subtarget.hasAVX512())
26277 return true;
26278
26279 bool LShift = VT.is128BitVector() || VT.is256BitVector();
26280 bool AShift = LShift && VT != MVT::v2i64 && VT != MVT::v4i64;
26281 return (Opcode == ISD::SRA) ? AShift : LShift;
26282}
26283
26284static SDValue LowerScalarImmediateShift(SDValue Op, SelectionDAG &DAG,
26285 const X86Subtarget &Subtarget) {
26286 MVT VT = Op.getSimpleValueType();
26287 SDLoc dl(Op);
26288 SDValue R = Op.getOperand(0);
26289 SDValue Amt = Op.getOperand(1);
26290 unsigned X86Opc = getTargetVShiftUniformOpcode(Op.getOpcode(), false);
26291
26292 auto ArithmeticShiftRight64 = [&](uint64_t ShiftAmt) {
26293 assert((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type")(((VT == MVT::v2i64 || VT == MVT::v4i64) && "Unexpected SRA type"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::v2i64 || VT == MVT::v4i64) && \"Unexpected SRA type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26293, __PRETTY_FUNCTION__));
26294 MVT ExVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() * 2);
26295 SDValue Ex = DAG.getBitcast(ExVT, R);
26296
26297 // ashr(R, 63) === cmp_slt(R, 0)
26298 if (ShiftAmt == 63 && Subtarget.hasSSE42()) {
26299 assert((VT != MVT::v4i64 || Subtarget.hasInt256()) &&(((VT != MVT::v4i64 || Subtarget.hasInt256()) && "Unsupported PCMPGT op"
) ? static_cast<void> (0) : __assert_fail ("(VT != MVT::v4i64 || Subtarget.hasInt256()) && \"Unsupported PCMPGT op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26300, __PRETTY_FUNCTION__))
26300 "Unsupported PCMPGT op")(((VT != MVT::v4i64 || Subtarget.hasInt256()) && "Unsupported PCMPGT op"
) ? static_cast<void> (0) : __assert_fail ("(VT != MVT::v4i64 || Subtarget.hasInt256()) && \"Unsupported PCMPGT op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26300, __PRETTY_FUNCTION__));
26301 return DAG.getNode(X86ISD::PCMPGT, dl, VT, DAG.getConstant(0, dl, VT), R);
26302 }
26303
26304 if (ShiftAmt >= 32) {
26305 // Splat sign to upper i32 dst, and SRA upper i32 src to lower i32.
26306 SDValue Upper =
26307 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex, 31, DAG);
26308 SDValue Lower = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
26309 ShiftAmt - 32, DAG);
26310 if (VT == MVT::v2i64)
26311 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {5, 1, 7, 3});
26312 if (VT == MVT::v4i64)
26313 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
26314 {9, 1, 11, 3, 13, 5, 15, 7});
26315 } else {
26316 // SRA upper i32, SRL whole i64 and select lower i32.
26317 SDValue Upper = getTargetVShiftByConstNode(X86ISD::VSRAI, dl, ExVT, Ex,
26318 ShiftAmt, DAG);
26319 SDValue Lower =
26320 getTargetVShiftByConstNode(X86ISD::VSRLI, dl, VT, R, ShiftAmt, DAG);
26321 Lower = DAG.getBitcast(ExVT, Lower);
26322 if (VT == MVT::v2i64)
26323 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower, {4, 1, 6, 3});
26324 if (VT == MVT::v4i64)
26325 Ex = DAG.getVectorShuffle(ExVT, dl, Upper, Lower,
26326 {8, 1, 10, 3, 12, 5, 14, 7});
26327 }
26328 return DAG.getBitcast(VT, Ex);
26329 };
26330
26331 // Optimize shl/srl/sra with constant shift amount.
26332 APInt APIntShiftAmt;
26333 if (!X86::isConstantSplat(Amt, APIntShiftAmt))
26334 return SDValue();
26335
26336 // If the shift amount is out of range, return undef.
26337 if (APIntShiftAmt.uge(VT.getScalarSizeInBits()))
26338 return DAG.getUNDEF(VT);
26339
26340 uint64_t ShiftAmt = APIntShiftAmt.getZExtValue();
26341
26342 if (SupportedVectorShiftWithImm(VT, Subtarget, Op.getOpcode()))
26343 return getTargetVShiftByConstNode(X86Opc, dl, VT, R, ShiftAmt, DAG);
26344
26345 // i64 SRA needs to be performed as partial shifts.
26346 if (((!Subtarget.hasXOP() && VT == MVT::v2i64) ||
26347 (Subtarget.hasInt256() && VT == MVT::v4i64)) &&
26348 Op.getOpcode() == ISD::SRA)
26349 return ArithmeticShiftRight64(ShiftAmt);
26350
26351 if (VT == MVT::v16i8 || (Subtarget.hasInt256() && VT == MVT::v32i8) ||
26352 VT == MVT::v64i8) {
26353 unsigned NumElts = VT.getVectorNumElements();
26354 MVT ShiftVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
26355
26356 // Simple i8 add case
26357 if (Op.getOpcode() == ISD::SHL && ShiftAmt == 1)
26358 return DAG.getNode(ISD::ADD, dl, VT, R, R);
26359
26360 // ashr(R, 7) === cmp_slt(R, 0)
26361 if (Op.getOpcode() == ISD::SRA && ShiftAmt == 7) {
26362 SDValue Zeros = DAG.getConstant(0, dl, VT);
26363 if (VT.is512BitVector()) {
26364 assert(VT == MVT::v64i8 && "Unexpected element type!")((VT == MVT::v64i8 && "Unexpected element type!") ? static_cast
<void> (0) : __assert_fail ("VT == MVT::v64i8 && \"Unexpected element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26364, __PRETTY_FUNCTION__));
26365 SDValue CMP = DAG.getSetCC(dl, MVT::v64i1, Zeros, R, ISD::SETGT);
26366 return DAG.getNode(ISD::SIGN_EXTEND, dl, VT, CMP);
26367 }
26368 return DAG.getNode(X86ISD::PCMPGT, dl, VT, Zeros, R);
26369 }
26370
26371 // XOP can shift v16i8 directly instead of as shift v8i16 + mask.
26372 if (VT == MVT::v16i8 && Subtarget.hasXOP())
26373 return SDValue();
26374
26375 if (Op.getOpcode() == ISD::SHL) {
26376 // Make a large shift.
26377 SDValue SHL = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ShiftVT, R,
26378 ShiftAmt, DAG);
26379 SHL = DAG.getBitcast(VT, SHL);
26380 // Zero out the rightmost bits.
26381 APInt Mask = APInt::getHighBitsSet(8, 8 - ShiftAmt);
26382 return DAG.getNode(ISD::AND, dl, VT, SHL, DAG.getConstant(Mask, dl, VT));
26383 }
26384 if (Op.getOpcode() == ISD::SRL) {
26385 // Make a large shift.
26386 SDValue SRL = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ShiftVT, R,
26387 ShiftAmt, DAG);
26388 SRL = DAG.getBitcast(VT, SRL);
26389 // Zero out the leftmost bits.
26390 return DAG.getNode(ISD::AND, dl, VT, SRL,
26391 DAG.getConstant(uint8_t(-1U) >> ShiftAmt, dl, VT));
26392 }
26393 if (Op.getOpcode() == ISD::SRA) {
26394 // ashr(R, Amt) === sub(xor(lshr(R, Amt), Mask), Mask)
26395 SDValue Res = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
26396
26397 SDValue Mask = DAG.getConstant(128 >> ShiftAmt, dl, VT);
26398 Res = DAG.getNode(ISD::XOR, dl, VT, Res, Mask);
26399 Res = DAG.getNode(ISD::SUB, dl, VT, Res, Mask);
26400 return Res;
26401 }
26402 llvm_unreachable("Unknown shift opcode.")::llvm::llvm_unreachable_internal("Unknown shift opcode.", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26402);
26403 }
26404
26405 return SDValue();
26406}
26407
26408static SDValue LowerScalarVariableShift(SDValue Op, SelectionDAG &DAG,
26409 const X86Subtarget &Subtarget) {
26410 MVT VT = Op.getSimpleValueType();
26411 SDLoc dl(Op);
26412 SDValue R = Op.getOperand(0);
26413 SDValue Amt = Op.getOperand(1);
26414 unsigned Opcode = Op.getOpcode();
26415 unsigned X86OpcI = getTargetVShiftUniformOpcode(Opcode, false);
26416 unsigned X86OpcV = getTargetVShiftUniformOpcode(Opcode, true);
26417
26418 if (SDValue BaseShAmt = DAG.getSplatValue(Amt)) {
26419 if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Opcode)) {
26420 MVT EltVT = VT.getVectorElementType();
26421 assert(EltVT.bitsLE(MVT::i64) && "Unexpected element type!")((EltVT.bitsLE(MVT::i64) && "Unexpected element type!"
) ? static_cast<void> (0) : __assert_fail ("EltVT.bitsLE(MVT::i64) && \"Unexpected element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26421, __PRETTY_FUNCTION__));
26422 if (EltVT != MVT::i64 && EltVT.bitsGT(MVT::i32))
26423 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, BaseShAmt);
26424 else if (EltVT.bitsLT(MVT::i32))
26425 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt);
26426
26427 return getTargetVShiftNode(X86OpcI, dl, VT, R, BaseShAmt, Subtarget, DAG);
26428 }
26429
26430 // vXi8 shifts - shift as v8i16 + mask result.
26431 if (((VT == MVT::v16i8 && !Subtarget.canExtendTo512DQ()) ||
26432 (VT == MVT::v32i8 && !Subtarget.canExtendTo512BW()) ||
26433 VT == MVT::v64i8) &&
26434 !Subtarget.hasXOP()) {
26435 unsigned NumElts = VT.getVectorNumElements();
26436 MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
26437 if (SupportedVectorShiftWithBaseAmnt(ExtVT, Subtarget, Opcode)) {
26438 unsigned LogicalOp = (Opcode == ISD::SHL ? ISD::SHL : ISD::SRL);
26439 unsigned LogicalX86Op = getTargetVShiftUniformOpcode(LogicalOp, false);
26440 BaseShAmt = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i32, BaseShAmt);
26441
26442 // Create the mask using vXi16 shifts. For shift-rights we need to move
26443 // the upper byte down before splatting the vXi8 mask.
26444 SDValue BitMask = DAG.getConstant(-1, dl, ExtVT);
26445 BitMask = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, BitMask,
26446 BaseShAmt, Subtarget, DAG);
26447 if (Opcode != ISD::SHL)
26448 BitMask = getTargetVShiftByConstNode(LogicalX86Op, dl, ExtVT, BitMask,
26449 8, DAG);
26450 BitMask = DAG.getBitcast(VT, BitMask);
26451 BitMask = DAG.getVectorShuffle(VT, dl, BitMask, BitMask,
26452 SmallVector<int, 64>(NumElts, 0));
26453
26454 SDValue Res = getTargetVShiftNode(LogicalX86Op, dl, ExtVT,
26455 DAG.getBitcast(ExtVT, R), BaseShAmt,
26456 Subtarget, DAG);
26457 Res = DAG.getBitcast(VT, Res);
26458 Res = DAG.getNode(ISD::AND, dl, VT, Res, BitMask);
26459
26460 if (Opcode == ISD::SRA) {
26461 // ashr(R, Amt) === sub(xor(lshr(R, Amt), SignMask), SignMask)
26462 // SignMask = lshr(SignBit, Amt) - safe to do this with PSRLW.
26463 SDValue SignMask = DAG.getConstant(0x8080, dl, ExtVT);
26464 SignMask = getTargetVShiftNode(LogicalX86Op, dl, ExtVT, SignMask,
26465 BaseShAmt, Subtarget, DAG);
26466 SignMask = DAG.getBitcast(VT, SignMask);
26467 Res = DAG.getNode(ISD::XOR, dl, VT, Res, SignMask);
26468 Res = DAG.getNode(ISD::SUB, dl, VT, Res, SignMask);
26469 }
26470 return Res;
26471 }
26472 }
26473 }
26474
26475 // Check cases (mainly 32-bit) where i64 is expanded into high and low parts.
26476 if (VT == MVT::v2i64 && Amt.getOpcode() == ISD::BITCAST &&
26477 Amt.getOperand(0).getOpcode() == ISD::BUILD_VECTOR) {
26478 Amt = Amt.getOperand(0);
26479 unsigned Ratio = 64 / Amt.getScalarValueSizeInBits();
26480 std::vector<SDValue> Vals(Ratio);
26481 for (unsigned i = 0; i != Ratio; ++i)
26482 Vals[i] = Amt.getOperand(i);
26483 for (unsigned i = Ratio, e = Amt.getNumOperands(); i != e; i += Ratio) {
26484 for (unsigned j = 0; j != Ratio; ++j)
26485 if (Vals[j] != Amt.getOperand(i + j))
26486 return SDValue();
26487 }
26488
26489 if (SupportedVectorShiftWithBaseAmnt(VT, Subtarget, Op.getOpcode()))
26490 return DAG.getNode(X86OpcV, dl, VT, R, Op.getOperand(1));
26491 }
26492 return SDValue();
26493}
26494
26495// Convert a shift/rotate left amount to a multiplication scale factor.
26496static SDValue convertShiftLeftToScale(SDValue Amt, const SDLoc &dl,
26497 const X86Subtarget &Subtarget,
26498 SelectionDAG &DAG) {
26499 MVT VT = Amt.getSimpleValueType();
26500 if (!(VT == MVT::v8i16 || VT == MVT::v4i32 ||
26501 (Subtarget.hasInt256() && VT == MVT::v16i16) ||
26502 (!Subtarget.hasAVX512() && VT == MVT::v16i8)))
26503 return SDValue();
26504
26505 if (ISD::isBuildVectorOfConstantSDNodes(Amt.getNode())) {
26506 SmallVector<SDValue, 8> Elts;
26507 MVT SVT = VT.getVectorElementType();
26508 unsigned SVTBits = SVT.getSizeInBits();
26509 APInt One(SVTBits, 1);
26510 unsigned NumElems = VT.getVectorNumElements();
26511
26512 for (unsigned i = 0; i != NumElems; ++i) {
26513 SDValue Op = Amt->getOperand(i);
26514 if (Op->isUndef()) {
26515 Elts.push_back(Op);
26516 continue;
26517 }
26518
26519 ConstantSDNode *ND = cast<ConstantSDNode>(Op);
26520 APInt C(SVTBits, ND->getZExtValue());
26521 uint64_t ShAmt = C.getZExtValue();
26522 if (ShAmt >= SVTBits) {
26523 Elts.push_back(DAG.getUNDEF(SVT));
26524 continue;
26525 }
26526 Elts.push_back(DAG.getConstant(One.shl(ShAmt), dl, SVT));
26527 }
26528 return DAG.getBuildVector(VT, dl, Elts);
26529 }
26530
26531 // If the target doesn't support variable shifts, use either FP conversion
26532 // or integer multiplication to avoid shifting each element individually.
26533 if (VT == MVT::v4i32) {
26534 Amt = DAG.getNode(ISD::SHL, dl, VT, Amt, DAG.getConstant(23, dl, VT));
26535 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt,
26536 DAG.getConstant(0x3f800000U, dl, VT));
26537 Amt = DAG.getBitcast(MVT::v4f32, Amt);
26538 return DAG.getNode(ISD::FP_TO_SINT, dl, VT, Amt);
26539 }
26540
26541 // AVX2 can more effectively perform this as a zext/trunc to/from v8i32.
26542 if (VT == MVT::v8i16 && !Subtarget.hasAVX2()) {
26543 SDValue Z = DAG.getConstant(0, dl, VT);
26544 SDValue Lo = DAG.getBitcast(MVT::v4i32, getUnpackl(DAG, dl, VT, Amt, Z));
26545 SDValue Hi = DAG.getBitcast(MVT::v4i32, getUnpackh(DAG, dl, VT, Amt, Z));
26546 Lo = convertShiftLeftToScale(Lo, dl, Subtarget, DAG);
26547 Hi = convertShiftLeftToScale(Hi, dl, Subtarget, DAG);
26548 if (Subtarget.hasSSE41())
26549 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
26550
26551 return DAG.getVectorShuffle(VT, dl, DAG.getBitcast(VT, Lo),
26552 DAG.getBitcast(VT, Hi),
26553 {0, 2, 4, 6, 8, 10, 12, 14});
26554 }
26555
26556 return SDValue();
26557}
26558
26559static SDValue LowerShift(SDValue Op, const X86Subtarget &Subtarget,
26560 SelectionDAG &DAG) {
26561 MVT VT = Op.getSimpleValueType();
26562 SDLoc dl(Op);
26563 SDValue R = Op.getOperand(0);
26564 SDValue Amt = Op.getOperand(1);
26565 unsigned EltSizeInBits = VT.getScalarSizeInBits();
26566 bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
26567
26568 unsigned Opc = Op.getOpcode();
26569 unsigned X86OpcV = getTargetVShiftUniformOpcode(Opc, true);
26570 unsigned X86OpcI = getTargetVShiftUniformOpcode(Opc, false);
26571
26572 assert(VT.isVector() && "Custom lowering only for vector shifts!")((VT.isVector() && "Custom lowering only for vector shifts!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Custom lowering only for vector shifts!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26572, __PRETTY_FUNCTION__));
26573 assert(Subtarget.hasSSE2() && "Only custom lower when we have SSE2!")((Subtarget.hasSSE2() && "Only custom lower when we have SSE2!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Only custom lower when we have SSE2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26573, __PRETTY_FUNCTION__));
26574
26575 if (SDValue V = LowerScalarImmediateShift(Op, DAG, Subtarget))
26576 return V;
26577
26578 if (SDValue V = LowerScalarVariableShift(Op, DAG, Subtarget))
26579 return V;
26580
26581 if (SupportedVectorVarShift(VT, Subtarget, Opc))
26582 return Op;
26583
26584 // XOP has 128-bit variable logical/arithmetic shifts.
26585 // +ve/-ve Amt = shift left/right.
26586 if (Subtarget.hasXOP() && (VT == MVT::v2i64 || VT == MVT::v4i32 ||
26587 VT == MVT::v8i16 || VT == MVT::v16i8)) {
26588 if (Opc == ISD::SRL || Opc == ISD::SRA) {
26589 SDValue Zero = DAG.getConstant(0, dl, VT);
26590 Amt = DAG.getNode(ISD::SUB, dl, VT, Zero, Amt);
26591 }
26592 if (Opc == ISD::SHL || Opc == ISD::SRL)
26593 return DAG.getNode(X86ISD::VPSHL, dl, VT, R, Amt);
26594 if (Opc == ISD::SRA)
26595 return DAG.getNode(X86ISD::VPSHA, dl, VT, R, Amt);
26596 }
26597
26598 // 2i64 vector logical shifts can efficiently avoid scalarization - do the
26599 // shifts per-lane and then shuffle the partial results back together.
26600 if (VT == MVT::v2i64 && Opc != ISD::SRA) {
26601 // Splat the shift amounts so the scalar shifts above will catch it.
26602 SDValue Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {0, 0});
26603 SDValue Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Amt, {1, 1});
26604 SDValue R0 = DAG.getNode(Opc, dl, VT, R, Amt0);
26605 SDValue R1 = DAG.getNode(Opc, dl, VT, R, Amt1);
26606 return DAG.getVectorShuffle(VT, dl, R0, R1, {0, 3});
26607 }
26608
26609 // i64 vector arithmetic shift can be emulated with the transform:
26610 // M = lshr(SIGN_MASK, Amt)
26611 // ashr(R, Amt) === sub(xor(lshr(R, Amt), M), M)
26612 if ((VT == MVT::v2i64 || (VT == MVT::v4i64 && Subtarget.hasInt256())) &&
26613 Opc == ISD::SRA) {
26614 SDValue S = DAG.getConstant(APInt::getSignMask(64), dl, VT);
26615 SDValue M = DAG.getNode(ISD::SRL, dl, VT, S, Amt);
26616 R = DAG.getNode(ISD::SRL, dl, VT, R, Amt);
26617 R = DAG.getNode(ISD::XOR, dl, VT, R, M);
26618 R = DAG.getNode(ISD::SUB, dl, VT, R, M);
26619 return R;
26620 }
26621
26622 // If possible, lower this shift as a sequence of two shifts by
26623 // constant plus a BLENDing shuffle instead of scalarizing it.
26624 // Example:
26625 // (v4i32 (srl A, (build_vector < X, Y, Y, Y>)))
26626 //
26627 // Could be rewritten as:
26628 // (v4i32 (MOVSS (srl A, <Y,Y,Y,Y>), (srl A, <X,X,X,X>)))
26629 //
26630 // The advantage is that the two shifts from the example would be
26631 // lowered as X86ISD::VSRLI nodes in parallel before blending.
26632 if (ConstantAmt && (VT == MVT::v8i16 || VT == MVT::v4i32 ||
26633 (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
26634 SDValue Amt1, Amt2;
26635 unsigned NumElts = VT.getVectorNumElements();
26636 SmallVector<int, 8> ShuffleMask;
26637 for (unsigned i = 0; i != NumElts; ++i) {
26638 SDValue A = Amt->getOperand(i);
26639 if (A.isUndef()) {
26640 ShuffleMask.push_back(SM_SentinelUndef);
26641 continue;
26642 }
26643 if (!Amt1 || Amt1 == A) {
26644 ShuffleMask.push_back(i);
26645 Amt1 = A;
26646 continue;
26647 }
26648 if (!Amt2 || Amt2 == A) {
26649 ShuffleMask.push_back(i + NumElts);
26650 Amt2 = A;
26651 continue;
26652 }
26653 break;
26654 }
26655
26656 // Only perform this blend if we can perform it without loading a mask.
26657 if (ShuffleMask.size() == NumElts && Amt1 && Amt2 &&
26658 (VT != MVT::v16i16 ||
26659 is128BitLaneRepeatedShuffleMask(VT, ShuffleMask)) &&
26660 (VT == MVT::v4i32 || Subtarget.hasSSE41() || Opc != ISD::SHL ||
26661 canWidenShuffleElements(ShuffleMask))) {
26662 auto *Cst1 = dyn_cast<ConstantSDNode>(Amt1);
26663 auto *Cst2 = dyn_cast<ConstantSDNode>(Amt2);
26664 if (Cst1 && Cst2 && Cst1->getAPIntValue().ult(EltSizeInBits) &&
26665 Cst2->getAPIntValue().ult(EltSizeInBits)) {
26666 SDValue Shift1 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
26667 Cst1->getZExtValue(), DAG);
26668 SDValue Shift2 = getTargetVShiftByConstNode(X86OpcI, dl, VT, R,
26669 Cst2->getZExtValue(), DAG);
26670 return DAG.getVectorShuffle(VT, dl, Shift1, Shift2, ShuffleMask);
26671 }
26672 }
26673 }
26674
26675 // If possible, lower this packed shift into a vector multiply instead of
26676 // expanding it into a sequence of scalar shifts.
26677 if (Opc == ISD::SHL)
26678 if (SDValue Scale = convertShiftLeftToScale(Amt, dl, Subtarget, DAG))
26679 return DAG.getNode(ISD::MUL, dl, VT, R, Scale);
26680
26681 // Constant ISD::SRL can be performed efficiently on vXi16 vectors as we
26682 // can replace with ISD::MULHU, creating scale factor from (NumEltBits - Amt).
26683 if (Opc == ISD::SRL && ConstantAmt &&
26684 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256()))) {
26685 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
26686 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
26687 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
26688 SDValue Zero = DAG.getConstant(0, dl, VT);
26689 SDValue ZAmt = DAG.getSetCC(dl, VT, Amt, Zero, ISD::SETEQ);
26690 SDValue Res = DAG.getNode(ISD::MULHU, dl, VT, R, Scale);
26691 return DAG.getSelect(dl, VT, ZAmt, R, Res);
26692 }
26693 }
26694
26695 // Constant ISD::SRA can be performed efficiently on vXi16 vectors as we
26696 // can replace with ISD::MULHS, creating scale factor from (NumEltBits - Amt).
26697 // TODO: Special case handling for shift by 0/1, really we can afford either
26698 // of these cases in pre-SSE41/XOP/AVX512 but not both.
26699 if (Opc == ISD::SRA && ConstantAmt &&
26700 (VT == MVT::v8i16 || (VT == MVT::v16i16 && Subtarget.hasInt256())) &&
26701 ((Subtarget.hasSSE41() && !Subtarget.hasXOP() &&
26702 !Subtarget.hasAVX512()) ||
26703 DAG.isKnownNeverZero(Amt))) {
26704 SDValue EltBits = DAG.getConstant(EltSizeInBits, dl, VT);
26705 SDValue RAmt = DAG.getNode(ISD::SUB, dl, VT, EltBits, Amt);
26706 if (SDValue Scale = convertShiftLeftToScale(RAmt, dl, Subtarget, DAG)) {
26707 SDValue Amt0 =
26708 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(0, dl, VT), ISD::SETEQ);
26709 SDValue Amt1 =
26710 DAG.getSetCC(dl, VT, Amt, DAG.getConstant(1, dl, VT), ISD::SETEQ);
26711 SDValue Sra1 =
26712 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, R, 1, DAG);
26713 SDValue Res = DAG.getNode(ISD::MULHS, dl, VT, R, Scale);
26714 Res = DAG.getSelect(dl, VT, Amt0, R, Res);
26715 return DAG.getSelect(dl, VT, Amt1, Sra1, Res);
26716 }
26717 }
26718
26719 // v4i32 Non Uniform Shifts.
26720 // If the shift amount is constant we can shift each lane using the SSE2
26721 // immediate shifts, else we need to zero-extend each lane to the lower i64
26722 // and shift using the SSE2 variable shifts.
26723 // The separate results can then be blended together.
26724 if (VT == MVT::v4i32) {
26725 SDValue Amt0, Amt1, Amt2, Amt3;
26726 if (ConstantAmt) {
26727 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {0, 0, 0, 0});
26728 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {1, 1, 1, 1});
26729 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {2, 2, 2, 2});
26730 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, DAG.getUNDEF(VT), {3, 3, 3, 3});
26731 } else {
26732 // The SSE2 shifts use the lower i64 as the same shift amount for
26733 // all lanes and the upper i64 is ignored. On AVX we're better off
26734 // just zero-extending, but for SSE just duplicating the top 16-bits is
26735 // cheaper and has the same effect for out of range values.
26736 if (Subtarget.hasAVX()) {
26737 SDValue Z = DAG.getConstant(0, dl, VT);
26738 Amt0 = DAG.getVectorShuffle(VT, dl, Amt, Z, {0, 4, -1, -1});
26739 Amt1 = DAG.getVectorShuffle(VT, dl, Amt, Z, {1, 5, -1, -1});
26740 Amt2 = DAG.getVectorShuffle(VT, dl, Amt, Z, {2, 6, -1, -1});
26741 Amt3 = DAG.getVectorShuffle(VT, dl, Amt, Z, {3, 7, -1, -1});
26742 } else {
26743 SDValue Amt01 = DAG.getBitcast(MVT::v8i16, Amt);
26744 SDValue Amt23 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
26745 {4, 5, 6, 7, -1, -1, -1, -1});
26746 Amt0 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
26747 {0, 1, 1, 1, -1, -1, -1, -1});
26748 Amt1 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt01, Amt01,
26749 {2, 3, 3, 3, -1, -1, -1, -1});
26750 Amt2 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt23, Amt23,
26751 {0, 1, 1, 1, -1, -1, -1, -1});
26752 Amt3 = DAG.getVectorShuffle(MVT::v8i16, dl, Amt23, Amt23,
26753 {2, 3, 3, 3, -1, -1, -1, -1});
26754 }
26755 }
26756
26757 unsigned ShOpc = ConstantAmt ? Opc : X86OpcV;
26758 SDValue R0 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt0));
26759 SDValue R1 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt1));
26760 SDValue R2 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt2));
26761 SDValue R3 = DAG.getNode(ShOpc, dl, VT, R, DAG.getBitcast(VT, Amt3));
26762
26763 // Merge the shifted lane results optimally with/without PBLENDW.
26764 // TODO - ideally shuffle combining would handle this.
26765 if (Subtarget.hasSSE41()) {
26766 SDValue R02 = DAG.getVectorShuffle(VT, dl, R0, R2, {0, -1, 6, -1});
26767 SDValue R13 = DAG.getVectorShuffle(VT, dl, R1, R3, {-1, 1, -1, 7});
26768 return DAG.getVectorShuffle(VT, dl, R02, R13, {0, 5, 2, 7});
26769 }
26770 SDValue R01 = DAG.getVectorShuffle(VT, dl, R0, R1, {0, -1, -1, 5});
26771 SDValue R23 = DAG.getVectorShuffle(VT, dl, R2, R3, {2, -1, -1, 7});
26772 return DAG.getVectorShuffle(VT, dl, R01, R23, {0, 3, 4, 7});
26773 }
26774
26775 // It's worth extending once and using the vXi16/vXi32 shifts for smaller
26776 // types, but without AVX512 the extra overheads to get from vXi8 to vXi32
26777 // make the existing SSE solution better.
26778 // NOTE: We honor prefered vector width before promoting to 512-bits.
26779 if ((Subtarget.hasInt256() && VT == MVT::v8i16) ||
26780 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i16) ||
26781 (Subtarget.canExtendTo512DQ() && VT == MVT::v16i8) ||
26782 (Subtarget.canExtendTo512BW() && VT == MVT::v32i8) ||
26783 (Subtarget.hasBWI() && Subtarget.hasVLX() && VT == MVT::v16i8)) {
26784 assert((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) &&(((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8
) && "Unexpected vector type") ? static_cast<void>
(0) : __assert_fail ("(!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26785, __PRETTY_FUNCTION__))
26785 "Unexpected vector type")(((!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8
) && "Unexpected vector type") ? static_cast<void>
(0) : __assert_fail ("(!Subtarget.hasBWI() || VT == MVT::v32i8 || VT == MVT::v16i8) && \"Unexpected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26785, __PRETTY_FUNCTION__));
26786 MVT EvtSVT = Subtarget.hasBWI() ? MVT::i16 : MVT::i32;
26787 MVT ExtVT = MVT::getVectorVT(EvtSVT, VT.getVectorNumElements());
26788 unsigned ExtOpc = Opc == ISD::SRA ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
26789 R = DAG.getNode(ExtOpc, dl, ExtVT, R);
26790 Amt = DAG.getNode(ISD::ZERO_EXTEND, dl, ExtVT, Amt);
26791 return DAG.getNode(ISD::TRUNCATE, dl, VT,
26792 DAG.getNode(Opc, dl, ExtVT, R, Amt));
26793 }
26794
26795 // Constant ISD::SRA/SRL can be performed efficiently on vXi8 vectors as we
26796 // extend to vXi16 to perform a MUL scale effectively as a MUL_LOHI.
26797 if (ConstantAmt && (Opc == ISD::SRA || Opc == ISD::SRL) &&
26798 (VT == MVT::v16i8 || VT == MVT::v64i8 ||
26799 (VT == MVT::v32i8 && Subtarget.hasInt256())) &&
26800 !Subtarget.hasXOP()) {
26801 int NumElts = VT.getVectorNumElements();
26802 SDValue Cst8 = DAG.getTargetConstant(8, dl, MVT::i8);
26803
26804 // Extend constant shift amount to vXi16 (it doesn't matter if the type
26805 // isn't legal).
26806 MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts);
26807 Amt = DAG.getZExtOrTrunc(Amt, dl, ExVT);
26808 Amt = DAG.getNode(ISD::SUB, dl, ExVT, DAG.getConstant(8, dl, ExVT), Amt);
26809 Amt = DAG.getNode(ISD::SHL, dl, ExVT, DAG.getConstant(1, dl, ExVT), Amt);
26810 assert(ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) &&((ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) &&
"Constant build vector expected") ? static_cast<void> (
0) : __assert_fail ("ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) && \"Constant build vector expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26811, __PRETTY_FUNCTION__))
26811 "Constant build vector expected")((ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) &&
"Constant build vector expected") ? static_cast<void> (
0) : __assert_fail ("ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()) && \"Constant build vector expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 26811, __PRETTY_FUNCTION__));
26812
26813 if (VT == MVT::v16i8 && Subtarget.hasInt256()) {
26814 R = Opc == ISD::SRA ? DAG.getSExtOrTrunc(R, dl, ExVT)
26815 : DAG.getZExtOrTrunc(R, dl, ExVT);
26816 R = DAG.getNode(ISD::MUL, dl, ExVT, R, Amt);
26817 R = DAG.getNode(X86ISD::VSRLI, dl, ExVT, R, Cst8);
26818 return DAG.getZExtOrTrunc(R, dl, VT);
26819 }
26820
26821 SmallVector<SDValue, 16> LoAmt, HiAmt;
26822 for (int i = 0; i != NumElts; i += 16) {
26823 for (int j = 0; j != 8; ++j) {
26824 LoAmt.push_back(Amt.getOperand(i + j));
26825 HiAmt.push_back(Amt.getOperand(i + j + 8));
26826 }
26827 }
26828
26829 MVT VT16 = MVT::getVectorVT(MVT::i16, NumElts / 2);
26830 SDValue LoA = DAG.getBuildVector(VT16, dl, LoAmt);
26831 SDValue HiA = DAG.getBuildVector(VT16, dl, HiAmt);
26832
26833 SDValue LoR = DAG.getBitcast(VT16, getUnpackl(DAG, dl, VT, R, R));
26834 SDValue HiR = DAG.getBitcast(VT16, getUnpackh(DAG, dl, VT, R, R));
26835 LoR = DAG.getNode(X86OpcI, dl, VT16, LoR, Cst8);
26836 HiR = DAG.getNode(X86OpcI, dl, VT16, HiR, Cst8);
26837 LoR = DAG.getNode(ISD::MUL, dl, VT16, LoR, LoA);
26838 HiR = DAG.getNode(ISD::MUL, dl, VT16, HiR, HiA);
26839 LoR = DAG.getNode(X86ISD::VSRLI, dl, VT16, LoR, Cst8);
26840 HiR = DAG.getNode(X86ISD::VSRLI, dl, VT16, HiR, Cst8);
26841 return DAG.getNode(X86ISD::PACKUS, dl, VT, LoR, HiR);
26842 }
26843
26844 if (VT == MVT::v16i8 ||
26845 (VT == MVT::v32i8 && Subtarget.hasInt256() && !Subtarget.hasXOP()) ||
26846 (VT == MVT::v64i8 && Subtarget.hasBWI())) {
26847 MVT ExtVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
26848
26849 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
26850 if (VT.is512BitVector()) {
26851 // On AVX512BW targets we make use of the fact that VSELECT lowers
26852 // to a masked blend which selects bytes based just on the sign bit
26853 // extracted to a mask.
26854 MVT MaskVT = MVT::getVectorVT(MVT::i1, VT.getVectorNumElements());
26855 V0 = DAG.getBitcast(VT, V0);
26856 V1 = DAG.getBitcast(VT, V1);
26857 Sel = DAG.getBitcast(VT, Sel);
26858 Sel = DAG.getSetCC(dl, MaskVT, DAG.getConstant(0, dl, VT), Sel,
26859 ISD::SETGT);
26860 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
26861 } else if (Subtarget.hasSSE41()) {
26862 // On SSE41 targets we make use of the fact that VSELECT lowers
26863 // to PBLENDVB which selects bytes based just on the sign bit.
26864 V0 = DAG.getBitcast(VT, V0);
26865 V1 = DAG.getBitcast(VT, V1);
26866 Sel = DAG.getBitcast(VT, Sel);
26867 return DAG.getBitcast(SelVT, DAG.getSelect(dl, VT, Sel, V0, V1));
26868 }
26869 // On pre-SSE41 targets we test for the sign bit by comparing to
26870 // zero - a negative value will set all bits of the lanes to true
26871 // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
26872 SDValue Z = DAG.getConstant(0, dl, SelVT);
26873 SDValue C = DAG.getNode(X86ISD::PCMPGT, dl, SelVT, Z, Sel);
26874 return DAG.getSelect(dl, SelVT, C, V0, V1);
26875 };
26876
26877 // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
26878 // We can safely do this using i16 shifts as we're only interested in
26879 // the 3 lower bits of each byte.
26880 Amt = DAG.getBitcast(ExtVT, Amt);
26881 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, ExtVT, Amt, 5, DAG);
26882 Amt = DAG.getBitcast(VT, Amt);
26883
26884 if (Opc == ISD::SHL || Opc == ISD::SRL) {
26885 // r = VSELECT(r, shift(r, 4), a);
26886 SDValue M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(4, dl, VT));
26887 R = SignBitSelect(VT, Amt, M, R);
26888
26889 // a += a
26890 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
26891
26892 // r = VSELECT(r, shift(r, 2), a);
26893 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(2, dl, VT));
26894 R = SignBitSelect(VT, Amt, M, R);
26895
26896 // a += a
26897 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
26898
26899 // return VSELECT(r, shift(r, 1), a);
26900 M = DAG.getNode(Opc, dl, VT, R, DAG.getConstant(1, dl, VT));
26901 R = SignBitSelect(VT, Amt, M, R);
26902 return R;
26903 }
26904
26905 if (Opc == ISD::SRA) {
26906 // For SRA we need to unpack each byte to the higher byte of a i16 vector
26907 // so we can correctly sign extend. We don't care what happens to the
26908 // lower byte.
26909 SDValue ALo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26910 SDValue AHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), Amt);
26911 SDValue RLo = getUnpackl(DAG, dl, VT, DAG.getUNDEF(VT), R);
26912 SDValue RHi = getUnpackh(DAG, dl, VT, DAG.getUNDEF(VT), R);
26913 ALo = DAG.getBitcast(ExtVT, ALo);
26914 AHi = DAG.getBitcast(ExtVT, AHi);
26915 RLo = DAG.getBitcast(ExtVT, RLo);
26916 RHi = DAG.getBitcast(ExtVT, RHi);
26917
26918 // r = VSELECT(r, shift(r, 4), a);
26919 SDValue MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 4, DAG);
26920 SDValue MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 4, DAG);
26921 RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
26922 RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
26923
26924 // a += a
26925 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
26926 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
26927
26928 // r = VSELECT(r, shift(r, 2), a);
26929 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 2, DAG);
26930 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 2, DAG);
26931 RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
26932 RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
26933
26934 // a += a
26935 ALo = DAG.getNode(ISD::ADD, dl, ExtVT, ALo, ALo);
26936 AHi = DAG.getNode(ISD::ADD, dl, ExtVT, AHi, AHi);
26937
26938 // r = VSELECT(r, shift(r, 1), a);
26939 MLo = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RLo, 1, DAG);
26940 MHi = getTargetVShiftByConstNode(X86OpcI, dl, ExtVT, RHi, 1, DAG);
26941 RLo = SignBitSelect(ExtVT, ALo, MLo, RLo);
26942 RHi = SignBitSelect(ExtVT, AHi, MHi, RHi);
26943
26944 // Logical shift the result back to the lower byte, leaving a zero upper
26945 // byte meaning that we can safely pack with PACKUSWB.
26946 RLo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RLo, 8, DAG);
26947 RHi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, RHi, 8, DAG);
26948 return DAG.getNode(X86ISD::PACKUS, dl, VT, RLo, RHi);
26949 }
26950 }
26951
26952 if (Subtarget.hasInt256() && !Subtarget.hasXOP() && VT == MVT::v16i16) {
26953 MVT ExtVT = MVT::v8i32;
26954 SDValue Z = DAG.getConstant(0, dl, VT);
26955 SDValue ALo = getUnpackl(DAG, dl, VT, Amt, Z);
26956 SDValue AHi = getUnpackh(DAG, dl, VT, Amt, Z);
26957 SDValue RLo = getUnpackl(DAG, dl, VT, Z, R);
26958 SDValue RHi = getUnpackh(DAG, dl, VT, Z, R);
26959 ALo = DAG.getBitcast(ExtVT, ALo);
26960 AHi = DAG.getBitcast(ExtVT, AHi);
26961 RLo = DAG.getBitcast(ExtVT, RLo);
26962 RHi = DAG.getBitcast(ExtVT, RHi);
26963 SDValue Lo = DAG.getNode(Opc, dl, ExtVT, RLo, ALo);
26964 SDValue Hi = DAG.getNode(Opc, dl, ExtVT, RHi, AHi);
26965 Lo = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Lo, 16, DAG);
26966 Hi = getTargetVShiftByConstNode(X86ISD::VSRLI, dl, ExtVT, Hi, 16, DAG);
26967 return DAG.getNode(X86ISD::PACKUS, dl, VT, Lo, Hi);
26968 }
26969
26970 if (VT == MVT::v8i16) {
26971 // If we have a constant shift amount, the non-SSE41 path is best as
26972 // avoiding bitcasts make it easier to constant fold and reduce to PBLENDW.
26973 bool UseSSE41 = Subtarget.hasSSE41() &&
26974 !ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
26975
26976 auto SignBitSelect = [&](SDValue Sel, SDValue V0, SDValue V1) {
26977 // On SSE41 targets we make use of the fact that VSELECT lowers
26978 // to PBLENDVB which selects bytes based just on the sign bit.
26979 if (UseSSE41) {
26980 MVT ExtVT = MVT::getVectorVT(MVT::i8, VT.getVectorNumElements() * 2);
26981 V0 = DAG.getBitcast(ExtVT, V0);
26982 V1 = DAG.getBitcast(ExtVT, V1);
26983 Sel = DAG.getBitcast(ExtVT, Sel);
26984 return DAG.getBitcast(VT, DAG.getSelect(dl, ExtVT, Sel, V0, V1));
26985 }
26986 // On pre-SSE41 targets we splat the sign bit - a negative value will
26987 // set all bits of the lanes to true and VSELECT uses that in
26988 // its OR(AND(V0,C),AND(V1,~C)) lowering.
26989 SDValue C =
26990 getTargetVShiftByConstNode(X86ISD::VSRAI, dl, VT, Sel, 15, DAG);
26991 return DAG.getSelect(dl, VT, C, V0, V1);
26992 };
26993
26994 // Turn 'a' into a mask suitable for VSELECT: a = a << 12;
26995 if (UseSSE41) {
26996 // On SSE41 targets we need to replicate the shift mask in both
26997 // bytes for PBLENDVB.
26998 Amt = DAG.getNode(
26999 ISD::OR, dl, VT,
27000 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 4, DAG),
27001 getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG));
27002 } else {
27003 Amt = getTargetVShiftByConstNode(X86ISD::VSHLI, dl, VT, Amt, 12, DAG);
27004 }
27005
27006 // r = VSELECT(r, shift(r, 8), a);
27007 SDValue M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 8, DAG);
27008 R = SignBitSelect(Amt, M, R);
27009
27010 // a += a
27011 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
27012
27013 // r = VSELECT(r, shift(r, 4), a);
27014 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 4, DAG);
27015 R = SignBitSelect(Amt, M, R);
27016
27017 // a += a
27018 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
27019
27020 // r = VSELECT(r, shift(r, 2), a);
27021 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 2, DAG);
27022 R = SignBitSelect(Amt, M, R);
27023
27024 // a += a
27025 Amt = DAG.getNode(ISD::ADD, dl, VT, Amt, Amt);
27026
27027 // return VSELECT(r, shift(r, 1), a);
27028 M = getTargetVShiftByConstNode(X86OpcI, dl, VT, R, 1, DAG);
27029 R = SignBitSelect(Amt, M, R);
27030 return R;
27031 }
27032
27033 // Decompose 256-bit shifts into 128-bit shifts.
27034 if (VT.is256BitVector())
27035 return split256IntArith(Op, DAG);
27036
27037 return SDValue();
27038}
27039
27040static SDValue LowerRotate(SDValue Op, const X86Subtarget &Subtarget,
27041 SelectionDAG &DAG) {
27042 MVT VT = Op.getSimpleValueType();
27043 assert(VT.isVector() && "Custom lowering only for vector rotates!")((VT.isVector() && "Custom lowering only for vector rotates!"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() && \"Custom lowering only for vector rotates!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27043, __PRETTY_FUNCTION__));
27044
27045 SDLoc DL(Op);
27046 SDValue R = Op.getOperand(0);
27047 SDValue Amt = Op.getOperand(1);
27048 unsigned Opcode = Op.getOpcode();
27049 unsigned EltSizeInBits = VT.getScalarSizeInBits();
27050 int NumElts = VT.getVectorNumElements();
27051
27052 // Check for constant splat rotation amount.
27053 APInt UndefElts;
27054 SmallVector<APInt, 32> EltBits;
27055 int CstSplatIndex = -1;
27056 if (getTargetConstantBitsFromNode(Amt, EltSizeInBits, UndefElts, EltBits))
27057 for (int i = 0; i != NumElts; ++i)
27058 if (!UndefElts[i]) {
27059 if (CstSplatIndex < 0 || EltBits[i] == EltBits[CstSplatIndex]) {
27060 CstSplatIndex = i;
27061 continue;
27062 }
27063 CstSplatIndex = -1;
27064 break;
27065 }
27066
27067 // AVX512 implicitly uses modulo rotation amounts.
27068 if (Subtarget.hasAVX512() && 32 <= EltSizeInBits) {
27069 // Attempt to rotate by immediate.
27070 if (0 <= CstSplatIndex) {
27071 unsigned Op = (Opcode == ISD::ROTL ? X86ISD::VROTLI : X86ISD::VROTRI);
27072 uint64_t RotateAmt = EltBits[CstSplatIndex].urem(EltSizeInBits);
27073 return DAG.getNode(Op, DL, VT, R,
27074 DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
27075 }
27076
27077 // Else, fall-back on VPROLV/VPRORV.
27078 return Op;
27079 }
27080
27081 assert((Opcode == ISD::ROTL) && "Only ROTL supported")(((Opcode == ISD::ROTL) && "Only ROTL supported") ? static_cast
<void> (0) : __assert_fail ("(Opcode == ISD::ROTL) && \"Only ROTL supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27081, __PRETTY_FUNCTION__));
27082
27083 // XOP has 128-bit vector variable + immediate rotates.
27084 // +ve/-ve Amt = rotate left/right - just need to handle ISD::ROTL.
27085 // XOP implicitly uses modulo rotation amounts.
27086 if (Subtarget.hasXOP()) {
27087 if (VT.is256BitVector())
27088 return split256IntArith(Op, DAG);
27089 assert(VT.is128BitVector() && "Only rotate 128-bit vectors!")((VT.is128BitVector() && "Only rotate 128-bit vectors!"
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Only rotate 128-bit vectors!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27089, __PRETTY_FUNCTION__));
27090
27091 // Attempt to rotate by immediate.
27092 if (0 <= CstSplatIndex) {
27093 uint64_t RotateAmt = EltBits[CstSplatIndex].urem(EltSizeInBits);
27094 return DAG.getNode(X86ISD::VROTLI, DL, VT, R,
27095 DAG.getTargetConstant(RotateAmt, DL, MVT::i8));
27096 }
27097
27098 // Use general rotate by variable (per-element).
27099 return Op;
27100 }
27101
27102 // Split 256-bit integers on pre-AVX2 targets.
27103 if (VT.is256BitVector() && !Subtarget.hasAVX2())
27104 return split256IntArith(Op, DAG);
27105
27106 assert((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||(((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8)
&& Subtarget.hasAVX2())) && "Only vXi32/vXi16/vXi8 vector rotates supported"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && Subtarget.hasAVX2())) && \"Only vXi32/vXi16/vXi8 vector rotates supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27109, __PRETTY_FUNCTION__))
27107 ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) &&(((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8)
&& Subtarget.hasAVX2())) && "Only vXi32/vXi16/vXi8 vector rotates supported"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && Subtarget.hasAVX2())) && \"Only vXi32/vXi16/vXi8 vector rotates supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27109, __PRETTY_FUNCTION__))
27108 Subtarget.hasAVX2())) &&(((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8)
&& Subtarget.hasAVX2())) && "Only vXi32/vXi16/vXi8 vector rotates supported"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && Subtarget.hasAVX2())) && \"Only vXi32/vXi16/vXi8 vector rotates supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27109, __PRETTY_FUNCTION__))
27109 "Only vXi32/vXi16/vXi8 vector rotates supported")(((VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 ||
((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8)
&& Subtarget.hasAVX2())) && "Only vXi32/vXi16/vXi8 vector rotates supported"
) ? static_cast<void> (0) : __assert_fail ("(VT == MVT::v4i32 || VT == MVT::v8i16 || VT == MVT::v16i8 || ((VT == MVT::v8i32 || VT == MVT::v16i16 || VT == MVT::v32i8) && Subtarget.hasAVX2())) && \"Only vXi32/vXi16/vXi8 vector rotates supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27109, __PRETTY_FUNCTION__));
27110
27111 // Rotate by an uniform constant - expand back to shifts.
27112 if (0 <= CstSplatIndex)
27113 return SDValue();
27114
27115 bool IsSplatAmt = DAG.isSplatValue(Amt);
27116
27117 // v16i8/v32i8: Split rotation into rot4/rot2/rot1 stages and select by
27118 // the amount bit.
27119 if (EltSizeInBits == 8 && !IsSplatAmt) {
27120 if (ISD::isBuildVectorOfConstantSDNodes(Amt.getNode()))
27121 return SDValue();
27122
27123 // We don't need ModuloAmt here as we just peek at individual bits.
27124 MVT ExtVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
27125
27126 auto SignBitSelect = [&](MVT SelVT, SDValue Sel, SDValue V0, SDValue V1) {
27127 if (Subtarget.hasSSE41()) {
27128 // On SSE41 targets we make use of the fact that VSELECT lowers
27129 // to PBLENDVB which selects bytes based just on the sign bit.
27130 V0 = DAG.getBitcast(VT, V0);
27131 V1 = DAG.getBitcast(VT, V1);
27132 Sel = DAG.getBitcast(VT, Sel);
27133 return DAG.getBitcast(SelVT, DAG.getSelect(DL, VT, Sel, V0, V1));
27134 }
27135 // On pre-SSE41 targets we test for the sign bit by comparing to
27136 // zero - a negative value will set all bits of the lanes to true
27137 // and VSELECT uses that in its OR(AND(V0,C),AND(V1,~C)) lowering.
27138 SDValue Z = DAG.getConstant(0, DL, SelVT);
27139 SDValue C = DAG.getNode(X86ISD::PCMPGT, DL, SelVT, Z, Sel);
27140 return DAG.getSelect(DL, SelVT, C, V0, V1);
27141 };
27142
27143 // Turn 'a' into a mask suitable for VSELECT: a = a << 5;
27144 // We can safely do this using i16 shifts as we're only interested in
27145 // the 3 lower bits of each byte.
27146 Amt = DAG.getBitcast(ExtVT, Amt);
27147 Amt = DAG.getNode(ISD::SHL, DL, ExtVT, Amt, DAG.getConstant(5, DL, ExtVT));
27148 Amt = DAG.getBitcast(VT, Amt);
27149
27150 // r = VSELECT(r, rot(r, 4), a);
27151 SDValue M;
27152 M = DAG.getNode(
27153 ISD::OR, DL, VT,
27154 DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(4, DL, VT)),
27155 DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(4, DL, VT)));
27156 R = SignBitSelect(VT, Amt, M, R);
27157
27158 // a += a
27159 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
27160
27161 // r = VSELECT(r, rot(r, 2), a);
27162 M = DAG.getNode(
27163 ISD::OR, DL, VT,
27164 DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(2, DL, VT)),
27165 DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(6, DL, VT)));
27166 R = SignBitSelect(VT, Amt, M, R);
27167
27168 // a += a
27169 Amt = DAG.getNode(ISD::ADD, DL, VT, Amt, Amt);
27170
27171 // return VSELECT(r, rot(r, 1), a);
27172 M = DAG.getNode(
27173 ISD::OR, DL, VT,
27174 DAG.getNode(ISD::SHL, DL, VT, R, DAG.getConstant(1, DL, VT)),
27175 DAG.getNode(ISD::SRL, DL, VT, R, DAG.getConstant(7, DL, VT)));
27176 return SignBitSelect(VT, Amt, M, R);
27177 }
27178
27179 // ISD::ROT* uses modulo rotate amounts.
27180 Amt = DAG.getNode(ISD::AND, DL, VT, Amt,
27181 DAG.getConstant(EltSizeInBits - 1, DL, VT));
27182
27183 bool ConstantAmt = ISD::isBuildVectorOfConstantSDNodes(Amt.getNode());
27184 bool LegalVarShifts = SupportedVectorVarShift(VT, Subtarget, ISD::SHL) &&
27185 SupportedVectorVarShift(VT, Subtarget, ISD::SRL);
27186
27187 // Fallback for splats + all supported variable shifts.
27188 // Fallback for non-constants AVX2 vXi16 as well.
27189 if (IsSplatAmt || LegalVarShifts || (Subtarget.hasAVX2() && !ConstantAmt)) {
27190 SDValue AmtR = DAG.getConstant(EltSizeInBits, DL, VT);
27191 AmtR = DAG.getNode(ISD::SUB, DL, VT, AmtR, Amt);
27192 SDValue SHL = DAG.getNode(ISD::SHL, DL, VT, R, Amt);
27193 SDValue SRL = DAG.getNode(ISD::SRL, DL, VT, R, AmtR);
27194 return DAG.getNode(ISD::OR, DL, VT, SHL, SRL);
27195 }
27196
27197 // As with shifts, convert the rotation amount to a multiplication factor.
27198 SDValue Scale = convertShiftLeftToScale(Amt, DL, Subtarget, DAG);
27199 assert(Scale && "Failed to convert ROTL amount to scale")((Scale && "Failed to convert ROTL amount to scale") ?
static_cast<void> (0) : __assert_fail ("Scale && \"Failed to convert ROTL amount to scale\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27199, __PRETTY_FUNCTION__));
27200
27201 // v8i16/v16i16: perform unsigned multiply hi/lo and OR the results.
27202 if (EltSizeInBits == 16) {
27203 SDValue Lo = DAG.getNode(ISD::MUL, DL, VT, R, Scale);
27204 SDValue Hi = DAG.getNode(ISD::MULHU, DL, VT, R, Scale);
27205 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
27206 }
27207
27208 // v4i32: make use of the PMULUDQ instruction to multiply 2 lanes of v4i32
27209 // to v2i64 results at a time. The upper 32-bits contain the wrapped bits
27210 // that can then be OR'd with the lower 32-bits.
27211 assert(VT == MVT::v4i32 && "Only v4i32 vector rotate expected")((VT == MVT::v4i32 && "Only v4i32 vector rotate expected"
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::v4i32 && \"Only v4i32 vector rotate expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27211, __PRETTY_FUNCTION__));
27212 static const int OddMask[] = {1, -1, 3, -1};
27213 SDValue R13 = DAG.getVectorShuffle(VT, DL, R, R, OddMask);
27214 SDValue Scale13 = DAG.getVectorShuffle(VT, DL, Scale, Scale, OddMask);
27215
27216 SDValue Res02 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
27217 DAG.getBitcast(MVT::v2i64, R),
27218 DAG.getBitcast(MVT::v2i64, Scale));
27219 SDValue Res13 = DAG.getNode(X86ISD::PMULUDQ, DL, MVT::v2i64,
27220 DAG.getBitcast(MVT::v2i64, R13),
27221 DAG.getBitcast(MVT::v2i64, Scale13));
27222 Res02 = DAG.getBitcast(VT, Res02);
27223 Res13 = DAG.getBitcast(VT, Res13);
27224
27225 return DAG.getNode(ISD::OR, DL, VT,
27226 DAG.getVectorShuffle(VT, DL, Res02, Res13, {0, 4, 2, 6}),
27227 DAG.getVectorShuffle(VT, DL, Res02, Res13, {1, 5, 3, 7}));
27228}
27229
27230/// Returns true if the operand type is exactly twice the native width, and
27231/// the corresponding cmpxchg8b or cmpxchg16b instruction is available.
27232/// Used to know whether to use cmpxchg8/16b when expanding atomic operations
27233/// (otherwise we leave them alone to become __sync_fetch_and_... calls).
27234bool X86TargetLowering::needsCmpXchgNb(Type *MemType) const {
27235 unsigned OpWidth = MemType->getPrimitiveSizeInBits();
27236
27237 if (OpWidth == 64)
27238 return Subtarget.hasCmpxchg8b() && !Subtarget.is64Bit();
27239 if (OpWidth == 128)
27240 return Subtarget.hasCmpxchg16b();
27241
27242 return false;
27243}
27244
27245// TODO: In 32-bit mode, use MOVLPS when SSE1 is available?
27246// TODO: In 32-bit mode, use FISTP when X87 is available?
27247bool X86TargetLowering::shouldExpandAtomicStoreInIR(StoreInst *SI) const {
27248 Type *MemType = SI->getValueOperand()->getType();
27249
27250 bool NoImplicitFloatOps =
27251 SI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
27252 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
27253 !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2())
27254 return false;
27255
27256 return needsCmpXchgNb(MemType);
27257}
27258
27259// Note: this turns large loads into lock cmpxchg8b/16b.
27260// TODO: In 32-bit mode, use MOVLPS when SSE1 is available?
27261TargetLowering::AtomicExpansionKind
27262X86TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const {
27263 Type *MemType = LI->getType();
27264
27265 // If this a 64 bit atomic load on a 32-bit target and SSE2 is enabled, we
27266 // can use movq to do the load. If we have X87 we can load into an 80-bit
27267 // X87 register and store it to a stack temporary.
27268 bool NoImplicitFloatOps =
27269 LI->getFunction()->hasFnAttribute(Attribute::NoImplicitFloat);
27270 if (MemType->getPrimitiveSizeInBits() == 64 && !Subtarget.is64Bit() &&
27271 !Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
27272 (Subtarget.hasSSE2() || Subtarget.hasX87()))
27273 return AtomicExpansionKind::None;
27274
27275 return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
27276 : AtomicExpansionKind::None;
27277}
27278
27279TargetLowering::AtomicExpansionKind
27280X86TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
27281 unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
27282 Type *MemType = AI->getType();
27283
27284 // If the operand is too big, we must see if cmpxchg8/16b is available
27285 // and default to library calls otherwise.
27286 if (MemType->getPrimitiveSizeInBits() > NativeWidth) {
27287 return needsCmpXchgNb(MemType) ? AtomicExpansionKind::CmpXChg
27288 : AtomicExpansionKind::None;
27289 }
27290
27291 AtomicRMWInst::BinOp Op = AI->getOperation();
27292 switch (Op) {
27293 default:
27294 llvm_unreachable("Unknown atomic operation")::llvm::llvm_unreachable_internal("Unknown atomic operation",
"/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27294);
27295 case AtomicRMWInst::Xchg:
27296 case AtomicRMWInst::Add:
27297 case AtomicRMWInst::Sub:
27298 // It's better to use xadd, xsub or xchg for these in all cases.
27299 return AtomicExpansionKind::None;
27300 case AtomicRMWInst::Or:
27301 case AtomicRMWInst::And:
27302 case AtomicRMWInst::Xor:
27303 // If the atomicrmw's result isn't actually used, we can just add a "lock"
27304 // prefix to a normal instruction for these operations.
27305 return !AI->use_empty() ? AtomicExpansionKind::CmpXChg
27306 : AtomicExpansionKind::None;
27307 case AtomicRMWInst::Nand:
27308 case AtomicRMWInst::Max:
27309 case AtomicRMWInst::Min:
27310 case AtomicRMWInst::UMax:
27311 case AtomicRMWInst::UMin:
27312 case AtomicRMWInst::FAdd:
27313 case AtomicRMWInst::FSub:
27314 // These always require a non-trivial set of data operations on x86. We must
27315 // use a cmpxchg loop.
27316 return AtomicExpansionKind::CmpXChg;
27317 }
27318}
27319
27320LoadInst *
27321X86TargetLowering::lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const {
27322 unsigned NativeWidth = Subtarget.is64Bit() ? 64 : 32;
27323 Type *MemType = AI->getType();
27324 // Accesses larger than the native width are turned into cmpxchg/libcalls, so
27325 // there is no benefit in turning such RMWs into loads, and it is actually
27326 // harmful as it introduces a mfence.
27327 if (MemType->getPrimitiveSizeInBits() > NativeWidth)
27328 return nullptr;
27329
27330 // If this is a canonical idempotent atomicrmw w/no uses, we have a better
27331 // lowering available in lowerAtomicArith.
27332 // TODO: push more cases through this path.
27333 if (auto *C = dyn_cast<ConstantInt>(AI->getValOperand()))
27334 if (AI->getOperation() == AtomicRMWInst::Or && C->isZero() &&
27335 AI->use_empty())
27336 return nullptr;
27337
27338 auto Builder = IRBuilder<>(AI);
27339 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
27340 auto SSID = AI->getSyncScopeID();
27341 // We must restrict the ordering to avoid generating loads with Release or
27342 // ReleaseAcquire orderings.
27343 auto Order = AtomicCmpXchgInst::getStrongestFailureOrdering(AI->getOrdering());
27344
27345 // Before the load we need a fence. Here is an example lifted from
27346 // http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf showing why a fence
27347 // is required:
27348 // Thread 0:
27349 // x.store(1, relaxed);
27350 // r1 = y.fetch_add(0, release);
27351 // Thread 1:
27352 // y.fetch_add(42, acquire);
27353 // r2 = x.load(relaxed);
27354 // r1 = r2 = 0 is impossible, but becomes possible if the idempotent rmw is
27355 // lowered to just a load without a fence. A mfence flushes the store buffer,
27356 // making the optimization clearly correct.
27357 // FIXME: it is required if isReleaseOrStronger(Order) but it is not clear
27358 // otherwise, we might be able to be more aggressive on relaxed idempotent
27359 // rmw. In practice, they do not look useful, so we don't try to be
27360 // especially clever.
27361 if (SSID == SyncScope::SingleThread)
27362 // FIXME: we could just insert an X86ISD::MEMBARRIER here, except we are at
27363 // the IR level, so we must wrap it in an intrinsic.
27364 return nullptr;
27365
27366 if (!Subtarget.hasMFence())
27367 // FIXME: it might make sense to use a locked operation here but on a
27368 // different cache-line to prevent cache-line bouncing. In practice it
27369 // is probably a small win, and x86 processors without mfence are rare
27370 // enough that we do not bother.
27371 return nullptr;
27372
27373 Function *MFence =
27374 llvm::Intrinsic::getDeclaration(M, Intrinsic::x86_sse2_mfence);
27375 Builder.CreateCall(MFence, {});
27376
27377 // Finally we can emit the atomic load.
27378 LoadInst *Loaded =
27379 Builder.CreateAlignedLoad(AI->getType(), AI->getPointerOperand(),
27380 AI->getType()->getPrimitiveSizeInBits());
27381 Loaded->setAtomic(Order, SSID);
27382 AI->replaceAllUsesWith(Loaded);
27383 AI->eraseFromParent();
27384 return Loaded;
27385}
27386
27387bool X86TargetLowering::lowerAtomicStoreAsStoreSDNode(const StoreInst &SI) const {
27388 if (!SI.isUnordered())
27389 return false;
27390 return ExperimentalUnorderedISEL;
27391}
27392bool X86TargetLowering::lowerAtomicLoadAsLoadSDNode(const LoadInst &LI) const {
27393 if (!LI.isUnordered())
27394 return false;
27395 return ExperimentalUnorderedISEL;
27396}
27397
27398
27399/// Emit a locked operation on a stack location which does not change any
27400/// memory location, but does involve a lock prefix. Location is chosen to be
27401/// a) very likely accessed only by a single thread to minimize cache traffic,
27402/// and b) definitely dereferenceable. Returns the new Chain result.
27403static SDValue emitLockedStackOp(SelectionDAG &DAG,
27404 const X86Subtarget &Subtarget,
27405 SDValue Chain, SDLoc DL) {
27406 // Implementation notes:
27407 // 1) LOCK prefix creates a full read/write reordering barrier for memory
27408 // operations issued by the current processor. As such, the location
27409 // referenced is not relevant for the ordering properties of the instruction.
27410 // See: Intel® 64 and IA-32 ArchitecturesSoftware Developer’s Manual,
27411 // 8.2.3.9 Loads and Stores Are Not Reordered with Locked Instructions
27412 // 2) Using an immediate operand appears to be the best encoding choice
27413 // here since it doesn't require an extra register.
27414 // 3) OR appears to be very slightly faster than ADD. (Though, the difference
27415 // is small enough it might just be measurement noise.)
27416 // 4) When choosing offsets, there are several contributing factors:
27417 // a) If there's no redzone, we default to TOS. (We could allocate a cache
27418 // line aligned stack object to improve this case.)
27419 // b) To minimize our chances of introducing a false dependence, we prefer
27420 // to offset the stack usage from TOS slightly.
27421 // c) To minimize concerns about cross thread stack usage - in particular,
27422 // the idiomatic MyThreadPool.run([&StackVars]() {...}) pattern which
27423 // captures state in the TOS frame and accesses it from many threads -
27424 // we want to use an offset such that the offset is in a distinct cache
27425 // line from the TOS frame.
27426 //
27427 // For a general discussion of the tradeoffs and benchmark results, see:
27428 // https://shipilev.net/blog/2014/on-the-fence-with-dependencies/
27429
27430 auto &MF = DAG.getMachineFunction();
27431 auto &TFL = *Subtarget.getFrameLowering();
27432 const unsigned SPOffset = TFL.has128ByteRedZone(MF) ? -64 : 0;
27433
27434 if (Subtarget.is64Bit()) {
27435 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
27436 SDValue Ops[] = {
27437 DAG.getRegister(X86::RSP, MVT::i64), // Base
27438 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
27439 DAG.getRegister(0, MVT::i64), // Index
27440 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
27441 DAG.getRegister(0, MVT::i16), // Segment.
27442 Zero,
27443 Chain};
27444 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
27445 MVT::Other, Ops);
27446 return SDValue(Res, 1);
27447 }
27448
27449 SDValue Zero = DAG.getTargetConstant(0, DL, MVT::i32);
27450 SDValue Ops[] = {
27451 DAG.getRegister(X86::ESP, MVT::i32), // Base
27452 DAG.getTargetConstant(1, DL, MVT::i8), // Scale
27453 DAG.getRegister(0, MVT::i32), // Index
27454 DAG.getTargetConstant(SPOffset, DL, MVT::i32), // Disp
27455 DAG.getRegister(0, MVT::i16), // Segment.
27456 Zero,
27457 Chain
27458 };
27459 SDNode *Res = DAG.getMachineNode(X86::OR32mi8Locked, DL, MVT::i32,
27460 MVT::Other, Ops);
27461 return SDValue(Res, 1);
27462}
27463
27464static SDValue LowerATOMIC_FENCE(SDValue Op, const X86Subtarget &Subtarget,
27465 SelectionDAG &DAG) {
27466 SDLoc dl(Op);
27467 AtomicOrdering FenceOrdering =
27468 static_cast<AtomicOrdering>(Op.getConstantOperandVal(1));
27469 SyncScope::ID FenceSSID =
27470 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
27471
27472 // The only fence that needs an instruction is a sequentially-consistent
27473 // cross-thread fence.
27474 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
27475 FenceSSID == SyncScope::System) {
27476 if (Subtarget.hasMFence())
27477 return DAG.getNode(X86ISD::MFENCE, dl, MVT::Other, Op.getOperand(0));
27478
27479 SDValue Chain = Op.getOperand(0);
27480 return emitLockedStackOp(DAG, Subtarget, Chain, dl);
27481 }
27482
27483 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
27484 return DAG.getNode(X86ISD::MEMBARRIER, dl, MVT::Other, Op.getOperand(0));
27485}
27486
27487static SDValue LowerCMP_SWAP(SDValue Op, const X86Subtarget &Subtarget,
27488 SelectionDAG &DAG) {
27489 MVT T = Op.getSimpleValueType();
27490 SDLoc DL(Op);
27491 unsigned Reg = 0;
27492 unsigned size = 0;
27493 switch(T.SimpleTy) {
27494 default: llvm_unreachable("Invalid value type!")::llvm::llvm_unreachable_internal("Invalid value type!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27494);
27495 case MVT::i8: Reg = X86::AL; size = 1; break;
27496 case MVT::i16: Reg = X86::AX; size = 2; break;
27497 case MVT::i32: Reg = X86::EAX; size = 4; break;
27498 case MVT::i64:
27499 assert(Subtarget.is64Bit() && "Node not type legal!")((Subtarget.is64Bit() && "Node not type legal!") ? static_cast
<void> (0) : __assert_fail ("Subtarget.is64Bit() && \"Node not type legal!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27499, __PRETTY_FUNCTION__));
27500 Reg = X86::RAX; size = 8;
27501 break;
27502 }
27503 SDValue cpIn = DAG.getCopyToReg(Op.getOperand(0), DL, Reg,
27504 Op.getOperand(2), SDValue());
27505 SDValue Ops[] = { cpIn.getValue(0),
27506 Op.getOperand(1),
27507 Op.getOperand(3),
27508 DAG.getTargetConstant(size, DL, MVT::i8),
27509 cpIn.getValue(1) };
27510 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
27511 MachineMemOperand *MMO = cast<AtomicSDNode>(Op)->getMemOperand();
27512 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::LCMPXCHG_DAG, DL, Tys,
27513 Ops, T, MMO);
27514
27515 SDValue cpOut =
27516 DAG.getCopyFromReg(Result.getValue(0), DL, Reg, T, Result.getValue(1));
27517 SDValue EFLAGS = DAG.getCopyFromReg(cpOut.getValue(1), DL, X86::EFLAGS,
27518 MVT::i32, cpOut.getValue(2));
27519 SDValue Success = getSETCC(X86::COND_E, EFLAGS, DL, DAG);
27520
27521 return DAG.getNode(ISD::MERGE_VALUES, DL, Op->getVTList(),
27522 cpOut, Success, EFLAGS.getValue(1));
27523}
27524
27525// Create MOVMSKB, taking into account whether we need to split for AVX1.
27526static SDValue getPMOVMSKB(const SDLoc &DL, SDValue V, SelectionDAG &DAG,
27527 const X86Subtarget &Subtarget) {
27528 MVT InVT = V.getSimpleValueType();
27529
27530 if (InVT == MVT::v64i8) {
27531 SDValue Lo, Hi;
27532 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
27533 Lo = getPMOVMSKB(DL, Lo, DAG, Subtarget);
27534 Hi = getPMOVMSKB(DL, Hi, DAG, Subtarget);
27535 Lo = DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, Lo);
27536 Hi = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Hi);
27537 Hi = DAG.getNode(ISD::SHL, DL, MVT::i64, Hi,
27538 DAG.getConstant(32, DL, MVT::i8));
27539 return DAG.getNode(ISD::OR, DL, MVT::i64, Lo, Hi);
27540 }
27541 if (InVT == MVT::v32i8 && !Subtarget.hasInt256()) {
27542 SDValue Lo, Hi;
27543 std::tie(Lo, Hi) = DAG.SplitVector(V, DL);
27544 Lo = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Lo);
27545 Hi = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Hi);
27546 Hi = DAG.getNode(ISD::SHL, DL, MVT::i32, Hi,
27547 DAG.getConstant(16, DL, MVT::i8));
27548 return DAG.getNode(ISD::OR, DL, MVT::i32, Lo, Hi);
27549 }
27550
27551 return DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
27552}
27553
27554static SDValue LowerBITCAST(SDValue Op, const X86Subtarget &Subtarget,
27555 SelectionDAG &DAG) {
27556 SDValue Src = Op.getOperand(0);
27557 MVT SrcVT = Src.getSimpleValueType();
27558 MVT DstVT = Op.getSimpleValueType();
27559
27560 // Legalize (v64i1 (bitcast i64 (X))) by splitting the i64, bitcasting each
27561 // half to v32i1 and concatenating the result.
27562 if (SrcVT == MVT::i64 && DstVT == MVT::v64i1) {
27563 assert(!Subtarget.is64Bit() && "Expected 32-bit mode")((!Subtarget.is64Bit() && "Expected 32-bit mode") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Expected 32-bit mode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27563, __PRETTY_FUNCTION__));
27564 assert(Subtarget.hasBWI() && "Expected BWI target")((Subtarget.hasBWI() && "Expected BWI target") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasBWI() && \"Expected BWI target\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27564, __PRETTY_FUNCTION__));
27565 SDLoc dl(Op);
27566 SDValue Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,
27567 DAG.getIntPtrConstant(0, dl));
27568 Lo = DAG.getBitcast(MVT::v32i1, Lo);
27569 SDValue Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, MVT::i32, Src,
27570 DAG.getIntPtrConstant(1, dl));
27571 Hi = DAG.getBitcast(MVT::v32i1, Hi);
27572 return DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v64i1, Lo, Hi);
27573 }
27574
27575 // Custom splitting for BWI types when AVX512F is available but BWI isn't.
27576 if ((SrcVT == MVT::v32i16 || SrcVT == MVT::v64i8) && DstVT.isVector() &&
27577 DAG.getTargetLoweringInfo().isTypeLegal(DstVT)) {
27578 SDLoc dl(Op);
27579 SDValue Lo, Hi;
27580 std::tie(Lo, Hi) = DAG.SplitVector(Op.getOperand(0), dl);
27581 MVT CastVT = DstVT.getHalfNumVectorElementsVT();
27582 Lo = DAG.getBitcast(CastVT, Lo);
27583 Hi = DAG.getBitcast(CastVT, Hi);
27584 return DAG.getNode(ISD::CONCAT_VECTORS, dl, DstVT, Lo, Hi);
27585 }
27586
27587 // Use MOVMSK for vector to scalar conversion to prevent scalarization.
27588 if ((SrcVT == MVT::v16i1 || SrcVT == MVT::v32i1) && DstVT.isScalarInteger()) {
27589 assert(!Subtarget.hasAVX512() && "Should use K-registers with AVX512")((!Subtarget.hasAVX512() && "Should use K-registers with AVX512"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.hasAVX512() && \"Should use K-registers with AVX512\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27589, __PRETTY_FUNCTION__));
27590 MVT SExtVT = SrcVT == MVT::v16i1 ? MVT::v16i8 : MVT::v32i8;
27591 SDLoc DL(Op);
27592 SDValue V = DAG.getSExtOrTrunc(Src, DL, SExtVT);
27593 V = getPMOVMSKB(DL, V, DAG, Subtarget);
27594 return DAG.getZExtOrTrunc(V, DL, DstVT);
27595 }
27596
27597 assert((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 ||(((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT
::v8i8 || SrcVT == MVT::i64) && "Unexpected VT!") ? static_cast
<void> (0) : __assert_fail ("(SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 || SrcVT == MVT::i64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27598, __PRETTY_FUNCTION__))
27598 SrcVT == MVT::i64) && "Unexpected VT!")(((SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT
::v8i8 || SrcVT == MVT::i64) && "Unexpected VT!") ? static_cast
<void> (0) : __assert_fail ("(SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8 || SrcVT == MVT::i64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27598, __PRETTY_FUNCTION__));
27599
27600 assert(Subtarget.hasSSE2() && "Requires at least SSE2!")((Subtarget.hasSSE2() && "Requires at least SSE2!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Requires at least SSE2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27600, __PRETTY_FUNCTION__));
27601 if (!(DstVT == MVT::f64 && SrcVT == MVT::i64) &&
27602 !(DstVT == MVT::x86mmx && SrcVT.isVector()))
27603 // This conversion needs to be expanded.
27604 return SDValue();
27605
27606 SDLoc dl(Op);
27607 if (SrcVT.isVector()) {
27608 // Widen the vector in input in the case of MVT::v2i32.
27609 // Example: from MVT::v2i32 to MVT::v4i32.
27610 MVT NewVT = MVT::getVectorVT(SrcVT.getVectorElementType(),
27611 SrcVT.getVectorNumElements() * 2);
27612 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, NewVT, Src,
27613 DAG.getUNDEF(SrcVT));
27614 } else {
27615 assert(SrcVT == MVT::i64 && !Subtarget.is64Bit() &&((SrcVT == MVT::i64 && !Subtarget.is64Bit() &&
"Unexpected source type in LowerBITCAST") ? static_cast<void
> (0) : __assert_fail ("SrcVT == MVT::i64 && !Subtarget.is64Bit() && \"Unexpected source type in LowerBITCAST\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27616, __PRETTY_FUNCTION__))
27616 "Unexpected source type in LowerBITCAST")((SrcVT == MVT::i64 && !Subtarget.is64Bit() &&
"Unexpected source type in LowerBITCAST") ? static_cast<void
> (0) : __assert_fail ("SrcVT == MVT::i64 && !Subtarget.is64Bit() && \"Unexpected source type in LowerBITCAST\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27616, __PRETTY_FUNCTION__));
27617 Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, Src);
27618 }
27619
27620 MVT V2X64VT = DstVT == MVT::f64 ? MVT::v2f64 : MVT::v2i64;
27621 Src = DAG.getNode(ISD::BITCAST, dl, V2X64VT, Src);
27622
27623 if (DstVT == MVT::x86mmx)
27624 return DAG.getNode(X86ISD::MOVDQ2Q, dl, DstVT, Src);
27625
27626 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, DstVT, Src,
27627 DAG.getIntPtrConstant(0, dl));
27628}
27629
27630/// Compute the horizontal sum of bytes in V for the elements of VT.
27631///
27632/// Requires V to be a byte vector and VT to be an integer vector type with
27633/// wider elements than V's type. The width of the elements of VT determines
27634/// how many bytes of V are summed horizontally to produce each element of the
27635/// result.
27636static SDValue LowerHorizontalByteSum(SDValue V, MVT VT,
27637 const X86Subtarget &Subtarget,
27638 SelectionDAG &DAG) {
27639 SDLoc DL(V);
27640 MVT ByteVecVT = V.getSimpleValueType();
27641 MVT EltVT = VT.getVectorElementType();
27642 assert(ByteVecVT.getVectorElementType() == MVT::i8 &&((ByteVecVT.getVectorElementType() == MVT::i8 && "Expected value to have byte element type."
) ? static_cast<void> (0) : __assert_fail ("ByteVecVT.getVectorElementType() == MVT::i8 && \"Expected value to have byte element type.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27643, __PRETTY_FUNCTION__))
27643 "Expected value to have byte element type.")((ByteVecVT.getVectorElementType() == MVT::i8 && "Expected value to have byte element type."
) ? static_cast<void> (0) : __assert_fail ("ByteVecVT.getVectorElementType() == MVT::i8 && \"Expected value to have byte element type.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27643, __PRETTY_FUNCTION__));
27644 assert(EltVT != MVT::i8 &&((EltVT != MVT::i8 && "Horizontal byte sum only makes sense for wider elements!"
) ? static_cast<void> (0) : __assert_fail ("EltVT != MVT::i8 && \"Horizontal byte sum only makes sense for wider elements!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27645, __PRETTY_FUNCTION__))
27645 "Horizontal byte sum only makes sense for wider elements!")((EltVT != MVT::i8 && "Horizontal byte sum only makes sense for wider elements!"
) ? static_cast<void> (0) : __assert_fail ("EltVT != MVT::i8 && \"Horizontal byte sum only makes sense for wider elements!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27645, __PRETTY_FUNCTION__));
27646 unsigned VecSize = VT.getSizeInBits();
27647 assert(ByteVecVT.getSizeInBits() == VecSize && "Cannot change vector size!")((ByteVecVT.getSizeInBits() == VecSize && "Cannot change vector size!"
) ? static_cast<void> (0) : __assert_fail ("ByteVecVT.getSizeInBits() == VecSize && \"Cannot change vector size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27647, __PRETTY_FUNCTION__));
27648
27649 // PSADBW instruction horizontally add all bytes and leave the result in i64
27650 // chunks, thus directly computes the pop count for v2i64 and v4i64.
27651 if (EltVT == MVT::i64) {
27652 SDValue Zeros = DAG.getConstant(0, DL, ByteVecVT);
27653 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
27654 V = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT, V, Zeros);
27655 return DAG.getBitcast(VT, V);
27656 }
27657
27658 if (EltVT == MVT::i32) {
27659 // We unpack the low half and high half into i32s interleaved with zeros so
27660 // that we can use PSADBW to horizontally sum them. The most useful part of
27661 // this is that it lines up the results of two PSADBW instructions to be
27662 // two v2i64 vectors which concatenated are the 4 population counts. We can
27663 // then use PACKUSWB to shrink and concatenate them into a v4i32 again.
27664 SDValue Zeros = DAG.getConstant(0, DL, VT);
27665 SDValue V32 = DAG.getBitcast(VT, V);
27666 SDValue Low = getUnpackl(DAG, DL, VT, V32, Zeros);
27667 SDValue High = getUnpackh(DAG, DL, VT, V32, Zeros);
27668
27669 // Do the horizontal sums into two v2i64s.
27670 Zeros = DAG.getConstant(0, DL, ByteVecVT);
27671 MVT SadVecVT = MVT::getVectorVT(MVT::i64, VecSize / 64);
27672 Low = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
27673 DAG.getBitcast(ByteVecVT, Low), Zeros);
27674 High = DAG.getNode(X86ISD::PSADBW, DL, SadVecVT,
27675 DAG.getBitcast(ByteVecVT, High), Zeros);
27676
27677 // Merge them together.
27678 MVT ShortVecVT = MVT::getVectorVT(MVT::i16, VecSize / 16);
27679 V = DAG.getNode(X86ISD::PACKUS, DL, ByteVecVT,
27680 DAG.getBitcast(ShortVecVT, Low),
27681 DAG.getBitcast(ShortVecVT, High));
27682
27683 return DAG.getBitcast(VT, V);
27684 }
27685
27686 // The only element type left is i16.
27687 assert(EltVT == MVT::i16 && "Unknown how to handle type")((EltVT == MVT::i16 && "Unknown how to handle type") ?
static_cast<void> (0) : __assert_fail ("EltVT == MVT::i16 && \"Unknown how to handle type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27687, __PRETTY_FUNCTION__));
27688
27689 // To obtain pop count for each i16 element starting from the pop count for
27690 // i8 elements, shift the i16s left by 8, sum as i8s, and then shift as i16s
27691 // right by 8. It is important to shift as i16s as i8 vector shift isn't
27692 // directly supported.
27693 SDValue ShifterV = DAG.getConstant(8, DL, VT);
27694 SDValue Shl = DAG.getNode(ISD::SHL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
27695 V = DAG.getNode(ISD::ADD, DL, ByteVecVT, DAG.getBitcast(ByteVecVT, Shl),
27696 DAG.getBitcast(ByteVecVT, V));
27697 return DAG.getNode(ISD::SRL, DL, VT, DAG.getBitcast(VT, V), ShifterV);
27698}
27699
27700static SDValue LowerVectorCTPOPInRegLUT(SDValue Op, const SDLoc &DL,
27701 const X86Subtarget &Subtarget,
27702 SelectionDAG &DAG) {
27703 MVT VT = Op.getSimpleValueType();
27704 MVT EltVT = VT.getVectorElementType();
27705 int NumElts = VT.getVectorNumElements();
27706 (void)EltVT;
27707 assert(EltVT == MVT::i8 && "Only vXi8 vector CTPOP lowering supported.")((EltVT == MVT::i8 && "Only vXi8 vector CTPOP lowering supported."
) ? static_cast<void> (0) : __assert_fail ("EltVT == MVT::i8 && \"Only vXi8 vector CTPOP lowering supported.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27707, __PRETTY_FUNCTION__));
27708
27709 // Implement a lookup table in register by using an algorithm based on:
27710 // http://wm.ite.pl/articles/sse-popcount.html
27711 //
27712 // The general idea is that every lower byte nibble in the input vector is an
27713 // index into a in-register pre-computed pop count table. We then split up the
27714 // input vector in two new ones: (1) a vector with only the shifted-right
27715 // higher nibbles for each byte and (2) a vector with the lower nibbles (and
27716 // masked out higher ones) for each byte. PSHUFB is used separately with both
27717 // to index the in-register table. Next, both are added and the result is a
27718 // i8 vector where each element contains the pop count for input byte.
27719 const int LUT[16] = {/* 0 */ 0, /* 1 */ 1, /* 2 */ 1, /* 3 */ 2,
27720 /* 4 */ 1, /* 5 */ 2, /* 6 */ 2, /* 7 */ 3,
27721 /* 8 */ 1, /* 9 */ 2, /* a */ 2, /* b */ 3,
27722 /* c */ 2, /* d */ 3, /* e */ 3, /* f */ 4};
27723
27724 SmallVector<SDValue, 64> LUTVec;
27725 for (int i = 0; i < NumElts; ++i)
27726 LUTVec.push_back(DAG.getConstant(LUT[i % 16], DL, MVT::i8));
27727 SDValue InRegLUT = DAG.getBuildVector(VT, DL, LUTVec);
27728 SDValue M0F = DAG.getConstant(0x0F, DL, VT);
27729
27730 // High nibbles
27731 SDValue FourV = DAG.getConstant(4, DL, VT);
27732 SDValue HiNibbles = DAG.getNode(ISD::SRL, DL, VT, Op, FourV);
27733
27734 // Low nibbles
27735 SDValue LoNibbles = DAG.getNode(ISD::AND, DL, VT, Op, M0F);
27736
27737 // The input vector is used as the shuffle mask that index elements into the
27738 // LUT. After counting low and high nibbles, add the vector to obtain the
27739 // final pop count per i8 element.
27740 SDValue HiPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, HiNibbles);
27741 SDValue LoPopCnt = DAG.getNode(X86ISD::PSHUFB, DL, VT, InRegLUT, LoNibbles);
27742 return DAG.getNode(ISD::ADD, DL, VT, HiPopCnt, LoPopCnt);
27743}
27744
27745// Please ensure that any codegen change from LowerVectorCTPOP is reflected in
27746// updated cost models in X86TTIImpl::getIntrinsicInstrCost.
27747static SDValue LowerVectorCTPOP(SDValue Op, const X86Subtarget &Subtarget,
27748 SelectionDAG &DAG) {
27749 MVT VT = Op.getSimpleValueType();
27750 assert((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) &&(((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector
()) && "Unknown CTPOP type to handle") ? static_cast<
void> (0) : __assert_fail ("(VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) && \"Unknown CTPOP type to handle\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27751, __PRETTY_FUNCTION__))
27751 "Unknown CTPOP type to handle")(((VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector
()) && "Unknown CTPOP type to handle") ? static_cast<
void> (0) : __assert_fail ("(VT.is512BitVector() || VT.is256BitVector() || VT.is128BitVector()) && \"Unknown CTPOP type to handle\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27751, __PRETTY_FUNCTION__));
27752 SDLoc DL(Op.getNode());
27753 SDValue Op0 = Op.getOperand(0);
27754
27755 // TRUNC(CTPOP(ZEXT(X))) to make use of vXi32/vXi64 VPOPCNT instructions.
27756 if (Subtarget.hasVPOPCNTDQ()) {
27757 unsigned NumElems = VT.getVectorNumElements();
27758 assert((VT.getVectorElementType() == MVT::i8 ||(((VT.getVectorElementType() == MVT::i8 || VT.getVectorElementType
() == MVT::i16) && "Unexpected type") ? static_cast<
void> (0) : __assert_fail ("(VT.getVectorElementType() == MVT::i8 || VT.getVectorElementType() == MVT::i16) && \"Unexpected type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27759, __PRETTY_FUNCTION__))
27759 VT.getVectorElementType() == MVT::i16) && "Unexpected type")(((VT.getVectorElementType() == MVT::i8 || VT.getVectorElementType
() == MVT::i16) && "Unexpected type") ? static_cast<
void> (0) : __assert_fail ("(VT.getVectorElementType() == MVT::i8 || VT.getVectorElementType() == MVT::i16) && \"Unexpected type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27759, __PRETTY_FUNCTION__));
27760 if (NumElems < 16 || (NumElems == 16 && Subtarget.canExtendTo512DQ())) {
27761 MVT NewVT = MVT::getVectorVT(MVT::i32, NumElems);
27762 Op = DAG.getNode(ISD::ZERO_EXTEND, DL, NewVT, Op0);
27763 Op = DAG.getNode(ISD::CTPOP, DL, NewVT, Op);
27764 return DAG.getNode(ISD::TRUNCATE, DL, VT, Op);
27765 }
27766 }
27767
27768 // Decompose 256-bit ops into smaller 128-bit ops.
27769 if (VT.is256BitVector() && !Subtarget.hasInt256())
27770 return Lower256IntUnary(Op, DAG);
27771
27772 // Decompose 512-bit ops into smaller 256-bit ops.
27773 if (VT.is512BitVector() && !Subtarget.hasBWI())
27774 return Lower512IntUnary(Op, DAG);
27775
27776 // For element types greater than i8, do vXi8 pop counts and a bytesum.
27777 if (VT.getScalarType() != MVT::i8) {
27778 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
27779 SDValue ByteOp = DAG.getBitcast(ByteVT, Op0);
27780 SDValue PopCnt8 = DAG.getNode(ISD::CTPOP, DL, ByteVT, ByteOp);
27781 return LowerHorizontalByteSum(PopCnt8, VT, Subtarget, DAG);
27782 }
27783
27784 // We can't use the fast LUT approach, so fall back on LegalizeDAG.
27785 if (!Subtarget.hasSSSE3())
27786 return SDValue();
27787
27788 return LowerVectorCTPOPInRegLUT(Op0, DL, Subtarget, DAG);
27789}
27790
27791static SDValue LowerCTPOP(SDValue Op, const X86Subtarget &Subtarget,
27792 SelectionDAG &DAG) {
27793 assert(Op.getSimpleValueType().isVector() &&((Op.getSimpleValueType().isVector() && "We only do custom lowering for vector population count."
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().isVector() && \"We only do custom lowering for vector population count.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27794, __PRETTY_FUNCTION__))
27794 "We only do custom lowering for vector population count.")((Op.getSimpleValueType().isVector() && "We only do custom lowering for vector population count."
) ? static_cast<void> (0) : __assert_fail ("Op.getSimpleValueType().isVector() && \"We only do custom lowering for vector population count.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27794, __PRETTY_FUNCTION__));
27795 return LowerVectorCTPOP(Op, Subtarget, DAG);
27796}
27797
27798static SDValue LowerBITREVERSE_XOP(SDValue Op, SelectionDAG &DAG) {
27799 MVT VT = Op.getSimpleValueType();
27800 SDValue In = Op.getOperand(0);
27801 SDLoc DL(Op);
27802
27803 // For scalars, its still beneficial to transfer to/from the SIMD unit to
27804 // perform the BITREVERSE.
27805 if (!VT.isVector()) {
27806 MVT VecVT = MVT::getVectorVT(VT, 128 / VT.getSizeInBits());
27807 SDValue Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, In);
27808 Res = DAG.getNode(ISD::BITREVERSE, DL, VecVT, Res);
27809 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Res,
27810 DAG.getIntPtrConstant(0, DL));
27811 }
27812
27813 int NumElts = VT.getVectorNumElements();
27814 int ScalarSizeInBytes = VT.getScalarSizeInBits() / 8;
27815
27816 // Decompose 256-bit ops into smaller 128-bit ops.
27817 if (VT.is256BitVector())
27818 return Lower256IntUnary(Op, DAG);
27819
27820 assert(VT.is128BitVector() &&((VT.is128BitVector() && "Only 128-bit vector bitreverse lowering supported."
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Only 128-bit vector bitreverse lowering supported.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27821, __PRETTY_FUNCTION__))
27821 "Only 128-bit vector bitreverse lowering supported.")((VT.is128BitVector() && "Only 128-bit vector bitreverse lowering supported."
) ? static_cast<void> (0) : __assert_fail ("VT.is128BitVector() && \"Only 128-bit vector bitreverse lowering supported.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27821, __PRETTY_FUNCTION__));
27822
27823 // VPPERM reverses the bits of a byte with the permute Op (2 << 5), and we
27824 // perform the BSWAP in the shuffle.
27825 // Its best to shuffle using the second operand as this will implicitly allow
27826 // memory folding for multiple vectors.
27827 SmallVector<SDValue, 16> MaskElts;
27828 for (int i = 0; i != NumElts; ++i) {
27829 for (int j = ScalarSizeInBytes - 1; j >= 0; --j) {
27830 int SourceByte = 16 + (i * ScalarSizeInBytes) + j;
27831 int PermuteByte = SourceByte | (2 << 5);
27832 MaskElts.push_back(DAG.getConstant(PermuteByte, DL, MVT::i8));
27833 }
27834 }
27835
27836 SDValue Mask = DAG.getBuildVector(MVT::v16i8, DL, MaskElts);
27837 SDValue Res = DAG.getBitcast(MVT::v16i8, In);
27838 Res = DAG.getNode(X86ISD::VPPERM, DL, MVT::v16i8, DAG.getUNDEF(MVT::v16i8),
27839 Res, Mask);
27840 return DAG.getBitcast(VT, Res);
27841}
27842
27843static SDValue LowerBITREVERSE(SDValue Op, const X86Subtarget &Subtarget,
27844 SelectionDAG &DAG) {
27845 MVT VT = Op.getSimpleValueType();
27846
27847 if (Subtarget.hasXOP() && !VT.is512BitVector())
27848 return LowerBITREVERSE_XOP(Op, DAG);
27849
27850 assert(Subtarget.hasSSSE3() && "SSSE3 required for BITREVERSE")((Subtarget.hasSSSE3() && "SSSE3 required for BITREVERSE"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasSSSE3() && \"SSSE3 required for BITREVERSE\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27850, __PRETTY_FUNCTION__));
27851
27852 SDValue In = Op.getOperand(0);
27853 SDLoc DL(Op);
27854
27855 // Split v8i64/v16i32 without BWI so that we can still use the PSHUFB
27856 // lowering.
27857 if (VT == MVT::v8i64 || VT == MVT::v16i32) {
27858 assert(!Subtarget.hasBWI() && "BWI should Expand BITREVERSE")((!Subtarget.hasBWI() && "BWI should Expand BITREVERSE"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.hasBWI() && \"BWI should Expand BITREVERSE\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27858, __PRETTY_FUNCTION__));
27859 return Lower512IntUnary(Op, DAG);
27860 }
27861
27862 unsigned NumElts = VT.getVectorNumElements();
27863 assert(VT.getScalarType() == MVT::i8 &&((VT.getScalarType() == MVT::i8 && "Only byte vector BITREVERSE supported"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i8 && \"Only byte vector BITREVERSE supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27864, __PRETTY_FUNCTION__))
27864 "Only byte vector BITREVERSE supported")((VT.getScalarType() == MVT::i8 && "Only byte vector BITREVERSE supported"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i8 && \"Only byte vector BITREVERSE supported\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27864, __PRETTY_FUNCTION__));
27865
27866 // Decompose 256-bit ops into smaller 128-bit ops on pre-AVX2.
27867 if (VT.is256BitVector() && !Subtarget.hasInt256())
27868 return Lower256IntUnary(Op, DAG);
27869
27870 // Perform BITREVERSE using PSHUFB lookups. Each byte is split into
27871 // two nibbles and a PSHUFB lookup to find the bitreverse of each
27872 // 0-15 value (moved to the other nibble).
27873 SDValue NibbleMask = DAG.getConstant(0xF, DL, VT);
27874 SDValue Lo = DAG.getNode(ISD::AND, DL, VT, In, NibbleMask);
27875 SDValue Hi = DAG.getNode(ISD::SRL, DL, VT, In, DAG.getConstant(4, DL, VT));
27876
27877 const int LoLUT[16] = {
27878 /* 0 */ 0x00, /* 1 */ 0x80, /* 2 */ 0x40, /* 3 */ 0xC0,
27879 /* 4 */ 0x20, /* 5 */ 0xA0, /* 6 */ 0x60, /* 7 */ 0xE0,
27880 /* 8 */ 0x10, /* 9 */ 0x90, /* a */ 0x50, /* b */ 0xD0,
27881 /* c */ 0x30, /* d */ 0xB0, /* e */ 0x70, /* f */ 0xF0};
27882 const int HiLUT[16] = {
27883 /* 0 */ 0x00, /* 1 */ 0x08, /* 2 */ 0x04, /* 3 */ 0x0C,
27884 /* 4 */ 0x02, /* 5 */ 0x0A, /* 6 */ 0x06, /* 7 */ 0x0E,
27885 /* 8 */ 0x01, /* 9 */ 0x09, /* a */ 0x05, /* b */ 0x0D,
27886 /* c */ 0x03, /* d */ 0x0B, /* e */ 0x07, /* f */ 0x0F};
27887
27888 SmallVector<SDValue, 16> LoMaskElts, HiMaskElts;
27889 for (unsigned i = 0; i < NumElts; ++i) {
27890 LoMaskElts.push_back(DAG.getConstant(LoLUT[i % 16], DL, MVT::i8));
27891 HiMaskElts.push_back(DAG.getConstant(HiLUT[i % 16], DL, MVT::i8));
27892 }
27893
27894 SDValue LoMask = DAG.getBuildVector(VT, DL, LoMaskElts);
27895 SDValue HiMask = DAG.getBuildVector(VT, DL, HiMaskElts);
27896 Lo = DAG.getNode(X86ISD::PSHUFB, DL, VT, LoMask, Lo);
27897 Hi = DAG.getNode(X86ISD::PSHUFB, DL, VT, HiMask, Hi);
27898 return DAG.getNode(ISD::OR, DL, VT, Lo, Hi);
27899}
27900
27901static SDValue lowerAtomicArithWithLOCK(SDValue N, SelectionDAG &DAG,
27902 const X86Subtarget &Subtarget) {
27903 unsigned NewOpc = 0;
27904 switch (N->getOpcode()) {
27905 case ISD::ATOMIC_LOAD_ADD:
27906 NewOpc = X86ISD::LADD;
27907 break;
27908 case ISD::ATOMIC_LOAD_SUB:
27909 NewOpc = X86ISD::LSUB;
27910 break;
27911 case ISD::ATOMIC_LOAD_OR:
27912 NewOpc = X86ISD::LOR;
27913 break;
27914 case ISD::ATOMIC_LOAD_XOR:
27915 NewOpc = X86ISD::LXOR;
27916 break;
27917 case ISD::ATOMIC_LOAD_AND:
27918 NewOpc = X86ISD::LAND;
27919 break;
27920 default:
27921 llvm_unreachable("Unknown ATOMIC_LOAD_ opcode")::llvm::llvm_unreachable_internal("Unknown ATOMIC_LOAD_ opcode"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27921);
27922 }
27923
27924 MachineMemOperand *MMO = cast<MemSDNode>(N)->getMemOperand();
27925
27926 return DAG.getMemIntrinsicNode(
27927 NewOpc, SDLoc(N), DAG.getVTList(MVT::i32, MVT::Other),
27928 {N->getOperand(0), N->getOperand(1), N->getOperand(2)},
27929 /*MemVT=*/N->getSimpleValueType(0), MMO);
27930}
27931
27932/// Lower atomic_load_ops into LOCK-prefixed operations.
27933static SDValue lowerAtomicArith(SDValue N, SelectionDAG &DAG,
27934 const X86Subtarget &Subtarget) {
27935 AtomicSDNode *AN = cast<AtomicSDNode>(N.getNode());
27936 SDValue Chain = N->getOperand(0);
27937 SDValue LHS = N->getOperand(1);
27938 SDValue RHS = N->getOperand(2);
27939 unsigned Opc = N->getOpcode();
27940 MVT VT = N->getSimpleValueType(0);
27941 SDLoc DL(N);
27942
27943 // We can lower atomic_load_add into LXADD. However, any other atomicrmw op
27944 // can only be lowered when the result is unused. They should have already
27945 // been transformed into a cmpxchg loop in AtomicExpand.
27946 if (N->hasAnyUseOfValue(0)) {
27947 // Handle (atomic_load_sub p, v) as (atomic_load_add p, -v), to be able to
27948 // select LXADD if LOCK_SUB can't be selected.
27949 if (Opc == ISD::ATOMIC_LOAD_SUB) {
27950 RHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), RHS);
27951 return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, VT, Chain, LHS,
27952 RHS, AN->getMemOperand());
27953 }
27954 assert(Opc == ISD::ATOMIC_LOAD_ADD &&((Opc == ISD::ATOMIC_LOAD_ADD && "Used AtomicRMW ops other than Add should have been expanded!"
) ? static_cast<void> (0) : __assert_fail ("Opc == ISD::ATOMIC_LOAD_ADD && \"Used AtomicRMW ops other than Add should have been expanded!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27955, __PRETTY_FUNCTION__))
27955 "Used AtomicRMW ops other than Add should have been expanded!")((Opc == ISD::ATOMIC_LOAD_ADD && "Used AtomicRMW ops other than Add should have been expanded!"
) ? static_cast<void> (0) : __assert_fail ("Opc == ISD::ATOMIC_LOAD_ADD && \"Used AtomicRMW ops other than Add should have been expanded!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27955, __PRETTY_FUNCTION__));
27956 return N;
27957 }
27958
27959 // Specialized lowering for the canonical form of an idemptotent atomicrmw.
27960 // The core idea here is that since the memory location isn't actually
27961 // changing, all we need is a lowering for the *ordering* impacts of the
27962 // atomicrmw. As such, we can chose a different operation and memory
27963 // location to minimize impact on other code.
27964 if (Opc == ISD::ATOMIC_LOAD_OR && isNullConstant(RHS)) {
27965 // On X86, the only ordering which actually requires an instruction is
27966 // seq_cst which isn't SingleThread, everything just needs to be preserved
27967 // during codegen and then dropped. Note that we expect (but don't assume),
27968 // that orderings other than seq_cst and acq_rel have been canonicalized to
27969 // a store or load.
27970 if (AN->getOrdering() == AtomicOrdering::SequentiallyConsistent &&
27971 AN->getSyncScopeID() == SyncScope::System) {
27972 // Prefer a locked operation against a stack location to minimize cache
27973 // traffic. This assumes that stack locations are very likely to be
27974 // accessed only by the owning thread.
27975 SDValue NewChain = emitLockedStackOp(DAG, Subtarget, Chain, DL);
27976 assert(!N->hasAnyUseOfValue(0))((!N->hasAnyUseOfValue(0)) ? static_cast<void> (0) :
__assert_fail ("!N->hasAnyUseOfValue(0)", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27976, __PRETTY_FUNCTION__));
27977 // NOTE: The getUNDEF is needed to give something for the unused result 0.
27978 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
27979 DAG.getUNDEF(VT), NewChain);
27980 }
27981 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
27982 SDValue NewChain = DAG.getNode(X86ISD::MEMBARRIER, DL, MVT::Other, Chain);
27983 assert(!N->hasAnyUseOfValue(0))((!N->hasAnyUseOfValue(0)) ? static_cast<void> (0) :
__assert_fail ("!N->hasAnyUseOfValue(0)", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27983, __PRETTY_FUNCTION__));
27984 // NOTE: The getUNDEF is needed to give something for the unused result 0.
27985 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
27986 DAG.getUNDEF(VT), NewChain);
27987 }
27988
27989 SDValue LockOp = lowerAtomicArithWithLOCK(N, DAG, Subtarget);
27990 // RAUW the chain, but don't worry about the result, as it's unused.
27991 assert(!N->hasAnyUseOfValue(0))((!N->hasAnyUseOfValue(0)) ? static_cast<void> (0) :
__assert_fail ("!N->hasAnyUseOfValue(0)", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 27991, __PRETTY_FUNCTION__));
27992 // NOTE: The getUNDEF is needed to give something for the unused result 0.
27993 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(),
27994 DAG.getUNDEF(VT), LockOp.getValue(1));
27995}
27996
27997static SDValue LowerATOMIC_STORE(SDValue Op, SelectionDAG &DAG,
27998 const X86Subtarget &Subtarget) {
27999 auto *Node = cast<AtomicSDNode>(Op.getNode());
28000 SDLoc dl(Node);
28001 EVT VT = Node->getMemoryVT();
28002
28003 bool IsSeqCst = Node->getOrdering() == AtomicOrdering::SequentiallyConsistent;
28004 bool IsTypeLegal = DAG.getTargetLoweringInfo().isTypeLegal(VT);
28005
28006 // If this store is not sequentially consistent and the type is legal
28007 // we can just keep it.
28008 if (!IsSeqCst && IsTypeLegal)
28009 return Op;
28010
28011 if (VT == MVT::i64 && !IsTypeLegal) {
28012 // For illegal i64 atomic_stores, we can try to use MOVQ if SSE2 is enabled.
28013 // FIXME: Use movlps with SSE1.
28014 // FIXME: Use fist with X87.
28015 bool NoImplicitFloatOps =
28016 DAG.getMachineFunction().getFunction().hasFnAttribute(
28017 Attribute::NoImplicitFloat);
28018 if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps &&
28019 Subtarget.hasSSE2()) {
28020 SDValue SclToVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
28021 Node->getOperand(2));
28022 SDVTList Tys = DAG.getVTList(MVT::Other);
28023 SDValue Ops[] = { Node->getChain(), SclToVec, Node->getBasePtr() };
28024 SDValue Chain = DAG.getMemIntrinsicNode(X86ISD::VEXTRACT_STORE, dl, Tys,
28025 Ops, MVT::i64,
28026 Node->getMemOperand());
28027
28028 // If this is a sequentially consistent store, also emit an appropriate
28029 // barrier.
28030 if (IsSeqCst)
28031 Chain = emitLockedStackOp(DAG, Subtarget, Chain, dl);
28032
28033 return Chain;
28034 }
28035 }
28036
28037 // Convert seq_cst store -> xchg
28038 // Convert wide store -> swap (-> cmpxchg8b/cmpxchg16b)
28039 // FIXME: 16-byte ATOMIC_SWAP isn't actually hooked up at the moment.
28040 SDValue Swap = DAG.getAtomic(ISD::ATOMIC_SWAP, dl,
28041 Node->getMemoryVT(),
28042 Node->getOperand(0),
28043 Node->getOperand(1), Node->getOperand(2),
28044 Node->getMemOperand());
28045 return Swap.getValue(1);
28046}
28047
28048static SDValue LowerADDSUBCARRY(SDValue Op, SelectionDAG &DAG) {
28049 SDNode *N = Op.getNode();
28050 MVT VT = N->getSimpleValueType(0);
28051
28052 // Let legalize expand this if it isn't a legal type yet.
28053 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
28054 return SDValue();
28055
28056 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
28057 SDLoc DL(N);
28058
28059 // Set the carry flag.
28060 SDValue Carry = Op.getOperand(2);
28061 EVT CarryVT = Carry.getValueType();
28062 APInt NegOne = APInt::getAllOnesValue(CarryVT.getScalarSizeInBits());
28063 Carry = DAG.getNode(X86ISD::ADD, DL, DAG.getVTList(CarryVT, MVT::i32),
28064 Carry, DAG.getConstant(NegOne, DL, CarryVT));
28065
28066 unsigned Opc = Op.getOpcode() == ISD::ADDCARRY ? X86ISD::ADC : X86ISD::SBB;
28067 SDValue Sum = DAG.getNode(Opc, DL, VTs, Op.getOperand(0),
28068 Op.getOperand(1), Carry.getValue(1));
28069
28070 SDValue SetCC = getSETCC(X86::COND_B, Sum.getValue(1), DL, DAG);
28071 if (N->getValueType(1) == MVT::i1)
28072 SetCC = DAG.getNode(ISD::TRUNCATE, DL, MVT::i1, SetCC);
28073
28074 return DAG.getNode(ISD::MERGE_VALUES, DL, N->getVTList(), Sum, SetCC);
28075}
28076
28077static SDValue LowerFSINCOS(SDValue Op, const X86Subtarget &Subtarget,
28078 SelectionDAG &DAG) {
28079 assert(Subtarget.isTargetDarwin() && Subtarget.is64Bit())((Subtarget.isTargetDarwin() && Subtarget.is64Bit()) ?
static_cast<void> (0) : __assert_fail ("Subtarget.isTargetDarwin() && Subtarget.is64Bit()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28079, __PRETTY_FUNCTION__));
28080
28081 // For MacOSX, we want to call an alternative entry point: __sincos_stret,
28082 // which returns the values as { float, float } (in XMM0) or
28083 // { double, double } (which is returned in XMM0, XMM1).
28084 SDLoc dl(Op);
28085 SDValue Arg = Op.getOperand(0);
28086 EVT ArgVT = Arg.getValueType();
28087 Type *ArgTy = ArgVT.getTypeForEVT(*DAG.getContext());
28088
28089 TargetLowering::ArgListTy Args;
28090 TargetLowering::ArgListEntry Entry;
28091
28092 Entry.Node = Arg;
28093 Entry.Ty = ArgTy;
28094 Entry.IsSExt = false;
28095 Entry.IsZExt = false;
28096 Args.push_back(Entry);
28097
28098 bool isF64 = ArgVT == MVT::f64;
28099 // Only optimize x86_64 for now. i386 is a bit messy. For f32,
28100 // the small struct {f32, f32} is returned in (eax, edx). For f64,
28101 // the results are returned via SRet in memory.
28102 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28103 RTLIB::Libcall LC = isF64 ? RTLIB::SINCOS_STRET_F64 : RTLIB::SINCOS_STRET_F32;
28104 const char *LibcallName = TLI.getLibcallName(LC);
28105 SDValue Callee =
28106 DAG.getExternalSymbol(LibcallName, TLI.getPointerTy(DAG.getDataLayout()));
28107
28108 Type *RetTy = isF64 ? (Type *)StructType::get(ArgTy, ArgTy)
28109 : (Type *)VectorType::get(ArgTy, 4);
28110
28111 TargetLowering::CallLoweringInfo CLI(DAG);
28112 CLI.setDebugLoc(dl)
28113 .setChain(DAG.getEntryNode())
28114 .setLibCallee(CallingConv::C, RetTy, Callee, std::move(Args));
28115
28116 std::pair<SDValue, SDValue> CallResult = TLI.LowerCallTo(CLI);
28117
28118 if (isF64)
28119 // Returned in xmm0 and xmm1.
28120 return CallResult.first;
28121
28122 // Returned in bits 0:31 and 32:64 xmm0.
28123 SDValue SinVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
28124 CallResult.first, DAG.getIntPtrConstant(0, dl));
28125 SDValue CosVal = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ArgVT,
28126 CallResult.first, DAG.getIntPtrConstant(1, dl));
28127 SDVTList Tys = DAG.getVTList(ArgVT, ArgVT);
28128 return DAG.getNode(ISD::MERGE_VALUES, dl, Tys, SinVal, CosVal);
28129}
28130
28131/// Widen a vector input to a vector of NVT. The
28132/// input vector must have the same element type as NVT.
28133static SDValue ExtendToType(SDValue InOp, MVT NVT, SelectionDAG &DAG,
28134 bool FillWithZeroes = false) {
28135 // Check if InOp already has the right width.
28136 MVT InVT = InOp.getSimpleValueType();
28137 if (InVT == NVT)
28138 return InOp;
28139
28140 if (InOp.isUndef())
28141 return DAG.getUNDEF(NVT);
28142
28143 assert(InVT.getVectorElementType() == NVT.getVectorElementType() &&((InVT.getVectorElementType() == NVT.getVectorElementType() &&
"input and widen element type must match") ? static_cast<
void> (0) : __assert_fail ("InVT.getVectorElementType() == NVT.getVectorElementType() && \"input and widen element type must match\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28144, __PRETTY_FUNCTION__))
28144 "input and widen element type must match")((InVT.getVectorElementType() == NVT.getVectorElementType() &&
"input and widen element type must match") ? static_cast<
void> (0) : __assert_fail ("InVT.getVectorElementType() == NVT.getVectorElementType() && \"input and widen element type must match\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28144, __PRETTY_FUNCTION__));
28145
28146 unsigned InNumElts = InVT.getVectorNumElements();
28147 unsigned WidenNumElts = NVT.getVectorNumElements();
28148 assert(WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0 &&((WidenNumElts > InNumElts && WidenNumElts % InNumElts
== 0 && "Unexpected request for vector widening") ? static_cast
<void> (0) : __assert_fail ("WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0 && \"Unexpected request for vector widening\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28149, __PRETTY_FUNCTION__))
28149 "Unexpected request for vector widening")((WidenNumElts > InNumElts && WidenNumElts % InNumElts
== 0 && "Unexpected request for vector widening") ? static_cast
<void> (0) : __assert_fail ("WidenNumElts > InNumElts && WidenNumElts % InNumElts == 0 && \"Unexpected request for vector widening\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28149, __PRETTY_FUNCTION__));
28150
28151 SDLoc dl(InOp);
28152 if (InOp.getOpcode() == ISD::CONCAT_VECTORS &&
28153 InOp.getNumOperands() == 2) {
28154 SDValue N1 = InOp.getOperand(1);
28155 if ((ISD::isBuildVectorAllZeros(N1.getNode()) && FillWithZeroes) ||
28156 N1.isUndef()) {
28157 InOp = InOp.getOperand(0);
28158 InVT = InOp.getSimpleValueType();
28159 InNumElts = InVT.getVectorNumElements();
28160 }
28161 }
28162 if (ISD::isBuildVectorOfConstantSDNodes(InOp.getNode()) ||
28163 ISD::isBuildVectorOfConstantFPSDNodes(InOp.getNode())) {
28164 SmallVector<SDValue, 16> Ops;
28165 for (unsigned i = 0; i < InNumElts; ++i)
28166 Ops.push_back(InOp.getOperand(i));
28167
28168 EVT EltVT = InOp.getOperand(0).getValueType();
28169
28170 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, EltVT) :
28171 DAG.getUNDEF(EltVT);
28172 for (unsigned i = 0; i < WidenNumElts - InNumElts; ++i)
28173 Ops.push_back(FillVal);
28174 return DAG.getBuildVector(NVT, dl, Ops);
28175 }
28176 SDValue FillVal = FillWithZeroes ? DAG.getConstant(0, dl, NVT) :
28177 DAG.getUNDEF(NVT);
28178 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, NVT, FillVal,
28179 InOp, DAG.getIntPtrConstant(0, dl));
28180}
28181
28182static SDValue LowerMSCATTER(SDValue Op, const X86Subtarget &Subtarget,
28183 SelectionDAG &DAG) {
28184 assert(Subtarget.hasAVX512() &&((Subtarget.hasAVX512() && "MGATHER/MSCATTER are supported on AVX-512 arch only"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"MGATHER/MSCATTER are supported on AVX-512 arch only\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28185, __PRETTY_FUNCTION__))
28185 "MGATHER/MSCATTER are supported on AVX-512 arch only")((Subtarget.hasAVX512() && "MGATHER/MSCATTER are supported on AVX-512 arch only"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"MGATHER/MSCATTER are supported on AVX-512 arch only\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28185, __PRETTY_FUNCTION__));
28186
28187 MaskedScatterSDNode *N = cast<MaskedScatterSDNode>(Op.getNode());
28188 SDValue Src = N->getValue();
28189 MVT VT = Src.getSimpleValueType();
28190 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op")((VT.getScalarSizeInBits() >= 32 && "Unsupported scatter op"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() >= 32 && \"Unsupported scatter op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28190, __PRETTY_FUNCTION__));
28191 SDLoc dl(Op);
28192
28193 SDValue Scale = N->getScale();
28194 SDValue Index = N->getIndex();
28195 SDValue Mask = N->getMask();
28196 SDValue Chain = N->getChain();
28197 SDValue BasePtr = N->getBasePtr();
28198
28199 if (VT == MVT::v2f32 || VT == MVT::v2i32) {
28200 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type")((Mask.getValueType() == MVT::v2i1 && "Unexpected mask type"
) ? static_cast<void> (0) : __assert_fail ("Mask.getValueType() == MVT::v2i1 && \"Unexpected mask type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28200, __PRETTY_FUNCTION__));
28201 // If the index is v2i64 and we have VLX we can use xmm for data and index.
28202 if (Index.getValueType() == MVT::v2i64 && Subtarget.hasVLX()) {
28203 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28204 EVT WideVT = TLI.getTypeToTransformTo(*DAG.getContext(), VT);
28205 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT, Src, DAG.getUNDEF(VT));
28206 SDVTList VTs = DAG.getVTList(MVT::v2i1, MVT::Other);
28207 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
28208 SDValue NewScatter = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
28209 VTs, Ops, dl, N->getMemoryVT(), N->getMemOperand());
28210 return SDValue(NewScatter.getNode(), 1);
28211 }
28212 return SDValue();
28213 }
28214
28215 MVT IndexVT = Index.getSimpleValueType();
28216 MVT MaskVT = Mask.getSimpleValueType();
28217
28218 // If the index is v2i32, we're being called by type legalization and we
28219 // should just let the default handling take care of it.
28220 if (IndexVT == MVT::v2i32)
28221 return SDValue();
28222
28223 // If we don't have VLX and neither the passthru or index is 512-bits, we
28224 // need to widen until one is.
28225 if (!Subtarget.hasVLX() && !VT.is512BitVector() &&
28226 !Index.getSimpleValueType().is512BitVector()) {
28227 // Determine how much we need to widen by to get a 512-bit type.
28228 unsigned Factor = std::min(512/VT.getSizeInBits(),
28229 512/IndexVT.getSizeInBits());
28230 unsigned NumElts = VT.getVectorNumElements() * Factor;
28231
28232 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
28233 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
28234 MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
28235
28236 Src = ExtendToType(Src, VT, DAG);
28237 Index = ExtendToType(Index, IndexVT, DAG);
28238 Mask = ExtendToType(Mask, MaskVT, DAG, true);
28239 }
28240
28241 SDVTList VTs = DAG.getVTList(MaskVT, MVT::Other);
28242 SDValue Ops[] = {Chain, Src, Mask, BasePtr, Index, Scale};
28243 SDValue NewScatter = DAG.getTargetMemSDNode<X86MaskedScatterSDNode>(
28244 VTs, Ops, dl, N->getMemoryVT(), N->getMemOperand());
28245 return SDValue(NewScatter.getNode(), 1);
28246}
28247
28248static SDValue LowerMLOAD(SDValue Op, const X86Subtarget &Subtarget,
28249 SelectionDAG &DAG) {
28250
28251 MaskedLoadSDNode *N = cast<MaskedLoadSDNode>(Op.getNode());
28252 MVT VT = Op.getSimpleValueType();
28253 MVT ScalarVT = VT.getScalarType();
28254 SDValue Mask = N->getMask();
28255 MVT MaskVT = Mask.getSimpleValueType();
28256 SDValue PassThru = N->getPassThru();
28257 SDLoc dl(Op);
28258
28259 // Handle AVX masked loads which don't support passthru other than 0.
28260 if (MaskVT.getVectorElementType() != MVT::i1) {
28261 // We also allow undef in the isel pattern.
28262 if (PassThru.isUndef() || ISD::isBuildVectorAllZeros(PassThru.getNode()))
28263 return Op;
28264
28265 SDValue NewLoad = DAG.getMaskedLoad(
28266 VT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
28267 getZeroVector(VT, Subtarget, DAG, dl), N->getMemoryVT(),
28268 N->getMemOperand(), N->getAddressingMode(), N->getExtensionType(),
28269 N->isExpandingLoad());
28270 // Emit a blend.
28271 SDValue Select = DAG.getNode(ISD::VSELECT, dl, MaskVT, Mask, NewLoad,
28272 PassThru);
28273 return DAG.getMergeValues({ Select, NewLoad.getValue(1) }, dl);
28274 }
28275
28276 assert((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&(((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&
"Expanding masked load is supported on AVX-512 target only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isExpandingLoad() || Subtarget.hasAVX512()) && \"Expanding masked load is supported on AVX-512 target only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28277, __PRETTY_FUNCTION__))
28277 "Expanding masked load is supported on AVX-512 target only!")(((!N->isExpandingLoad() || Subtarget.hasAVX512()) &&
"Expanding masked load is supported on AVX-512 target only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isExpandingLoad() || Subtarget.hasAVX512()) && \"Expanding masked load is supported on AVX-512 target only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28277, __PRETTY_FUNCTION__));
28278
28279 assert((!N->isExpandingLoad() || ScalarVT.getSizeInBits() >= 32) &&(((!N->isExpandingLoad() || ScalarVT.getSizeInBits() >=
32) && "Expanding masked load is supported for 32 and 64-bit types only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isExpandingLoad() || ScalarVT.getSizeInBits() >= 32) && \"Expanding masked load is supported for 32 and 64-bit types only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28280, __PRETTY_FUNCTION__))
28280 "Expanding masked load is supported for 32 and 64-bit types only!")(((!N->isExpandingLoad() || ScalarVT.getSizeInBits() >=
32) && "Expanding masked load is supported for 32 and 64-bit types only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isExpandingLoad() || ScalarVT.getSizeInBits() >= 32) && \"Expanding masked load is supported for 32 and 64-bit types only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28280, __PRETTY_FUNCTION__));
28281
28282 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&((Subtarget.hasAVX512() && !Subtarget.hasVLX() &&
!VT.is512BitVector() && "Cannot lower masked load op."
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && \"Cannot lower masked load op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28283, __PRETTY_FUNCTION__))
28283 "Cannot lower masked load op.")((Subtarget.hasAVX512() && !Subtarget.hasVLX() &&
!VT.is512BitVector() && "Cannot lower masked load op."
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && \"Cannot lower masked load op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28283, __PRETTY_FUNCTION__));
28284
28285 assert((ScalarVT.getSizeInBits() >= 32 ||(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked load op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked load op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28288, __PRETTY_FUNCTION__))
28286 (Subtarget.hasBWI() &&(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked load op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked load op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28288, __PRETTY_FUNCTION__))
28287 (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked load op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked load op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28288, __PRETTY_FUNCTION__))
28288 "Unsupported masked load op.")(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked load op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked load op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28288, __PRETTY_FUNCTION__));
28289
28290 // This operation is legal for targets with VLX, but without
28291 // VLX the vector should be widened to 512 bit
28292 unsigned NumEltsInWideVec = 512 / VT.getScalarSizeInBits();
28293 MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
28294 PassThru = ExtendToType(PassThru, WideDataVT, DAG);
28295
28296 // Mask element has to be i1.
28297 assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&((Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
"Unexpected mask type") ? static_cast<void> (0) : __assert_fail
("Mask.getSimpleValueType().getScalarType() == MVT::i1 && \"Unexpected mask type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28298, __PRETTY_FUNCTION__))
28298 "Unexpected mask type")((Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
"Unexpected mask type") ? static_cast<void> (0) : __assert_fail
("Mask.getSimpleValueType().getScalarType() == MVT::i1 && \"Unexpected mask type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28298, __PRETTY_FUNCTION__));
28299
28300 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
28301
28302 Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
28303 SDValue NewLoad = DAG.getMaskedLoad(
28304 WideDataVT, dl, N->getChain(), N->getBasePtr(), N->getOffset(), Mask,
28305 PassThru, N->getMemoryVT(), N->getMemOperand(), N->getAddressingMode(),
28306 N->getExtensionType(), N->isExpandingLoad());
28307
28308 SDValue Exract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT,
28309 NewLoad.getValue(0),
28310 DAG.getIntPtrConstant(0, dl));
28311 SDValue RetOps[] = {Exract, NewLoad.getValue(1)};
28312 return DAG.getMergeValues(RetOps, dl);
28313}
28314
28315static SDValue LowerMSTORE(SDValue Op, const X86Subtarget &Subtarget,
28316 SelectionDAG &DAG) {
28317 MaskedStoreSDNode *N = cast<MaskedStoreSDNode>(Op.getNode());
28318 SDValue DataToStore = N->getValue();
28319 MVT VT = DataToStore.getSimpleValueType();
28320 MVT ScalarVT = VT.getScalarType();
28321 SDValue Mask = N->getMask();
28322 SDLoc dl(Op);
28323
28324 assert((!N->isCompressingStore() || Subtarget.hasAVX512()) &&(((!N->isCompressingStore() || Subtarget.hasAVX512()) &&
"Expanding masked load is supported on AVX-512 target only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isCompressingStore() || Subtarget.hasAVX512()) && \"Expanding masked load is supported on AVX-512 target only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28325, __PRETTY_FUNCTION__))
28325 "Expanding masked load is supported on AVX-512 target only!")(((!N->isCompressingStore() || Subtarget.hasAVX512()) &&
"Expanding masked load is supported on AVX-512 target only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isCompressingStore() || Subtarget.hasAVX512()) && \"Expanding masked load is supported on AVX-512 target only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28325, __PRETTY_FUNCTION__));
28326
28327 assert((!N->isCompressingStore() || ScalarVT.getSizeInBits() >= 32) &&(((!N->isCompressingStore() || ScalarVT.getSizeInBits() >=
32) && "Expanding masked load is supported for 32 and 64-bit types only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isCompressingStore() || ScalarVT.getSizeInBits() >= 32) && \"Expanding masked load is supported for 32 and 64-bit types only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28328, __PRETTY_FUNCTION__))
28328 "Expanding masked load is supported for 32 and 64-bit types only!")(((!N->isCompressingStore() || ScalarVT.getSizeInBits() >=
32) && "Expanding masked load is supported for 32 and 64-bit types only!"
) ? static_cast<void> (0) : __assert_fail ("(!N->isCompressingStore() || ScalarVT.getSizeInBits() >= 32) && \"Expanding masked load is supported for 32 and 64-bit types only!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28328, __PRETTY_FUNCTION__));
28329
28330 assert(Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&((Subtarget.hasAVX512() && !Subtarget.hasVLX() &&
!VT.is512BitVector() && "Cannot lower masked store op."
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && \"Cannot lower masked store op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28331, __PRETTY_FUNCTION__))
28331 "Cannot lower masked store op.")((Subtarget.hasAVX512() && !Subtarget.hasVLX() &&
!VT.is512BitVector() && "Cannot lower masked store op."
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() && \"Cannot lower masked store op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28331, __PRETTY_FUNCTION__));
28332
28333 assert((ScalarVT.getSizeInBits() >= 32 ||(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked store op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked store op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28336, __PRETTY_FUNCTION__))
28334 (Subtarget.hasBWI() &&(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked store op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked store op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28336, __PRETTY_FUNCTION__))
28335 (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) &&(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked store op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked store op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28336, __PRETTY_FUNCTION__))
28336 "Unsupported masked store op.")(((ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() &&
(ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && "Unsupported masked store op."
) ? static_cast<void> (0) : __assert_fail ("(ScalarVT.getSizeInBits() >= 32 || (Subtarget.hasBWI() && (ScalarVT == MVT::i8 || ScalarVT == MVT::i16))) && \"Unsupported masked store op.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28336, __PRETTY_FUNCTION__));
28337
28338 // This operation is legal for targets with VLX, but without
28339 // VLX the vector should be widened to 512 bit
28340 unsigned NumEltsInWideVec = 512/VT.getScalarSizeInBits();
28341 MVT WideDataVT = MVT::getVectorVT(ScalarVT, NumEltsInWideVec);
28342
28343 // Mask element has to be i1.
28344 assert(Mask.getSimpleValueType().getScalarType() == MVT::i1 &&((Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
"Unexpected mask type") ? static_cast<void> (0) : __assert_fail
("Mask.getSimpleValueType().getScalarType() == MVT::i1 && \"Unexpected mask type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28345, __PRETTY_FUNCTION__))
28345 "Unexpected mask type")((Mask.getSimpleValueType().getScalarType() == MVT::i1 &&
"Unexpected mask type") ? static_cast<void> (0) : __assert_fail
("Mask.getSimpleValueType().getScalarType() == MVT::i1 && \"Unexpected mask type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28345, __PRETTY_FUNCTION__));
28346
28347 MVT WideMaskVT = MVT::getVectorVT(MVT::i1, NumEltsInWideVec);
28348
28349 DataToStore = ExtendToType(DataToStore, WideDataVT, DAG);
28350 Mask = ExtendToType(Mask, WideMaskVT, DAG, true);
28351 return DAG.getMaskedStore(N->getChain(), dl, DataToStore, N->getBasePtr(),
28352 N->getOffset(), Mask, N->getMemoryVT(),
28353 N->getMemOperand(), N->getAddressingMode(),
28354 N->isTruncatingStore(), N->isCompressingStore());
28355}
28356
28357static SDValue LowerMGATHER(SDValue Op, const X86Subtarget &Subtarget,
28358 SelectionDAG &DAG) {
28359 assert(Subtarget.hasAVX2() &&((Subtarget.hasAVX2() && "MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28360, __PRETTY_FUNCTION__))
28360 "MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only")((Subtarget.hasAVX2() && "MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX2() && \"MGATHER/MSCATTER are supported on AVX-512/AVX-2 arch only\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28360, __PRETTY_FUNCTION__));
28361
28362 MaskedGatherSDNode *N = cast<MaskedGatherSDNode>(Op.getNode());
28363 SDLoc dl(Op);
28364 MVT VT = Op.getSimpleValueType();
28365 SDValue Index = N->getIndex();
28366 SDValue Mask = N->getMask();
28367 SDValue PassThru = N->getPassThru();
28368 MVT IndexVT = Index.getSimpleValueType();
28369 MVT MaskVT = Mask.getSimpleValueType();
28370
28371 assert(VT.getScalarSizeInBits() >= 32 && "Unsupported gather op")((VT.getScalarSizeInBits() >= 32 && "Unsupported gather op"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarSizeInBits() >= 32 && \"Unsupported gather op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28371, __PRETTY_FUNCTION__));
28372
28373 // If the index is v2i32, we're being called by type legalization.
28374 if (IndexVT == MVT::v2i32)
28375 return SDValue();
28376
28377 // If we don't have VLX and neither the passthru or index is 512-bits, we
28378 // need to widen until one is.
28379 MVT OrigVT = VT;
28380 if (Subtarget.hasAVX512() && !Subtarget.hasVLX() && !VT.is512BitVector() &&
28381 !IndexVT.is512BitVector()) {
28382 // Determine how much we need to widen by to get a 512-bit type.
28383 unsigned Factor = std::min(512/VT.getSizeInBits(),
28384 512/IndexVT.getSizeInBits());
28385
28386 unsigned NumElts = VT.getVectorNumElements() * Factor;
28387
28388 VT = MVT::getVectorVT(VT.getVectorElementType(), NumElts);
28389 IndexVT = MVT::getVectorVT(IndexVT.getVectorElementType(), NumElts);
28390 MaskVT = MVT::getVectorVT(MVT::i1, NumElts);
28391
28392 PassThru = ExtendToType(PassThru, VT, DAG);
28393 Index = ExtendToType(Index, IndexVT, DAG);
28394 Mask = ExtendToType(Mask, MaskVT, DAG, true);
28395 }
28396
28397 SDValue Ops[] = { N->getChain(), PassThru, Mask, N->getBasePtr(), Index,
28398 N->getScale() };
28399 SDValue NewGather = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
28400 DAG.getVTList(VT, MaskVT, MVT::Other), Ops, dl, N->getMemoryVT(),
28401 N->getMemOperand());
28402 SDValue Extract = DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, OrigVT,
28403 NewGather, DAG.getIntPtrConstant(0, dl));
28404 return DAG.getMergeValues({Extract, NewGather.getValue(2)}, dl);
28405}
28406
28407static SDValue LowerADDRSPACECAST(SDValue Op, SelectionDAG &DAG) {
28408 SDLoc dl(Op);
28409 SDValue Src = Op.getOperand(0);
28410 MVT DstVT = Op.getSimpleValueType();
28411
28412 AddrSpaceCastSDNode *N = cast<AddrSpaceCastSDNode>(Op.getNode());
28413 unsigned SrcAS = N->getSrcAddressSpace();
28414
28415 assert(SrcAS != N->getDestAddressSpace() &&((SrcAS != N->getDestAddressSpace() && "addrspacecast must be between different address spaces"
) ? static_cast<void> (0) : __assert_fail ("SrcAS != N->getDestAddressSpace() && \"addrspacecast must be between different address spaces\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28416, __PRETTY_FUNCTION__))
28416 "addrspacecast must be between different address spaces")((SrcAS != N->getDestAddressSpace() && "addrspacecast must be between different address spaces"
) ? static_cast<void> (0) : __assert_fail ("SrcAS != N->getDestAddressSpace() && \"addrspacecast must be between different address spaces\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28416, __PRETTY_FUNCTION__));
28417
28418 if (SrcAS == X86AS::PTR32_UPTR && DstVT == MVT::i64) {
28419 Op = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src);
28420 } else if (DstVT == MVT::i64) {
28421 Op = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src);
28422 } else if (DstVT == MVT::i32) {
28423 Op = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src);
28424 } else {
28425 report_fatal_error("Bad address space in addrspacecast");
28426 }
28427 return Op;
28428}
28429
28430SDValue X86TargetLowering::LowerGC_TRANSITION(SDValue Op,
28431 SelectionDAG &DAG) const {
28432 // TODO: Eventually, the lowering of these nodes should be informed by or
28433 // deferred to the GC strategy for the function in which they appear. For
28434 // now, however, they must be lowered to something. Since they are logically
28435 // no-ops in the case of a null GC strategy (or a GC strategy which does not
28436 // require special handling for these nodes), lower them as literal NOOPs for
28437 // the time being.
28438 SmallVector<SDValue, 2> Ops;
28439
28440 Ops.push_back(Op.getOperand(0));
28441 if (Op->getGluedNode())
28442 Ops.push_back(Op->getOperand(Op->getNumOperands() - 1));
28443
28444 SDLoc OpDL(Op);
28445 SDVTList VTs = DAG.getVTList(MVT::Other, MVT::Glue);
28446 SDValue NOOP(DAG.getMachineNode(X86::NOOP, SDLoc(Op), VTs, Ops), 0);
28447
28448 return NOOP;
28449}
28450
28451SDValue X86TargetLowering::LowerF128Call(SDValue Op, SelectionDAG &DAG,
28452 RTLIB::Libcall Call) const {
28453
28454 bool IsStrict = Op->isStrictFPOpcode();
28455 unsigned Offset = IsStrict ? 1 : 0;
28456 SmallVector<SDValue, 2> Ops(Op->op_begin() + Offset, Op->op_end());
28457
28458 SDLoc dl(Op);
28459 SDValue Chain = IsStrict ? Op.getOperand(0) : SDValue();
28460 MakeLibCallOptions CallOptions;
28461 std::pair<SDValue, SDValue> Tmp = makeLibCall(DAG, Call, MVT::f128, Ops,
28462 CallOptions, dl, Chain);
28463
28464 if (IsStrict)
28465 return DAG.getMergeValues({ Tmp.first, Tmp.second }, dl);
28466
28467 return Tmp.first;
28468}
28469
28470/// Provide custom lowering hooks for some operations.
28471SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
28472 switch (Op.getOpcode()) {
28473 default: llvm_unreachable("Should not custom lower this!")::llvm::llvm_unreachable_internal("Should not custom lower this!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28473);
28474 case ISD::ATOMIC_FENCE: return LowerATOMIC_FENCE(Op, Subtarget, DAG);
28475 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
28476 return LowerCMP_SWAP(Op, Subtarget, DAG);
28477 case ISD::CTPOP: return LowerCTPOP(Op, Subtarget, DAG);
28478 case ISD::ATOMIC_LOAD_ADD:
28479 case ISD::ATOMIC_LOAD_SUB:
28480 case ISD::ATOMIC_LOAD_OR:
28481 case ISD::ATOMIC_LOAD_XOR:
28482 case ISD::ATOMIC_LOAD_AND: return lowerAtomicArith(Op, DAG, Subtarget);
28483 case ISD::ATOMIC_STORE: return LowerATOMIC_STORE(Op, DAG, Subtarget);
28484 case ISD::BITREVERSE: return LowerBITREVERSE(Op, Subtarget, DAG);
28485 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
28486 case ISD::CONCAT_VECTORS: return LowerCONCAT_VECTORS(Op, Subtarget, DAG);
28487 case ISD::VECTOR_SHUFFLE: return lowerVECTOR_SHUFFLE(Op, Subtarget, DAG);
28488 case ISD::VSELECT: return LowerVSELECT(Op, DAG);
28489 case ISD::EXTRACT_VECTOR_ELT: return LowerEXTRACT_VECTOR_ELT(Op, DAG);
28490 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
28491 case ISD::INSERT_SUBVECTOR: return LowerINSERT_SUBVECTOR(Op, Subtarget,DAG);
28492 case ISD::EXTRACT_SUBVECTOR: return LowerEXTRACT_SUBVECTOR(Op,Subtarget,DAG);
28493 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, Subtarget,DAG);
28494 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
28495 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
28496 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
28497 case ISD::ExternalSymbol: return LowerExternalSymbol(Op, DAG);
28498 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
28499 case ISD::SHL_PARTS:
28500 case ISD::SRA_PARTS:
28501 case ISD::SRL_PARTS: return LowerShiftParts(Op, DAG);
28502 case ISD::FSHL:
28503 case ISD::FSHR: return LowerFunnelShift(Op, Subtarget, DAG);
28504 case ISD::STRICT_SINT_TO_FP:
28505 case ISD::SINT_TO_FP: return LowerSINT_TO_FP(Op, DAG);
28506 case ISD::STRICT_UINT_TO_FP:
28507 case ISD::UINT_TO_FP: return LowerUINT_TO_FP(Op, DAG);
28508 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
28509 case ISD::ZERO_EXTEND: return LowerZERO_EXTEND(Op, Subtarget, DAG);
28510 case ISD::SIGN_EXTEND: return LowerSIGN_EXTEND(Op, Subtarget, DAG);
28511 case ISD::ANY_EXTEND: return LowerANY_EXTEND(Op, Subtarget, DAG);
28512 case ISD::ZERO_EXTEND_VECTOR_INREG:
28513 case ISD::SIGN_EXTEND_VECTOR_INREG:
28514 return LowerEXTEND_VECTOR_INREG(Op, Subtarget, DAG);
28515 case ISD::FP_TO_SINT:
28516 case ISD::STRICT_FP_TO_SINT:
28517 case ISD::FP_TO_UINT:
28518 case ISD::STRICT_FP_TO_UINT: return LowerFP_TO_INT(Op, DAG);
28519 case ISD::FP_EXTEND:
28520 case ISD::STRICT_FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
28521 case ISD::FP_ROUND:
28522 case ISD::STRICT_FP_ROUND: return LowerFP_ROUND(Op, DAG);
28523 case ISD::LOAD: return LowerLoad(Op, Subtarget, DAG);
28524 case ISD::STORE: return LowerStore(Op, Subtarget, DAG);
28525 case ISD::FADD:
28526 case ISD::FSUB: return lowerFaddFsub(Op, DAG);
28527 case ISD::FABS:
28528 case ISD::FNEG: return LowerFABSorFNEG(Op, DAG);
28529 case ISD::FCOPYSIGN: return LowerFCOPYSIGN(Op, DAG);
28530 case ISD::FGETSIGN: return LowerFGETSIGN(Op, DAG);
28531 case ISD::SETCC:
28532 case ISD::STRICT_FSETCC:
28533 case ISD::STRICT_FSETCCS: return LowerSETCC(Op, DAG);
28534 case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
28535 case ISD::SELECT: return LowerSELECT(Op, DAG);
28536 case ISD::BRCOND: return LowerBRCOND(Op, DAG);
28537 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
28538 case ISD::VASTART: return LowerVASTART(Op, DAG);
28539 case ISD::VAARG: return LowerVAARG(Op, DAG);
28540 case ISD::VACOPY: return LowerVACOPY(Op, Subtarget, DAG);
28541 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
28542 case ISD::INTRINSIC_VOID:
28543 case ISD::INTRINSIC_W_CHAIN: return LowerINTRINSIC_W_CHAIN(Op, Subtarget, DAG);
28544 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
28545 case ISD::ADDROFRETURNADDR: return LowerADDROFRETURNADDR(Op, DAG);
28546 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
28547 case ISD::FRAME_TO_ARGS_OFFSET:
28548 return LowerFRAME_TO_ARGS_OFFSET(Op, DAG);
28549 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
28550 case ISD::EH_RETURN: return LowerEH_RETURN(Op, DAG);
28551 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
28552 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
28553 case ISD::EH_SJLJ_SETUP_DISPATCH:
28554 return lowerEH_SJLJ_SETUP_DISPATCH(Op, DAG);
28555 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
28556 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
28557 case ISD::FLT_ROUNDS_: return LowerFLT_ROUNDS_(Op, DAG);
28558 case ISD::CTLZ:
28559 case ISD::CTLZ_ZERO_UNDEF: return LowerCTLZ(Op, Subtarget, DAG);
28560 case ISD::CTTZ:
28561 case ISD::CTTZ_ZERO_UNDEF: return LowerCTTZ(Op, Subtarget, DAG);
28562 case ISD::MUL: return LowerMUL(Op, Subtarget, DAG);
28563 case ISD::MULHS:
28564 case ISD::MULHU: return LowerMULH(Op, Subtarget, DAG);
28565 case ISD::ROTL:
28566 case ISD::ROTR: return LowerRotate(Op, Subtarget, DAG);
28567 case ISD::SRA:
28568 case ISD::SRL:
28569 case ISD::SHL: return LowerShift(Op, Subtarget, DAG);
28570 case ISD::SADDO:
28571 case ISD::UADDO:
28572 case ISD::SSUBO:
28573 case ISD::USUBO:
28574 case ISD::SMULO:
28575 case ISD::UMULO: return LowerXALUO(Op, DAG);
28576 case ISD::READCYCLECOUNTER: return LowerREADCYCLECOUNTER(Op, Subtarget,DAG);
28577 case ISD::BITCAST: return LowerBITCAST(Op, Subtarget, DAG);
28578 case ISD::ADDCARRY:
28579 case ISD::SUBCARRY: return LowerADDSUBCARRY(Op, DAG);
28580 case ISD::ADD:
28581 case ISD::SUB: return lowerAddSub(Op, DAG, Subtarget);
28582 case ISD::UADDSAT:
28583 case ISD::SADDSAT:
28584 case ISD::USUBSAT:
28585 case ISD::SSUBSAT: return LowerADDSAT_SUBSAT(Op, DAG, Subtarget);
28586 case ISD::SMAX:
28587 case ISD::SMIN:
28588 case ISD::UMAX:
28589 case ISD::UMIN: return LowerMINMAX(Op, DAG);
28590 case ISD::ABS: return LowerABS(Op, Subtarget, DAG);
28591 case ISD::FSINCOS: return LowerFSINCOS(Op, Subtarget, DAG);
28592 case ISD::MLOAD: return LowerMLOAD(Op, Subtarget, DAG);
28593 case ISD::MSTORE: return LowerMSTORE(Op, Subtarget, DAG);
28594 case ISD::MGATHER: return LowerMGATHER(Op, Subtarget, DAG);
28595 case ISD::MSCATTER: return LowerMSCATTER(Op, Subtarget, DAG);
28596 case ISD::GC_TRANSITION_START:
28597 case ISD::GC_TRANSITION_END: return LowerGC_TRANSITION(Op, DAG);
28598 case ISD::ADDRSPACECAST:
28599 return LowerADDRSPACECAST(Op, DAG);
28600 }
28601}
28602
28603/// Places new result values for the node in Results (their number
28604/// and types must exactly match those of the original return values of
28605/// the node), or leaves Results empty, which indicates that the node is not
28606/// to be custom lowered after all.
28607void X86TargetLowering::LowerOperationWrapper(SDNode *N,
28608 SmallVectorImpl<SDValue> &Results,
28609 SelectionDAG &DAG) const {
28610 SDValue Res = LowerOperation(SDValue(N, 0), DAG);
28611
28612 if (!Res.getNode())
28613 return;
28614
28615 // If the original node has one result, take the return value from
28616 // LowerOperation as is. It might not be result number 0.
28617 if (N->getNumValues() == 1) {
28618 Results.push_back(Res);
28619 return;
28620 }
28621
28622 // If the original node has multiple results, then the return node should
28623 // have the same number of results.
28624 assert((N->getNumValues() == Res->getNumValues()) &&(((N->getNumValues() == Res->getNumValues()) &&
"Lowering returned the wrong number of results!") ? static_cast
<void> (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28625, __PRETTY_FUNCTION__))
28625 "Lowering returned the wrong number of results!")(((N->getNumValues() == Res->getNumValues()) &&
"Lowering returned the wrong number of results!") ? static_cast
<void> (0) : __assert_fail ("(N->getNumValues() == Res->getNumValues()) && \"Lowering returned the wrong number of results!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28625, __PRETTY_FUNCTION__));
28626
28627 // Places new result values base on N result number.
28628 for (unsigned I = 0, E = N->getNumValues(); I != E; ++I)
28629 Results.push_back(Res.getValue(I));
28630}
28631
28632/// Replace a node with an illegal result type with a new node built out of
28633/// custom code.
28634void X86TargetLowering::ReplaceNodeResults(SDNode *N,
28635 SmallVectorImpl<SDValue>&Results,
28636 SelectionDAG &DAG) const {
28637 SDLoc dl(N);
28638 switch (N->getOpcode()) {
28639 default:
28640#ifndef NDEBUG
28641 dbgs() << "ReplaceNodeResults: ";
28642 N->dump(&DAG);
28643#endif
28644 llvm_unreachable("Do not know how to custom type legalize this operation!")::llvm::llvm_unreachable_internal("Do not know how to custom type legalize this operation!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28644);
28645 case ISD::CTPOP: {
28646 assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!")((N->getValueType(0) == MVT::i64 && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28646, __PRETTY_FUNCTION__));
28647 // Use a v2i64 if possible.
28648 bool NoImplicitFloatOps =
28649 DAG.getMachineFunction().getFunction().hasFnAttribute(
28650 Attribute::NoImplicitFloat);
28651 if (isTypeLegal(MVT::v2i64) && !NoImplicitFloatOps) {
28652 SDValue Wide =
28653 DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64, N->getOperand(0));
28654 Wide = DAG.getNode(ISD::CTPOP, dl, MVT::v2i64, Wide);
28655 // Bit count should fit in 32-bits, extract it as that and then zero
28656 // extend to i64. Otherwise we end up extracting bits 63:32 separately.
28657 Wide = DAG.getNode(ISD::BITCAST, dl, MVT::v4i32, Wide);
28658 Wide = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i32, Wide,
28659 DAG.getIntPtrConstant(0, dl));
28660 Wide = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64, Wide);
28661 Results.push_back(Wide);
28662 }
28663 return;
28664 }
28665 case ISD::MUL: {
28666 EVT VT = N->getValueType(0);
28667 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
VT.getVectorElementType() == MVT::i8 && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && VT.getVectorElementType() == MVT::i8 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28668, __PRETTY_FUNCTION__))
28668 VT.getVectorElementType() == MVT::i8 && "Unexpected VT!")((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
VT.getVectorElementType() == MVT::i8 && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && VT.getVectorElementType() == MVT::i8 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28668, __PRETTY_FUNCTION__));
28669 // Pre-promote these to vXi16 to avoid op legalization thinking all 16
28670 // elements are needed.
28671 MVT MulVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements());
28672 SDValue Op0 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(0));
28673 SDValue Op1 = DAG.getNode(ISD::ANY_EXTEND, dl, MulVT, N->getOperand(1));
28674 SDValue Res = DAG.getNode(ISD::MUL, dl, MulVT, Op0, Op1);
28675 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
28676 unsigned NumConcats = 16 / VT.getVectorNumElements();
28677 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
28678 ConcatOps[0] = Res;
28679 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i8, ConcatOps);
28680 Results.push_back(Res);
28681 return;
28682 }
28683 case X86ISD::VPMADDWD:
28684 case X86ISD::AVG: {
28685 // Legalize types for ISD::UADDSAT/SADDSAT/USUBSAT/SSUBSAT and
28686 // X86ISD::AVG/VPMADDWD by widening.
28687 assert(Subtarget.hasSSE2() && "Requires at least SSE2!")((Subtarget.hasSSE2() && "Requires at least SSE2!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Requires at least SSE2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28687, __PRETTY_FUNCTION__));
28688
28689 EVT VT = N->getValueType(0);
28690 EVT InVT = N->getOperand(0).getValueType();
28691 assert(VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 &&((VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits
() == 0 && "Expected a VT that divides into 128 bits."
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 && \"Expected a VT that divides into 128 bits.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28692, __PRETTY_FUNCTION__))
28692 "Expected a VT that divides into 128 bits.")((VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits
() == 0 && "Expected a VT that divides into 128 bits."
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() < 128 && 128 % VT.getSizeInBits() == 0 && \"Expected a VT that divides into 128 bits.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28692, __PRETTY_FUNCTION__));
28693 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28694, __PRETTY_FUNCTION__))
28694 "Unexpected type action!")((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28694, __PRETTY_FUNCTION__));
28695 unsigned NumConcat = 128 / InVT.getSizeInBits();
28696
28697 EVT InWideVT = EVT::getVectorVT(*DAG.getContext(),
28698 InVT.getVectorElementType(),
28699 NumConcat * InVT.getVectorNumElements());
28700 EVT WideVT = EVT::getVectorVT(*DAG.getContext(),
28701 VT.getVectorElementType(),
28702 NumConcat * VT.getVectorNumElements());
28703
28704 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getUNDEF(InVT));
28705 Ops[0] = N->getOperand(0);
28706 SDValue InVec0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
28707 Ops[0] = N->getOperand(1);
28708 SDValue InVec1 = DAG.getNode(ISD::CONCAT_VECTORS, dl, InWideVT, Ops);
28709
28710 SDValue Res = DAG.getNode(N->getOpcode(), dl, WideVT, InVec0, InVec1);
28711 Results.push_back(Res);
28712 return;
28713 }
28714 case ISD::ABS: {
28715 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
28716 assert(N->getValueType(0) == MVT::i64 &&((N->getValueType(0) == MVT::i64 && "Unexpected type (!= i64) on ABS."
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"Unexpected type (!= i64) on ABS.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28717, __PRETTY_FUNCTION__))
28717 "Unexpected type (!= i64) on ABS.")((N->getValueType(0) == MVT::i64 && "Unexpected type (!= i64) on ABS."
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"Unexpected type (!= i64) on ABS.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28717, __PRETTY_FUNCTION__));
28718 MVT HalfT = MVT::i32;
28719 SDValue Lo, Hi, Tmp;
28720 SDVTList VTList = DAG.getVTList(HalfT, MVT::i1);
28721
28722 Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
28723 DAG.getConstant(0, dl, HalfT));
28724 Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(0),
28725 DAG.getConstant(1, dl, HalfT));
28726 Tmp = DAG.getNode(
28727 ISD::SRA, dl, HalfT, Hi,
28728 DAG.getConstant(HalfT.getSizeInBits() - 1, dl,
28729 TLI.getShiftAmountTy(HalfT, DAG.getDataLayout())));
28730 Lo = DAG.getNode(ISD::UADDO, dl, VTList, Tmp, Lo);
28731 Hi = DAG.getNode(ISD::ADDCARRY, dl, VTList, Tmp, Hi,
28732 SDValue(Lo.getNode(), 1));
28733 Hi = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Hi);
28734 Lo = DAG.getNode(ISD::XOR, dl, HalfT, Tmp, Lo);
28735 Results.push_back(Lo);
28736 Results.push_back(Hi);
28737 return;
28738 }
28739 // We might have generated v2f32 FMIN/FMAX operations. Widen them to v4f32.
28740 case X86ISD::FMINC:
28741 case X86ISD::FMIN:
28742 case X86ISD::FMAXC:
28743 case X86ISD::FMAX: {
28744 EVT VT = N->getValueType(0);
28745 assert(VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX.")((VT == MVT::v2f32 && "Unexpected type (!= v2f32) on FMIN/FMAX."
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::v2f32 && \"Unexpected type (!= v2f32) on FMIN/FMAX.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28745, __PRETTY_FUNCTION__));
28746 SDValue UNDEF = DAG.getUNDEF(VT);
28747 SDValue LHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
28748 N->getOperand(0), UNDEF);
28749 SDValue RHS = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32,
28750 N->getOperand(1), UNDEF);
28751 Results.push_back(DAG.getNode(N->getOpcode(), dl, MVT::v4f32, LHS, RHS));
28752 return;
28753 }
28754 case ISD::SDIV:
28755 case ISD::UDIV:
28756 case ISD::SREM:
28757 case ISD::UREM: {
28758 EVT VT = N->getValueType(0);
28759 if (VT.isVector()) {
28760 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28761, __PRETTY_FUNCTION__))
28761 "Unexpected type action!")((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28761, __PRETTY_FUNCTION__));
28762 // If this RHS is a constant splat vector we can widen this and let
28763 // division/remainder by constant optimize it.
28764 // TODO: Can we do something for non-splat?
28765 APInt SplatVal;
28766 if (ISD::isConstantSplatVector(N->getOperand(1).getNode(), SplatVal)) {
28767 unsigned NumConcats = 128 / VT.getSizeInBits();
28768 SmallVector<SDValue, 8> Ops0(NumConcats, DAG.getUNDEF(VT));
28769 Ops0[0] = N->getOperand(0);
28770 EVT ResVT = getTypeToTransformTo(*DAG.getContext(), VT);
28771 SDValue N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, ResVT, Ops0);
28772 SDValue N1 = DAG.getConstant(SplatVal, dl, ResVT);
28773 SDValue Res = DAG.getNode(N->getOpcode(), dl, ResVT, N0, N1);
28774 Results.push_back(Res);
28775 }
28776 return;
28777 }
28778
28779 LLVM_FALLTHROUGH[[gnu::fallthrough]];
28780 }
28781 case ISD::SDIVREM:
28782 case ISD::UDIVREM: {
28783 SDValue V = LowerWin64_i128OP(SDValue(N,0), DAG);
28784 Results.push_back(V);
28785 return;
28786 }
28787 case ISD::TRUNCATE: {
28788 MVT VT = N->getSimpleValueType(0);
28789 if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
28790 return;
28791
28792 // The generic legalizer will try to widen the input type to the same
28793 // number of elements as the widened result type. But this isn't always
28794 // the best thing so do some custom legalization to avoid some cases.
28795 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();
28796 SDValue In = N->getOperand(0);
28797 EVT InVT = In.getValueType();
28798
28799 unsigned InBits = InVT.getSizeInBits();
28800 if (128 % InBits == 0) {
28801 // 128 bit and smaller inputs should avoid truncate all together and
28802 // just use a build_vector that will become a shuffle.
28803 // TODO: Widen and use a shuffle directly?
28804 MVT InEltVT = InVT.getSimpleVT().getVectorElementType();
28805 EVT EltVT = VT.getVectorElementType();
28806 unsigned WidenNumElts = WidenVT.getVectorNumElements();
28807 SmallVector<SDValue, 16> Ops(WidenNumElts, DAG.getUNDEF(EltVT));
28808 // Use the original element count so we don't do more scalar opts than
28809 // necessary.
28810 unsigned MinElts = VT.getVectorNumElements();
28811 for (unsigned i=0; i < MinElts; ++i) {
28812 SDValue Val = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, InEltVT, In,
28813 DAG.getIntPtrConstant(i, dl));
28814 Ops[i] = DAG.getNode(ISD::TRUNCATE, dl, EltVT, Val);
28815 }
28816 Results.push_back(DAG.getBuildVector(WidenVT, dl, Ops));
28817 return;
28818 }
28819 // With AVX512 there are some cases that can use a target specific
28820 // truncate node to go from 256/512 to less than 128 with zeros in the
28821 // upper elements of the 128 bit result.
28822 if (Subtarget.hasAVX512() && isTypeLegal(InVT)) {
28823 // We can use VTRUNC directly if for 256 bits with VLX or for any 512.
28824 if ((InBits == 256 && Subtarget.hasVLX()) || InBits == 512) {
28825 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
28826 return;
28827 }
28828 // There's one case we can widen to 512 bits and use VTRUNC.
28829 if (InVT == MVT::v4i64 && VT == MVT::v4i8 && isTypeLegal(MVT::v8i64)) {
28830 In = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i64, In,
28831 DAG.getUNDEF(MVT::v4i64));
28832 Results.push_back(DAG.getNode(X86ISD::VTRUNC, dl, WidenVT, In));
28833 return;
28834 }
28835 }
28836 if (Subtarget.hasVLX() && InVT == MVT::v8i64 && VT == MVT::v8i8 &&
28837 getTypeAction(*DAG.getContext(), InVT) == TypeSplitVector &&
28838 isTypeLegal(MVT::v4i64)) {
28839 // Input needs to be split and output needs to widened. Let's use two
28840 // VTRUNCs, and shuffle their results together into the wider type.
28841 SDValue Lo, Hi;
28842 std::tie(Lo, Hi) = DAG.SplitVector(In, dl);
28843
28844 Lo = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Lo);
28845 Hi = DAG.getNode(X86ISD::VTRUNC, dl, MVT::v16i8, Hi);
28846 SDValue Res = DAG.getVectorShuffle(MVT::v16i8, dl, Lo, Hi,
28847 { 0, 1, 2, 3, 16, 17, 18, 19,
28848 -1, -1, -1, -1, -1, -1, -1, -1 });
28849 Results.push_back(Res);
28850 return;
28851 }
28852
28853 return;
28854 }
28855 case ISD::ANY_EXTEND:
28856 // Right now, only MVT::v8i8 has Custom action for an illegal type.
28857 // It's intended to custom handle the input type.
28858 assert(N->getValueType(0) == MVT::v8i8 &&((N->getValueType(0) == MVT::v8i8 && "Do not know how to legalize this Node"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::v8i8 && \"Do not know how to legalize this Node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28859, __PRETTY_FUNCTION__))
28859 "Do not know how to legalize this Node")((N->getValueType(0) == MVT::v8i8 && "Do not know how to legalize this Node"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::v8i8 && \"Do not know how to legalize this Node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28859, __PRETTY_FUNCTION__));
28860 return;
28861 case ISD::SIGN_EXTEND:
28862 case ISD::ZERO_EXTEND: {
28863 EVT VT = N->getValueType(0);
28864 SDValue In = N->getOperand(0);
28865 EVT InVT = In.getValueType();
28866 if (!Subtarget.hasSSE41() && VT == MVT::v4i64 &&
28867 (InVT == MVT::v4i16 || InVT == MVT::v4i8)){
28868 assert(getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28869, __PRETTY_FUNCTION__))
28869 "Unexpected type action!")((getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), InVT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28869, __PRETTY_FUNCTION__));
28870 assert(N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode")((N->getOpcode() == ISD::SIGN_EXTEND && "Unexpected opcode"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28870, __PRETTY_FUNCTION__));
28871 // Custom split this so we can extend i8/i16->i32 invec. This is better
28872 // since sign_extend_inreg i8/i16->i64 requires an extend to i32 using
28873 // sra. Then extending from i32 to i64 using pcmpgt. By custom splitting
28874 // we allow the sra from the extend to i32 to be shared by the split.
28875 In = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, In);
28876
28877 // Fill a vector with sign bits for each element.
28878 SDValue Zero = DAG.getConstant(0, dl, MVT::v4i32);
28879 SDValue SignBits = DAG.getSetCC(dl, MVT::v4i32, Zero, In, ISD::SETGT);
28880
28881 // Create an unpackl and unpackh to interleave the sign bits then bitcast
28882 // to v2i64.
28883 SDValue Lo = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
28884 {0, 4, 1, 5});
28885 Lo = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Lo);
28886 SDValue Hi = DAG.getVectorShuffle(MVT::v4i32, dl, In, SignBits,
28887 {2, 6, 3, 7});
28888 Hi = DAG.getNode(ISD::BITCAST, dl, MVT::v2i64, Hi);
28889
28890 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
28891 Results.push_back(Res);
28892 return;
28893 }
28894
28895 if (VT == MVT::v16i32 || VT == MVT::v8i64) {
28896 if (!InVT.is128BitVector()) {
28897 // Not a 128 bit vector, but maybe type legalization will promote
28898 // it to 128 bits.
28899 if (getTypeAction(*DAG.getContext(), InVT) != TypePromoteInteger)
28900 return;
28901 InVT = getTypeToTransformTo(*DAG.getContext(), InVT);
28902 if (!InVT.is128BitVector())
28903 return;
28904
28905 // Promote the input to 128 bits. Type legalization will turn this into
28906 // zext_inreg/sext_inreg.
28907 In = DAG.getNode(N->getOpcode(), dl, InVT, In);
28908 }
28909
28910 // Perform custom splitting instead of the two stage extend we would get
28911 // by default.
28912 EVT LoVT, HiVT;
28913 std::tie(LoVT, HiVT) = DAG.GetSplitDestVTs(N->getValueType(0));
28914 assert(isTypeLegal(LoVT) && "Split VT not legal?")((isTypeLegal(LoVT) && "Split VT not legal?") ? static_cast
<void> (0) : __assert_fail ("isTypeLegal(LoVT) && \"Split VT not legal?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28914, __PRETTY_FUNCTION__));
28915
28916 SDValue Lo = getExtendInVec(N->getOpcode(), dl, LoVT, In, DAG);
28917
28918 // We need to shift the input over by half the number of elements.
28919 unsigned NumElts = InVT.getVectorNumElements();
28920 unsigned HalfNumElts = NumElts / 2;
28921 SmallVector<int, 16> ShufMask(NumElts, SM_SentinelUndef);
28922 for (unsigned i = 0; i != HalfNumElts; ++i)
28923 ShufMask[i] = i + HalfNumElts;
28924
28925 SDValue Hi = DAG.getVectorShuffle(InVT, dl, In, In, ShufMask);
28926 Hi = getExtendInVec(N->getOpcode(), dl, HiVT, Hi, DAG);
28927
28928 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, VT, Lo, Hi);
28929 Results.push_back(Res);
28930 }
28931 return;
28932 }
28933 case ISD::FP_TO_SINT:
28934 case ISD::STRICT_FP_TO_SINT:
28935 case ISD::FP_TO_UINT:
28936 case ISD::STRICT_FP_TO_UINT: {
28937 bool IsStrict = N->isStrictFPOpcode();
28938 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
28939 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
28940 EVT VT = N->getValueType(0);
28941 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
28942 EVT SrcVT = Src.getValueType();
28943
28944 if (VT.isVector() && VT.getScalarSizeInBits() < 32) {
28945 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28946, __PRETTY_FUNCTION__))
28946 "Unexpected type action!")((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28946, __PRETTY_FUNCTION__));
28947
28948 // Try to create a 128 bit vector, but don't exceed a 32 bit element.
28949 unsigned NewEltWidth = std::min(128 / VT.getVectorNumElements(), 32U);
28950 MVT PromoteVT = MVT::getVectorVT(MVT::getIntegerVT(NewEltWidth),
28951 VT.getVectorNumElements());
28952 SDValue Res;
28953 SDValue Chain;
28954 if (IsStrict) {
28955 Res = DAG.getNode(ISD::STRICT_FP_TO_SINT, dl, {PromoteVT, MVT::Other},
28956 {N->getOperand(0), Src});
28957 Chain = Res.getValue(1);
28958 } else
28959 Res = DAG.getNode(ISD::FP_TO_SINT, dl, PromoteVT, Src);
28960
28961 // Preserve what we know about the size of the original result. Except
28962 // when the result is v2i32 since we can't widen the assert.
28963 if (PromoteVT != MVT::v2i32)
28964 Res = DAG.getNode(!IsSigned ? ISD::AssertZext : ISD::AssertSext,
28965 dl, PromoteVT, Res,
28966 DAG.getValueType(VT.getVectorElementType()));
28967
28968 // Truncate back to the original width.
28969 Res = DAG.getNode(ISD::TRUNCATE, dl, VT, Res);
28970
28971 // Now widen to 128 bits.
28972 unsigned NumConcats = 128 / VT.getSizeInBits();
28973 MVT ConcatVT = MVT::getVectorVT(VT.getSimpleVT().getVectorElementType(),
28974 VT.getVectorNumElements() * NumConcats);
28975 SmallVector<SDValue, 8> ConcatOps(NumConcats, DAG.getUNDEF(VT));
28976 ConcatOps[0] = Res;
28977 Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, ConcatVT, ConcatOps);
28978 Results.push_back(Res);
28979 if (IsStrict)
28980 Results.push_back(Chain);
28981 return;
28982 }
28983
28984
28985 if (VT == MVT::v2i32) {
28986 assert((IsSigned || Subtarget.hasAVX512()) &&(((IsSigned || Subtarget.hasAVX512()) && "Can only handle signed conversion without AVX512"
) ? static_cast<void> (0) : __assert_fail ("(IsSigned || Subtarget.hasAVX512()) && \"Can only handle signed conversion without AVX512\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28987, __PRETTY_FUNCTION__))
28987 "Can only handle signed conversion without AVX512")(((IsSigned || Subtarget.hasAVX512()) && "Can only handle signed conversion without AVX512"
) ? static_cast<void> (0) : __assert_fail ("(IsSigned || Subtarget.hasAVX512()) && \"Can only handle signed conversion without AVX512\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28987, __PRETTY_FUNCTION__));
28988 assert(Subtarget.hasSSE2() && "Requires at least SSE2!")((Subtarget.hasSSE2() && "Requires at least SSE2!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Requires at least SSE2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28988, __PRETTY_FUNCTION__));
28989 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28990, __PRETTY_FUNCTION__))
28990 "Unexpected type action!")((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 28990, __PRETTY_FUNCTION__));
28991 if (Src.getValueType() == MVT::v2f64) {
28992 unsigned Opc;
28993 if (IsStrict)
28994 Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
28995 else
28996 Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
28997
28998 // If we have VLX we can emit a target specific FP_TO_UINT node,.
28999 if (!IsSigned && !Subtarget.hasVLX()) {
29000 // Otherwise we can defer to the generic legalizer which will widen
29001 // the input as well. This will be further widened during op
29002 // legalization to v8i32<-v8f64.
29003 // For strict nodes we'll need to widen ourselves.
29004 // FIXME: Fix the type legalizer to safely widen strict nodes?
29005 if (!IsStrict)
29006 return;
29007 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f64, Src,
29008 DAG.getConstantFP(0.0, dl, MVT::v2f64));
29009 Opc = N->getOpcode();
29010 }
29011 SDValue Res;
29012 SDValue Chain;
29013 if (IsStrict) {
29014 Res = DAG.getNode(Opc, dl, {MVT::v4i32, MVT::Other},
29015 {N->getOperand(0), Src});
29016 Chain = Res.getValue(1);
29017 } else {
29018 Res = DAG.getNode(Opc, dl, MVT::v4i32, Src);
29019 }
29020 Results.push_back(Res);
29021 if (IsStrict)
29022 Results.push_back(Chain);
29023 return;
29024 }
29025
29026 // Custom widen strict v2f32->v2i32 by padding with zeros.
29027 // FIXME: Should generic type legalizer do this?
29028 if (Src.getValueType() == MVT::v2f32 && IsStrict) {
29029 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4f32, Src,
29030 DAG.getConstantFP(0.0, dl, MVT::v2f32));
29031 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4i32, MVT::Other},
29032 {N->getOperand(0), Src});
29033 Results.push_back(Res);
29034 Results.push_back(Res.getValue(1));
29035 return;
29036 }
29037
29038 // The FP_TO_INTHelper below only handles f32/f64/f80 scalar inputs,
29039 // so early out here.
29040 return;
29041 }
29042
29043 assert(!VT.isVector() && "Vectors should have been handled above!")((!VT.isVector() && "Vectors should have been handled above!"
) ? static_cast<void> (0) : __assert_fail ("!VT.isVector() && \"Vectors should have been handled above!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29043, __PRETTY_FUNCTION__));
29044
29045 if (Subtarget.hasDQI() && VT == MVT::i64 &&
29046 (SrcVT == MVT::f32 || SrcVT == MVT::f64)) {
29047 assert(!Subtarget.is64Bit() && "i64 should be legal")((!Subtarget.is64Bit() && "i64 should be legal") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"i64 should be legal\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29047, __PRETTY_FUNCTION__));
29048 unsigned NumElts = Subtarget.hasVLX() ? 2 : 8;
29049 // If we use a 128-bit result we might need to use a target specific node.
29050 unsigned SrcElts =
29051 std::max(NumElts, 128U / (unsigned)SrcVT.getSizeInBits());
29052 MVT VecVT = MVT::getVectorVT(MVT::i64, NumElts);
29053 MVT VecInVT = MVT::getVectorVT(SrcVT.getSimpleVT(), SrcElts);
29054 unsigned Opc = N->getOpcode();
29055 if (NumElts != SrcElts) {
29056 if (IsStrict)
29057 Opc = IsSigned ? X86ISD::STRICT_CVTTP2SI : X86ISD::STRICT_CVTTP2UI;
29058 else
29059 Opc = IsSigned ? X86ISD::CVTTP2SI : X86ISD::CVTTP2UI;
29060 }
29061
29062 SDValue ZeroIdx = DAG.getIntPtrConstant(0, dl);
29063 SDValue Res = DAG.getNode(ISD::INSERT_VECTOR_ELT, dl, VecInVT,
29064 DAG.getConstantFP(0.0, dl, VecInVT), Src,
29065 ZeroIdx);
29066 SDValue Chain;
29067 if (IsStrict) {
29068 SDVTList Tys = DAG.getVTList(VecVT, MVT::Other);
29069 Res = DAG.getNode(Opc, SDLoc(N), Tys, N->getOperand(0), Res);
29070 Chain = Res.getValue(1);
29071 } else
29072 Res = DAG.getNode(Opc, SDLoc(N), VecVT, Res);
29073 Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, VT, Res, ZeroIdx);
29074 Results.push_back(Res);
29075 if (IsStrict)
29076 Results.push_back(Chain);
29077 return;
29078 }
29079
29080 SDValue Chain;
29081 if (SDValue V = FP_TO_INTHelper(SDValue(N, 0), DAG, IsSigned, Chain)) {
29082 Results.push_back(V);
29083 if (IsStrict)
29084 Results.push_back(Chain);
29085 }
29086 return;
29087 }
29088 case ISD::SINT_TO_FP:
29089 case ISD::STRICT_SINT_TO_FP:
29090 case ISD::UINT_TO_FP:
29091 case ISD::STRICT_UINT_TO_FP: {
29092 bool IsStrict = N->isStrictFPOpcode();
29093 bool IsSigned = N->getOpcode() == ISD::SINT_TO_FP ||
29094 N->getOpcode() == ISD::STRICT_SINT_TO_FP;
29095 EVT VT = N->getValueType(0);
29096 if (VT != MVT::v2f32)
29097 return;
29098 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
29099 EVT SrcVT = Src.getValueType();
29100 if (Subtarget.hasDQI() && Subtarget.hasVLX() && SrcVT == MVT::v2i64) {
29101 if (IsStrict) {
29102 unsigned Opc = IsSigned ? X86ISD::STRICT_CVTSI2P
29103 : X86ISD::STRICT_CVTUI2P;
29104 SDValue Res = DAG.getNode(Opc, dl, {MVT::v4f32, MVT::Other},
29105 {N->getOperand(0), Src});
29106 Results.push_back(Res);
29107 Results.push_back(Res.getValue(1));
29108 } else {
29109 unsigned Opc = IsSigned ? X86ISD::CVTSI2P : X86ISD::CVTUI2P;
29110 Results.push_back(DAG.getNode(Opc, dl, MVT::v4f32, Src));
29111 }
29112 return;
29113 }
29114 if (SrcVT == MVT::v2i64 && !IsSigned && Subtarget.is64Bit() &&
29115 Subtarget.hasSSE41() && !Subtarget.hasAVX512()) {
29116 SDValue Zero = DAG.getConstant(0, dl, SrcVT);
29117 SDValue One = DAG.getConstant(1, dl, SrcVT);
29118 SDValue Sign = DAG.getNode(ISD::OR, dl, SrcVT,
29119 DAG.getNode(ISD::SRL, dl, SrcVT, Src, One),
29120 DAG.getNode(ISD::AND, dl, SrcVT, Src, One));
29121 SDValue IsNeg = DAG.getSetCC(dl, MVT::v2i64, Src, Zero, ISD::SETLT);
29122 SDValue SignSrc = DAG.getSelect(dl, SrcVT, IsNeg, Sign, Src);
29123 SmallVector<SDValue, 4> SignCvts(4, DAG.getConstantFP(0.0, dl, MVT::f32));
29124 for (int i = 0; i != 2; ++i) {
29125 SDValue Src = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64,
29126 SignSrc, DAG.getIntPtrConstant(i, dl));
29127 if (IsStrict)
29128 SignCvts[i] =
29129 DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {MVT::f32, MVT::Other},
29130 {N->getOperand(0), Src});
29131 else
29132 SignCvts[i] = DAG.getNode(ISD::SINT_TO_FP, dl, MVT::f32, Src);
29133 };
29134 SDValue SignCvt = DAG.getBuildVector(MVT::v4f32, dl, SignCvts);
29135 SDValue Slow, Chain;
29136 if (IsStrict) {
29137 Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
29138 SignCvts[0].getValue(1), SignCvts[1].getValue(1));
29139 Slow = DAG.getNode(ISD::STRICT_FADD, dl, {MVT::v4f32, MVT::Other},
29140 {Chain, SignCvt, SignCvt});
29141 Chain = Slow.getValue(1);
29142 } else {
29143 Slow = DAG.getNode(ISD::FADD, dl, MVT::v4f32, SignCvt, SignCvt);
29144 }
29145 IsNeg = DAG.getBitcast(MVT::v4i32, IsNeg);
29146 IsNeg =
29147 DAG.getVectorShuffle(MVT::v4i32, dl, IsNeg, IsNeg, {1, 3, -1, -1});
29148 SDValue Cvt = DAG.getSelect(dl, MVT::v4f32, IsNeg, Slow, SignCvt);
29149 Results.push_back(Cvt);
29150 if (IsStrict)
29151 Results.push_back(Chain);
29152 return;
29153 }
29154
29155 if (SrcVT != MVT::v2i32)
29156 return;
29157
29158 if (IsSigned || Subtarget.hasAVX512()) {
29159 if (!IsStrict)
29160 return;
29161
29162 // Custom widen strict v2i32->v2f32 to avoid scalarization.
29163 // FIXME: Should generic type legalizer do this?
29164 Src = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i32, Src,
29165 DAG.getConstant(0, dl, MVT::v2i32));
29166 SDValue Res = DAG.getNode(N->getOpcode(), dl, {MVT::v4f32, MVT::Other},
29167 {N->getOperand(0), Src});
29168 Results.push_back(Res);
29169 Results.push_back(Res.getValue(1));
29170 return;
29171 }
29172
29173 assert(Subtarget.hasSSE2() && "Requires at least SSE2!")((Subtarget.hasSSE2() && "Requires at least SSE2!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Requires at least SSE2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29173, __PRETTY_FUNCTION__));
29174 SDValue ZExtIn = DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::v2i64, Src);
29175 SDValue VBias =
29176 DAG.getConstantFP(BitsToDouble(0x4330000000000000ULL), dl, MVT::v2f64);
29177 SDValue Or = DAG.getNode(ISD::OR, dl, MVT::v2i64, ZExtIn,
29178 DAG.getBitcast(MVT::v2i64, VBias));
29179 Or = DAG.getBitcast(MVT::v2f64, Or);
29180 if (IsStrict) {
29181 SDValue Sub = DAG.getNode(ISD::STRICT_FSUB, dl, {MVT::v2f64, MVT::Other},
29182 {N->getOperand(0), Or, VBias});
29183 SDValue Res = DAG.getNode(X86ISD::STRICT_VFPROUND, dl,
29184 {MVT::v4f32, MVT::Other},
29185 {Sub.getValue(1), Sub});
29186 Results.push_back(Res);
29187 Results.push_back(Res.getValue(1));
29188 } else {
29189 // TODO: Are there any fast-math-flags to propagate here?
29190 SDValue Sub = DAG.getNode(ISD::FSUB, dl, MVT::v2f64, Or, VBias);
29191 Results.push_back(DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, Sub));
29192 }
29193 return;
29194 }
29195 case ISD::STRICT_FP_ROUND:
29196 case ISD::FP_ROUND: {
29197 bool IsStrict = N->isStrictFPOpcode();
29198 SDValue Src = N->getOperand(IsStrict ? 1 : 0);
29199 if (!isTypeLegal(Src.getValueType()))
29200 return;
29201 SDValue V;
29202 if (IsStrict)
29203 V = DAG.getNode(X86ISD::STRICT_VFPROUND, dl, {MVT::v4f32, MVT::Other},
29204 {N->getOperand(0), N->getOperand(1)});
29205 else
29206 V = DAG.getNode(X86ISD::VFPROUND, dl, MVT::v4f32, N->getOperand(0));
29207 Results.push_back(V);
29208 if (IsStrict)
29209 Results.push_back(V.getValue(1));
29210 return;
29211 }
29212 case ISD::FP_EXTEND: {
29213 // Right now, only MVT::v2f32 has OperationAction for FP_EXTEND.
29214 // No other ValueType for FP_EXTEND should reach this point.
29215 assert(N->getValueType(0) == MVT::v2f32 &&((N->getValueType(0) == MVT::v2f32 && "Do not know how to legalize this Node"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::v2f32 && \"Do not know how to legalize this Node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29216, __PRETTY_FUNCTION__))
29216 "Do not know how to legalize this Node")((N->getValueType(0) == MVT::v2f32 && "Do not know how to legalize this Node"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::v2f32 && \"Do not know how to legalize this Node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29216, __PRETTY_FUNCTION__));
29217 return;
29218 }
29219 case ISD::INTRINSIC_W_CHAIN: {
29220 unsigned IntNo = N->getConstantOperandVal(1);
29221 switch (IntNo) {
29222 default : llvm_unreachable("Do not know how to custom type "::llvm::llvm_unreachable_internal("Do not know how to custom type "
"legalize this intrinsic operation!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29223)
29223 "legalize this intrinsic operation!")::llvm::llvm_unreachable_internal("Do not know how to custom type "
"legalize this intrinsic operation!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29223);
29224 case Intrinsic::x86_rdtsc:
29225 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget,
29226 Results);
29227 case Intrinsic::x86_rdtscp:
29228 return getReadTimeStampCounter(N, dl, X86::RDTSCP, DAG, Subtarget,
29229 Results);
29230 case Intrinsic::x86_rdpmc:
29231 expandIntrinsicWChainHelper(N, dl, DAG, X86::RDPMC, X86::ECX, Subtarget,
29232 Results);
29233 return;
29234 case Intrinsic::x86_xgetbv:
29235 expandIntrinsicWChainHelper(N, dl, DAG, X86::XGETBV, X86::ECX, Subtarget,
29236 Results);
29237 return;
29238 }
29239 }
29240 case ISD::READCYCLECOUNTER: {
29241 return getReadTimeStampCounter(N, dl, X86::RDTSC, DAG, Subtarget, Results);
29242 }
29243 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS: {
29244 EVT T = N->getValueType(0);
29245 assert((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair")(((T == MVT::i64 || T == MVT::i128) && "can only expand cmpxchg pair"
) ? static_cast<void> (0) : __assert_fail ("(T == MVT::i64 || T == MVT::i128) && \"can only expand cmpxchg pair\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29245, __PRETTY_FUNCTION__));
29246 bool Regs64bit = T == MVT::i128;
29247 assert((!Regs64bit || Subtarget.hasCmpxchg16b()) &&(((!Regs64bit || Subtarget.hasCmpxchg16b()) && "64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B"
) ? static_cast<void> (0) : __assert_fail ("(!Regs64bit || Subtarget.hasCmpxchg16b()) && \"64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29248, __PRETTY_FUNCTION__))
29248 "64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B")(((!Regs64bit || Subtarget.hasCmpxchg16b()) && "64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B"
) ? static_cast<void> (0) : __assert_fail ("(!Regs64bit || Subtarget.hasCmpxchg16b()) && \"64-bit ATOMIC_CMP_SWAP_WITH_SUCCESS requires CMPXCHG16B\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29248, __PRETTY_FUNCTION__));
29249 MVT HalfT = Regs64bit ? MVT::i64 : MVT::i32;
29250 SDValue cpInL, cpInH;
29251 cpInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
29252 DAG.getConstant(0, dl, HalfT));
29253 cpInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(2),
29254 DAG.getConstant(1, dl, HalfT));
29255 cpInL = DAG.getCopyToReg(N->getOperand(0), dl,
29256 Regs64bit ? X86::RAX : X86::EAX,
29257 cpInL, SDValue());
29258 cpInH = DAG.getCopyToReg(cpInL.getValue(0), dl,
29259 Regs64bit ? X86::RDX : X86::EDX,
29260 cpInH, cpInL.getValue(1));
29261 SDValue swapInL, swapInH;
29262 swapInL = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
29263 DAG.getConstant(0, dl, HalfT));
29264 swapInH = DAG.getNode(ISD::EXTRACT_ELEMENT, dl, HalfT, N->getOperand(3),
29265 DAG.getConstant(1, dl, HalfT));
29266 swapInH =
29267 DAG.getCopyToReg(cpInH.getValue(0), dl, Regs64bit ? X86::RCX : X86::ECX,
29268 swapInH, cpInH.getValue(1));
29269 // If the current function needs the base pointer, RBX,
29270 // we shouldn't use cmpxchg directly.
29271 // Indeed the lowering of that instruction will clobber
29272 // that register and since RBX will be a reserved register
29273 // the register allocator will not make sure its value will
29274 // be properly saved and restored around this live-range.
29275 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
29276 SDValue Result;
29277 SDVTList Tys = DAG.getVTList(MVT::Other, MVT::Glue);
29278 Register BasePtr = TRI->getBaseRegister();
29279 MachineMemOperand *MMO = cast<AtomicSDNode>(N)->getMemOperand();
29280 if (TRI->hasBasePointer(DAG.getMachineFunction()) &&
29281 (BasePtr == X86::RBX || BasePtr == X86::EBX)) {
29282 // ISel prefers the LCMPXCHG64 variant.
29283 // If that assert breaks, that means it is not the case anymore,
29284 // and we need to teach LCMPXCHG8_SAVE_EBX_DAG how to save RBX,
29285 // not just EBX. This is a matter of accepting i64 input for that
29286 // pseudo, and restoring into the register of the right wide
29287 // in expand pseudo. Everything else should just work.
29288 assert(((Regs64bit == (BasePtr == X86::RBX)) || BasePtr == X86::EBX) &&((((Regs64bit == (BasePtr == X86::RBX)) || BasePtr == X86::EBX
) && "Saving only half of the RBX") ? static_cast<
void> (0) : __assert_fail ("((Regs64bit == (BasePtr == X86::RBX)) || BasePtr == X86::EBX) && \"Saving only half of the RBX\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29289, __PRETTY_FUNCTION__))
29289 "Saving only half of the RBX")((((Regs64bit == (BasePtr == X86::RBX)) || BasePtr == X86::EBX
) && "Saving only half of the RBX") ? static_cast<
void> (0) : __assert_fail ("((Regs64bit == (BasePtr == X86::RBX)) || BasePtr == X86::EBX) && \"Saving only half of the RBX\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29289, __PRETTY_FUNCTION__));
29290 unsigned Opcode = Regs64bit ? X86ISD::LCMPXCHG16_SAVE_RBX_DAG
29291 : X86ISD::LCMPXCHG8_SAVE_EBX_DAG;
29292 SDValue RBXSave = DAG.getCopyFromReg(swapInH.getValue(0), dl,
29293 Regs64bit ? X86::RBX : X86::EBX,
29294 HalfT, swapInH.getValue(1));
29295 SDValue Ops[] = {/*Chain*/ RBXSave.getValue(1), N->getOperand(1), swapInL,
29296 RBXSave,
29297 /*Glue*/ RBXSave.getValue(2)};
29298 Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, T, MMO);
29299 } else {
29300 unsigned Opcode =
29301 Regs64bit ? X86ISD::LCMPXCHG16_DAG : X86ISD::LCMPXCHG8_DAG;
29302 swapInL = DAG.getCopyToReg(swapInH.getValue(0), dl,
29303 Regs64bit ? X86::RBX : X86::EBX, swapInL,
29304 swapInH.getValue(1));
29305 SDValue Ops[] = {swapInL.getValue(0), N->getOperand(1),
29306 swapInL.getValue(1)};
29307 Result = DAG.getMemIntrinsicNode(Opcode, dl, Tys, Ops, T, MMO);
29308 }
29309 SDValue cpOutL = DAG.getCopyFromReg(Result.getValue(0), dl,
29310 Regs64bit ? X86::RAX : X86::EAX,
29311 HalfT, Result.getValue(1));
29312 SDValue cpOutH = DAG.getCopyFromReg(cpOutL.getValue(1), dl,
29313 Regs64bit ? X86::RDX : X86::EDX,
29314 HalfT, cpOutL.getValue(2));
29315 SDValue OpsF[] = { cpOutL.getValue(0), cpOutH.getValue(0)};
29316
29317 SDValue EFLAGS = DAG.getCopyFromReg(cpOutH.getValue(1), dl, X86::EFLAGS,
29318 MVT::i32, cpOutH.getValue(2));
29319 SDValue Success = getSETCC(X86::COND_E, EFLAGS, dl, DAG);
29320 Success = DAG.getZExtOrTrunc(Success, dl, N->getValueType(1));
29321
29322 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, dl, T, OpsF));
29323 Results.push_back(Success);
29324 Results.push_back(EFLAGS.getValue(1));
29325 return;
29326 }
29327 case ISD::ATOMIC_LOAD: {
29328 assert(N->getValueType(0) == MVT::i64 && "Unexpected VT!")((N->getValueType(0) == MVT::i64 && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i64 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29328, __PRETTY_FUNCTION__));
29329 bool NoImplicitFloatOps =
29330 DAG.getMachineFunction().getFunction().hasFnAttribute(
29331 Attribute::NoImplicitFloat);
29332 if (!Subtarget.useSoftFloat() && !NoImplicitFloatOps) {
29333 auto *Node = cast<AtomicSDNode>(N);
29334 if (Subtarget.hasSSE2()) {
29335 // Use a VZEXT_LOAD which will be selected as MOVQ. Then extract the
29336 // lower 64-bits.
29337 SDVTList Tys = DAG.getVTList(MVT::v2i64, MVT::Other);
29338 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
29339 SDValue Ld = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
29340 MVT::i64, Node->getMemOperand());
29341 SDValue Res = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::i64, Ld,
29342 DAG.getIntPtrConstant(0, dl));
29343 Results.push_back(Res);
29344 Results.push_back(Ld.getValue(1));
29345 return;
29346 }
29347 if (Subtarget.hasX87()) {
29348 // First load this into an 80-bit X87 register. This will put the whole
29349 // integer into the significand.
29350 // FIXME: Do we need to glue? See FIXME comment in BuildFILD.
29351 SDVTList Tys = DAG.getVTList(MVT::f80, MVT::Other, MVT::Glue);
29352 SDValue Ops[] = { Node->getChain(), Node->getBasePtr() };
29353 SDValue Result = DAG.getMemIntrinsicNode(X86ISD::FILD_FLAG,
29354 dl, Tys, Ops, MVT::i64,
29355 Node->getMemOperand());
29356 SDValue Chain = Result.getValue(1);
29357 SDValue InFlag = Result.getValue(2);
29358
29359 // Now store the X87 register to a stack temporary and convert to i64.
29360 // This store is not atomic and doesn't need to be.
29361 // FIXME: We don't need a stack temporary if the result of the load
29362 // is already being stored. We could just directly store there.
29363 SDValue StackPtr = DAG.CreateStackTemporary(MVT::i64);
29364 int SPFI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
29365 MachinePointerInfo MPI =
29366 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), SPFI);
29367 SDValue StoreOps[] = { Chain, Result, StackPtr, InFlag };
29368 Chain = DAG.getMemIntrinsicNode(X86ISD::FIST, dl,
29369 DAG.getVTList(MVT::Other), StoreOps,
29370 MVT::i64, MPI, 0 /*Align*/,
29371 MachineMemOperand::MOStore);
29372
29373 // Finally load the value back from the stack temporary and return it.
29374 // This load is not atomic and doesn't need to be.
29375 // This load will be further type legalized.
29376 Result = DAG.getLoad(MVT::i64, dl, Chain, StackPtr, MPI);
29377 Results.push_back(Result);
29378 Results.push_back(Result.getValue(1));
29379 return;
29380 }
29381 }
29382 // TODO: Use MOVLPS when SSE1 is available?
29383 // Delegate to generic TypeLegalization. Situations we can really handle
29384 // should have already been dealt with by AtomicExpandPass.cpp.
29385 break;
29386 }
29387 case ISD::ATOMIC_SWAP:
29388 case ISD::ATOMIC_LOAD_ADD:
29389 case ISD::ATOMIC_LOAD_SUB:
29390 case ISD::ATOMIC_LOAD_AND:
29391 case ISD::ATOMIC_LOAD_OR:
29392 case ISD::ATOMIC_LOAD_XOR:
29393 case ISD::ATOMIC_LOAD_NAND:
29394 case ISD::ATOMIC_LOAD_MIN:
29395 case ISD::ATOMIC_LOAD_MAX:
29396 case ISD::ATOMIC_LOAD_UMIN:
29397 case ISD::ATOMIC_LOAD_UMAX:
29398 // Delegate to generic TypeLegalization. Situations we can really handle
29399 // should have already been dealt with by AtomicExpandPass.cpp.
29400 break;
29401
29402 case ISD::BITCAST: {
29403 assert(Subtarget.hasSSE2() && "Requires at least SSE2!")((Subtarget.hasSSE2() && "Requires at least SSE2!") ?
static_cast<void> (0) : __assert_fail ("Subtarget.hasSSE2() && \"Requires at least SSE2!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29403, __PRETTY_FUNCTION__));
29404 EVT DstVT = N->getValueType(0);
29405 EVT SrcVT = N->getOperand(0).getValueType();
29406
29407 // If this is a bitcast from a v64i1 k-register to a i64 on a 32-bit target
29408 // we can split using the k-register rather than memory.
29409 if (SrcVT == MVT::v64i1 && DstVT == MVT::i64 && Subtarget.hasBWI()) {
29410 assert(!Subtarget.is64Bit() && "Expected 32-bit mode")((!Subtarget.is64Bit() && "Expected 32-bit mode") ? static_cast
<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Expected 32-bit mode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29410, __PRETTY_FUNCTION__));
29411 SDValue Lo, Hi;
29412 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
29413 Lo = DAG.getBitcast(MVT::i32, Lo);
29414 Hi = DAG.getBitcast(MVT::i32, Hi);
29415 SDValue Res = DAG.getNode(ISD::BUILD_PAIR, dl, MVT::i64, Lo, Hi);
29416 Results.push_back(Res);
29417 return;
29418 }
29419
29420 // Custom splitting for BWI types when AVX512F is available but BWI isn't.
29421 if ((DstVT == MVT::v32i16 || DstVT == MVT::v64i8) &&
29422 SrcVT.isVector() && isTypeLegal(SrcVT)) {
29423 SDValue Lo, Hi;
29424 std::tie(Lo, Hi) = DAG.SplitVectorOperand(N, 0);
29425 MVT CastVT = (DstVT == MVT::v32i16) ? MVT::v16i16 : MVT::v32i8;
29426 Lo = DAG.getBitcast(CastVT, Lo);
29427 Hi = DAG.getBitcast(CastVT, Hi);
29428 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, dl, DstVT, Lo, Hi);
29429 Results.push_back(Res);
29430 return;
29431 }
29432
29433 if (DstVT.isVector() && SrcVT == MVT::x86mmx) {
29434 assert(getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29435, __PRETTY_FUNCTION__))
29435 "Unexpected type action!")((getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), DstVT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29435, __PRETTY_FUNCTION__));
29436 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), DstVT);
29437 SDValue Res = DAG.getNode(X86ISD::MOVQ2DQ, dl, WideVT, N->getOperand(0));
29438 Results.push_back(Res);
29439 return;
29440 }
29441
29442 return;
29443 }
29444 case ISD::MGATHER: {
29445 EVT VT = N->getValueType(0);
29446 if ((VT == MVT::v2f32 || VT == MVT::v2i32) &&
29447 (Subtarget.hasVLX() || !Subtarget.hasAVX512())) {
29448 auto *Gather = cast<MaskedGatherSDNode>(N);
29449 SDValue Index = Gather->getIndex();
29450 if (Index.getValueType() != MVT::v2i64)
29451 return;
29452 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29453, __PRETTY_FUNCTION__))
29453 "Unexpected type action!")((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29453, __PRETTY_FUNCTION__));
29454 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
29455 SDValue Mask = Gather->getMask();
29456 assert(Mask.getValueType() == MVT::v2i1 && "Unexpected mask type")((Mask.getValueType() == MVT::v2i1 && "Unexpected mask type"
) ? static_cast<void> (0) : __assert_fail ("Mask.getValueType() == MVT::v2i1 && \"Unexpected mask type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29456, __PRETTY_FUNCTION__));
29457 SDValue PassThru = DAG.getNode(ISD::CONCAT_VECTORS, dl, WideVT,
29458 Gather->getPassThru(),
29459 DAG.getUNDEF(VT));
29460 if (!Subtarget.hasVLX()) {
29461 // We need to widen the mask, but the instruction will only use 2
29462 // of its elements. So we can use undef.
29463 Mask = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v4i1, Mask,
29464 DAG.getUNDEF(MVT::v2i1));
29465 Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i32, Mask);
29466 }
29467 SDValue Ops[] = { Gather->getChain(), PassThru, Mask,
29468 Gather->getBasePtr(), Index, Gather->getScale() };
29469 SDValue Res = DAG.getTargetMemSDNode<X86MaskedGatherSDNode>(
29470 DAG.getVTList(WideVT, Mask.getValueType(), MVT::Other), Ops, dl,
29471 Gather->getMemoryVT(), Gather->getMemOperand());
29472 Results.push_back(Res);
29473 Results.push_back(Res.getValue(2));
29474 return;
29475 }
29476 return;
29477 }
29478 case ISD::LOAD: {
29479 // Use an f64/i64 load and a scalar_to_vector for v2f32/v2i32 loads. This
29480 // avoids scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
29481 // cast since type legalization will try to use an i64 load.
29482 MVT VT = N->getSimpleValueType(0);
29483 assert(VT.isVector() && VT.getSizeInBits() == 64 && "Unexpected VT")((VT.isVector() && VT.getSizeInBits() == 64 &&
"Unexpected VT") ? static_cast<void> (0) : __assert_fail
("VT.isVector() && VT.getSizeInBits() == 64 && \"Unexpected VT\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29483, __PRETTY_FUNCTION__));
29484 assert(getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29485, __PRETTY_FUNCTION__))
29485 "Unexpected type action!")((getTypeAction(*DAG.getContext(), VT) == TypeWidenVector &&
"Unexpected type action!") ? static_cast<void> (0) : __assert_fail
("getTypeAction(*DAG.getContext(), VT) == TypeWidenVector && \"Unexpected type action!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29485, __PRETTY_FUNCTION__));
29486 if (!ISD::isNON_EXTLoad(N))
29487 return;
29488 auto *Ld = cast<LoadSDNode>(N);
29489 if (Subtarget.hasSSE2()) {
29490 MVT LdVT = Subtarget.is64Bit() && VT.isInteger() ? MVT::i64 : MVT::f64;
29491 SDValue Res = DAG.getLoad(LdVT, dl, Ld->getChain(), Ld->getBasePtr(),
29492 Ld->getPointerInfo(), Ld->getAlignment(),
29493 Ld->getMemOperand()->getFlags());
29494 SDValue Chain = Res.getValue(1);
29495 MVT VecVT = MVT::getVectorVT(LdVT, 2);
29496 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VecVT, Res);
29497 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
29498 Res = DAG.getBitcast(WideVT, Res);
29499 Results.push_back(Res);
29500 Results.push_back(Chain);
29501 return;
29502 }
29503 assert(Subtarget.hasSSE1() && "Expected SSE")((Subtarget.hasSSE1() && "Expected SSE") ? static_cast
<void> (0) : __assert_fail ("Subtarget.hasSSE1() && \"Expected SSE\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29503, __PRETTY_FUNCTION__));
29504 SDVTList Tys = DAG.getVTList(MVT::v4f32, MVT::Other);
29505 SDValue Ops[] = {Ld->getChain(), Ld->getBasePtr()};
29506 SDValue Res = DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
29507 MVT::i64, Ld->getMemOperand());
29508 Results.push_back(Res);
29509 Results.push_back(Res.getValue(1));
29510 return;
29511 }
29512 case ISD::ADDRSPACECAST: {
29513 SDValue Src = N->getOperand(0);
29514 EVT DstVT = N->getValueType(0);
29515 AddrSpaceCastSDNode *CastN = cast<AddrSpaceCastSDNode>(N);
29516 unsigned SrcAS = CastN->getSrcAddressSpace();
29517
29518 assert(SrcAS != CastN->getDestAddressSpace() &&((SrcAS != CastN->getDestAddressSpace() && "addrspacecast must be between different address spaces"
) ? static_cast<void> (0) : __assert_fail ("SrcAS != CastN->getDestAddressSpace() && \"addrspacecast must be between different address spaces\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29519, __PRETTY_FUNCTION__))
29519 "addrspacecast must be between different address spaces")((SrcAS != CastN->getDestAddressSpace() && "addrspacecast must be between different address spaces"
) ? static_cast<void> (0) : __assert_fail ("SrcAS != CastN->getDestAddressSpace() && \"addrspacecast must be between different address spaces\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 29519, __PRETTY_FUNCTION__));
29520
29521 SDValue Res;
29522 if (SrcAS == X86AS::PTR32_UPTR && DstVT == MVT::i64)
29523 Res = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Src);
29524 else if (DstVT == MVT::i64)
29525 Res = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Src);
29526 else if (DstVT == MVT::i32)
29527 Res = DAG.getNode(ISD::TRUNCATE, dl, DstVT, Src);
29528 else
29529 report_fatal_error("Unrecognized addrspacecast type legalization");
29530
29531 Results.push_back(Res);
29532 return;
29533 }
29534 }
29535}
29536
29537const char *X86TargetLowering::getTargetNodeName(unsigned Opcode) const {
29538 switch ((X86ISD::NodeType)Opcode) {
29539 case X86ISD::FIRST_NUMBER: break;
29540 case X86ISD::BSF: return "X86ISD::BSF";
29541 case X86ISD::BSR: return "X86ISD::BSR";
29542 case X86ISD::SHLD: return "X86ISD::SHLD";
29543 case X86ISD::SHRD: return "X86ISD::SHRD";
29544 case X86ISD::FAND: return "X86ISD::FAND";
29545 case X86ISD::FANDN: return "X86ISD::FANDN";
29546 case X86ISD::FOR: return "X86ISD::FOR";
29547 case X86ISD::FXOR: return "X86ISD::FXOR";
29548 case X86ISD::FILD: return "X86ISD::FILD";
29549 case X86ISD::FILD_FLAG: return "X86ISD::FILD_FLAG";
29550 case X86ISD::FIST: return "X86ISD::FIST";
29551 case X86ISD::FP_TO_INT_IN_MEM: return "X86ISD::FP_TO_INT_IN_MEM";
29552 case X86ISD::FLD: return "X86ISD::FLD";
29553 case X86ISD::FST: return "X86ISD::FST";
29554 case X86ISD::CALL: return "X86ISD::CALL";
29555 case X86ISD::BT: return "X86ISD::BT";
29556 case X86ISD::CMP: return "X86ISD::CMP";
29557 case X86ISD::STRICT_FCMP: return "X86ISD::STRICT_FCMP";
29558 case X86ISD::STRICT_FCMPS: return "X86ISD::STRICT_FCMPS";
29559 case X86ISD::COMI: return "X86ISD::COMI";
29560 case X86ISD::UCOMI: return "X86ISD::UCOMI";
29561 case X86ISD::CMPM: return "X86ISD::CMPM";
29562 case X86ISD::STRICT_CMPM: return "X86ISD::STRICT_CMPM";
29563 case X86ISD::CMPM_SAE: return "X86ISD::CMPM_SAE";
29564 case X86ISD::SETCC: return "X86ISD::SETCC";
29565 case X86ISD::SETCC_CARRY: return "X86ISD::SETCC_CARRY";
29566 case X86ISD::FSETCC: return "X86ISD::FSETCC";
29567 case X86ISD::FSETCCM: return "X86ISD::FSETCCM";
29568 case X86ISD::FSETCCM_SAE: return "X86ISD::FSETCCM_SAE";
29569 case X86ISD::CMOV: return "X86ISD::CMOV";
29570 case X86ISD::BRCOND: return "X86ISD::BRCOND";
29571 case X86ISD::RET_FLAG: return "X86ISD::RET_FLAG";
29572 case X86ISD::IRET: return "X86ISD::IRET";
29573 case X86ISD::REP_STOS: return "X86ISD::REP_STOS";
29574 case X86ISD::REP_MOVS: return "X86ISD::REP_MOVS";
29575 case X86ISD::GlobalBaseReg: return "X86ISD::GlobalBaseReg";
29576 case X86ISD::Wrapper: return "X86ISD::Wrapper";
29577 case X86ISD::WrapperRIP: return "X86ISD::WrapperRIP";
29578 case X86ISD::MOVQ2DQ: return "X86ISD::MOVQ2DQ";
29579 case X86ISD::MOVDQ2Q: return "X86ISD::MOVDQ2Q";
29580 case X86ISD::MMX_MOVD2W: return "X86ISD::MMX_MOVD2W";
29581 case X86ISD::MMX_MOVW2D: return "X86ISD::MMX_MOVW2D";
29582 case X86ISD::PEXTRB: return "X86ISD::PEXTRB";
29583 case X86ISD::PEXTRW: return "X86ISD::PEXTRW";
29584 case X86ISD::INSERTPS: return "X86ISD::INSERTPS";
29585 case X86ISD::PINSRB: return "X86ISD::PINSRB";
29586 case X86ISD::PINSRW: return "X86ISD::PINSRW";
29587 case X86ISD::PSHUFB: return "X86ISD::PSHUFB";
29588 case X86ISD::ANDNP: return "X86ISD::ANDNP";
29589 case X86ISD::BLENDI: return "X86ISD::BLENDI";
29590 case X86ISD::BLENDV: return "X86ISD::BLENDV";
29591 case X86ISD::HADD: return "X86ISD::HADD";
29592 case X86ISD::HSUB: return "X86ISD::HSUB";
29593 case X86ISD::FHADD: return "X86ISD::FHADD";
29594 case X86ISD::FHSUB: return "X86ISD::FHSUB";
29595 case X86ISD::CONFLICT: return "X86ISD::CONFLICT";
29596 case X86ISD::FMAX: return "X86ISD::FMAX";
29597 case X86ISD::FMAXS: return "X86ISD::FMAXS";
29598 case X86ISD::FMAX_SAE: return "X86ISD::FMAX_SAE";
29599 case X86ISD::FMAXS_SAE: return "X86ISD::FMAXS_SAE";
29600 case X86ISD::FMIN: return "X86ISD::FMIN";
29601 case X86ISD::FMINS: return "X86ISD::FMINS";
29602 case X86ISD::FMIN_SAE: return "X86ISD::FMIN_SAE";
29603 case X86ISD::FMINS_SAE: return "X86ISD::FMINS_SAE";
29604 case X86ISD::FMAXC: return "X86ISD::FMAXC";
29605 case X86ISD::FMINC: return "X86ISD::FMINC";
29606 case X86ISD::FRSQRT: return "X86ISD::FRSQRT";
29607 case X86ISD::FRCP: return "X86ISD::FRCP";
29608 case X86ISD::EXTRQI: return "X86ISD::EXTRQI";
29609 case X86ISD::INSERTQI: return "X86ISD::INSERTQI";
29610 case X86ISD::TLSADDR: return "X86ISD::TLSADDR";
29611 case X86ISD::TLSBASEADDR: return "X86ISD::TLSBASEADDR";
29612 case X86ISD::TLSCALL: return "X86ISD::TLSCALL";
29613 case X86ISD::EH_SJLJ_SETJMP: return "X86ISD::EH_SJLJ_SETJMP";
29614 case X86ISD::EH_SJLJ_LONGJMP: return "X86ISD::EH_SJLJ_LONGJMP";
29615 case X86ISD::EH_SJLJ_SETUP_DISPATCH:
29616 return "X86ISD::EH_SJLJ_SETUP_DISPATCH";
29617 case X86ISD::EH_RETURN: return "X86ISD::EH_RETURN";
29618 case X86ISD::TC_RETURN: return "X86ISD::TC_RETURN";
29619 case X86ISD::FNSTCW16m: return "X86ISD::FNSTCW16m";
29620 case X86ISD::FNSTSW16r: return "X86ISD::FNSTSW16r";
29621 case X86ISD::LCMPXCHG_DAG: return "X86ISD::LCMPXCHG_DAG";
29622 case X86ISD::LCMPXCHG8_DAG: return "X86ISD::LCMPXCHG8_DAG";
29623 case X86ISD::LCMPXCHG16_DAG: return "X86ISD::LCMPXCHG16_DAG";
29624 case X86ISD::LCMPXCHG8_SAVE_EBX_DAG:
29625 return "X86ISD::LCMPXCHG8_SAVE_EBX_DAG";
29626 case X86ISD::LCMPXCHG16_SAVE_RBX_DAG:
29627 return "X86ISD::LCMPXCHG16_SAVE_RBX_DAG";
29628 case X86ISD::LADD: return "X86ISD::LADD";
29629 case X86ISD::LSUB: return "X86ISD::LSUB";
29630 case X86ISD::LOR: return "X86ISD::LOR";
29631 case X86ISD::LXOR: return "X86ISD::LXOR";
29632 case X86ISD::LAND: return "X86ISD::LAND";
29633 case X86ISD::VZEXT_MOVL: return "X86ISD::VZEXT_MOVL";
29634 case X86ISD::VZEXT_LOAD: return "X86ISD::VZEXT_LOAD";
29635 case X86ISD::VEXTRACT_STORE: return "X86ISD::VEXTRACT_STORE";
29636 case X86ISD::VTRUNC: return "X86ISD::VTRUNC";
29637 case X86ISD::VTRUNCS: return "X86ISD::VTRUNCS";
29638 case X86ISD::VTRUNCUS: return "X86ISD::VTRUNCUS";
29639 case X86ISD::VMTRUNC: return "X86ISD::VMTRUNC";
29640 case X86ISD::VMTRUNCS: return "X86ISD::VMTRUNCS";
29641 case X86ISD::VMTRUNCUS: return "X86ISD::VMTRUNCUS";
29642 case X86ISD::VTRUNCSTORES: return "X86ISD::VTRUNCSTORES";
29643 case X86ISD::VTRUNCSTOREUS: return "X86ISD::VTRUNCSTOREUS";
29644 case X86ISD::VMTRUNCSTORES: return "X86ISD::VMTRUNCSTORES";
29645 case X86ISD::VMTRUNCSTOREUS: return "X86ISD::VMTRUNCSTOREUS";
29646 case X86ISD::VFPEXT: return "X86ISD::VFPEXT";
29647 case X86ISD::STRICT_VFPEXT: return "X86ISD::STRICT_VFPEXT";
29648 case X86ISD::VFPEXT_SAE: return "X86ISD::VFPEXT_SAE";
29649 case X86ISD::VFPEXTS: return "X86ISD::VFPEXTS";
29650 case X86ISD::VFPEXTS_SAE: return "X86ISD::VFPEXTS_SAE";
29651 case X86ISD::VFPROUND: return "X86ISD::VFPROUND";
29652 case X86ISD::STRICT_VFPROUND: return "X86ISD::STRICT_VFPROUND";
29653 case X86ISD::VMFPROUND: return "X86ISD::VMFPROUND";
29654 case X86ISD::VFPROUND_RND: return "X86ISD::VFPROUND_RND";
29655 case X86ISD::VFPROUNDS: return "X86ISD::VFPROUNDS";
29656 case X86ISD::VFPROUNDS_RND: return "X86ISD::VFPROUNDS_RND";
29657 case X86ISD::VSHLDQ: return "X86ISD::VSHLDQ";
29658 case X86ISD::VSRLDQ: return "X86ISD::VSRLDQ";
29659 case X86ISD::VSHL: return "X86ISD::VSHL";
29660 case X86ISD::VSRL: return "X86ISD::VSRL";
29661 case X86ISD::VSRA: return "X86ISD::VSRA";
29662 case X86ISD::VSHLI: return "X86ISD::VSHLI";
29663 case X86ISD::VSRLI: return "X86ISD::VSRLI";
29664 case X86ISD::VSRAI: return "X86ISD::VSRAI";
29665 case X86ISD::VSHLV: return "X86ISD::VSHLV";
29666 case X86ISD::VSRLV: return "X86ISD::VSRLV";
29667 case X86ISD::VSRAV: return "X86ISD::VSRAV";
29668 case X86ISD::VROTLI: return "X86ISD::VROTLI";
29669 case X86ISD::VROTRI: return "X86ISD::VROTRI";
29670 case X86ISD::VPPERM: return "X86ISD::VPPERM";
29671 case X86ISD::CMPP: return "X86ISD::CMPP";
29672 case X86ISD::STRICT_CMPP: return "X86ISD::STRICT_CMPP";
29673 case X86ISD::PCMPEQ: return "X86ISD::PCMPEQ";
29674 case X86ISD::PCMPGT: return "X86ISD::PCMPGT";
29675 case X86ISD::PHMINPOS: return "X86ISD::PHMINPOS";
29676 case X86ISD::ADD: return "X86ISD::ADD";
29677 case X86ISD::SUB: return "X86ISD::SUB";
29678 case X86ISD::ADC: return "X86ISD::ADC";
29679 case X86ISD::SBB: return "X86ISD::SBB";
29680 case X86ISD::SMUL: return "X86ISD::SMUL";
29681 case X86ISD::UMUL: return "X86ISD::UMUL";
29682 case X86ISD::OR: return "X86ISD::OR";
29683 case X86ISD::XOR: return "X86ISD::XOR";
29684 case X86ISD::AND: return "X86ISD::AND";
29685 case X86ISD::BEXTR: return "X86ISD::BEXTR";
29686 case X86ISD::BZHI: return "X86ISD::BZHI";
29687 case X86ISD::MUL_IMM: return "X86ISD::MUL_IMM";
29688 case X86ISD::MOVMSK: return "X86ISD::MOVMSK";
29689 case X86ISD::PTEST: return "X86ISD::PTEST";
29690 case X86ISD::TESTP: return "X86ISD::TESTP";
29691 case X86ISD::KORTEST: return "X86ISD::KORTEST";
29692 case X86ISD::KTEST: return "X86ISD::KTEST";
29693 case X86ISD::KADD: return "X86ISD::KADD";
29694 case X86ISD::KSHIFTL: return "X86ISD::KSHIFTL";
29695 case X86ISD::KSHIFTR: return "X86ISD::KSHIFTR";
29696 case X86ISD::PACKSS: return "X86ISD::PACKSS";
29697 case X86ISD::PACKUS: return "X86ISD::PACKUS";
29698 case X86ISD::PALIGNR: return "X86ISD::PALIGNR";
29699 case X86ISD::VALIGN: return "X86ISD::VALIGN";
29700 case X86ISD::VSHLD: return "X86ISD::VSHLD";
29701 case X86ISD::VSHRD: return "X86ISD::VSHRD";
29702 case X86ISD::VSHLDV: return "X86ISD::VSHLDV";
29703 case X86ISD::VSHRDV: return "X86ISD::VSHRDV";
29704 case X86ISD::PSHUFD: return "X86ISD::PSHUFD";
29705 case X86ISD::PSHUFHW: return "X86ISD::PSHUFHW";
29706 case X86ISD::PSHUFLW: return "X86ISD::PSHUFLW";
29707 case X86ISD::SHUFP: return "X86ISD::SHUFP";
29708 case X86ISD::SHUF128: return "X86ISD::SHUF128";
29709 case X86ISD::MOVLHPS: return "X86ISD::MOVLHPS";
29710 case X86ISD::MOVHLPS: return "X86ISD::MOVHLPS";
29711 case X86ISD::MOVDDUP: return "X86ISD::MOVDDUP";
29712 case X86ISD::MOVSHDUP: return "X86ISD::MOVSHDUP";
29713 case X86ISD::MOVSLDUP: return "X86ISD::MOVSLDUP";
29714 case X86ISD::MOVSD: return "X86ISD::MOVSD";
29715 case X86ISD::MOVSS: return "X86ISD::MOVSS";
29716 case X86ISD::UNPCKL: return "X86ISD::UNPCKL";
29717 case X86ISD::UNPCKH: return "X86ISD::UNPCKH";
29718 case X86ISD::VBROADCAST: return "X86ISD::VBROADCAST";
29719 case X86ISD::VBROADCAST_LOAD: return "X86ISD::VBROADCAST_LOAD";
29720 case X86ISD::VBROADCASTM: return "X86ISD::VBROADCASTM";
29721 case X86ISD::SUBV_BROADCAST: return "X86ISD::SUBV_BROADCAST";
29722 case X86ISD::VPERMILPV: return "X86ISD::VPERMILPV";
29723 case X86ISD::VPERMILPI: return "X86ISD::VPERMILPI";
29724 case X86ISD::VPERM2X128: return "X86ISD::VPERM2X128";
29725 case X86ISD::VPERMV: return "X86ISD::VPERMV";
29726 case X86ISD::VPERMV3: return "X86ISD::VPERMV3";
29727 case X86ISD::VPERMI: return "X86ISD::VPERMI";
29728 case X86ISD::VPTERNLOG: return "X86ISD::VPTERNLOG";
29729 case X86ISD::VFIXUPIMM: return "X86ISD::VFIXUPIMM";
29730 case X86ISD::VFIXUPIMM_SAE: return "X86ISD::VFIXUPIMM_SAE";
29731 case X86ISD::VFIXUPIMMS: return "X86ISD::VFIXUPIMMS";
29732 case X86ISD::VFIXUPIMMS_SAE: return "X86ISD::VFIXUPIMMS_SAE";
29733 case X86ISD::VRANGE: return "X86ISD::VRANGE";
29734 case X86ISD::VRANGE_SAE: return "X86ISD::VRANGE_SAE";
29735 case X86ISD::VRANGES: return "X86ISD::VRANGES";
29736 case X86ISD::VRANGES_SAE: return "X86ISD::VRANGES_SAE";
29737 case X86ISD::PMULUDQ: return "X86ISD::PMULUDQ";
29738 case X86ISD::PMULDQ: return "X86ISD::PMULDQ";
29739 case X86ISD::PSADBW: return "X86ISD::PSADBW";
29740 case X86ISD::DBPSADBW: return "X86ISD::DBPSADBW";
29741 case X86ISD::VASTART_SAVE_XMM_REGS: return "X86ISD::VASTART_SAVE_XMM_REGS";
29742 case X86ISD::VAARG_64: return "X86ISD::VAARG_64";
29743 case X86ISD::WIN_ALLOCA: return "X86ISD::WIN_ALLOCA";
29744 case X86ISD::MEMBARRIER: return "X86ISD::MEMBARRIER";
29745 case X86ISD::MFENCE: return "X86ISD::MFENCE";
29746 case X86ISD::SEG_ALLOCA: return "X86ISD::SEG_ALLOCA";
29747 case X86ISD::SAHF: return "X86ISD::SAHF";
29748 case X86ISD::RDRAND: return "X86ISD::RDRAND";
29749 case X86ISD::RDSEED: return "X86ISD::RDSEED";
29750 case X86ISD::RDPKRU: return "X86ISD::RDPKRU";
29751 case X86ISD::WRPKRU: return "X86ISD::WRPKRU";
29752 case X86ISD::VPMADDUBSW: return "X86ISD::VPMADDUBSW";
29753 case X86ISD::VPMADDWD: return "X86ISD::VPMADDWD";
29754 case X86ISD::VPSHA: return "X86ISD::VPSHA";
29755 case X86ISD::VPSHL: return "X86ISD::VPSHL";
29756 case X86ISD::VPCOM: return "X86ISD::VPCOM";
29757 case X86ISD::VPCOMU: return "X86ISD::VPCOMU";
29758 case X86ISD::VPERMIL2: return "X86ISD::VPERMIL2";
29759 case X86ISD::FMSUB: return "X86ISD::FMSUB";
29760 case X86ISD::FNMADD: return "X86ISD::FNMADD";
29761 case X86ISD::FNMSUB: return "X86ISD::FNMSUB";
29762 case X86ISD::FMADDSUB: return "X86ISD::FMADDSUB";
29763 case X86ISD::FMSUBADD: return "X86ISD::FMSUBADD";
29764 case X86ISD::FMADD_RND: return "X86ISD::FMADD_RND";
29765 case X86ISD::FNMADD_RND: return "X86ISD::FNMADD_RND";
29766 case X86ISD::FMSUB_RND: return "X86ISD::FMSUB_RND";
29767 case X86ISD::FNMSUB_RND: return "X86ISD::FNMSUB_RND";
29768 case X86ISD::FMADDSUB_RND: return "X86ISD::FMADDSUB_RND";
29769 case X86ISD::FMSUBADD_RND: return "X86ISD::FMSUBADD_RND";
29770 case X86ISD::VPMADD52H: return "X86ISD::VPMADD52H";
29771 case X86ISD::VPMADD52L: return "X86ISD::VPMADD52L";
29772 case X86ISD::VRNDSCALE: return "X86ISD::VRNDSCALE";
29773 case X86ISD::STRICT_VRNDSCALE: return "X86ISD::STRICT_VRNDSCALE";
29774 case X86ISD::VRNDSCALE_SAE: return "X86ISD::VRNDSCALE_SAE";
29775 case X86ISD::VRNDSCALES: return "X86ISD::VRNDSCALES";
29776 case X86ISD::VRNDSCALES_SAE: return "X86ISD::VRNDSCALES_SAE";
29777 case X86ISD::VREDUCE: return "X86ISD::VREDUCE";
29778 case X86ISD::VREDUCE_SAE: return "X86ISD::VREDUCE_SAE";
29779 case X86ISD::VREDUCES: return "X86ISD::VREDUCES";
29780 case X86ISD::VREDUCES_SAE: return "X86ISD::VREDUCES_SAE";
29781 case X86ISD::VGETMANT: return "X86ISD::VGETMANT";
29782 case X86ISD::VGETMANT_SAE: return "X86ISD::VGETMANT_SAE";
29783 case X86ISD::VGETMANTS: return "X86ISD::VGETMANTS";
29784 case X86ISD::VGETMANTS_SAE: return "X86ISD::VGETMANTS_SAE";
29785 case X86ISD::PCMPESTR: return "X86ISD::PCMPESTR";
29786 case X86ISD::PCMPISTR: return "X86ISD::PCMPISTR";
29787 case X86ISD::XTEST: return "X86ISD::XTEST";
29788 case X86ISD::COMPRESS: return "X86ISD::COMPRESS";
29789 case X86ISD::EXPAND: return "X86ISD::EXPAND";
29790 case X86ISD::SELECTS: return "X86ISD::SELECTS";
29791 case X86ISD::ADDSUB: return "X86ISD::ADDSUB";
29792 case X86ISD::RCP14: return "X86ISD::RCP14";
29793 case X86ISD::RCP14S: return "X86ISD::RCP14S";
29794 case X86ISD::RCP28: return "X86ISD::RCP28";
29795 case X86ISD::RCP28_SAE: return "X86ISD::RCP28_SAE";
29796 case X86ISD::RCP28S: return "X86ISD::RCP28S";
29797 case X86ISD::RCP28S_SAE: return "X86ISD::RCP28S_SAE";
29798 case X86ISD::EXP2: return "X86ISD::EXP2";
29799 case X86ISD::EXP2_SAE: return "X86ISD::EXP2_SAE";
29800 case X86ISD::RSQRT14: return "X86ISD::RSQRT14";
29801 case X86ISD::RSQRT14S: return "X86ISD::RSQRT14S";
29802 case X86ISD::RSQRT28: return "X86ISD::RSQRT28";
29803 case X86ISD::RSQRT28_SAE: return "X86ISD::RSQRT28_SAE";
29804 case X86ISD::RSQRT28S: return "X86ISD::RSQRT28S";
29805 case X86ISD::RSQRT28S_SAE: return "X86ISD::RSQRT28S_SAE";
29806 case X86ISD::FADD_RND: return "X86ISD::FADD_RND";
29807 case X86ISD::FADDS: return "X86ISD::FADDS";
29808 case X86ISD::FADDS_RND: return "X86ISD::FADDS_RND";
29809 case X86ISD::FSUB_RND: return "X86ISD::FSUB_RND";
29810 case X86ISD::FSUBS: return "X86ISD::FSUBS";
29811 case X86ISD::FSUBS_RND: return "X86ISD::FSUBS_RND";
29812 case X86ISD::FMUL_RND: return "X86ISD::FMUL_RND";
29813 case X86ISD::FMULS: return "X86ISD::FMULS";
29814 case X86ISD::FMULS_RND: return "X86ISD::FMULS_RND";
29815 case X86ISD::FDIV_RND: return "X86ISD::FDIV_RND";
29816 case X86ISD::FDIVS: return "X86ISD::FDIVS";
29817 case X86ISD::FDIVS_RND: return "X86ISD::FDIVS_RND";
29818 case X86ISD::FSQRT_RND: return "X86ISD::FSQRT_RND";
29819 case X86ISD::FSQRTS: return "X86ISD::FSQRTS";
29820 case X86ISD::FSQRTS_RND: return "X86ISD::FSQRTS_RND";
29821 case X86ISD::FGETEXP: return "X86ISD::FGETEXP";
29822 case X86ISD::FGETEXP_SAE: return "X86ISD::FGETEXP_SAE";
29823 case X86ISD::FGETEXPS: return "X86ISD::FGETEXPS";
29824 case X86ISD::FGETEXPS_SAE: return "X86ISD::FGETEXPS_SAE";
29825 case X86ISD::SCALEF: return "X86ISD::SCALEF";
29826 case X86ISD::SCALEF_RND: return "X86ISD::SCALEF_RND";
29827 case X86ISD::SCALEFS: return "X86ISD::SCALEFS";
29828 case X86ISD::SCALEFS_RND: return "X86ISD::SCALEFS_RND";
29829 case X86ISD::AVG: return "X86ISD::AVG";
29830 case X86ISD::MULHRS: return "X86ISD::MULHRS";
29831 case X86ISD::SINT_TO_FP_RND: return "X86ISD::SINT_TO_FP_RND";
29832 case X86ISD::UINT_TO_FP_RND: return "X86ISD::UINT_TO_FP_RND";
29833 case X86ISD::CVTTP2SI: return "X86ISD::CVTTP2SI";
29834 case X86ISD::CVTTP2UI: return "X86ISD::CVTTP2UI";
29835 case X86ISD::STRICT_CVTTP2SI: return "X86ISD::STRICT_CVTTP2SI";
29836 case X86ISD::STRICT_CVTTP2UI: return "X86ISD::STRICT_CVTTP2UI";
29837 case X86ISD::MCVTTP2SI: return "X86ISD::MCVTTP2SI";
29838 case X86ISD::MCVTTP2UI: return "X86ISD::MCVTTP2UI";
29839 case X86ISD::CVTTP2SI_SAE: return "X86ISD::CVTTP2SI_SAE";
29840 case X86ISD::CVTTP2UI_SAE: return "X86ISD::CVTTP2UI_SAE";
29841 case X86ISD::CVTTS2SI: return "X86ISD::CVTTS2SI";
29842 case X86ISD::CVTTS2UI: return "X86ISD::CVTTS2UI";
29843 case X86ISD::CVTTS2SI_SAE: return "X86ISD::CVTTS2SI_SAE";
29844 case X86ISD::CVTTS2UI_SAE: return "X86ISD::CVTTS2UI_SAE";
29845 case X86ISD::CVTSI2P: return "X86ISD::CVTSI2P";
29846 case X86ISD::CVTUI2P: return "X86ISD::CVTUI2P";
29847 case X86ISD::STRICT_CVTSI2P: return "X86ISD::STRICT_CVTSI2P";
29848 case X86ISD::STRICT_CVTUI2P: return "X86ISD::STRICT_CVTUI2P";
29849 case X86ISD::MCVTSI2P: return "X86ISD::MCVTSI2P";
29850 case X86ISD::MCVTUI2P: return "X86ISD::MCVTUI2P";
29851 case X86ISD::VFPCLASS: return "X86ISD::VFPCLASS";
29852 case X86ISD::VFPCLASSS: return "X86ISD::VFPCLASSS";
29853 case X86ISD::MULTISHIFT: return "X86ISD::MULTISHIFT";
29854 case X86ISD::SCALAR_SINT_TO_FP: return "X86ISD::SCALAR_SINT_TO_FP";
29855 case X86ISD::SCALAR_SINT_TO_FP_RND: return "X86ISD::SCALAR_SINT_TO_FP_RND";
29856 case X86ISD::SCALAR_UINT_TO_FP: return "X86ISD::SCALAR_UINT_TO_FP";
29857 case X86ISD::SCALAR_UINT_TO_FP_RND: return "X86ISD::SCALAR_UINT_TO_FP_RND";
29858 case X86ISD::CVTPS2PH: return "X86ISD::CVTPS2PH";
29859 case X86ISD::MCVTPS2PH: return "X86ISD::MCVTPS2PH";
29860 case X86ISD::CVTPH2PS: return "X86ISD::CVTPH2PS";
29861 case X86ISD::CVTPH2PS_SAE: return "X86ISD::CVTPH2PS_SAE";
29862 case X86ISD::CVTP2SI: return "X86ISD::CVTP2SI";
29863 case X86ISD::CVTP2UI: return "X86ISD::CVTP2UI";
29864 case X86ISD::MCVTP2SI: return "X86ISD::MCVTP2SI";
29865 case X86ISD::MCVTP2UI: return "X86ISD::MCVTP2UI";
29866 case X86ISD::CVTP2SI_RND: return "X86ISD::CVTP2SI_RND";
29867 case X86ISD::CVTP2UI_RND: return "X86ISD::CVTP2UI_RND";
29868 case X86ISD::CVTS2SI: return "X86ISD::CVTS2SI";
29869 case X86ISD::CVTS2UI: return "X86ISD::CVTS2UI";
29870 case X86ISD::CVTS2SI_RND: return "X86ISD::CVTS2SI_RND";
29871 case X86ISD::CVTS2UI_RND: return "X86ISD::CVTS2UI_RND";
29872 case X86ISD::CVTNE2PS2BF16: return "X86ISD::CVTNE2PS2BF16";
29873 case X86ISD::CVTNEPS2BF16: return "X86ISD::CVTNEPS2BF16";
29874 case X86ISD::MCVTNEPS2BF16: return "X86ISD::MCVTNEPS2BF16";
29875 case X86ISD::DPBF16PS: return "X86ISD::DPBF16PS";
29876 case X86ISD::LWPINS: return "X86ISD::LWPINS";
29877 case X86ISD::MGATHER: return "X86ISD::MGATHER";
29878 case X86ISD::MSCATTER: return "X86ISD::MSCATTER";
29879 case X86ISD::VPDPBUSD: return "X86ISD::VPDPBUSD";
29880 case X86ISD::VPDPBUSDS: return "X86ISD::VPDPBUSDS";
29881 case X86ISD::VPDPWSSD: return "X86ISD::VPDPWSSD";
29882 case X86ISD::VPDPWSSDS: return "X86ISD::VPDPWSSDS";
29883 case X86ISD::VPSHUFBITQMB: return "X86ISD::VPSHUFBITQMB";
29884 case X86ISD::GF2P8MULB: return "X86ISD::GF2P8MULB";
29885 case X86ISD::GF2P8AFFINEQB: return "X86ISD::GF2P8AFFINEQB";
29886 case X86ISD::GF2P8AFFINEINVQB: return "X86ISD::GF2P8AFFINEINVQB";
29887 case X86ISD::NT_CALL: return "X86ISD::NT_CALL";
29888 case X86ISD::NT_BRIND: return "X86ISD::NT_BRIND";
29889 case X86ISD::UMWAIT: return "X86ISD::UMWAIT";
29890 case X86ISD::TPAUSE: return "X86ISD::TPAUSE";
29891 case X86ISD::ENQCMD: return "X86ISD:ENQCMD";
29892 case X86ISD::ENQCMDS: return "X86ISD:ENQCMDS";
29893 case X86ISD::VP2INTERSECT: return "X86ISD::VP2INTERSECT";
29894 }
29895 return nullptr;
29896}
29897
29898/// Return true if the addressing mode represented by AM is legal for this
29899/// target, for a load/store of the specified type.
29900bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
29901 const AddrMode &AM, Type *Ty,
29902 unsigned AS,
29903 Instruction *I) const {
29904 // X86 supports extremely general addressing modes.
29905 CodeModel::Model M = getTargetMachine().getCodeModel();
29906
29907 // X86 allows a sign-extended 32-bit immediate field as a displacement.
29908 if (!X86::isOffsetSuitableForCodeModel(AM.BaseOffs, M, AM.BaseGV != nullptr))
29909 return false;
29910
29911 if (AM.BaseGV) {
29912 unsigned GVFlags = Subtarget.classifyGlobalReference(AM.BaseGV);
29913
29914 // If a reference to this global requires an extra load, we can't fold it.
29915 if (isGlobalStubReference(GVFlags))
29916 return false;
29917
29918 // If BaseGV requires a register for the PIC base, we cannot also have a
29919 // BaseReg specified.
29920 if (AM.HasBaseReg && isGlobalRelativeToPICBase(GVFlags))
29921 return false;
29922
29923 // If lower 4G is not available, then we must use rip-relative addressing.
29924 if ((M != CodeModel::Small || isPositionIndependent()) &&
29925 Subtarget.is64Bit() && (AM.BaseOffs || AM.Scale > 1))
29926 return false;
29927 }
29928
29929 switch (AM.Scale) {
29930 case 0:
29931 case 1:
29932 case 2:
29933 case 4:
29934 case 8:
29935 // These scales always work.
29936 break;
29937 case 3:
29938 case 5:
29939 case 9:
29940 // These scales are formed with basereg+scalereg. Only accept if there is
29941 // no basereg yet.
29942 if (AM.HasBaseReg)
29943 return false;
29944 break;
29945 default: // Other stuff never works.
29946 return false;
29947 }
29948
29949 return true;
29950}
29951
29952bool X86TargetLowering::isVectorShiftByScalarCheap(Type *Ty) const {
29953 unsigned Bits = Ty->getScalarSizeInBits();
29954
29955 // 8-bit shifts are always expensive, but versions with a scalar amount aren't
29956 // particularly cheaper than those without.
29957 if (Bits == 8)
29958 return false;
29959
29960 // XOP has v16i8/v8i16/v4i32/v2i64 variable vector shifts.
29961 if (Subtarget.hasXOP() && Ty->getPrimitiveSizeInBits() == 128 &&
29962 (Bits == 8 || Bits == 16 || Bits == 32 || Bits == 64))
29963 return false;
29964
29965 // AVX2 has vpsllv[dq] instructions (and other shifts) that make variable
29966 // shifts just as cheap as scalar ones.
29967 if (Subtarget.hasAVX2() && (Bits == 32 || Bits == 64))
29968 return false;
29969
29970 // AVX512BW has shifts such as vpsllvw.
29971 if (Subtarget.hasBWI() && Bits == 16)
29972 return false;
29973
29974 // Otherwise, it's significantly cheaper to shift by a scalar amount than by a
29975 // fully general vector.
29976 return true;
29977}
29978
29979bool X86TargetLowering::isBinOp(unsigned Opcode) const {
29980 switch (Opcode) {
29981 // These are non-commutative binops.
29982 // TODO: Add more X86ISD opcodes once we have test coverage.
29983 case X86ISD::ANDNP:
29984 case X86ISD::PCMPGT:
29985 case X86ISD::FMAX:
29986 case X86ISD::FMIN:
29987 case X86ISD::FANDN:
29988 return true;
29989 }
29990
29991 return TargetLoweringBase::isBinOp(Opcode);
29992}
29993
29994bool X86TargetLowering::isCommutativeBinOp(unsigned Opcode) const {
29995 switch (Opcode) {
29996 // TODO: Add more X86ISD opcodes once we have test coverage.
29997 case X86ISD::PCMPEQ:
29998 case X86ISD::PMULDQ:
29999 case X86ISD::PMULUDQ:
30000 case X86ISD::FMAXC:
30001 case X86ISD::FMINC:
30002 case X86ISD::FAND:
30003 case X86ISD::FOR:
30004 case X86ISD::FXOR:
30005 return true;
30006 }
30007
30008 return TargetLoweringBase::isCommutativeBinOp(Opcode);
30009}
30010
30011bool X86TargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
30012 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
30013 return false;
30014 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
30015 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
30016 return NumBits1 > NumBits2;
30017}
30018
30019bool X86TargetLowering::allowTruncateForTailCall(Type *Ty1, Type *Ty2) const {
30020 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
30021 return false;
30022
30023 if (!isTypeLegal(EVT::getEVT(Ty1)))
30024 return false;
30025
30026 assert(Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop")((Ty1->getPrimitiveSizeInBits() <= 64 && "i128 is probably not a noop"
) ? static_cast<void> (0) : __assert_fail ("Ty1->getPrimitiveSizeInBits() <= 64 && \"i128 is probably not a noop\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30026, __PRETTY_FUNCTION__));
30027
30028 // Assuming the caller doesn't have a zeroext or signext return parameter,
30029 // truncation all the way down to i1 is valid.
30030 return true;
30031}
30032
30033bool X86TargetLowering::isLegalICmpImmediate(int64_t Imm) const {
30034 return isInt<32>(Imm);
30035}
30036
30037bool X86TargetLowering::isLegalAddImmediate(int64_t Imm) const {
30038 // Can also use sub to handle negated immediates.
30039 return isInt<32>(Imm);
30040}
30041
30042bool X86TargetLowering::isLegalStoreImmediate(int64_t Imm) const {
30043 return isInt<32>(Imm);
30044}
30045
30046bool X86TargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
30047 if (!VT1.isInteger() || !VT2.isInteger())
30048 return false;
30049 unsigned NumBits1 = VT1.getSizeInBits();
30050 unsigned NumBits2 = VT2.getSizeInBits();
30051 return NumBits1 > NumBits2;
30052}
30053
30054bool X86TargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
30055 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
30056 return Ty1->isIntegerTy(32) && Ty2->isIntegerTy(64) && Subtarget.is64Bit();
30057}
30058
30059bool X86TargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
30060 // x86-64 implicitly zero-extends 32-bit results in 64-bit registers.
30061 return VT1 == MVT::i32 && VT2 == MVT::i64 && Subtarget.is64Bit();
30062}
30063
30064bool X86TargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
30065 EVT VT1 = Val.getValueType();
30066 if (isZExtFree(VT1, VT2))
30067 return true;
30068
30069 if (Val.getOpcode() != ISD::LOAD)
30070 return false;
30071
30072 if (!VT1.isSimple() || !VT1.isInteger() ||
30073 !VT2.isSimple() || !VT2.isInteger())
30074 return false;
30075
30076 switch (VT1.getSimpleVT().SimpleTy) {
30077 default: break;
30078 case MVT::i8:
30079 case MVT::i16:
30080 case MVT::i32:
30081 // X86 has 8, 16, and 32-bit zero-extending loads.
30082 return true;
30083 }
30084
30085 return false;
30086}
30087
30088bool X86TargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
30089 if (isa<MaskedLoadSDNode>(ExtVal.getOperand(0)))
30090 return false;
30091
30092 EVT SrcVT = ExtVal.getOperand(0).getValueType();
30093
30094 // There is no extending load for vXi1.
30095 if (SrcVT.getScalarType() == MVT::i1)
30096 return false;
30097
30098 return true;
30099}
30100
30101bool X86TargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
30102 EVT VT) const {
30103 if (!Subtarget.hasAnyFMA())
30104 return false;
30105
30106 VT = VT.getScalarType();
30107
30108 if (!VT.isSimple())
30109 return false;
30110
30111 switch (VT.getSimpleVT().SimpleTy) {
30112 case MVT::f32:
30113 case MVT::f64:
30114 return true;
30115 default:
30116 break;
30117 }
30118
30119 return false;
30120}
30121
30122bool X86TargetLowering::isNarrowingProfitable(EVT VT1, EVT VT2) const {
30123 // i16 instructions are longer (0x66 prefix) and potentially slower.
30124 return !(VT1 == MVT::i32 && VT2 == MVT::i16);
30125}
30126
30127/// Targets can use this to indicate that they only support *some*
30128/// VECTOR_SHUFFLE operations, those with specific masks.
30129/// By default, if a target supports the VECTOR_SHUFFLE node, all mask values
30130/// are assumed to be legal.
30131bool X86TargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
30132 if (!VT.isSimple())
30133 return false;
30134
30135 // Not for i1 vectors
30136 if (VT.getSimpleVT().getScalarType() == MVT::i1)
30137 return false;
30138
30139 // Very little shuffling can be done for 64-bit vectors right now.
30140 if (VT.getSimpleVT().getSizeInBits() == 64)
30141 return false;
30142
30143 // We only care that the types being shuffled are legal. The lowering can
30144 // handle any possible shuffle mask that results.
30145 return isTypeLegal(VT.getSimpleVT());
30146}
30147
30148bool X86TargetLowering::isVectorClearMaskLegal(ArrayRef<int> Mask,
30149 EVT VT) const {
30150 // Don't convert an 'and' into a shuffle that we don't directly support.
30151 // vpblendw and vpshufb for 256-bit vectors are not available on AVX1.
30152 if (!Subtarget.hasAVX2())
30153 if (VT == MVT::v32i8 || VT == MVT::v16i16)
30154 return false;
30155
30156 // Just delegate to the generic legality, clear masks aren't special.
30157 return isShuffleMaskLegal(Mask, VT);
30158}
30159
30160bool X86TargetLowering::areJTsAllowed(const Function *Fn) const {
30161 // If the subtarget is using retpolines, we need to not generate jump tables.
30162 if (Subtarget.useRetpolineIndirectBranches())
30163 return false;
30164
30165 // Otherwise, fallback on the generic logic.
30166 return TargetLowering::areJTsAllowed(Fn);
30167}
30168
30169//===----------------------------------------------------------------------===//
30170// X86 Scheduler Hooks
30171//===----------------------------------------------------------------------===//
30172
30173/// Utility function to emit xbegin specifying the start of an RTM region.
30174static MachineBasicBlock *emitXBegin(MachineInstr &MI, MachineBasicBlock *MBB,
30175 const TargetInstrInfo *TII) {
30176 DebugLoc DL = MI.getDebugLoc();
30177
30178 const BasicBlock *BB = MBB->getBasicBlock();
30179 MachineFunction::iterator I = ++MBB->getIterator();
30180
30181 // For the v = xbegin(), we generate
30182 //
30183 // thisMBB:
30184 // xbegin sinkMBB
30185 //
30186 // mainMBB:
30187 // s0 = -1
30188 //
30189 // fallBB:
30190 // eax = # XABORT_DEF
30191 // s1 = eax
30192 //
30193 // sinkMBB:
30194 // v = phi(s0/mainBB, s1/fallBB)
30195
30196 MachineBasicBlock *thisMBB = MBB;
30197 MachineFunction *MF = MBB->getParent();
30198 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
30199 MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
30200 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
30201 MF->insert(I, mainMBB);
30202 MF->insert(I, fallMBB);
30203 MF->insert(I, sinkMBB);
30204
30205 // Transfer the remainder of BB and its successor edges to sinkMBB.
30206 sinkMBB->splice(sinkMBB->begin(), MBB,
30207 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
30208 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
30209
30210 MachineRegisterInfo &MRI = MF->getRegInfo();
30211 Register DstReg = MI.getOperand(0).getReg();
30212 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
30213 Register mainDstReg = MRI.createVirtualRegister(RC);
30214 Register fallDstReg = MRI.createVirtualRegister(RC);
30215
30216 // thisMBB:
30217 // xbegin fallMBB
30218 // # fallthrough to mainMBB
30219 // # abortion to fallMBB
30220 BuildMI(thisMBB, DL, TII->get(X86::XBEGIN_4)).addMBB(fallMBB);
30221 thisMBB->addSuccessor(mainMBB);
30222 thisMBB->addSuccessor(fallMBB);
30223
30224 // mainMBB:
30225 // mainDstReg := -1
30226 BuildMI(mainMBB, DL, TII->get(X86::MOV32ri), mainDstReg).addImm(-1);
30227 BuildMI(mainMBB, DL, TII->get(X86::JMP_1)).addMBB(sinkMBB);
30228 mainMBB->addSuccessor(sinkMBB);
30229
30230 // fallMBB:
30231 // ; pseudo instruction to model hardware's definition from XABORT
30232 // EAX := XABORT_DEF
30233 // fallDstReg := EAX
30234 BuildMI(fallMBB, DL, TII->get(X86::XABORT_DEF));
30235 BuildMI(fallMBB, DL, TII->get(TargetOpcode::COPY), fallDstReg)
30236 .addReg(X86::EAX);
30237 fallMBB->addSuccessor(sinkMBB);
30238
30239 // sinkMBB:
30240 // DstReg := phi(mainDstReg/mainBB, fallDstReg/fallBB)
30241 BuildMI(*sinkMBB, sinkMBB->begin(), DL, TII->get(X86::PHI), DstReg)
30242 .addReg(mainDstReg).addMBB(mainMBB)
30243 .addReg(fallDstReg).addMBB(fallMBB);
30244
30245 MI.eraseFromParent();
30246 return sinkMBB;
30247}
30248
30249
30250
30251MachineBasicBlock *
30252X86TargetLowering::EmitVAARG64WithCustomInserter(MachineInstr &MI,
30253 MachineBasicBlock *MBB) const {
30254 // Emit va_arg instruction on X86-64.
30255
30256 // Operands to this pseudo-instruction:
30257 // 0 ) Output : destination address (reg)
30258 // 1-5) Input : va_list address (addr, i64mem)
30259 // 6 ) ArgSize : Size (in bytes) of vararg type
30260 // 7 ) ArgMode : 0=overflow only, 1=use gp_offset, 2=use fp_offset
30261 // 8 ) Align : Alignment of type
30262 // 9 ) EFLAGS (implicit-def)
30263
30264 assert(MI.getNumOperands() == 10 && "VAARG_64 should have 10 operands!")((MI.getNumOperands() == 10 && "VAARG_64 should have 10 operands!"
) ? static_cast<void> (0) : __assert_fail ("MI.getNumOperands() == 10 && \"VAARG_64 should have 10 operands!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30264, __PRETTY_FUNCTION__));
30265 static_assert(X86::AddrNumOperands == 5,
30266 "VAARG_64 assumes 5 address operands");
30267
30268 Register DestReg = MI.getOperand(0).getReg();
30269 MachineOperand &Base = MI.getOperand(1);
30270 MachineOperand &Scale = MI.getOperand(2);
30271 MachineOperand &Index = MI.getOperand(3);
30272 MachineOperand &Disp = MI.getOperand(4);
30273 MachineOperand &Segment = MI.getOperand(5);
30274 unsigned ArgSize = MI.getOperand(6).getImm();
30275 unsigned ArgMode = MI.getOperand(7).getImm();
30276 unsigned Align = MI.getOperand(8).getImm();
30277
30278 MachineFunction *MF = MBB->getParent();
30279
30280 // Memory Reference
30281 assert(MI.hasOneMemOperand() && "Expected VAARG_64 to have one memoperand")((MI.hasOneMemOperand() && "Expected VAARG_64 to have one memoperand"
) ? static_cast<void> (0) : __assert_fail ("MI.hasOneMemOperand() && \"Expected VAARG_64 to have one memoperand\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30281, __PRETTY_FUNCTION__));
30282
30283 MachineMemOperand *OldMMO = MI.memoperands().front();
30284
30285 // Clone the MMO into two separate MMOs for loading and storing
30286 MachineMemOperand *LoadOnlyMMO = MF->getMachineMemOperand(
30287 OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOStore);
30288 MachineMemOperand *StoreOnlyMMO = MF->getMachineMemOperand(
30289 OldMMO, OldMMO->getFlags() & ~MachineMemOperand::MOLoad);
30290
30291 // Machine Information
30292 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
30293 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
30294 const TargetRegisterClass *AddrRegClass = getRegClassFor(MVT::i64);
30295 const TargetRegisterClass *OffsetRegClass = getRegClassFor(MVT::i32);
30296 DebugLoc DL = MI.getDebugLoc();
30297
30298 // struct va_list {
30299 // i32 gp_offset
30300 // i32 fp_offset
30301 // i64 overflow_area (address)
30302 // i64 reg_save_area (address)
30303 // }
30304 // sizeof(va_list) = 24
30305 // alignment(va_list) = 8
30306
30307 unsigned TotalNumIntRegs = 6;
30308 unsigned TotalNumXMMRegs = 8;
30309 bool UseGPOffset = (ArgMode == 1);
30310 bool UseFPOffset = (ArgMode == 2);
30311 unsigned MaxOffset = TotalNumIntRegs * 8 +
30312 (UseFPOffset ? TotalNumXMMRegs * 16 : 0);
30313
30314 /* Align ArgSize to a multiple of 8 */
30315 unsigned ArgSizeA8 = (ArgSize + 7) & ~7;
30316 bool NeedsAlign = (Align > 8);
30317
30318 MachineBasicBlock *thisMBB = MBB;
30319 MachineBasicBlock *overflowMBB;
30320 MachineBasicBlock *offsetMBB;
30321 MachineBasicBlock *endMBB;
30322
30323 unsigned OffsetDestReg = 0; // Argument address computed by offsetMBB
30324 unsigned OverflowDestReg = 0; // Argument address computed by overflowMBB
30325 unsigned OffsetReg = 0;
30326
30327 if (!UseGPOffset && !UseFPOffset) {
30328 // If we only pull from the overflow region, we don't create a branch.
30329 // We don't need to alter control flow.
30330 OffsetDestReg = 0; // unused
30331 OverflowDestReg = DestReg;
30332
30333 offsetMBB = nullptr;
30334 overflowMBB = thisMBB;
30335 endMBB = thisMBB;
30336 } else {
30337 // First emit code to check if gp_offset (or fp_offset) is below the bound.
30338 // If so, pull the argument from reg_save_area. (branch to offsetMBB)
30339 // If not, pull from overflow_area. (branch to overflowMBB)
30340 //
30341 // thisMBB
30342 // | .
30343 // | .
30344 // offsetMBB overflowMBB
30345 // | .
30346 // | .
30347 // endMBB
30348
30349 // Registers for the PHI in endMBB
30350 OffsetDestReg = MRI.createVirtualRegister(AddrRegClass);
30351 OverflowDestReg = MRI.createVirtualRegister(AddrRegClass);
30352
30353 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
30354 overflowMBB = MF->CreateMachineBasicBlock(LLVM_BB);
30355 offsetMBB = MF->CreateMachineBasicBlock(LLVM_BB);
30356 endMBB = MF->CreateMachineBasicBlock(LLVM_BB);
30357
30358 MachineFunction::iterator MBBIter = ++MBB->getIterator();
30359
30360 // Insert the new basic blocks
30361 MF->insert(MBBIter, offsetMBB);
30362 MF->insert(MBBIter, overflowMBB);
30363 MF->insert(MBBIter, endMBB);
30364
30365 // Transfer the remainder of MBB and its successor edges to endMBB.
30366 endMBB->splice(endMBB->begin(), thisMBB,
30367 std::next(MachineBasicBlock::iterator(MI)), thisMBB->end());
30368 endMBB->transferSuccessorsAndUpdatePHIs(thisMBB);
30369
30370 // Make offsetMBB and overflowMBB successors of thisMBB
30371 thisMBB->addSuccessor(offsetMBB);
30372 thisMBB->addSuccessor(overflowMBB);
30373
30374 // endMBB is a successor of both offsetMBB and overflowMBB
30375 offsetMBB->addSuccessor(endMBB);
30376 overflowMBB->addSuccessor(endMBB);
30377
30378 // Load the offset value into a register
30379 OffsetReg = MRI.createVirtualRegister(OffsetRegClass);
30380 BuildMI(thisMBB, DL, TII->get(X86::MOV32rm), OffsetReg)
30381 .add(Base)
30382 .add(Scale)
30383 .add(Index)
30384 .addDisp(Disp, UseFPOffset ? 4 : 0)
30385 .add(Segment)
30386 .setMemRefs(LoadOnlyMMO);
30387
30388 // Check if there is enough room left to pull this argument.
30389 BuildMI(thisMBB, DL, TII->get(X86::CMP32ri))
30390 .addReg(OffsetReg)
30391 .addImm(MaxOffset + 8 - ArgSizeA8);
30392
30393 // Branch to "overflowMBB" if offset >= max
30394 // Fall through to "offsetMBB" otherwise
30395 BuildMI(thisMBB, DL, TII->get(X86::JCC_1))
30396 .addMBB(overflowMBB).addImm(X86::COND_AE);
30397 }
30398
30399 // In offsetMBB, emit code to use the reg_save_area.
30400 if (offsetMBB) {
30401 assert(OffsetReg != 0)((OffsetReg != 0) ? static_cast<void> (0) : __assert_fail
("OffsetReg != 0", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30401, __PRETTY_FUNCTION__));
30402
30403 // Read the reg_save_area address.
30404 Register RegSaveReg = MRI.createVirtualRegister(AddrRegClass);
30405 BuildMI(offsetMBB, DL, TII->get(X86::MOV64rm), RegSaveReg)
30406 .add(Base)
30407 .add(Scale)
30408 .add(Index)
30409 .addDisp(Disp, 16)
30410 .add(Segment)
30411 .setMemRefs(LoadOnlyMMO);
30412
30413 // Zero-extend the offset
30414 Register OffsetReg64 = MRI.createVirtualRegister(AddrRegClass);
30415 BuildMI(offsetMBB, DL, TII->get(X86::SUBREG_TO_REG), OffsetReg64)
30416 .addImm(0)
30417 .addReg(OffsetReg)
30418 .addImm(X86::sub_32bit);
30419
30420 // Add the offset to the reg_save_area to get the final address.
30421 BuildMI(offsetMBB, DL, TII->get(X86::ADD64rr), OffsetDestReg)
30422 .addReg(OffsetReg64)
30423 .addReg(RegSaveReg);
30424
30425 // Compute the offset for the next argument
30426 Register NextOffsetReg = MRI.createVirtualRegister(OffsetRegClass);
30427 BuildMI(offsetMBB, DL, TII->get(X86::ADD32ri), NextOffsetReg)
30428 .addReg(OffsetReg)
30429 .addImm(UseFPOffset ? 16 : 8);
30430
30431 // Store it back into the va_list.
30432 BuildMI(offsetMBB, DL, TII->get(X86::MOV32mr))
30433 .add(Base)
30434 .add(Scale)
30435 .add(Index)
30436 .addDisp(Disp, UseFPOffset ? 4 : 0)
30437 .add(Segment)
30438 .addReg(NextOffsetReg)
30439 .setMemRefs(StoreOnlyMMO);
30440
30441 // Jump to endMBB
30442 BuildMI(offsetMBB, DL, TII->get(X86::JMP_1))
30443 .addMBB(endMBB);
30444 }
30445
30446 //
30447 // Emit code to use overflow area
30448 //
30449
30450 // Load the overflow_area address into a register.
30451 Register OverflowAddrReg = MRI.createVirtualRegister(AddrRegClass);
30452 BuildMI(overflowMBB, DL, TII->get(X86::MOV64rm), OverflowAddrReg)
30453 .add(Base)
30454 .add(Scale)
30455 .add(Index)
30456 .addDisp(Disp, 8)
30457 .add(Segment)
30458 .setMemRefs(LoadOnlyMMO);
30459
30460 // If we need to align it, do so. Otherwise, just copy the address
30461 // to OverflowDestReg.
30462 if (NeedsAlign) {
30463 // Align the overflow address
30464 assert(isPowerOf2_32(Align) && "Alignment must be a power of 2")((isPowerOf2_32(Align) && "Alignment must be a power of 2"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(Align) && \"Alignment must be a power of 2\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30464, __PRETTY_FUNCTION__));
30465 Register TmpReg = MRI.createVirtualRegister(AddrRegClass);
30466
30467 // aligned_addr = (addr + (align-1)) & ~(align-1)
30468 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), TmpReg)
30469 .addReg(OverflowAddrReg)
30470 .addImm(Align-1);
30471
30472 BuildMI(overflowMBB, DL, TII->get(X86::AND64ri32), OverflowDestReg)
30473 .addReg(TmpReg)
30474 .addImm(~(uint64_t)(Align-1));
30475 } else {
30476 BuildMI(overflowMBB, DL, TII->get(TargetOpcode::COPY), OverflowDestReg)
30477 .addReg(OverflowAddrReg);
30478 }
30479
30480 // Compute the next overflow address after this argument.
30481 // (the overflow address should be kept 8-byte aligned)
30482 Register NextAddrReg = MRI.createVirtualRegister(AddrRegClass);
30483 BuildMI(overflowMBB, DL, TII->get(X86::ADD64ri32), NextAddrReg)
30484 .addReg(OverflowDestReg)
30485 .addImm(ArgSizeA8);
30486
30487 // Store the new overflow address.
30488 BuildMI(overflowMBB, DL, TII->get(X86::MOV64mr))
30489 .add(Base)
30490 .add(Scale)
30491 .add(Index)
30492 .addDisp(Disp, 8)
30493 .add(Segment)
30494 .addReg(NextAddrReg)
30495 .setMemRefs(StoreOnlyMMO);
30496
30497 // If we branched, emit the PHI to the front of endMBB.
30498 if (offsetMBB) {
30499 BuildMI(*endMBB, endMBB->begin(), DL,
30500 TII->get(X86::PHI), DestReg)
30501 .addReg(OffsetDestReg).addMBB(offsetMBB)
30502 .addReg(OverflowDestReg).addMBB(overflowMBB);
30503 }
30504
30505 // Erase the pseudo instruction
30506 MI.eraseFromParent();
30507
30508 return endMBB;
30509}
30510
30511MachineBasicBlock *X86TargetLowering::EmitVAStartSaveXMMRegsWithCustomInserter(
30512 MachineInstr &MI, MachineBasicBlock *MBB) const {
30513 // Emit code to save XMM registers to the stack. The ABI says that the
30514 // number of registers to save is given in %al, so it's theoretically
30515 // possible to do an indirect jump trick to avoid saving all of them,
30516 // however this code takes a simpler approach and just executes all
30517 // of the stores if %al is non-zero. It's less code, and it's probably
30518 // easier on the hardware branch predictor, and stores aren't all that
30519 // expensive anyway.
30520
30521 // Create the new basic blocks. One block contains all the XMM stores,
30522 // and one block is the final destination regardless of whether any
30523 // stores were performed.
30524 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
30525 MachineFunction *F = MBB->getParent();
30526 MachineFunction::iterator MBBIter = ++MBB->getIterator();
30527 MachineBasicBlock *XMMSaveMBB = F->CreateMachineBasicBlock(LLVM_BB);
30528 MachineBasicBlock *EndMBB = F->CreateMachineBasicBlock(LLVM_BB);
30529 F->insert(MBBIter, XMMSaveMBB);
30530 F->insert(MBBIter, EndMBB);
30531
30532 // Transfer the remainder of MBB and its successor edges to EndMBB.
30533 EndMBB->splice(EndMBB->begin(), MBB,
30534 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
30535 EndMBB->transferSuccessorsAndUpdatePHIs(MBB);
30536
30537 // The original block will now fall through to the XMM save block.
30538 MBB->addSuccessor(XMMSaveMBB);
30539 // The XMMSaveMBB will fall through to the end block.
30540 XMMSaveMBB->addSuccessor(EndMBB);
30541
30542 // Now add the instructions.
30543 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
30544 DebugLoc DL = MI.getDebugLoc();
30545
30546 Register CountReg = MI.getOperand(0).getReg();
30547 int64_t RegSaveFrameIndex = MI.getOperand(1).getImm();
30548 int64_t VarArgsFPOffset = MI.getOperand(2).getImm();
30549
30550 if (!Subtarget.isCallingConvWin64(F->getFunction().getCallingConv())) {
30551 // If %al is 0, branch around the XMM save block.
30552 BuildMI(MBB, DL, TII->get(X86::TEST8rr)).addReg(CountReg).addReg(CountReg);
30553 BuildMI(MBB, DL, TII->get(X86::JCC_1)).addMBB(EndMBB).addImm(X86::COND_E);
30554 MBB->addSuccessor(EndMBB);
30555 }
30556
30557 // Make sure the last operand is EFLAGS, which gets clobbered by the branch
30558 // that was just emitted, but clearly shouldn't be "saved".
30559 assert((MI.getNumOperands() <= 3 ||(((MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands
() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg
() == X86::EFLAGS) && "Expected last argument to be EFLAGS"
) ? static_cast<void> (0) : __assert_fail ("(MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg() == X86::EFLAGS) && \"Expected last argument to be EFLAGS\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30562, __PRETTY_FUNCTION__))
30560 !MI.getOperand(MI.getNumOperands() - 1).isReg() ||(((MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands
() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg
() == X86::EFLAGS) && "Expected last argument to be EFLAGS"
) ? static_cast<void> (0) : __assert_fail ("(MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg() == X86::EFLAGS) && \"Expected last argument to be EFLAGS\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30562, __PRETTY_FUNCTION__))
30561 MI.getOperand(MI.getNumOperands() - 1).getReg() == X86::EFLAGS) &&(((MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands
() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg
() == X86::EFLAGS) && "Expected last argument to be EFLAGS"
) ? static_cast<void> (0) : __assert_fail ("(MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg() == X86::EFLAGS) && \"Expected last argument to be EFLAGS\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30562, __PRETTY_FUNCTION__))
30562 "Expected last argument to be EFLAGS")(((MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands
() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg
() == X86::EFLAGS) && "Expected last argument to be EFLAGS"
) ? static_cast<void> (0) : __assert_fail ("(MI.getNumOperands() <= 3 || !MI.getOperand(MI.getNumOperands() - 1).isReg() || MI.getOperand(MI.getNumOperands() - 1).getReg() == X86::EFLAGS) && \"Expected last argument to be EFLAGS\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 30562, __PRETTY_FUNCTION__));
30563 unsigned MOVOpc = Subtarget.hasAVX() ? X86::VMOVAPSmr : X86::MOVAPSmr;
30564 // In the XMM save block, save all the XMM argument registers.
30565 for (int i = 3, e = MI.getNumOperands() - 1; i != e; ++i) {
30566 int64_t Offset = (i - 3) * 16 + VarArgsFPOffset;
30567 MachineMemOperand *MMO = F->getMachineMemOperand(
30568 MachinePointerInfo::getFixedStack(*F, RegSaveFrameIndex, Offset),
30569 MachineMemOperand::MOStore,
30570 /*Size=*/16, /*Align=*/16);
30571 BuildMI(XMMSaveMBB, DL, TII->get(MOVOpc))
30572 .addFrameIndex(RegSaveFrameIndex)
30573 .addImm(/*Scale=*/1)
30574 .addReg(/*IndexReg=*/0)
30575 .addImm(/*Disp=*/Offset)
30576 .addReg(/*Segment=*/0)
30577 .addReg(MI.getOperand(i).getReg())
30578 .addMemOperand(MMO);
30579 }
30580
30581 MI.eraseFromParent(); // The pseudo instruction is gone now.
30582
30583 return EndMBB;
30584}
30585
30586// The EFLAGS operand of SelectItr might be missing a kill marker
30587// because there were multiple uses of EFLAGS, and ISel didn't know
30588// which to mark. Figure out whether SelectItr should have had a
30589// kill marker, and set it if it should. Returns the correct kill
30590// marker value.
30591static bool checkAndUpdateEFLAGSKill(MachineBasicBlock::iterator SelectItr,
30592 MachineBasicBlock* BB,
30593 const TargetRegisterInfo* TRI) {
30594 // Scan forward through BB for a use/def of EFLAGS.
30595 MachineBasicBlock::iterator miI(std::next(SelectItr));
30596 for (MachineBasicBlock::iterator miE = BB->end(); miI != miE; ++miI) {
30597 const MachineInstr& mi = *miI;
30598 if (mi.readsRegister(X86::EFLAGS))
30599 return false;
30600 if (mi.definesRegister(X86::EFLAGS))
30601 break; // Should have kill-flag - update below.
30602 }
30603
30604 // If we hit the end of the block, check whether EFLAGS is live into a
30605 // successor.
30606 if (miI == BB->end()) {
30607 for (MachineBasicBlock::succ_iterator sItr = BB->succ_begin(),
30608 sEnd = BB->succ_end();
30609 sItr != sEnd; ++sItr) {
30610 MachineBasicBlock* succ = *sItr;
30611 if (succ->isLiveIn(X86::EFLAGS))
30612 return false;
30613 }
30614 }
30615
30616 // We found a def, or hit the end of the basic block and EFLAGS wasn't live
30617 // out. SelectMI should have a kill flag on EFLAGS.
30618 SelectItr->addRegisterKilled(X86::EFLAGS, TRI);
30619 return true;
30620}
30621
30622// Return true if it is OK for this CMOV pseudo-opcode to be cascaded
30623// together with other CMOV pseudo-opcodes into a single basic-block with
30624// conditional jump around it.
30625static bool isCMOVPseudo(MachineInstr &MI) {
30626 switch (MI.getOpcode()) {
30627 case X86::CMOV_FR32:
30628 case X86::CMOV_FR32X:
30629 case X86::CMOV_FR64:
30630 case X86::CMOV_FR64X:
30631 case X86::CMOV_GR8:
30632 case X86::CMOV_GR16:
30633 case X86::CMOV_GR32:
30634 case X86::CMOV_RFP32:
30635 case X86::CMOV_RFP64:
30636 case X86::CMOV_RFP80:
30637 case X86::CMOV_VR128:
30638 case X86::CMOV_VR128X:
30639 case X86::CMOV_VR256:
30640 case X86::CMOV_VR256X:
30641 case X86::CMOV_VR512:
30642 case X86::CMOV_VK2:
30643 case X86::CMOV_VK4:
30644 case X86::CMOV_VK8:
30645 case X86::CMOV_VK16:
30646 case X86::CMOV_VK32:
30647 case X86::CMOV_VK64:
30648 return true;
30649
30650 default:
30651 return false;
30652 }
30653}
30654
30655// Helper function, which inserts PHI functions into SinkMBB:
30656// %Result(i) = phi [ %FalseValue(i), FalseMBB ], [ %TrueValue(i), TrueMBB ],
30657// where %FalseValue(i) and %TrueValue(i) are taken from the consequent CMOVs
30658// in [MIItBegin, MIItEnd) range. It returns the last MachineInstrBuilder for
30659// the last PHI function inserted.
30660static MachineInstrBuilder createPHIsForCMOVsInSinkBB(
30661 MachineBasicBlock::iterator MIItBegin, MachineBasicBlock::iterator MIItEnd,
30662 MachineBasicBlock *TrueMBB, MachineBasicBlock *FalseMBB,
30663 MachineBasicBlock *SinkMBB) {
30664 MachineFunction *MF = TrueMBB->getParent();
30665 const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
30666 DebugLoc DL = MIItBegin->getDebugLoc();
30667
30668 X86::CondCode CC = X86::CondCode(MIItBegin->getOperand(3).getImm());
30669 X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
30670
30671 MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
30672
30673 // As we are creating the PHIs, we have to be careful if there is more than
30674 // one. Later CMOVs may reference the results of earlier CMOVs, but later
30675 // PHIs have to reference the individual true/false inputs from earlier PHIs.
30676 // That also means that PHI construction must work forward from earlier to
30677 // later, and that the code must maintain a mapping from earlier PHI's
30678 // destination registers, and the registers that went into the PHI.
30679 DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
30680 MachineInstrBuilder MIB;
30681
30682 for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) {
30683 Register DestReg = MIIt->getOperand(0).getReg();
30684 Register Op1Reg = MIIt->getOperand(1).getReg();
30685 Register Op2Reg = MIIt->getOperand(2).getReg();
30686
30687 // If this CMOV we are generating is the opposite condition from
30688 // the jump we generated, then we have to swap the operands for the
30689 // PHI that is going to be generated.
30690 if (MIIt->getOperand(3).getImm() == OppCC)
30691 std::swap(Op1Reg, Op2Reg);
30692
30693 if (RegRewriteTable.find(Op1Reg) != RegRewriteTable.end())
30694 Op1Reg = RegRewriteTable[Op1Reg].first;
30695
30696 if (RegRewriteTable.find(Op2Reg) != RegRewriteTable.end())
30697 Op2Reg = RegRewriteTable[Op2Reg].second;
30698
30699 MIB = BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(X86::PHI), DestReg)
30700 .addReg(Op1Reg)
30701 .addMBB(FalseMBB)
30702 .addReg(Op2Reg)
30703 .addMBB(TrueMBB);
30704
30705 // Add this PHI to the rewrite table.
30706 RegRewriteTable[DestReg] = std::make_pair(Op1Reg, Op2Reg);
30707 }
30708
30709 return MIB;
30710}
30711
30712// Lower cascaded selects in form of (SecondCmov (FirstCMOV F, T, cc1), T, cc2).
30713MachineBasicBlock *
30714X86TargetLowering::EmitLoweredCascadedSelect(MachineInstr &FirstCMOV,
30715 MachineInstr &SecondCascadedCMOV,
30716 MachineBasicBlock *ThisMBB) const {
30717 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
30718 DebugLoc DL = FirstCMOV.getDebugLoc();
30719
30720 // We lower cascaded CMOVs such as
30721 //
30722 // (SecondCascadedCMOV (FirstCMOV F, T, cc1), T, cc2)
30723 //
30724 // to two successive branches.
30725 //
30726 // Without this, we would add a PHI between the two jumps, which ends up
30727 // creating a few copies all around. For instance, for
30728 //
30729 // (sitofp (zext (fcmp une)))
30730 //
30731 // we would generate:
30732 //
30733 // ucomiss %xmm1, %xmm0
30734 // movss <1.0f>, %xmm0
30735 // movaps %xmm0, %xmm1
30736 // jne .LBB5_2
30737 // xorps %xmm1, %xmm1
30738 // .LBB5_2:
30739 // jp .LBB5_4
30740 // movaps %xmm1, %xmm0
30741 // .LBB5_4:
30742 // retq
30743 //
30744 // because this custom-inserter would have generated:
30745 //
30746 // A
30747 // | \
30748 // | B
30749 // | /
30750 // C
30751 // | \
30752 // | D
30753 // | /
30754 // E
30755 //
30756 // A: X = ...; Y = ...
30757 // B: empty
30758 // C: Z = PHI [X, A], [Y, B]
30759 // D: empty
30760 // E: PHI [X, C], [Z, D]
30761 //
30762 // If we lower both CMOVs in a single step, we can instead generate:
30763 //
30764 // A
30765 // | \
30766 // | C
30767 // | /|
30768 // |/ |
30769 // | |
30770 // | D
30771 // | /
30772 // E
30773 //
30774 // A: X = ...; Y = ...
30775 // D: empty
30776 // E: PHI [X, A], [X, C], [Y, D]
30777 //
30778 // Which, in our sitofp/fcmp example, gives us something like:
30779 //
30780 // ucomiss %xmm1, %xmm0
30781 // movss <1.0f>, %xmm0
30782 // jne .LBB5_4
30783 // jp .LBB5_4
30784 // xorps %xmm0, %xmm0
30785 // .LBB5_4:
30786 // retq
30787 //
30788
30789 // We lower cascaded CMOV into two successive branches to the same block.
30790 // EFLAGS is used by both, so mark it as live in the second.
30791 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
30792 MachineFunction *F = ThisMBB->getParent();
30793 MachineBasicBlock *FirstInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
30794 MachineBasicBlock *SecondInsertedMBB = F->CreateMachineBasicBlock(LLVM_BB);
30795 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
30796
30797 MachineFunction::iterator It = ++ThisMBB->getIterator();
30798 F->insert(It, FirstInsertedMBB);
30799 F->insert(It, SecondInsertedMBB);
30800 F->insert(It, SinkMBB);
30801
30802 // For a cascaded CMOV, we lower it to two successive branches to
30803 // the same block (SinkMBB). EFLAGS is used by both, so mark it as live in
30804 // the FirstInsertedMBB.
30805 FirstInsertedMBB->addLiveIn(X86::EFLAGS);
30806
30807 // If the EFLAGS register isn't dead in the terminator, then claim that it's
30808 // live into the sink and copy blocks.
30809 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
30810 if (!SecondCascadedCMOV.killsRegister(X86::EFLAGS) &&
30811 !checkAndUpdateEFLAGSKill(SecondCascadedCMOV, ThisMBB, TRI)) {
30812 SecondInsertedMBB->addLiveIn(X86::EFLAGS);
30813 SinkMBB->addLiveIn(X86::EFLAGS);
30814 }
30815
30816 // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
30817 SinkMBB->splice(SinkMBB->begin(), ThisMBB,
30818 std::next(MachineBasicBlock::iterator(FirstCMOV)),
30819 ThisMBB->end());
30820 SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
30821
30822 // Fallthrough block for ThisMBB.
30823 ThisMBB->addSuccessor(FirstInsertedMBB);
30824 // The true block target of the first branch is always SinkMBB.
30825 ThisMBB->addSuccessor(SinkMBB);
30826 // Fallthrough block for FirstInsertedMBB.
30827 FirstInsertedMBB->addSuccessor(SecondInsertedMBB);
30828 // The true block for the branch of FirstInsertedMBB.
30829 FirstInsertedMBB->addSuccessor(SinkMBB);
30830 // This is fallthrough.
30831 SecondInsertedMBB->addSuccessor(SinkMBB);
30832
30833 // Create the conditional branch instructions.
30834 X86::CondCode FirstCC = X86::CondCode(FirstCMOV.getOperand(3).getImm());
30835 BuildMI(ThisMBB, DL, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(FirstCC);
30836
30837 X86::CondCode SecondCC =
30838 X86::CondCode(SecondCascadedCMOV.getOperand(3).getImm());
30839 BuildMI(FirstInsertedMBB, DL, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(SecondCC);
30840
30841 // SinkMBB:
30842 // %Result = phi [ %FalseValue, SecondInsertedMBB ], [ %TrueValue, ThisMBB ]
30843 Register DestReg = FirstCMOV.getOperand(0).getReg();
30844 Register Op1Reg = FirstCMOV.getOperand(1).getReg();
30845 Register Op2Reg = FirstCMOV.getOperand(2).getReg();
30846 MachineInstrBuilder MIB =
30847 BuildMI(*SinkMBB, SinkMBB->begin(), DL, TII->get(X86::PHI), DestReg)
30848 .addReg(Op1Reg)
30849 .addMBB(SecondInsertedMBB)
30850 .addReg(Op2Reg)
30851 .addMBB(ThisMBB);
30852
30853 // The second SecondInsertedMBB provides the same incoming value as the
30854 // FirstInsertedMBB (the True operand of the SELECT_CC/CMOV nodes).
30855 MIB.addReg(FirstCMOV.getOperand(2).getReg()).addMBB(FirstInsertedMBB);
30856 // Copy the PHI result to the register defined by the second CMOV.
30857 BuildMI(*SinkMBB, std::next(MachineBasicBlock::iterator(MIB.getInstr())), DL,
30858 TII->get(TargetOpcode::COPY),
30859 SecondCascadedCMOV.getOperand(0).getReg())
30860 .addReg(FirstCMOV.getOperand(0).getReg());
30861
30862 // Now remove the CMOVs.
30863 FirstCMOV.eraseFromParent();
30864 SecondCascadedCMOV.eraseFromParent();
30865
30866 return SinkMBB;
30867}
30868
30869MachineBasicBlock *
30870X86TargetLowering::EmitLoweredSelect(MachineInstr &MI,
30871 MachineBasicBlock *ThisMBB) const {
30872 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
30873 DebugLoc DL = MI.getDebugLoc();
30874
30875 // To "insert" a SELECT_CC instruction, we actually have to insert the
30876 // diamond control-flow pattern. The incoming instruction knows the
30877 // destination vreg to set, the condition code register to branch on, the
30878 // true/false values to select between and a branch opcode to use.
30879
30880 // ThisMBB:
30881 // ...
30882 // TrueVal = ...
30883 // cmpTY ccX, r1, r2
30884 // bCC copy1MBB
30885 // fallthrough --> FalseMBB
30886
30887 // This code lowers all pseudo-CMOV instructions. Generally it lowers these
30888 // as described above, by inserting a BB, and then making a PHI at the join
30889 // point to select the true and false operands of the CMOV in the PHI.
30890 //
30891 // The code also handles two different cases of multiple CMOV opcodes
30892 // in a row.
30893 //
30894 // Case 1:
30895 // In this case, there are multiple CMOVs in a row, all which are based on
30896 // the same condition setting (or the exact opposite condition setting).
30897 // In this case we can lower all the CMOVs using a single inserted BB, and
30898 // then make a number of PHIs at the join point to model the CMOVs. The only
30899 // trickiness here, is that in a case like:
30900 //
30901 // t2 = CMOV cond1 t1, f1
30902 // t3 = CMOV cond1 t2, f2
30903 //
30904 // when rewriting this into PHIs, we have to perform some renaming on the
30905 // temps since you cannot have a PHI operand refer to a PHI result earlier
30906 // in the same block. The "simple" but wrong lowering would be:
30907 //
30908 // t2 = PHI t1(BB1), f1(BB2)
30909 // t3 = PHI t2(BB1), f2(BB2)
30910 //
30911 // but clearly t2 is not defined in BB1, so that is incorrect. The proper
30912 // renaming is to note that on the path through BB1, t2 is really just a
30913 // copy of t1, and do that renaming, properly generating:
30914 //
30915 // t2 = PHI t1(BB1), f1(BB2)
30916 // t3 = PHI t1(BB1), f2(BB2)
30917 //
30918 // Case 2:
30919 // CMOV ((CMOV F, T, cc1), T, cc2) is checked here and handled by a separate
30920 // function - EmitLoweredCascadedSelect.
30921
30922 X86::CondCode CC = X86::CondCode(MI.getOperand(3).getImm());
30923 X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
30924 MachineInstr *LastCMOV = &MI;
30925 MachineBasicBlock::iterator NextMIIt = MachineBasicBlock::iterator(MI);
30926
30927 // Check for case 1, where there are multiple CMOVs with the same condition
30928 // first. Of the two cases of multiple CMOV lowerings, case 1 reduces the
30929 // number of jumps the most.
30930
30931 if (isCMOVPseudo(MI)) {
30932 // See if we have a string of CMOVS with the same condition. Skip over
30933 // intervening debug insts.
30934 while (NextMIIt != ThisMBB->end() && isCMOVPseudo(*NextMIIt) &&
30935 (NextMIIt->getOperand(3).getImm() == CC ||
30936 NextMIIt->getOperand(3).getImm() == OppCC)) {
30937 LastCMOV = &*NextMIIt;
30938 ++NextMIIt;
30939 NextMIIt = skipDebugInstructionsForward(NextMIIt, ThisMBB->end());
30940 }
30941 }
30942
30943 // This checks for case 2, but only do this if we didn't already find
30944 // case 1, as indicated by LastCMOV == MI.
30945 if (LastCMOV == &MI && NextMIIt != ThisMBB->end() &&
30946 NextMIIt->getOpcode() == MI.getOpcode() &&
30947 NextMIIt->getOperand(2).getReg() == MI.getOperand(2).getReg() &&
30948 NextMIIt->getOperand(1).getReg() == MI.getOperand(0).getReg() &&
30949 NextMIIt->getOperand(1).isKill()) {
30950 return EmitLoweredCascadedSelect(MI, *NextMIIt, ThisMBB);
30951 }
30952
30953 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
30954 MachineFunction *F = ThisMBB->getParent();
30955 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
30956 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
30957
30958 MachineFunction::iterator It = ++ThisMBB->getIterator();
30959 F->insert(It, FalseMBB);
30960 F->insert(It, SinkMBB);
30961
30962 // If the EFLAGS register isn't dead in the terminator, then claim that it's
30963 // live into the sink and copy blocks.
30964 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
30965 if (!LastCMOV->killsRegister(X86::EFLAGS) &&
30966 !checkAndUpdateEFLAGSKill(LastCMOV, ThisMBB, TRI)) {
30967 FalseMBB->addLiveIn(X86::EFLAGS);
30968 SinkMBB->addLiveIn(X86::EFLAGS);
30969 }
30970
30971 // Transfer any debug instructions inside the CMOV sequence to the sunk block.
30972 auto DbgEnd = MachineBasicBlock::iterator(LastCMOV);
30973 auto DbgIt = MachineBasicBlock::iterator(MI);
30974 while (DbgIt != DbgEnd) {
30975 auto Next = std::next(DbgIt);
30976 if (DbgIt->isDebugInstr())
30977 SinkMBB->push_back(DbgIt->removeFromParent());
30978 DbgIt = Next;
30979 }
30980
30981 // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
30982 SinkMBB->splice(SinkMBB->end(), ThisMBB,
30983 std::next(MachineBasicBlock::iterator(LastCMOV)),
30984 ThisMBB->end());
30985 SinkMBB->transferSuccessorsAndUpdatePHIs(ThisMBB);
30986
30987 // Fallthrough block for ThisMBB.
30988 ThisMBB->addSuccessor(FalseMBB);
30989 // The true block target of the first (or only) branch is always a SinkMBB.
30990 ThisMBB->addSuccessor(SinkMBB);
30991 // Fallthrough block for FalseMBB.
30992 FalseMBB->addSuccessor(SinkMBB);
30993
30994 // Create the conditional branch instruction.
30995 BuildMI(ThisMBB, DL, TII->get(X86::JCC_1)).addMBB(SinkMBB).addImm(CC);
30996
30997 // SinkMBB:
30998 // %Result = phi [ %FalseValue, FalseMBB ], [ %TrueValue, ThisMBB ]
30999 // ...
31000 MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
31001 MachineBasicBlock::iterator MIItEnd =
31002 std::next(MachineBasicBlock::iterator(LastCMOV));
31003 createPHIsForCMOVsInSinkBB(MIItBegin, MIItEnd, ThisMBB, FalseMBB, SinkMBB);
31004
31005 // Now remove the CMOV(s).
31006 ThisMBB->erase(MIItBegin, MIItEnd);
31007
31008 return SinkMBB;
31009}
31010
31011MachineBasicBlock *
31012X86TargetLowering::EmitLoweredSegAlloca(MachineInstr &MI,
31013 MachineBasicBlock *BB) const {
31014 MachineFunction *MF = BB->getParent();
31015 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
31016 DebugLoc DL = MI.getDebugLoc();
31017 const BasicBlock *LLVM_BB = BB->getBasicBlock();
31018
31019 assert(MF->shouldSplitStack())((MF->shouldSplitStack()) ? static_cast<void> (0) : __assert_fail
("MF->shouldSplitStack()", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31019, __PRETTY_FUNCTION__));
31020
31021 const bool Is64Bit = Subtarget.is64Bit();
31022 const bool IsLP64 = Subtarget.isTarget64BitLP64();
31023
31024 const unsigned TlsReg = Is64Bit ? X86::FS : X86::GS;
31025 const unsigned TlsOffset = IsLP64 ? 0x70 : Is64Bit ? 0x40 : 0x30;
31026
31027 // BB:
31028 // ... [Till the alloca]
31029 // If stacklet is not large enough, jump to mallocMBB
31030 //
31031 // bumpMBB:
31032 // Allocate by subtracting from RSP
31033 // Jump to continueMBB
31034 //
31035 // mallocMBB:
31036 // Allocate by call to runtime
31037 //
31038 // continueMBB:
31039 // ...
31040 // [rest of original BB]
31041 //
31042
31043 MachineBasicBlock *mallocMBB = MF->CreateMachineBasicBlock(LLVM_BB);
31044 MachineBasicBlock *bumpMBB = MF->CreateMachineBasicBlock(LLVM_BB);
31045 MachineBasicBlock *continueMBB = MF->CreateMachineBasicBlock(LLVM_BB);
31046
31047 MachineRegisterInfo &MRI = MF->getRegInfo();
31048 const TargetRegisterClass *AddrRegClass =
31049 getRegClassFor(getPointerTy(MF->getDataLayout()));
31050
31051 unsigned mallocPtrVReg = MRI.createVirtualRegister(AddrRegClass),
31052 bumpSPPtrVReg = MRI.createVirtualRegister(AddrRegClass),
31053 tmpSPVReg = MRI.createVirtualRegister(AddrRegClass),
31054 SPLimitVReg = MRI.createVirtualRegister(AddrRegClass),
31055 sizeVReg = MI.getOperand(1).getReg(),
31056 physSPReg =
31057 IsLP64 || Subtarget.isTargetNaCl64() ? X86::RSP : X86::ESP;
31058
31059 MachineFunction::iterator MBBIter = ++BB->getIterator();
31060
31061 MF->insert(MBBIter, bumpMBB);
31062 MF->insert(MBBIter, mallocMBB);
31063 MF->insert(MBBIter, continueMBB);
31064
31065 continueMBB->splice(continueMBB->begin(), BB,
31066 std::next(MachineBasicBlock::iterator(MI)), BB->end());
31067 continueMBB->transferSuccessorsAndUpdatePHIs(BB);
31068
31069 // Add code to the main basic block to check if the stack limit has been hit,
31070 // and if so, jump to mallocMBB otherwise to bumpMBB.
31071 BuildMI(BB, DL, TII->get(TargetOpcode::COPY), tmpSPVReg).addReg(physSPReg);
31072 BuildMI(BB, DL, TII->get(IsLP64 ? X86::SUB64rr:X86::SUB32rr), SPLimitVReg)
31073 .addReg(tmpSPVReg).addReg(sizeVReg);
31074 BuildMI(BB, DL, TII->get(IsLP64 ? X86::CMP64mr:X86::CMP32mr))
31075 .addReg(0).addImm(1).addReg(0).addImm(TlsOffset).addReg(TlsReg)
31076 .addReg(SPLimitVReg);
31077 BuildMI(BB, DL, TII->get(X86::JCC_1)).addMBB(mallocMBB).addImm(X86::COND_G);
31078
31079 // bumpMBB simply decreases the stack pointer, since we know the current
31080 // stacklet has enough space.
31081 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), physSPReg)
31082 .addReg(SPLimitVReg);
31083 BuildMI(bumpMBB, DL, TII->get(TargetOpcode::COPY), bumpSPPtrVReg)
31084 .addReg(SPLimitVReg);
31085 BuildMI(bumpMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
31086
31087 // Calls into a routine in libgcc to allocate more space from the heap.
31088 const uint32_t *RegMask =
31089 Subtarget.getRegisterInfo()->getCallPreservedMask(*MF, CallingConv::C);
31090 if (IsLP64) {
31091 BuildMI(mallocMBB, DL, TII->get(X86::MOV64rr), X86::RDI)
31092 .addReg(sizeVReg);
31093 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
31094 .addExternalSymbol("__morestack_allocate_stack_space")
31095 .addRegMask(RegMask)
31096 .addReg(X86::RDI, RegState::Implicit)
31097 .addReg(X86::RAX, RegState::ImplicitDefine);
31098 } else if (Is64Bit) {
31099 BuildMI(mallocMBB, DL, TII->get(X86::MOV32rr), X86::EDI)
31100 .addReg(sizeVReg);
31101 BuildMI(mallocMBB, DL, TII->get(X86::CALL64pcrel32))
31102 .addExternalSymbol("__morestack_allocate_stack_space")
31103 .addRegMask(RegMask)
31104 .addReg(X86::EDI, RegState::Implicit)
31105 .addReg(X86::EAX, RegState::ImplicitDefine);
31106 } else {
31107 BuildMI(mallocMBB, DL, TII->get(X86::SUB32ri), physSPReg).addReg(physSPReg)
31108 .addImm(12);
31109 BuildMI(mallocMBB, DL, TII->get(X86::PUSH32r)).addReg(sizeVReg);
31110 BuildMI(mallocMBB, DL, TII->get(X86::CALLpcrel32))
31111 .addExternalSymbol("__morestack_allocate_stack_space")
31112 .addRegMask(RegMask)
31113 .addReg(X86::EAX, RegState::ImplicitDefine);
31114 }
31115
31116 if (!Is64Bit)
31117 BuildMI(mallocMBB, DL, TII->get(X86::ADD32ri), physSPReg).addReg(physSPReg)
31118 .addImm(16);
31119
31120 BuildMI(mallocMBB, DL, TII->get(TargetOpcode::COPY), mallocPtrVReg)
31121 .addReg(IsLP64 ? X86::RAX : X86::EAX);
31122 BuildMI(mallocMBB, DL, TII->get(X86::JMP_1)).addMBB(continueMBB);
31123
31124 // Set up the CFG correctly.
31125 BB->addSuccessor(bumpMBB);
31126 BB->addSuccessor(mallocMBB);
31127 mallocMBB->addSuccessor(continueMBB);
31128 bumpMBB->addSuccessor(continueMBB);
31129
31130 // Take care of the PHI nodes.
31131 BuildMI(*continueMBB, continueMBB->begin(), DL, TII->get(X86::PHI),
31132 MI.getOperand(0).getReg())
31133 .addReg(mallocPtrVReg)
31134 .addMBB(mallocMBB)
31135 .addReg(bumpSPPtrVReg)
31136 .addMBB(bumpMBB);
31137
31138 // Delete the original pseudo instruction.
31139 MI.eraseFromParent();
31140
31141 // And we're done.
31142 return continueMBB;
31143}
31144
31145MachineBasicBlock *
31146X86TargetLowering::EmitLoweredCatchRet(MachineInstr &MI,
31147 MachineBasicBlock *BB) const {
31148 MachineFunction *MF = BB->getParent();
31149 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
31150 MachineBasicBlock *TargetMBB = MI.getOperand(0).getMBB();
31151 DebugLoc DL = MI.getDebugLoc();
31152
31153 assert(!isAsynchronousEHPersonality(((!isAsynchronousEHPersonality( classifyEHPersonality(MF->
getFunction().getPersonalityFn())) && "SEH does not use catchret!"
) ? static_cast<void> (0) : __assert_fail ("!isAsynchronousEHPersonality( classifyEHPersonality(MF->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31155, __PRETTY_FUNCTION__))
31154 classifyEHPersonality(MF->getFunction().getPersonalityFn())) &&((!isAsynchronousEHPersonality( classifyEHPersonality(MF->
getFunction().getPersonalityFn())) && "SEH does not use catchret!"
) ? static_cast<void> (0) : __assert_fail ("!isAsynchronousEHPersonality( classifyEHPersonality(MF->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31155, __PRETTY_FUNCTION__))
31155 "SEH does not use catchret!")((!isAsynchronousEHPersonality( classifyEHPersonality(MF->
getFunction().getPersonalityFn())) && "SEH does not use catchret!"
) ? static_cast<void> (0) : __assert_fail ("!isAsynchronousEHPersonality( classifyEHPersonality(MF->getFunction().getPersonalityFn())) && \"SEH does not use catchret!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31155, __PRETTY_FUNCTION__));
31156
31157 // Only 32-bit EH needs to worry about manually restoring stack pointers.
31158 if (!Subtarget.is32Bit())
31159 return BB;
31160
31161 // C++ EH creates a new target block to hold the restore code, and wires up
31162 // the new block to the return destination with a normal JMP_4.
31163 MachineBasicBlock *RestoreMBB =
31164 MF->CreateMachineBasicBlock(BB->getBasicBlock());
31165 assert(BB->succ_size() == 1)((BB->succ_size() == 1) ? static_cast<void> (0) : __assert_fail
("BB->succ_size() == 1", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31165, __PRETTY_FUNCTION__));
31166 MF->insert(std::next(BB->getIterator()), RestoreMBB);
31167 RestoreMBB->transferSuccessorsAndUpdatePHIs(BB);
31168 BB->addSuccessor(RestoreMBB);
31169 MI.getOperand(0).setMBB(RestoreMBB);
31170
31171 auto RestoreMBBI = RestoreMBB->begin();
31172 BuildMI(*RestoreMBB, RestoreMBBI, DL, TII.get(X86::EH_RESTORE));
31173 BuildMI(*RestoreMBB, RestoreMBBI, DL, TII.get(X86::JMP_4)).addMBB(TargetMBB);
31174 return BB;
31175}
31176
31177MachineBasicBlock *
31178X86TargetLowering::EmitLoweredCatchPad(MachineInstr &MI,
31179 MachineBasicBlock *BB) const {
31180 MachineFunction *MF = BB->getParent();
31181 const Constant *PerFn = MF->getFunction().getPersonalityFn();
31182 bool IsSEH = isAsynchronousEHPersonality(classifyEHPersonality(PerFn));
31183 // Only 32-bit SEH requires special handling for catchpad.
31184 if (IsSEH && Subtarget.is32Bit()) {
31185 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
31186 DebugLoc DL = MI.getDebugLoc();
31187 BuildMI(*BB, MI, DL, TII.get(X86::EH_RESTORE));
31188 }
31189 MI.eraseFromParent();
31190 return BB;
31191}
31192
31193MachineBasicBlock *
31194X86TargetLowering::EmitLoweredTLSAddr(MachineInstr &MI,
31195 MachineBasicBlock *BB) const {
31196 // So, here we replace TLSADDR with the sequence:
31197 // adjust_stackdown -> TLSADDR -> adjust_stackup.
31198 // We need this because TLSADDR is lowered into calls
31199 // inside MC, therefore without the two markers shrink-wrapping
31200 // may push the prologue/epilogue pass them.
31201 const TargetInstrInfo &TII = *Subtarget.getInstrInfo();
31202 DebugLoc DL = MI.getDebugLoc();
31203 MachineFunction &MF = *BB->getParent();
31204
31205 // Emit CALLSEQ_START right before the instruction.
31206 unsigned AdjStackDown = TII.getCallFrameSetupOpcode();
31207 MachineInstrBuilder CallseqStart =
31208 BuildMI(MF, DL, TII.get(AdjStackDown)).addImm(0).addImm(0).addImm(0);
31209 BB->insert(MachineBasicBlock::iterator(MI), CallseqStart);
31210
31211 // Emit CALLSEQ_END right after the instruction.
31212 // We don't call erase from parent because we want to keep the
31213 // original instruction around.
31214 unsigned AdjStackUp = TII.getCallFrameDestroyOpcode();
31215 MachineInstrBuilder CallseqEnd =
31216 BuildMI(MF, DL, TII.get(AdjStackUp)).addImm(0).addImm(0);
31217 BB->insertAfter(MachineBasicBlock::iterator(MI), CallseqEnd);
31218
31219 return BB;
31220}
31221
31222MachineBasicBlock *
31223X86TargetLowering::EmitLoweredTLSCall(MachineInstr &MI,
31224 MachineBasicBlock *BB) const {
31225 // This is pretty easy. We're taking the value that we received from
31226 // our load from the relocation, sticking it in either RDI (x86-64)
31227 // or EAX and doing an indirect call. The return value will then
31228 // be in the normal return register.
31229 MachineFunction *F = BB->getParent();
31230 const X86InstrInfo *TII = Subtarget.getInstrInfo();
31231 DebugLoc DL = MI.getDebugLoc();
31232
31233 assert(Subtarget.isTargetDarwin() && "Darwin only instr emitted?")((Subtarget.isTargetDarwin() && "Darwin only instr emitted?"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.isTargetDarwin() && \"Darwin only instr emitted?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31233, __PRETTY_FUNCTION__));
31234 assert(MI.getOperand(3).isGlobal() && "This should be a global")((MI.getOperand(3).isGlobal() && "This should be a global"
) ? static_cast<void> (0) : __assert_fail ("MI.getOperand(3).isGlobal() && \"This should be a global\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31234, __PRETTY_FUNCTION__));
31235
31236 // Get a register mask for the lowered call.
31237 // FIXME: The 32-bit calls have non-standard calling conventions. Use a
31238 // proper register mask.
31239 const uint32_t *RegMask =
31240 Subtarget.is64Bit() ?
31241 Subtarget.getRegisterInfo()->getDarwinTLSCallPreservedMask() :
31242 Subtarget.getRegisterInfo()->getCallPreservedMask(*F, CallingConv::C);
31243 if (Subtarget.is64Bit()) {
31244 MachineInstrBuilder MIB =
31245 BuildMI(*BB, MI, DL, TII->get(X86::MOV64rm), X86::RDI)
31246 .addReg(X86::RIP)
31247 .addImm(0)
31248 .addReg(0)
31249 .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
31250 MI.getOperand(3).getTargetFlags())
31251 .addReg(0);
31252 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL64m));
31253 addDirectMem(MIB, X86::RDI);
31254 MIB.addReg(X86::RAX, RegState::ImplicitDefine).addRegMask(RegMask);
31255 } else if (!isPositionIndependent()) {
31256 MachineInstrBuilder MIB =
31257 BuildMI(*BB, MI, DL, TII->get(X86::MOV32rm), X86::EAX)
31258 .addReg(0)
31259 .addImm(0)
31260 .addReg(0)
31261 .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
31262 MI.getOperand(3).getTargetFlags())
31263 .addReg(0);
31264 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
31265 addDirectMem(MIB, X86::EAX);
31266 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
31267 } else {
31268 MachineInstrBuilder MIB =
31269 BuildMI(*BB, MI, DL, TII->get(X86::MOV32rm), X86::EAX)
31270 .addReg(TII->getGlobalBaseReg(F))
31271 .addImm(0)
31272 .addReg(0)
31273 .addGlobalAddress(MI.getOperand(3).getGlobal(), 0,
31274 MI.getOperand(3).getTargetFlags())
31275 .addReg(0);
31276 MIB = BuildMI(*BB, MI, DL, TII->get(X86::CALL32m));
31277 addDirectMem(MIB, X86::EAX);
31278 MIB.addReg(X86::EAX, RegState::ImplicitDefine).addRegMask(RegMask);
31279 }
31280
31281 MI.eraseFromParent(); // The pseudo instruction is gone now.
31282 return BB;
31283}
31284
31285static unsigned getOpcodeForRetpoline(unsigned RPOpc) {
31286 switch (RPOpc) {
31287 case X86::RETPOLINE_CALL32:
31288 return X86::CALLpcrel32;
31289 case X86::RETPOLINE_CALL64:
31290 return X86::CALL64pcrel32;
31291 case X86::RETPOLINE_TCRETURN32:
31292 return X86::TCRETURNdi;
31293 case X86::RETPOLINE_TCRETURN64:
31294 return X86::TCRETURNdi64;
31295 }
31296 llvm_unreachable("not retpoline opcode")::llvm::llvm_unreachable_internal("not retpoline opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31296);
31297}
31298
31299static const char *getRetpolineSymbol(const X86Subtarget &Subtarget,
31300 unsigned Reg) {
31301 if (Subtarget.useRetpolineExternalThunk()) {
31302 // When using an external thunk for retpolines, we pick names that match the
31303 // names GCC happens to use as well. This helps simplify the implementation
31304 // of the thunks for kernels where they have no easy ability to create
31305 // aliases and are doing non-trivial configuration of the thunk's body. For
31306 // example, the Linux kernel will do boot-time hot patching of the thunk
31307 // bodies and cannot easily export aliases of these to loaded modules.
31308 //
31309 // Note that at any point in the future, we may need to change the semantics
31310 // of how we implement retpolines and at that time will likely change the
31311 // name of the called thunk. Essentially, there is no hard guarantee that
31312 // LLVM will generate calls to specific thunks, we merely make a best-effort
31313 // attempt to help out kernels and other systems where duplicating the
31314 // thunks is costly.
31315 switch (Reg) {
31316 case X86::EAX:
31317 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31317, __PRETTY_FUNCTION__));
31318 return "__x86_indirect_thunk_eax";
31319 case X86::ECX:
31320 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31320, __PRETTY_FUNCTION__));
31321 return "__x86_indirect_thunk_ecx";
31322 case X86::EDX:
31323 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31323, __PRETTY_FUNCTION__));
31324 return "__x86_indirect_thunk_edx";
31325 case X86::EDI:
31326 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31326, __PRETTY_FUNCTION__));
31327 return "__x86_indirect_thunk_edi";
31328 case X86::R11:
31329 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!")((Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.is64Bit() && \"Should not be using a 64-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31329, __PRETTY_FUNCTION__));
31330 return "__x86_indirect_thunk_r11";
31331 }
31332 llvm_unreachable("unexpected reg for retpoline")::llvm::llvm_unreachable_internal("unexpected reg for retpoline"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31332);
31333 }
31334
31335 // When targeting an internal COMDAT thunk use an LLVM-specific name.
31336 switch (Reg) {
31337 case X86::EAX:
31338 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31338, __PRETTY_FUNCTION__));
31339 return "__llvm_retpoline_eax";
31340 case X86::ECX:
31341 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31341, __PRETTY_FUNCTION__));
31342 return "__llvm_retpoline_ecx";
31343 case X86::EDX:
31344 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31344, __PRETTY_FUNCTION__));
31345 return "__llvm_retpoline_edx";
31346 case X86::EDI:
31347 assert(!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!")((!Subtarget.is64Bit() && "Should not be using a 32-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"Should not be using a 32-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31347, __PRETTY_FUNCTION__));
31348 return "__llvm_retpoline_edi";
31349 case X86::R11:
31350 assert(Subtarget.is64Bit() && "Should not be using a 64-bit thunk!")((Subtarget.is64Bit() && "Should not be using a 64-bit thunk!"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.is64Bit() && \"Should not be using a 64-bit thunk!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31350, __PRETTY_FUNCTION__));
31351 return "__llvm_retpoline_r11";
31352 }
31353 llvm_unreachable("unexpected reg for retpoline")::llvm::llvm_unreachable_internal("unexpected reg for retpoline"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31353);
31354}
31355
31356MachineBasicBlock *
31357X86TargetLowering::EmitLoweredRetpoline(MachineInstr &MI,
31358 MachineBasicBlock *BB) const {
31359 // Copy the virtual register into the R11 physical register and
31360 // call the retpoline thunk.
31361 DebugLoc DL = MI.getDebugLoc();
31362 const X86InstrInfo *TII = Subtarget.getInstrInfo();
31363 Register CalleeVReg = MI.getOperand(0).getReg();
31364 unsigned Opc = getOpcodeForRetpoline(MI.getOpcode());
31365
31366 // Find an available scratch register to hold the callee. On 64-bit, we can
31367 // just use R11, but we scan for uses anyway to ensure we don't generate
31368 // incorrect code. On 32-bit, we use one of EAX, ECX, or EDX that isn't
31369 // already a register use operand to the call to hold the callee. If none
31370 // are available, use EDI instead. EDI is chosen because EBX is the PIC base
31371 // register and ESI is the base pointer to realigned stack frames with VLAs.
31372 SmallVector<unsigned, 3> AvailableRegs;
31373 if (Subtarget.is64Bit())
31374 AvailableRegs.push_back(X86::R11);
31375 else
31376 AvailableRegs.append({X86::EAX, X86::ECX, X86::EDX, X86::EDI});
31377
31378 // Zero out any registers that are already used.
31379 for (const auto &MO : MI.operands()) {
31380 if (MO.isReg() && MO.isUse())
31381 for (unsigned &Reg : AvailableRegs)
31382 if (Reg == MO.getReg())
31383 Reg = 0;
31384 }
31385
31386 // Choose the first remaining non-zero available register.
31387 unsigned AvailableReg = 0;
31388 for (unsigned MaybeReg : AvailableRegs) {
31389 if (MaybeReg) {
31390 AvailableReg = MaybeReg;
31391 break;
31392 }
31393 }
31394 if (!AvailableReg)
31395 report_fatal_error("calling convention incompatible with retpoline, no "
31396 "available registers");
31397
31398 const char *Symbol = getRetpolineSymbol(Subtarget, AvailableReg);
31399
31400 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), AvailableReg)
31401 .addReg(CalleeVReg);
31402 MI.getOperand(0).ChangeToES(Symbol);
31403 MI.setDesc(TII->get(Opc));
31404 MachineInstrBuilder(*BB->getParent(), &MI)
31405 .addReg(AvailableReg, RegState::Implicit | RegState::Kill);
31406 return BB;
31407}
31408
31409/// SetJmp implies future control flow change upon calling the corresponding
31410/// LongJmp.
31411/// Instead of using the 'return' instruction, the long jump fixes the stack and
31412/// performs an indirect branch. To do so it uses the registers that were stored
31413/// in the jump buffer (when calling SetJmp).
31414/// In case the shadow stack is enabled we need to fix it as well, because some
31415/// return addresses will be skipped.
31416/// The function will save the SSP for future fixing in the function
31417/// emitLongJmpShadowStackFix.
31418/// \sa emitLongJmpShadowStackFix
31419/// \param [in] MI The temporary Machine Instruction for the builtin.
31420/// \param [in] MBB The Machine Basic Block that will be modified.
31421void X86TargetLowering::emitSetJmpShadowStackFix(MachineInstr &MI,
31422 MachineBasicBlock *MBB) const {
31423 DebugLoc DL = MI.getDebugLoc();
31424 MachineFunction *MF = MBB->getParent();
31425 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
31426 MachineRegisterInfo &MRI = MF->getRegInfo();
31427 MachineInstrBuilder MIB;
31428
31429 // Memory Reference.
31430 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
31431 MI.memoperands_end());
31432
31433 // Initialize a register with zero.
31434 MVT PVT = getPointerTy(MF->getDataLayout());
31435 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
31436 Register ZReg = MRI.createVirtualRegister(PtrRC);
31437 unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;
31438 BuildMI(*MBB, MI, DL, TII->get(XorRROpc))
31439 .addDef(ZReg)
31440 .addReg(ZReg, RegState::Undef)
31441 .addReg(ZReg, RegState::Undef);
31442
31443 // Read the current SSP Register value to the zeroed register.
31444 Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
31445 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
31446 BuildMI(*MBB, MI, DL, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
31447
31448 // Write the SSP register value to offset 3 in input memory buffer.
31449 unsigned PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
31450 MIB = BuildMI(*MBB, MI, DL, TII->get(PtrStoreOpc));
31451 const int64_t SSPOffset = 3 * PVT.getStoreSize();
31452 const unsigned MemOpndSlot = 1;
31453 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
31454 if (i == X86::AddrDisp)
31455 MIB.addDisp(MI.getOperand(MemOpndSlot + i), SSPOffset);
31456 else
31457 MIB.add(MI.getOperand(MemOpndSlot + i));
31458 }
31459 MIB.addReg(SSPCopyReg);
31460 MIB.setMemRefs(MMOs);
31461}
31462
31463MachineBasicBlock *
31464X86TargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
31465 MachineBasicBlock *MBB) const {
31466 DebugLoc DL = MI.getDebugLoc();
31467 MachineFunction *MF = MBB->getParent();
31468 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
31469 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
31470 MachineRegisterInfo &MRI = MF->getRegInfo();
31471
31472 const BasicBlock *BB = MBB->getBasicBlock();
31473 MachineFunction::iterator I = ++MBB->getIterator();
31474
31475 // Memory Reference
31476 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
31477 MI.memoperands_end());
31478
31479 unsigned DstReg;
31480 unsigned MemOpndSlot = 0;
31481
31482 unsigned CurOp = 0;
31483
31484 DstReg = MI.getOperand(CurOp++).getReg();
31485 const TargetRegisterClass *RC = MRI.getRegClass(DstReg);
31486 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!")((TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!"
) ? static_cast<void> (0) : __assert_fail ("TRI->isTypeLegalForClass(*RC, MVT::i32) && \"Invalid destination!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31486, __PRETTY_FUNCTION__));
31487 (void)TRI;
31488 Register mainDstReg = MRI.createVirtualRegister(RC);
31489 Register restoreDstReg = MRI.createVirtualRegister(RC);
31490
31491 MemOpndSlot = CurOp;
31492
31493 MVT PVT = getPointerTy(MF->getDataLayout());
31494 assert((PVT == MVT::i64 || PVT == MVT::i32) &&(((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"
) ? static_cast<void> (0) : __assert_fail ("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31495, __PRETTY_FUNCTION__))
31495 "Invalid Pointer Size!")(((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"
) ? static_cast<void> (0) : __assert_fail ("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31495, __PRETTY_FUNCTION__));
31496
31497 // For v = setjmp(buf), we generate
31498 //
31499 // thisMBB:
31500 // buf[LabelOffset] = restoreMBB <-- takes address of restoreMBB
31501 // SjLjSetup restoreMBB
31502 //
31503 // mainMBB:
31504 // v_main = 0
31505 //
31506 // sinkMBB:
31507 // v = phi(main, restore)
31508 //
31509 // restoreMBB:
31510 // if base pointer being used, load it from frame
31511 // v_restore = 1
31512
31513 MachineBasicBlock *thisMBB = MBB;
31514 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
31515 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
31516 MachineBasicBlock *restoreMBB = MF->CreateMachineBasicBlock(BB);
31517 MF->insert(I, mainMBB);
31518 MF->insert(I, sinkMBB);
31519 MF->push_back(restoreMBB);
31520 restoreMBB->setHasAddressTaken();
31521
31522 MachineInstrBuilder MIB;
31523
31524 // Transfer the remainder of BB and its successor edges to sinkMBB.
31525 sinkMBB->splice(sinkMBB->begin(), MBB,
31526 std::next(MachineBasicBlock::iterator(MI)), MBB->end());
31527 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
31528
31529 // thisMBB:
31530 unsigned PtrStoreOpc = 0;
31531 unsigned LabelReg = 0;
31532 const int64_t LabelOffset = 1 * PVT.getStoreSize();
31533 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
31534 !isPositionIndependent();
31535
31536 // Prepare IP either in reg or imm.
31537 if (!UseImmLabel) {
31538 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
31539 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
31540 LabelReg = MRI.createVirtualRegister(PtrRC);
31541 if (Subtarget.is64Bit()) {
31542 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA64r), LabelReg)
31543 .addReg(X86::RIP)
31544 .addImm(0)
31545 .addReg(0)
31546 .addMBB(restoreMBB)
31547 .addReg(0);
31548 } else {
31549 const X86InstrInfo *XII = static_cast<const X86InstrInfo*>(TII);
31550 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::LEA32r), LabelReg)
31551 .addReg(XII->getGlobalBaseReg(MF))
31552 .addImm(0)
31553 .addReg(0)
31554 .addMBB(restoreMBB, Subtarget.classifyBlockAddressReference())
31555 .addReg(0);
31556 }
31557 } else
31558 PtrStoreOpc = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
31559 // Store IP
31560 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrStoreOpc));
31561 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
31562 if (i == X86::AddrDisp)
31563 MIB.addDisp(MI.getOperand(MemOpndSlot + i), LabelOffset);
31564 else
31565 MIB.add(MI.getOperand(MemOpndSlot + i));
31566 }
31567 if (!UseImmLabel)
31568 MIB.addReg(LabelReg);
31569 else
31570 MIB.addMBB(restoreMBB);
31571 MIB.setMemRefs(MMOs);
31572
31573 if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
31574 emitSetJmpShadowStackFix(MI, thisMBB);
31575 }
31576
31577 // Setup
31578 MIB = BuildMI(*thisMBB, MI, DL, TII->get(X86::EH_SjLj_Setup))
31579 .addMBB(restoreMBB);
31580
31581 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
31582 MIB.addRegMask(RegInfo->getNoPreservedMask());
31583 thisMBB->addSuccessor(mainMBB);
31584 thisMBB->addSuccessor(restoreMBB);
31585
31586 // mainMBB:
31587 // EAX = 0
31588 BuildMI(mainMBB, DL, TII->get(X86::MOV32r0), mainDstReg);
31589 mainMBB->addSuccessor(sinkMBB);
31590
31591 // sinkMBB:
31592 BuildMI(*sinkMBB, sinkMBB->begin(), DL,
31593 TII->get(X86::PHI), DstReg)
31594 .addReg(mainDstReg).addMBB(mainMBB)
31595 .addReg(restoreDstReg).addMBB(restoreMBB);
31596
31597 // restoreMBB:
31598 if (RegInfo->hasBasePointer(*MF)) {
31599 const bool Uses64BitFramePtr =
31600 Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
31601 X86MachineFunctionInfo *X86FI = MF->getInfo<X86MachineFunctionInfo>();
31602 X86FI->setRestoreBasePointer(MF);
31603 Register FramePtr = RegInfo->getFrameRegister(*MF);
31604 Register BasePtr = RegInfo->getBaseRegister();
31605 unsigned Opm = Uses64BitFramePtr ? X86::MOV64rm : X86::MOV32rm;
31606 addRegOffset(BuildMI(restoreMBB, DL, TII->get(Opm), BasePtr),
31607 FramePtr, true, X86FI->getRestoreBasePointerOffset())
31608 .setMIFlag(MachineInstr::FrameSetup);
31609 }
31610 BuildMI(restoreMBB, DL, TII->get(X86::MOV32ri), restoreDstReg).addImm(1);
31611 BuildMI(restoreMBB, DL, TII->get(X86::JMP_1)).addMBB(sinkMBB);
31612 restoreMBB->addSuccessor(sinkMBB);
31613
31614 MI.eraseFromParent();
31615 return sinkMBB;
31616}
31617
31618/// Fix the shadow stack using the previously saved SSP pointer.
31619/// \sa emitSetJmpShadowStackFix
31620/// \param [in] MI The temporary Machine Instruction for the builtin.
31621/// \param [in] MBB The Machine Basic Block that will be modified.
31622/// \return The sink MBB that will perform the future indirect branch.
31623MachineBasicBlock *
31624X86TargetLowering::emitLongJmpShadowStackFix(MachineInstr &MI,
31625 MachineBasicBlock *MBB) const {
31626 DebugLoc DL = MI.getDebugLoc();
31627 MachineFunction *MF = MBB->getParent();
31628 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
31629 MachineRegisterInfo &MRI = MF->getRegInfo();
31630
31631 // Memory Reference
31632 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
31633 MI.memoperands_end());
31634
31635 MVT PVT = getPointerTy(MF->getDataLayout());
31636 const TargetRegisterClass *PtrRC = getRegClassFor(PVT);
31637
31638 // checkSspMBB:
31639 // xor vreg1, vreg1
31640 // rdssp vreg1
31641 // test vreg1, vreg1
31642 // je sinkMBB # Jump if Shadow Stack is not supported
31643 // fallMBB:
31644 // mov buf+24/12(%rip), vreg2
31645 // sub vreg1, vreg2
31646 // jbe sinkMBB # No need to fix the Shadow Stack
31647 // fixShadowMBB:
31648 // shr 3/2, vreg2
31649 // incssp vreg2 # fix the SSP according to the lower 8 bits
31650 // shr 8, vreg2
31651 // je sinkMBB
31652 // fixShadowLoopPrepareMBB:
31653 // shl vreg2
31654 // mov 128, vreg3
31655 // fixShadowLoopMBB:
31656 // incssp vreg3
31657 // dec vreg2
31658 // jne fixShadowLoopMBB # Iterate until you finish fixing
31659 // # the Shadow Stack
31660 // sinkMBB:
31661
31662 MachineFunction::iterator I = ++MBB->getIterator();
31663 const BasicBlock *BB = MBB->getBasicBlock();
31664
31665 MachineBasicBlock *checkSspMBB = MF->CreateMachineBasicBlock(BB);
31666 MachineBasicBlock *fallMBB = MF->CreateMachineBasicBlock(BB);
31667 MachineBasicBlock *fixShadowMBB = MF->CreateMachineBasicBlock(BB);
31668 MachineBasicBlock *fixShadowLoopPrepareMBB = MF->CreateMachineBasicBlock(BB);
31669 MachineBasicBlock *fixShadowLoopMBB = MF->CreateMachineBasicBlock(BB);
31670 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
31671 MF->insert(I, checkSspMBB);
31672 MF->insert(I, fallMBB);
31673 MF->insert(I, fixShadowMBB);
31674 MF->insert(I, fixShadowLoopPrepareMBB);
31675 MF->insert(I, fixShadowLoopMBB);
31676 MF->insert(I, sinkMBB);
31677
31678 // Transfer the remainder of BB and its successor edges to sinkMBB.
31679 sinkMBB->splice(sinkMBB->begin(), MBB, MachineBasicBlock::iterator(MI),
31680 MBB->end());
31681 sinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
31682
31683 MBB->addSuccessor(checkSspMBB);
31684
31685 // Initialize a register with zero.
31686 Register ZReg = MRI.createVirtualRegister(PtrRC);
31687 unsigned XorRROpc = (PVT == MVT::i64) ? X86::XOR64rr : X86::XOR32rr;
31688 BuildMI(checkSspMBB, DL, TII->get(XorRROpc))
31689 .addDef(ZReg)
31690 .addReg(ZReg, RegState::Undef)
31691 .addReg(ZReg, RegState::Undef);
31692
31693 // Read the current SSP Register value to the zeroed register.
31694 Register SSPCopyReg = MRI.createVirtualRegister(PtrRC);
31695 unsigned RdsspOpc = (PVT == MVT::i64) ? X86::RDSSPQ : X86::RDSSPD;
31696 BuildMI(checkSspMBB, DL, TII->get(RdsspOpc), SSPCopyReg).addReg(ZReg);
31697
31698 // Check whether the result of the SSP register is zero and jump directly
31699 // to the sink.
31700 unsigned TestRROpc = (PVT == MVT::i64) ? X86::TEST64rr : X86::TEST32rr;
31701 BuildMI(checkSspMBB, DL, TII->get(TestRROpc))
31702 .addReg(SSPCopyReg)
31703 .addReg(SSPCopyReg);
31704 BuildMI(checkSspMBB, DL, TII->get(X86::JCC_1)).addMBB(sinkMBB).addImm(X86::COND_E);
31705 checkSspMBB->addSuccessor(sinkMBB);
31706 checkSspMBB->addSuccessor(fallMBB);
31707
31708 // Reload the previously saved SSP register value.
31709 Register PrevSSPReg = MRI.createVirtualRegister(PtrRC);
31710 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
31711 const int64_t SPPOffset = 3 * PVT.getStoreSize();
31712 MachineInstrBuilder MIB =
31713 BuildMI(fallMBB, DL, TII->get(PtrLoadOpc), PrevSSPReg);
31714 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
31715 const MachineOperand &MO = MI.getOperand(i);
31716 if (i == X86::AddrDisp)
31717 MIB.addDisp(MO, SPPOffset);
31718 else if (MO.isReg()) // Don't add the whole operand, we don't want to
31719 // preserve kill flags.
31720 MIB.addReg(MO.getReg());
31721 else
31722 MIB.add(MO);
31723 }
31724 MIB.setMemRefs(MMOs);
31725
31726 // Subtract the current SSP from the previous SSP.
31727 Register SspSubReg = MRI.createVirtualRegister(PtrRC);
31728 unsigned SubRROpc = (PVT == MVT::i64) ? X86::SUB64rr : X86::SUB32rr;
31729 BuildMI(fallMBB, DL, TII->get(SubRROpc), SspSubReg)
31730 .addReg(PrevSSPReg)
31731 .addReg(SSPCopyReg);
31732
31733 // Jump to sink in case PrevSSPReg <= SSPCopyReg.
31734 BuildMI(fallMBB, DL, TII->get(X86::JCC_1)).addMBB(sinkMBB).addImm(X86::COND_BE);
31735 fallMBB->addSuccessor(sinkMBB);
31736 fallMBB->addSuccessor(fixShadowMBB);
31737
31738 // Shift right by 2/3 for 32/64 because incssp multiplies the argument by 4/8.
31739 unsigned ShrRIOpc = (PVT == MVT::i64) ? X86::SHR64ri : X86::SHR32ri;
31740 unsigned Offset = (PVT == MVT::i64) ? 3 : 2;
31741 Register SspFirstShrReg = MRI.createVirtualRegister(PtrRC);
31742 BuildMI(fixShadowMBB, DL, TII->get(ShrRIOpc), SspFirstShrReg)
31743 .addReg(SspSubReg)
31744 .addImm(Offset);
31745
31746 // Increase SSP when looking only on the lower 8 bits of the delta.
31747 unsigned IncsspOpc = (PVT == MVT::i64) ? X86::INCSSPQ : X86::INCSSPD;
31748 BuildMI(fixShadowMBB, DL, TII->get(IncsspOpc)).addReg(SspFirstShrReg);
31749
31750 // Reset the lower 8 bits.
31751 Register SspSecondShrReg = MRI.createVirtualRegister(PtrRC);
31752 BuildMI(fixShadowMBB, DL, TII->get(ShrRIOpc), SspSecondShrReg)
31753 .addReg(SspFirstShrReg)
31754 .addImm(8);
31755
31756 // Jump if the result of the shift is zero.
31757 BuildMI(fixShadowMBB, DL, TII->get(X86::JCC_1)).addMBB(sinkMBB).addImm(X86::COND_E);
31758 fixShadowMBB->addSuccessor(sinkMBB);
31759 fixShadowMBB->addSuccessor(fixShadowLoopPrepareMBB);
31760
31761 // Do a single shift left.
31762 unsigned ShlR1Opc = (PVT == MVT::i64) ? X86::SHL64r1 : X86::SHL32r1;
31763 Register SspAfterShlReg = MRI.createVirtualRegister(PtrRC);
31764 BuildMI(fixShadowLoopPrepareMBB, DL, TII->get(ShlR1Opc), SspAfterShlReg)
31765 .addReg(SspSecondShrReg);
31766
31767 // Save the value 128 to a register (will be used next with incssp).
31768 Register Value128InReg = MRI.createVirtualRegister(PtrRC);
31769 unsigned MovRIOpc = (PVT == MVT::i64) ? X86::MOV64ri32 : X86::MOV32ri;
31770 BuildMI(fixShadowLoopPrepareMBB, DL, TII->get(MovRIOpc), Value128InReg)
31771 .addImm(128);
31772 fixShadowLoopPrepareMBB->addSuccessor(fixShadowLoopMBB);
31773
31774 // Since incssp only looks at the lower 8 bits, we might need to do several
31775 // iterations of incssp until we finish fixing the shadow stack.
31776 Register DecReg = MRI.createVirtualRegister(PtrRC);
31777 Register CounterReg = MRI.createVirtualRegister(PtrRC);
31778 BuildMI(fixShadowLoopMBB, DL, TII->get(X86::PHI), CounterReg)
31779 .addReg(SspAfterShlReg)
31780 .addMBB(fixShadowLoopPrepareMBB)
31781 .addReg(DecReg)
31782 .addMBB(fixShadowLoopMBB);
31783
31784 // Every iteration we increase the SSP by 128.
31785 BuildMI(fixShadowLoopMBB, DL, TII->get(IncsspOpc)).addReg(Value128InReg);
31786
31787 // Every iteration we decrement the counter by 1.
31788 unsigned DecROpc = (PVT == MVT::i64) ? X86::DEC64r : X86::DEC32r;
31789 BuildMI(fixShadowLoopMBB, DL, TII->get(DecROpc), DecReg).addReg(CounterReg);
31790
31791 // Jump if the counter is not zero yet.
31792 BuildMI(fixShadowLoopMBB, DL, TII->get(X86::JCC_1)).addMBB(fixShadowLoopMBB).addImm(X86::COND_NE);
31793 fixShadowLoopMBB->addSuccessor(sinkMBB);
31794 fixShadowLoopMBB->addSuccessor(fixShadowLoopMBB);
31795
31796 return sinkMBB;
31797}
31798
31799MachineBasicBlock *
31800X86TargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
31801 MachineBasicBlock *MBB) const {
31802 DebugLoc DL = MI.getDebugLoc();
31803 MachineFunction *MF = MBB->getParent();
31804 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
31805 MachineRegisterInfo &MRI = MF->getRegInfo();
31806
31807 // Memory Reference
31808 SmallVector<MachineMemOperand *, 2> MMOs(MI.memoperands_begin(),
31809 MI.memoperands_end());
31810
31811 MVT PVT = getPointerTy(MF->getDataLayout());
31812 assert((PVT == MVT::i64 || PVT == MVT::i32) &&(((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"
) ? static_cast<void> (0) : __assert_fail ("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31813, __PRETTY_FUNCTION__))
31813 "Invalid Pointer Size!")(((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"
) ? static_cast<void> (0) : __assert_fail ("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31813, __PRETTY_FUNCTION__));
31814
31815 const TargetRegisterClass *RC =
31816 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
31817 Register Tmp = MRI.createVirtualRegister(RC);
31818 // Since FP is only updated here but NOT referenced, it's treated as GPR.
31819 const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
31820 unsigned FP = (PVT == MVT::i64) ? X86::RBP : X86::EBP;
31821 Register SP = RegInfo->getStackRegister();
31822
31823 MachineInstrBuilder MIB;
31824
31825 const int64_t LabelOffset = 1 * PVT.getStoreSize();
31826 const int64_t SPOffset = 2 * PVT.getStoreSize();
31827
31828 unsigned PtrLoadOpc = (PVT == MVT::i64) ? X86::MOV64rm : X86::MOV32rm;
31829 unsigned IJmpOpc = (PVT == MVT::i64) ? X86::JMP64r : X86::JMP32r;
31830
31831 MachineBasicBlock *thisMBB = MBB;
31832
31833 // When CET and shadow stack is enabled, we need to fix the Shadow Stack.
31834 if (MF->getMMI().getModule()->getModuleFlag("cf-protection-return")) {
31835 thisMBB = emitLongJmpShadowStackFix(MI, thisMBB);
31836 }
31837
31838 // Reload FP
31839 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrLoadOpc), FP);
31840 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
31841 const MachineOperand &MO = MI.getOperand(i);
31842 if (MO.isReg()) // Don't add the whole operand, we don't want to
31843 // preserve kill flags.
31844 MIB.addReg(MO.getReg());
31845 else
31846 MIB.add(MO);
31847 }
31848 MIB.setMemRefs(MMOs);
31849
31850 // Reload IP
31851 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrLoadOpc), Tmp);
31852 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
31853 const MachineOperand &MO = MI.getOperand(i);
31854 if (i == X86::AddrDisp)
31855 MIB.addDisp(MO, LabelOffset);
31856 else if (MO.isReg()) // Don't add the whole operand, we don't want to
31857 // preserve kill flags.
31858 MIB.addReg(MO.getReg());
31859 else
31860 MIB.add(MO);
31861 }
31862 MIB.setMemRefs(MMOs);
31863
31864 // Reload SP
31865 MIB = BuildMI(*thisMBB, MI, DL, TII->get(PtrLoadOpc), SP);
31866 for (unsigned i = 0; i < X86::AddrNumOperands; ++i) {
31867 if (i == X86::AddrDisp)
31868 MIB.addDisp(MI.getOperand(i), SPOffset);
31869 else
31870 MIB.add(MI.getOperand(i)); // We can preserve the kill flags here, it's
31871 // the last instruction of the expansion.
31872 }
31873 MIB.setMemRefs(MMOs);
31874
31875 // Jump
31876 BuildMI(*thisMBB, MI, DL, TII->get(IJmpOpc)).addReg(Tmp);
31877
31878 MI.eraseFromParent();
31879 return thisMBB;
31880}
31881
31882void X86TargetLowering::SetupEntryBlockForSjLj(MachineInstr &MI,
31883 MachineBasicBlock *MBB,
31884 MachineBasicBlock *DispatchBB,
31885 int FI) const {
31886 DebugLoc DL = MI.getDebugLoc();
31887 MachineFunction *MF = MBB->getParent();
31888 MachineRegisterInfo *MRI = &MF->getRegInfo();
31889 const X86InstrInfo *TII = Subtarget.getInstrInfo();
31890
31891 MVT PVT = getPointerTy(MF->getDataLayout());
31892 assert((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!")(((PVT == MVT::i64 || PVT == MVT::i32) && "Invalid Pointer Size!"
) ? static_cast<void> (0) : __assert_fail ("(PVT == MVT::i64 || PVT == MVT::i32) && \"Invalid Pointer Size!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31892, __PRETTY_FUNCTION__));
31893
31894 unsigned Op = 0;
31895 unsigned VR = 0;
31896
31897 bool UseImmLabel = (MF->getTarget().getCodeModel() == CodeModel::Small) &&
31898 !isPositionIndependent();
31899
31900 if (UseImmLabel) {
31901 Op = (PVT == MVT::i64) ? X86::MOV64mi32 : X86::MOV32mi;
31902 } else {
31903 const TargetRegisterClass *TRC =
31904 (PVT == MVT::i64) ? &X86::GR64RegClass : &X86::GR32RegClass;
31905 VR = MRI->createVirtualRegister(TRC);
31906 Op = (PVT == MVT::i64) ? X86::MOV64mr : X86::MOV32mr;
31907
31908 if (Subtarget.is64Bit())
31909 BuildMI(*MBB, MI, DL, TII->get(X86::LEA64r), VR)
31910 .addReg(X86::RIP)
31911 .addImm(1)
31912 .addReg(0)
31913 .addMBB(DispatchBB)
31914 .addReg(0);
31915 else
31916 BuildMI(*MBB, MI, DL, TII->get(X86::LEA32r), VR)
31917 .addReg(0) /* TII->getGlobalBaseReg(MF) */
31918 .addImm(1)
31919 .addReg(0)
31920 .addMBB(DispatchBB, Subtarget.classifyBlockAddressReference())
31921 .addReg(0);
31922 }
31923
31924 MachineInstrBuilder MIB = BuildMI(*MBB, MI, DL, TII->get(Op));
31925 addFrameReference(MIB, FI, Subtarget.is64Bit() ? 56 : 36);
31926 if (UseImmLabel)
31927 MIB.addMBB(DispatchBB);
31928 else
31929 MIB.addReg(VR);
31930}
31931
31932MachineBasicBlock *
31933X86TargetLowering::EmitSjLjDispatchBlock(MachineInstr &MI,
31934 MachineBasicBlock *BB) const {
31935 DebugLoc DL = MI.getDebugLoc();
31936 MachineFunction *MF = BB->getParent();
31937 MachineRegisterInfo *MRI = &MF->getRegInfo();
31938 const X86InstrInfo *TII = Subtarget.getInstrInfo();
31939 int FI = MF->getFrameInfo().getFunctionContextIndex();
31940
31941 // Get a mapping of the call site numbers to all of the landing pads they're
31942 // associated with.
31943 DenseMap<unsigned, SmallVector<MachineBasicBlock *, 2>> CallSiteNumToLPad;
31944 unsigned MaxCSNum = 0;
31945 for (auto &MBB : *MF) {
31946 if (!MBB.isEHPad())
31947 continue;
31948
31949 MCSymbol *Sym = nullptr;
31950 for (const auto &MI : MBB) {
31951 if (MI.isDebugInstr())
31952 continue;
31953
31954 assert(MI.isEHLabel() && "expected EH_LABEL")((MI.isEHLabel() && "expected EH_LABEL") ? static_cast
<void> (0) : __assert_fail ("MI.isEHLabel() && \"expected EH_LABEL\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31954, __PRETTY_FUNCTION__));
31955 Sym = MI.getOperand(0).getMCSymbol();
31956 break;
31957 }
31958
31959 if (!MF->hasCallSiteLandingPad(Sym))
31960 continue;
31961
31962 for (unsigned CSI : MF->getCallSiteLandingPad(Sym)) {
31963 CallSiteNumToLPad[CSI].push_back(&MBB);
31964 MaxCSNum = std::max(MaxCSNum, CSI);
31965 }
31966 }
31967
31968 // Get an ordered list of the machine basic blocks for the jump table.
31969 std::vector<MachineBasicBlock *> LPadList;
31970 SmallPtrSet<MachineBasicBlock *, 32> InvokeBBs;
31971 LPadList.reserve(CallSiteNumToLPad.size());
31972
31973 for (unsigned CSI = 1; CSI <= MaxCSNum; ++CSI) {
31974 for (auto &LP : CallSiteNumToLPad[CSI]) {
31975 LPadList.push_back(LP);
31976 InvokeBBs.insert(LP->pred_begin(), LP->pred_end());
31977 }
31978 }
31979
31980 assert(!LPadList.empty() &&((!LPadList.empty() && "No landing pad destinations for the dispatch jump table!"
) ? static_cast<void> (0) : __assert_fail ("!LPadList.empty() && \"No landing pad destinations for the dispatch jump table!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31981, __PRETTY_FUNCTION__))
31981 "No landing pad destinations for the dispatch jump table!")((!LPadList.empty() && "No landing pad destinations for the dispatch jump table!"
) ? static_cast<void> (0) : __assert_fail ("!LPadList.empty() && \"No landing pad destinations for the dispatch jump table!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 31981, __PRETTY_FUNCTION__));
31982
31983 // Create the MBBs for the dispatch code.
31984
31985 // Shove the dispatch's address into the return slot in the function context.
31986 MachineBasicBlock *DispatchBB = MF->CreateMachineBasicBlock();
31987 DispatchBB->setIsEHPad(true);
31988
31989 MachineBasicBlock *TrapBB = MF->CreateMachineBasicBlock();
31990 BuildMI(TrapBB, DL, TII->get(X86::TRAP));
31991 DispatchBB->addSuccessor(TrapBB);
31992
31993 MachineBasicBlock *DispContBB = MF->CreateMachineBasicBlock();
31994 DispatchBB->addSuccessor(DispContBB);
31995
31996 // Insert MBBs.
31997 MF->push_back(DispatchBB);
31998 MF->push_back(DispContBB);
31999 MF->push_back(TrapBB);
32000
32001 // Insert code into the entry block that creates and registers the function
32002 // context.
32003 SetupEntryBlockForSjLj(MI, BB, DispatchBB, FI);
32004
32005 // Create the jump table and associated information
32006 unsigned JTE = getJumpTableEncoding();
32007 MachineJumpTableInfo *JTI = MF->getOrCreateJumpTableInfo(JTE);
32008 unsigned MJTI = JTI->createJumpTableIndex(LPadList);
32009
32010 const X86RegisterInfo &RI = TII->getRegisterInfo();
32011 // Add a register mask with no preserved registers. This results in all
32012 // registers being marked as clobbered.
32013 if (RI.hasBasePointer(*MF)) {
32014 const bool FPIs64Bit =
32015 Subtarget.isTarget64BitLP64() || Subtarget.isTargetNaCl64();
32016 X86MachineFunctionInfo *MFI = MF->getInfo<X86MachineFunctionInfo>();
32017 MFI->setRestoreBasePointer(MF);
32018
32019 Register FP = RI.getFrameRegister(*MF);
32020 Register BP = RI.getBaseRegister();
32021 unsigned Op = FPIs64Bit ? X86::MOV64rm : X86::MOV32rm;
32022 addRegOffset(BuildMI(DispatchBB, DL, TII->get(Op), BP), FP, true,
32023 MFI->getRestoreBasePointerOffset())
32024 .addRegMask(RI.getNoPreservedMask());
32025 } else {
32026 BuildMI(DispatchBB, DL, TII->get(X86::NOOP))
32027 .addRegMask(RI.getNoPreservedMask());
32028 }
32029
32030 // IReg is used as an index in a memory operand and therefore can't be SP
32031 Register IReg = MRI->createVirtualRegister(&X86::GR32_NOSPRegClass);
32032 addFrameReference(BuildMI(DispatchBB, DL, TII->get(X86::MOV32rm), IReg), FI,
32033 Subtarget.is64Bit() ? 8 : 4);
32034 BuildMI(DispatchBB, DL, TII->get(X86::CMP32ri))
32035 .addReg(IReg)
32036 .addImm(LPadList.size());
32037 BuildMI(DispatchBB, DL, TII->get(X86::JCC_1)).addMBB(TrapBB).addImm(X86::COND_AE);
32038
32039 if (Subtarget.is64Bit()) {
32040 Register BReg = MRI->createVirtualRegister(&X86::GR64RegClass);
32041 Register IReg64 = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);
32042
32043 // leaq .LJTI0_0(%rip), BReg
32044 BuildMI(DispContBB, DL, TII->get(X86::LEA64r), BReg)
32045 .addReg(X86::RIP)
32046 .addImm(1)
32047 .addReg(0)
32048 .addJumpTableIndex(MJTI)
32049 .addReg(0);
32050 // movzx IReg64, IReg
32051 BuildMI(DispContBB, DL, TII->get(TargetOpcode::SUBREG_TO_REG), IReg64)
32052 .addImm(0)
32053 .addReg(IReg)
32054 .addImm(X86::sub_32bit);
32055
32056 switch (JTE) {
32057 case MachineJumpTableInfo::EK_BlockAddress:
32058 // jmpq *(BReg,IReg64,8)
32059 BuildMI(DispContBB, DL, TII->get(X86::JMP64m))
32060 .addReg(BReg)
32061 .addImm(8)
32062 .addReg(IReg64)
32063 .addImm(0)
32064 .addReg(0);
32065 break;
32066 case MachineJumpTableInfo::EK_LabelDifference32: {
32067 Register OReg = MRI->createVirtualRegister(&X86::GR32RegClass);
32068 Register OReg64 = MRI->createVirtualRegister(&X86::GR64RegClass);
32069 Register TReg = MRI->createVirtualRegister(&X86::GR64RegClass);
32070
32071 // movl (BReg,IReg64,4), OReg
32072 BuildMI(DispContBB, DL, TII->get(X86::MOV32rm), OReg)
32073 .addReg(BReg)
32074 .addImm(4)
32075 .addReg(IReg64)
32076 .addImm(0)
32077 .addReg(0);
32078 // movsx OReg64, OReg
32079 BuildMI(DispContBB, DL, TII->get(X86::MOVSX64rr32), OReg64).addReg(OReg);
32080 // addq BReg, OReg64, TReg
32081 BuildMI(DispContBB, DL, TII->get(X86::ADD64rr), TReg)
32082 .addReg(OReg64)
32083 .addReg(BReg);
32084 // jmpq *TReg
32085 BuildMI(DispContBB, DL, TII->get(X86::JMP64r)).addReg(TReg);
32086 break;
32087 }
32088 default:
32089 llvm_unreachable("Unexpected jump table encoding")::llvm::llvm_unreachable_internal("Unexpected jump table encoding"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32089);
32090 }
32091 } else {
32092 // jmpl *.LJTI0_0(,IReg,4)
32093 BuildMI(DispContBB, DL, TII->get(X86::JMP32m))
32094 .addReg(0)
32095 .addImm(4)
32096 .addReg(IReg)
32097 .addJumpTableIndex(MJTI)
32098 .addReg(0);
32099 }
32100
32101 // Add the jump table entries as successors to the MBB.
32102 SmallPtrSet<MachineBasicBlock *, 8> SeenMBBs;
32103 for (auto &LP : LPadList)
32104 if (SeenMBBs.insert(LP).second)
32105 DispContBB->addSuccessor(LP);
32106
32107 // N.B. the order the invoke BBs are processed in doesn't matter here.
32108 SmallVector<MachineBasicBlock *, 64> MBBLPads;
32109 const MCPhysReg *SavedRegs = MF->getRegInfo().getCalleeSavedRegs();
32110 for (MachineBasicBlock *MBB : InvokeBBs) {
32111 // Remove the landing pad successor from the invoke block and replace it
32112 // with the new dispatch block.
32113 // Keep a copy of Successors since it's modified inside the loop.
32114 SmallVector<MachineBasicBlock *, 8> Successors(MBB->succ_rbegin(),
32115 MBB->succ_rend());
32116 // FIXME: Avoid quadratic complexity.
32117 for (auto MBBS : Successors) {
32118 if (MBBS->isEHPad()) {
32119 MBB->removeSuccessor(MBBS);
32120 MBBLPads.push_back(MBBS);
32121 }
32122 }
32123
32124 MBB->addSuccessor(DispatchBB);
32125
32126 // Find the invoke call and mark all of the callee-saved registers as
32127 // 'implicit defined' so that they're spilled. This prevents code from
32128 // moving instructions to before the EH block, where they will never be
32129 // executed.
32130 for (auto &II : reverse(*MBB)) {
32131 if (!II.isCall())
32132 continue;
32133
32134 DenseMap<unsigned, bool> DefRegs;
32135 for (auto &MOp : II.operands())
32136 if (MOp.isReg())
32137 DefRegs[MOp.getReg()] = true;
32138
32139 MachineInstrBuilder MIB(*MF, &II);
32140 for (unsigned RegIdx = 0; SavedRegs[RegIdx]; ++RegIdx) {
32141 unsigned Reg = SavedRegs[RegIdx];
32142 if (!DefRegs[Reg])
32143 MIB.addReg(Reg, RegState::ImplicitDefine | RegState::Dead);
32144 }
32145
32146 break;
32147 }
32148 }
32149
32150 // Mark all former landing pads as non-landing pads. The dispatch is the only
32151 // landing pad now.
32152 for (auto &LP : MBBLPads)
32153 LP->setIsEHPad(false);
32154
32155 // The instruction is gone now.
32156 MI.eraseFromParent();
32157 return BB;
32158}
32159
32160MachineBasicBlock *
32161X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
32162 MachineBasicBlock *BB) const {
32163 MachineFunction *MF = BB->getParent();
32164 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
32165 DebugLoc DL = MI.getDebugLoc();
32166
32167 switch (MI.getOpcode()) {
32168 default: llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32168);
32169 case X86::TLS_addr32:
32170 case X86::TLS_addr64:
32171 case X86::TLS_base_addr32:
32172 case X86::TLS_base_addr64:
32173 return EmitLoweredTLSAddr(MI, BB);
32174 case X86::RETPOLINE_CALL32:
32175 case X86::RETPOLINE_CALL64:
32176 case X86::RETPOLINE_TCRETURN32:
32177 case X86::RETPOLINE_TCRETURN64:
32178 return EmitLoweredRetpoline(MI, BB);
32179 case X86::CATCHRET:
32180 return EmitLoweredCatchRet(MI, BB);
32181 case X86::CATCHPAD:
32182 return EmitLoweredCatchPad(MI, BB);
32183 case X86::SEG_ALLOCA_32:
32184 case X86::SEG_ALLOCA_64:
32185 return EmitLoweredSegAlloca(MI, BB);
32186 case X86::TLSCall_32:
32187 case X86::TLSCall_64:
32188 return EmitLoweredTLSCall(MI, BB);
32189 case X86::CMOV_FR32:
32190 case X86::CMOV_FR32X:
32191 case X86::CMOV_FR64:
32192 case X86::CMOV_FR64X:
32193 case X86::CMOV_GR8:
32194 case X86::CMOV_GR16:
32195 case X86::CMOV_GR32:
32196 case X86::CMOV_RFP32:
32197 case X86::CMOV_RFP64:
32198 case X86::CMOV_RFP80:
32199 case X86::CMOV_VR128:
32200 case X86::CMOV_VR128X:
32201 case X86::CMOV_VR256:
32202 case X86::CMOV_VR256X:
32203 case X86::CMOV_VR512:
32204 case X86::CMOV_VK2:
32205 case X86::CMOV_VK4:
32206 case X86::CMOV_VK8:
32207 case X86::CMOV_VK16:
32208 case X86::CMOV_VK32:
32209 case X86::CMOV_VK64:
32210 return EmitLoweredSelect(MI, BB);
32211
32212 case X86::RDFLAGS32:
32213 case X86::RDFLAGS64: {
32214 unsigned PushF =
32215 MI.getOpcode() == X86::RDFLAGS32 ? X86::PUSHF32 : X86::PUSHF64;
32216 unsigned Pop = MI.getOpcode() == X86::RDFLAGS32 ? X86::POP32r : X86::POP64r;
32217 MachineInstr *Push = BuildMI(*BB, MI, DL, TII->get(PushF));
32218 // Permit reads of the EFLAGS and DF registers without them being defined.
32219 // This intrinsic exists to read external processor state in flags, such as
32220 // the trap flag, interrupt flag, and direction flag, none of which are
32221 // modeled by the backend.
32222 assert(Push->getOperand(2).getReg() == X86::EFLAGS &&((Push->getOperand(2).getReg() == X86::EFLAGS && "Unexpected register in operand!"
) ? static_cast<void> (0) : __assert_fail ("Push->getOperand(2).getReg() == X86::EFLAGS && \"Unexpected register in operand!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32223, __PRETTY_FUNCTION__))
32223 "Unexpected register in operand!")((Push->getOperand(2).getReg() == X86::EFLAGS && "Unexpected register in operand!"
) ? static_cast<void> (0) : __assert_fail ("Push->getOperand(2).getReg() == X86::EFLAGS && \"Unexpected register in operand!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32223, __PRETTY_FUNCTION__));
32224 Push->getOperand(2).setIsUndef();
32225 assert(Push->getOperand(3).getReg() == X86::DF &&((Push->getOperand(3).getReg() == X86::DF && "Unexpected register in operand!"
) ? static_cast<void> (0) : __assert_fail ("Push->getOperand(3).getReg() == X86::DF && \"Unexpected register in operand!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32226, __PRETTY_FUNCTION__))
32226 "Unexpected register in operand!")((Push->getOperand(3).getReg() == X86::DF && "Unexpected register in operand!"
) ? static_cast<void> (0) : __assert_fail ("Push->getOperand(3).getReg() == X86::DF && \"Unexpected register in operand!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32226, __PRETTY_FUNCTION__));
32227 Push->getOperand(3).setIsUndef();
32228 BuildMI(*BB, MI, DL, TII->get(Pop), MI.getOperand(0).getReg());
32229
32230 MI.eraseFromParent(); // The pseudo is gone now.
32231 return BB;
32232 }
32233
32234 case X86::WRFLAGS32:
32235 case X86::WRFLAGS64: {
32236 unsigned Push =
32237 MI.getOpcode() == X86::WRFLAGS32 ? X86::PUSH32r : X86::PUSH64r;
32238 unsigned PopF =
32239 MI.getOpcode() == X86::WRFLAGS32 ? X86::POPF32 : X86::POPF64;
32240 BuildMI(*BB, MI, DL, TII->get(Push)).addReg(MI.getOperand(0).getReg());
32241 BuildMI(*BB, MI, DL, TII->get(PopF));
32242
32243 MI.eraseFromParent(); // The pseudo is gone now.
32244 return BB;
32245 }
32246
32247 case X86::FP32_TO_INT16_IN_MEM:
32248 case X86::FP32_TO_INT32_IN_MEM:
32249 case X86::FP32_TO_INT64_IN_MEM:
32250 case X86::FP64_TO_INT16_IN_MEM:
32251 case X86::FP64_TO_INT32_IN_MEM:
32252 case X86::FP64_TO_INT64_IN_MEM:
32253 case X86::FP80_TO_INT16_IN_MEM:
32254 case X86::FP80_TO_INT32_IN_MEM:
32255 case X86::FP80_TO_INT64_IN_MEM: {
32256 // Change the floating point control register to use "round towards zero"
32257 // mode when truncating to an integer value.
32258 int OrigCWFrameIdx = MF->getFrameInfo().CreateStackObject(2, 2, false);
32259 addFrameReference(BuildMI(*BB, MI, DL,
32260 TII->get(X86::FNSTCW16m)), OrigCWFrameIdx);
32261
32262 // Load the old value of the control word...
32263 Register OldCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
32264 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOVZX32rm16), OldCW),
32265 OrigCWFrameIdx);
32266
32267 // OR 0b11 into bit 10 and 11. 0b11 is the encoding for round toward zero.
32268 Register NewCW = MF->getRegInfo().createVirtualRegister(&X86::GR32RegClass);
32269 BuildMI(*BB, MI, DL, TII->get(X86::OR32ri), NewCW)
32270 .addReg(OldCW, RegState::Kill).addImm(0xC00);
32271
32272 // Extract to 16 bits.
32273 Register NewCW16 =
32274 MF->getRegInfo().createVirtualRegister(&X86::GR16RegClass);
32275 BuildMI(*BB, MI, DL, TII->get(TargetOpcode::COPY), NewCW16)
32276 .addReg(NewCW, RegState::Kill, X86::sub_16bit);
32277
32278 // Prepare memory for FLDCW.
32279 int NewCWFrameIdx = MF->getFrameInfo().CreateStackObject(2, 2, false);
32280 addFrameReference(BuildMI(*BB, MI, DL, TII->get(X86::MOV16mr)),
32281 NewCWFrameIdx)
32282 .addReg(NewCW16, RegState::Kill);
32283
32284 // Reload the modified control word now...
32285 addFrameReference(BuildMI(*BB, MI, DL,
32286 TII->get(X86::FLDCW16m)), NewCWFrameIdx);
32287
32288 // Get the X86 opcode to use.
32289 unsigned Opc;
32290 switch (MI.getOpcode()) {
32291 default: llvm_unreachable("illegal opcode!")::llvm::llvm_unreachable_internal("illegal opcode!", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32291);
32292 case X86::FP32_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m32; break;
32293 case X86::FP32_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m32; break;
32294 case X86::FP32_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m32; break;
32295 case X86::FP64_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m64; break;
32296 case X86::FP64_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m64; break;
32297 case X86::FP64_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m64; break;
32298 case X86::FP80_TO_INT16_IN_MEM: Opc = X86::IST_Fp16m80; break;
32299 case X86::FP80_TO_INT32_IN_MEM: Opc = X86::IST_Fp32m80; break;
32300 case X86::FP80_TO_INT64_IN_MEM: Opc = X86::IST_Fp64m80; break;
32301 }
32302
32303 X86AddressMode AM = getAddressFromInstr(&MI, 0);
32304 addFullAddress(BuildMI(*BB, MI, DL, TII->get(Opc)), AM)
32305 .addReg(MI.getOperand(X86::AddrNumOperands).getReg());
32306
32307 // Reload the original control word now.
32308 addFrameReference(BuildMI(*BB, MI, DL,
32309 TII->get(X86::FLDCW16m)), OrigCWFrameIdx);
32310
32311 MI.eraseFromParent(); // The pseudo instruction is gone now.
32312 return BB;
32313 }
32314
32315 // xbegin
32316 case X86::XBEGIN:
32317 return emitXBegin(MI, BB, Subtarget.getInstrInfo());
32318
32319 case X86::VASTART_SAVE_XMM_REGS:
32320 return EmitVAStartSaveXMMRegsWithCustomInserter(MI, BB);
32321
32322 case X86::VAARG_64:
32323 return EmitVAARG64WithCustomInserter(MI, BB);
32324
32325 case X86::EH_SjLj_SetJmp32:
32326 case X86::EH_SjLj_SetJmp64:
32327 return emitEHSjLjSetJmp(MI, BB);
32328
32329 case X86::EH_SjLj_LongJmp32:
32330 case X86::EH_SjLj_LongJmp64:
32331 return emitEHSjLjLongJmp(MI, BB);
32332
32333 case X86::Int_eh_sjlj_setup_dispatch:
32334 return EmitSjLjDispatchBlock(MI, BB);
32335
32336 case TargetOpcode::STATEPOINT:
32337 // As an implementation detail, STATEPOINT shares the STACKMAP format at
32338 // this point in the process. We diverge later.
32339 return emitPatchPoint(MI, BB);
32340
32341 case TargetOpcode::STACKMAP:
32342 case TargetOpcode::PATCHPOINT:
32343 return emitPatchPoint(MI, BB);
32344
32345 case TargetOpcode::PATCHABLE_EVENT_CALL:
32346 return emitXRayCustomEvent(MI, BB);
32347
32348 case TargetOpcode::PATCHABLE_TYPED_EVENT_CALL:
32349 return emitXRayTypedEvent(MI, BB);
32350
32351 case X86::LCMPXCHG8B: {
32352 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
32353 // In addition to 4 E[ABCD] registers implied by encoding, CMPXCHG8B
32354 // requires a memory operand. If it happens that current architecture is
32355 // i686 and for current function we need a base pointer
32356 // - which is ESI for i686 - register allocator would not be able to
32357 // allocate registers for an address in form of X(%reg, %reg, Y)
32358 // - there never would be enough unreserved registers during regalloc
32359 // (without the need for base ptr the only option would be X(%edi, %esi, Y).
32360 // We are giving a hand to register allocator by precomputing the address in
32361 // a new vreg using LEA.
32362
32363 // If it is not i686 or there is no base pointer - nothing to do here.
32364 if (!Subtarget.is32Bit() || !TRI->hasBasePointer(*MF))
32365 return BB;
32366
32367 // Even though this code does not necessarily needs the base pointer to
32368 // be ESI, we check for that. The reason: if this assert fails, there are
32369 // some changes happened in the compiler base pointer handling, which most
32370 // probably have to be addressed somehow here.
32371 assert(TRI->getBaseRegister() == X86::ESI &&((TRI->getBaseRegister() == X86::ESI && "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "
"base pointer in mind") ? static_cast<void> (0) : __assert_fail
("TRI->getBaseRegister() == X86::ESI && \"LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a \" \"base pointer in mind\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32373, __PRETTY_FUNCTION__))
32372 "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "((TRI->getBaseRegister() == X86::ESI && "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "
"base pointer in mind") ? static_cast<void> (0) : __assert_fail
("TRI->getBaseRegister() == X86::ESI && \"LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a \" \"base pointer in mind\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32373, __PRETTY_FUNCTION__))
32373 "base pointer in mind")((TRI->getBaseRegister() == X86::ESI && "LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a "
"base pointer in mind") ? static_cast<void> (0) : __assert_fail
("TRI->getBaseRegister() == X86::ESI && \"LCMPXCHG8B custom insertion for i686 is written with X86::ESI as a \" \"base pointer in mind\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32373, __PRETTY_FUNCTION__));
32374
32375 MachineRegisterInfo &MRI = MF->getRegInfo();
32376 MVT SPTy = getPointerTy(MF->getDataLayout());
32377 const TargetRegisterClass *AddrRegClass = getRegClassFor(SPTy);
32378 Register computedAddrVReg = MRI.createVirtualRegister(AddrRegClass);
32379
32380 X86AddressMode AM = getAddressFromInstr(&MI, 0);
32381 // Regalloc does not need any help when the memory operand of CMPXCHG8B
32382 // does not use index register.
32383 if (AM.IndexReg == X86::NoRegister)
32384 return BB;
32385
32386 // After X86TargetLowering::ReplaceNodeResults CMPXCHG8B is glued to its
32387 // four operand definitions that are E[ABCD] registers. We skip them and
32388 // then insert the LEA.
32389 MachineBasicBlock::reverse_iterator RMBBI(MI.getReverseIterator());
32390 while (RMBBI != BB->rend() && (RMBBI->definesRegister(X86::EAX) ||
32391 RMBBI->definesRegister(X86::EBX) ||
32392 RMBBI->definesRegister(X86::ECX) ||
32393 RMBBI->definesRegister(X86::EDX))) {
32394 ++RMBBI;
32395 }
32396 MachineBasicBlock::iterator MBBI(RMBBI);
32397 addFullAddress(
32398 BuildMI(*BB, *MBBI, DL, TII->get(X86::LEA32r), computedAddrVReg), AM);
32399
32400 setDirectAddressInInstr(&MI, 0, computedAddrVReg);
32401
32402 return BB;
32403 }
32404 case X86::LCMPXCHG16B:
32405 return BB;
32406 case X86::LCMPXCHG8B_SAVE_EBX:
32407 case X86::LCMPXCHG16B_SAVE_RBX: {
32408 unsigned BasePtr =
32409 MI.getOpcode() == X86::LCMPXCHG8B_SAVE_EBX ? X86::EBX : X86::RBX;
32410 if (!BB->isLiveIn(BasePtr))
32411 BB->addLiveIn(BasePtr);
32412 return BB;
32413 }
32414 }
32415}
32416
32417//===----------------------------------------------------------------------===//
32418// X86 Optimization Hooks
32419//===----------------------------------------------------------------------===//
32420
32421bool
32422X86TargetLowering::targetShrinkDemandedConstant(SDValue Op,
32423 const APInt &Demanded,
32424 TargetLoweringOpt &TLO) const {
32425 // Only optimize Ands to prevent shrinking a constant that could be
32426 // matched by movzx.
32427 if (Op.getOpcode() != ISD::AND)
32428 return false;
32429
32430 EVT VT = Op.getValueType();
32431
32432 // Ignore vectors.
32433 if (VT.isVector())
32434 return false;
32435
32436 unsigned Size = VT.getSizeInBits();
32437
32438 // Make sure the RHS really is a constant.
32439 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op.getOperand(1));
32440 if (!C)
32441 return false;
32442
32443 const APInt &Mask = C->getAPIntValue();
32444
32445 // Clear all non-demanded bits initially.
32446 APInt ShrunkMask = Mask & Demanded;
32447
32448 // Find the width of the shrunk mask.
32449 unsigned Width = ShrunkMask.getActiveBits();
32450
32451 // If the mask is all 0s there's nothing to do here.
32452 if (Width == 0)
32453 return false;
32454
32455 // Find the next power of 2 width, rounding up to a byte.
32456 Width = PowerOf2Ceil(std::max(Width, 8U));
32457 // Truncate the width to size to handle illegal types.
32458 Width = std::min(Width, Size);
32459
32460 // Calculate a possible zero extend mask for this constant.
32461 APInt ZeroExtendMask = APInt::getLowBitsSet(Size, Width);
32462
32463 // If we aren't changing the mask, just return true to keep it and prevent
32464 // the caller from optimizing.
32465 if (ZeroExtendMask == Mask)
32466 return true;
32467
32468 // Make sure the new mask can be represented by a combination of mask bits
32469 // and non-demanded bits.
32470 if (!ZeroExtendMask.isSubsetOf(Mask | ~Demanded))
32471 return false;
32472
32473 // Replace the constant with the zero extend mask.
32474 SDLoc DL(Op);
32475 SDValue NewC = TLO.DAG.getConstant(ZeroExtendMask, DL, VT);
32476 SDValue NewOp = TLO.DAG.getNode(ISD::AND, DL, VT, Op.getOperand(0), NewC);
32477 return TLO.CombineTo(Op, NewOp);
32478}
32479
32480void X86TargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
32481 KnownBits &Known,
32482 const APInt &DemandedElts,
32483 const SelectionDAG &DAG,
32484 unsigned Depth) const {
32485 unsigned BitWidth = Known.getBitWidth();
32486 unsigned Opc = Op.getOpcode();
32487 EVT VT = Op.getValueType();
32488 assert((Opc >= ISD::BUILTIN_OP_END ||(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
|| Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32493, __PRETTY_FUNCTION__))
32489 Opc == ISD::INTRINSIC_WO_CHAIN ||(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
|| Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32493, __PRETTY_FUNCTION__))
32490 Opc == ISD::INTRINSIC_W_CHAIN ||(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
|| Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32493, __PRETTY_FUNCTION__))
32491 Opc == ISD::INTRINSIC_VOID) &&(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
|| Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32493, __PRETTY_FUNCTION__))
32492 "Should use MaskedValueIsZero if you don't know whether Op"(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
|| Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32493, __PRETTY_FUNCTION__))
32493 " is a target node!")(((Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN
|| Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID
) && "Should use MaskedValueIsZero if you don't know whether Op"
" is a target node!") ? static_cast<void> (0) : __assert_fail
("(Opc >= ISD::BUILTIN_OP_END || Opc == ISD::INTRINSIC_WO_CHAIN || Opc == ISD::INTRINSIC_W_CHAIN || Opc == ISD::INTRINSIC_VOID) && \"Should use MaskedValueIsZero if you don't know whether Op\" \" is a target node!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32493, __PRETTY_FUNCTION__));
32494
32495 Known.resetAll();
32496 switch (Opc) {
32497 default: break;
32498 case X86ISD::SETCC:
32499 Known.Zero.setBitsFrom(1);
32500 break;
32501 case X86ISD::MOVMSK: {
32502 unsigned NumLoBits = Op.getOperand(0).getValueType().getVectorNumElements();
32503 Known.Zero.setBitsFrom(NumLoBits);
32504 break;
32505 }
32506 case X86ISD::PEXTRB:
32507 case X86ISD::PEXTRW: {
32508 SDValue Src = Op.getOperand(0);
32509 EVT SrcVT = Src.getValueType();
32510 APInt DemandedElt = APInt::getOneBitSet(SrcVT.getVectorNumElements(),
32511 Op.getConstantOperandVal(1));
32512 Known = DAG.computeKnownBits(Src, DemandedElt, Depth + 1);
32513 Known = Known.zextOrTrunc(BitWidth, false);
32514 Known.Zero.setBitsFrom(SrcVT.getScalarSizeInBits());
32515 break;
32516 }
32517 case X86ISD::VSRAI:
32518 case X86ISD::VSHLI:
32519 case X86ISD::VSRLI: {
32520 unsigned ShAmt = Op.getConstantOperandVal(1);
32521 if (ShAmt >= VT.getScalarSizeInBits()) {
32522 Known.setAllZero();
32523 break;
32524 }
32525
32526 Known = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
32527 if (Opc == X86ISD::VSHLI) {
32528 Known.Zero <<= ShAmt;
32529 Known.One <<= ShAmt;
32530 // Low bits are known zero.
32531 Known.Zero.setLowBits(ShAmt);
32532 } else if (Opc == X86ISD::VSRLI) {
32533 Known.Zero.lshrInPlace(ShAmt);
32534 Known.One.lshrInPlace(ShAmt);
32535 // High bits are known zero.
32536 Known.Zero.setHighBits(ShAmt);
32537 } else {
32538 Known.Zero.ashrInPlace(ShAmt);
32539 Known.One.ashrInPlace(ShAmt);
32540 }
32541 break;
32542 }
32543 case X86ISD::PACKUS: {
32544 // PACKUS is just a truncation if the upper half is zero.
32545 APInt DemandedLHS, DemandedRHS;
32546 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
32547
32548 Known.One = APInt::getAllOnesValue(BitWidth * 2);
32549 Known.Zero = APInt::getAllOnesValue(BitWidth * 2);
32550
32551 KnownBits Known2;
32552 if (!!DemandedLHS) {
32553 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedLHS, Depth + 1);
32554 Known.One &= Known2.One;
32555 Known.Zero &= Known2.Zero;
32556 }
32557 if (!!DemandedRHS) {
32558 Known2 = DAG.computeKnownBits(Op.getOperand(1), DemandedRHS, Depth + 1);
32559 Known.One &= Known2.One;
32560 Known.Zero &= Known2.Zero;
32561 }
32562
32563 if (Known.countMinLeadingZeros() < BitWidth)
32564 Known.resetAll();
32565 Known = Known.trunc(BitWidth);
32566 break;
32567 }
32568 case X86ISD::ANDNP: {
32569 KnownBits Known2;
32570 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
32571 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
32572
32573 // ANDNP = (~X & Y);
32574 Known.One &= Known2.Zero;
32575 Known.Zero |= Known2.One;
32576 break;
32577 }
32578 case X86ISD::FOR: {
32579 KnownBits Known2;
32580 Known = DAG.computeKnownBits(Op.getOperand(1), DemandedElts, Depth + 1);
32581 Known2 = DAG.computeKnownBits(Op.getOperand(0), DemandedElts, Depth + 1);
32582
32583 // Output known-0 bits are only known if clear in both the LHS & RHS.
32584 Known.Zero &= Known2.Zero;
32585 // Output known-1 are known to be set if set in either the LHS | RHS.
32586 Known.One |= Known2.One;
32587 break;
32588 }
32589 case X86ISD::PSADBW: {
32590 assert(VT.getScalarType() == MVT::i64 &&((VT.getScalarType() == MVT::i64 && Op.getOperand(0).
getValueType().getScalarType() == MVT::i8 && "Unexpected PSADBW types"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i64 && Op.getOperand(0).getValueType().getScalarType() == MVT::i8 && \"Unexpected PSADBW types\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32592, __PRETTY_FUNCTION__))
32591 Op.getOperand(0).getValueType().getScalarType() == MVT::i8 &&((VT.getScalarType() == MVT::i64 && Op.getOperand(0).
getValueType().getScalarType() == MVT::i8 && "Unexpected PSADBW types"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i64 && Op.getOperand(0).getValueType().getScalarType() == MVT::i8 && \"Unexpected PSADBW types\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32592, __PRETTY_FUNCTION__))
32592 "Unexpected PSADBW types")((VT.getScalarType() == MVT::i64 && Op.getOperand(0).
getValueType().getScalarType() == MVT::i8 && "Unexpected PSADBW types"
) ? static_cast<void> (0) : __assert_fail ("VT.getScalarType() == MVT::i64 && Op.getOperand(0).getValueType().getScalarType() == MVT::i8 && \"Unexpected PSADBW types\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32592, __PRETTY_FUNCTION__));
32593
32594 // PSADBW - fills low 16 bits and zeros upper 48 bits of each i64 result.
32595 Known.Zero.setBitsFrom(16);
32596 break;
32597 }
32598 case X86ISD::CMOV: {
32599 Known = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
32600 // If we don't know any bits, early out.
32601 if (Known.isUnknown())
32602 break;
32603 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
32604
32605 // Only known if known in both the LHS and RHS.
32606 Known.One &= Known2.One;
32607 Known.Zero &= Known2.Zero;
32608 break;
32609 }
32610 }
32611
32612 // Handle target shuffles.
32613 // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
32614 if (isTargetShuffle(Opc)) {
32615 bool IsUnary;
32616 SmallVector<int, 64> Mask;
32617 SmallVector<SDValue, 2> Ops;
32618 if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask,
32619 IsUnary)) {
32620 unsigned NumOps = Ops.size();
32621 unsigned NumElts = VT.getVectorNumElements();
32622 if (Mask.size() == NumElts) {
32623 SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
32624 Known.Zero.setAllBits(); Known.One.setAllBits();
32625 for (unsigned i = 0; i != NumElts; ++i) {
32626 if (!DemandedElts[i])
32627 continue;
32628 int M = Mask[i];
32629 if (M == SM_SentinelUndef) {
32630 // For UNDEF elements, we don't know anything about the common state
32631 // of the shuffle result.
32632 Known.resetAll();
32633 break;
32634 } else if (M == SM_SentinelZero) {
32635 Known.One.clearAllBits();
32636 continue;
32637 }
32638 assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&((0 <= M && (unsigned)M < (NumOps * NumElts) &&
"Shuffle index out of range") ? static_cast<void> (0) :
__assert_fail ("0 <= M && (unsigned)M < (NumOps * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32639, __PRETTY_FUNCTION__))
32639 "Shuffle index out of range")((0 <= M && (unsigned)M < (NumOps * NumElts) &&
"Shuffle index out of range") ? static_cast<void> (0) :
__assert_fail ("0 <= M && (unsigned)M < (NumOps * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32639, __PRETTY_FUNCTION__));
32640
32641 unsigned OpIdx = (unsigned)M / NumElts;
32642 unsigned EltIdx = (unsigned)M % NumElts;
32643 if (Ops[OpIdx].getValueType() != VT) {
32644 // TODO - handle target shuffle ops with different value types.
32645 Known.resetAll();
32646 break;
32647 }
32648 DemandedOps[OpIdx].setBit(EltIdx);
32649 }
32650 // Known bits are the values that are shared by every demanded element.
32651 for (unsigned i = 0; i != NumOps && !Known.isUnknown(); ++i) {
32652 if (!DemandedOps[i])
32653 continue;
32654 KnownBits Known2 =
32655 DAG.computeKnownBits(Ops[i], DemandedOps[i], Depth + 1);
32656 Known.One &= Known2.One;
32657 Known.Zero &= Known2.Zero;
32658 }
32659 }
32660 }
32661 }
32662}
32663
32664unsigned X86TargetLowering::ComputeNumSignBitsForTargetNode(
32665 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
32666 unsigned Depth) const {
32667 EVT VT = Op.getValueType();
32668 unsigned VTBits = VT.getScalarSizeInBits();
32669 unsigned Opcode = Op.getOpcode();
32670 switch (Opcode) {
32671 case X86ISD::SETCC_CARRY:
32672 // SETCC_CARRY sets the dest to ~0 for true or 0 for false.
32673 return VTBits;
32674
32675 case X86ISD::VTRUNC: {
32676 // TODO: Add DemandedElts support.
32677 SDValue Src = Op.getOperand(0);
32678 unsigned NumSrcBits = Src.getScalarValueSizeInBits();
32679 assert(VTBits < NumSrcBits && "Illegal truncation input type")((VTBits < NumSrcBits && "Illegal truncation input type"
) ? static_cast<void> (0) : __assert_fail ("VTBits < NumSrcBits && \"Illegal truncation input type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32679, __PRETTY_FUNCTION__));
32680 unsigned Tmp = DAG.ComputeNumSignBits(Src, Depth + 1);
32681 if (Tmp > (NumSrcBits - VTBits))
32682 return Tmp - (NumSrcBits - VTBits);
32683 return 1;
32684 }
32685
32686 case X86ISD::PACKSS: {
32687 // PACKSS is just a truncation if the sign bits extend to the packed size.
32688 APInt DemandedLHS, DemandedRHS;
32689 getPackDemandedElts(Op.getValueType(), DemandedElts, DemandedLHS,
32690 DemandedRHS);
32691
32692 unsigned SrcBits = Op.getOperand(0).getScalarValueSizeInBits();
32693 unsigned Tmp0 = SrcBits, Tmp1 = SrcBits;
32694 if (!!DemandedLHS)
32695 Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), DemandedLHS, Depth + 1);
32696 if (!!DemandedRHS)
32697 Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), DemandedRHS, Depth + 1);
32698 unsigned Tmp = std::min(Tmp0, Tmp1);
32699 if (Tmp > (SrcBits - VTBits))
32700 return Tmp - (SrcBits - VTBits);
32701 return 1;
32702 }
32703
32704 case X86ISD::VSHLI: {
32705 SDValue Src = Op.getOperand(0);
32706 const APInt &ShiftVal = Op.getConstantOperandAPInt(1);
32707 if (ShiftVal.uge(VTBits))
32708 return VTBits; // Shifted all bits out --> zero.
32709 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
32710 if (ShiftVal.uge(Tmp))
32711 return 1; // Shifted all sign bits out --> unknown.
32712 return Tmp - ShiftVal.getZExtValue();
32713 }
32714
32715 case X86ISD::VSRAI: {
32716 SDValue Src = Op.getOperand(0);
32717 APInt ShiftVal = Op.getConstantOperandAPInt(1);
32718 if (ShiftVal.uge(VTBits - 1))
32719 return VTBits; // Sign splat.
32720 unsigned Tmp = DAG.ComputeNumSignBits(Src, DemandedElts, Depth + 1);
32721 ShiftVal += Tmp;
32722 return ShiftVal.uge(VTBits) ? VTBits : ShiftVal.getZExtValue();
32723 }
32724
32725 case X86ISD::PCMPGT:
32726 case X86ISD::PCMPEQ:
32727 case X86ISD::CMPP:
32728 case X86ISD::VPCOM:
32729 case X86ISD::VPCOMU:
32730 // Vector compares return zero/all-bits result values.
32731 return VTBits;
32732
32733 case X86ISD::ANDNP: {
32734 unsigned Tmp0 =
32735 DAG.ComputeNumSignBits(Op.getOperand(0), DemandedElts, Depth + 1);
32736 if (Tmp0 == 1) return 1; // Early out.
32737 unsigned Tmp1 =
32738 DAG.ComputeNumSignBits(Op.getOperand(1), DemandedElts, Depth + 1);
32739 return std::min(Tmp0, Tmp1);
32740 }
32741
32742 case X86ISD::CMOV: {
32743 unsigned Tmp0 = DAG.ComputeNumSignBits(Op.getOperand(0), Depth+1);
32744 if (Tmp0 == 1) return 1; // Early out.
32745 unsigned Tmp1 = DAG.ComputeNumSignBits(Op.getOperand(1), Depth+1);
32746 return std::min(Tmp0, Tmp1);
32747 }
32748 }
32749
32750 // Handle target shuffles.
32751 // TODO - use resolveTargetShuffleInputs once we can limit recursive depth.
32752 if (isTargetShuffle(Opcode)) {
32753 bool IsUnary;
32754 SmallVector<int, 64> Mask;
32755 SmallVector<SDValue, 2> Ops;
32756 if (getTargetShuffleMask(Op.getNode(), VT.getSimpleVT(), true, Ops, Mask,
32757 IsUnary)) {
32758 unsigned NumOps = Ops.size();
32759 unsigned NumElts = VT.getVectorNumElements();
32760 if (Mask.size() == NumElts) {
32761 SmallVector<APInt, 2> DemandedOps(NumOps, APInt(NumElts, 0));
32762 for (unsigned i = 0; i != NumElts; ++i) {
32763 if (!DemandedElts[i])
32764 continue;
32765 int M = Mask[i];
32766 if (M == SM_SentinelUndef) {
32767 // For UNDEF elements, we don't know anything about the common state
32768 // of the shuffle result.
32769 return 1;
32770 } else if (M == SM_SentinelZero) {
32771 // Zero = all sign bits.
32772 continue;
32773 }
32774 assert(0 <= M && (unsigned)M < (NumOps * NumElts) &&((0 <= M && (unsigned)M < (NumOps * NumElts) &&
"Shuffle index out of range") ? static_cast<void> (0) :
__assert_fail ("0 <= M && (unsigned)M < (NumOps * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32775, __PRETTY_FUNCTION__))
32775 "Shuffle index out of range")((0 <= M && (unsigned)M < (NumOps * NumElts) &&
"Shuffle index out of range") ? static_cast<void> (0) :
__assert_fail ("0 <= M && (unsigned)M < (NumOps * NumElts) && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32775, __PRETTY_FUNCTION__));
32776
32777 unsigned OpIdx = (unsigned)M / NumElts;
32778 unsigned EltIdx = (unsigned)M % NumElts;
32779 if (Ops[OpIdx].getValueType() != VT) {
32780 // TODO - handle target shuffle ops with different value types.
32781 return 1;
32782 }
32783 DemandedOps[OpIdx].setBit(EltIdx);
32784 }
32785 unsigned Tmp0 = VTBits;
32786 for (unsigned i = 0; i != NumOps && Tmp0 > 1; ++i) {
32787 if (!DemandedOps[i])
32788 continue;
32789 unsigned Tmp1 =
32790 DAG.ComputeNumSignBits(Ops[i], DemandedOps[i], Depth + 1);
32791 Tmp0 = std::min(Tmp0, Tmp1);
32792 }
32793 return Tmp0;
32794 }
32795 }
32796 }
32797
32798 // Fallback case.
32799 return 1;
32800}
32801
32802SDValue X86TargetLowering::unwrapAddress(SDValue N) const {
32803 if (N->getOpcode() == X86ISD::Wrapper || N->getOpcode() == X86ISD::WrapperRIP)
32804 return N->getOperand(0);
32805 return N;
32806}
32807
32808// Attempt to match a combined shuffle mask against supported unary shuffle
32809// instructions.
32810// TODO: Investigate sharing more of this with shuffle lowering.
32811static bool matchUnaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
32812 bool AllowFloatDomain, bool AllowIntDomain,
32813 SDValue &V1, const SDLoc &DL, SelectionDAG &DAG,
32814 const X86Subtarget &Subtarget, unsigned &Shuffle,
32815 MVT &SrcVT, MVT &DstVT) {
32816 unsigned NumMaskElts = Mask.size();
32817 unsigned MaskEltSize = MaskVT.getScalarSizeInBits();
32818
32819 // Match against a VZEXT_MOVL vXi32 zero-extending instruction.
32820 if (MaskEltSize == 32 && isUndefOrEqual(Mask[0], 0) &&
32821 isUndefOrZero(Mask[1]) && isUndefInRange(Mask, 2, NumMaskElts - 2)) {
32822 Shuffle = X86ISD::VZEXT_MOVL;
32823 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
32824 return true;
32825 }
32826
32827 // Match against a ANY/ZERO_EXTEND_VECTOR_INREG instruction.
32828 // TODO: Add 512-bit vector support (split AVX512F and AVX512BW).
32829 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE41()) ||
32830 (MaskVT.is256BitVector() && Subtarget.hasInt256()))) {
32831 unsigned MaxScale = 64 / MaskEltSize;
32832 for (unsigned Scale = 2; Scale <= MaxScale; Scale *= 2) {
32833 bool MatchAny = true;
32834 bool MatchZero = true;
32835 unsigned NumDstElts = NumMaskElts / Scale;
32836 for (unsigned i = 0; i != NumDstElts && (MatchAny || MatchZero); ++i) {
32837 if (!isUndefOrEqual(Mask[i * Scale], (int)i)) {
32838 MatchAny = MatchZero = false;
32839 break;
32840 }
32841 MatchAny &= isUndefInRange(Mask, (i * Scale) + 1, Scale - 1);
32842 MatchZero &= isUndefOrZeroInRange(Mask, (i * Scale) + 1, Scale - 1);
32843 }
32844 if (MatchAny || MatchZero) {
32845 assert(MatchZero && "Failed to match zext but matched aext?")((MatchZero && "Failed to match zext but matched aext?"
) ? static_cast<void> (0) : __assert_fail ("MatchZero && \"Failed to match zext but matched aext?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32845, __PRETTY_FUNCTION__));
32846 unsigned SrcSize = std::max(128u, NumDstElts * MaskEltSize);
32847 MVT ScalarTy = MaskVT.isInteger() ? MaskVT.getScalarType() :
32848 MVT::getIntegerVT(MaskEltSize);
32849 SrcVT = MVT::getVectorVT(ScalarTy, SrcSize / MaskEltSize);
32850
32851 if (SrcVT.getSizeInBits() != MaskVT.getSizeInBits())
32852 V1 = extractSubVector(V1, 0, DAG, DL, SrcSize);
32853
32854 Shuffle = unsigned(MatchAny ? ISD::ANY_EXTEND : ISD::ZERO_EXTEND);
32855 if (SrcVT.getVectorNumElements() != NumDstElts)
32856 Shuffle = getOpcode_EXTEND_VECTOR_INREG(Shuffle);
32857
32858 DstVT = MVT::getIntegerVT(Scale * MaskEltSize);
32859 DstVT = MVT::getVectorVT(DstVT, NumDstElts);
32860 return true;
32861 }
32862 }
32863 }
32864
32865 // Match against a VZEXT_MOVL instruction, SSE1 only supports 32-bits (MOVSS).
32866 if (((MaskEltSize == 32) || (MaskEltSize == 64 && Subtarget.hasSSE2())) &&
32867 isUndefOrEqual(Mask[0], 0) &&
32868 isUndefOrZeroInRange(Mask, 1, NumMaskElts - 1)) {
32869 Shuffle = X86ISD::VZEXT_MOVL;
32870 SrcVT = DstVT = !Subtarget.hasSSE2() ? MVT::v4f32 : MaskVT;
32871 return true;
32872 }
32873
32874 // Check if we have SSE3 which will let us use MOVDDUP etc. The
32875 // instructions are no slower than UNPCKLPD but has the option to
32876 // fold the input operand into even an unaligned memory load.
32877 if (MaskVT.is128BitVector() && Subtarget.hasSSE3() && AllowFloatDomain) {
32878 if (isTargetShuffleEquivalent(Mask, {0, 0})) {
32879 Shuffle = X86ISD::MOVDDUP;
32880 SrcVT = DstVT = MVT::v2f64;
32881 return true;
32882 }
32883 if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2})) {
32884 Shuffle = X86ISD::MOVSLDUP;
32885 SrcVT = DstVT = MVT::v4f32;
32886 return true;
32887 }
32888 if (isTargetShuffleEquivalent(Mask, {1, 1, 3, 3})) {
32889 Shuffle = X86ISD::MOVSHDUP;
32890 SrcVT = DstVT = MVT::v4f32;
32891 return true;
32892 }
32893 }
32894
32895 if (MaskVT.is256BitVector() && AllowFloatDomain) {
32896 assert(Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles")((Subtarget.hasAVX() && "AVX required for 256-bit vector shuffles"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX() && \"AVX required for 256-bit vector shuffles\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32896, __PRETTY_FUNCTION__));
32897 if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2})) {
32898 Shuffle = X86ISD::MOVDDUP;
32899 SrcVT = DstVT = MVT::v4f64;
32900 return true;
32901 }
32902 if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6})) {
32903 Shuffle = X86ISD::MOVSLDUP;
32904 SrcVT = DstVT = MVT::v8f32;
32905 return true;
32906 }
32907 if (isTargetShuffleEquivalent(Mask, {1, 1, 3, 3, 5, 5, 7, 7})) {
32908 Shuffle = X86ISD::MOVSHDUP;
32909 SrcVT = DstVT = MVT::v8f32;
32910 return true;
32911 }
32912 }
32913
32914 if (MaskVT.is512BitVector() && AllowFloatDomain) {
32915 assert(Subtarget.hasAVX512() &&((Subtarget.hasAVX512() && "AVX512 required for 512-bit vector shuffles"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"AVX512 required for 512-bit vector shuffles\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32916, __PRETTY_FUNCTION__))
32916 "AVX512 required for 512-bit vector shuffles")((Subtarget.hasAVX512() && "AVX512 required for 512-bit vector shuffles"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX512() && \"AVX512 required for 512-bit vector shuffles\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32916, __PRETTY_FUNCTION__));
32917 if (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2, 4, 4, 6, 6})) {
32918 Shuffle = X86ISD::MOVDDUP;
32919 SrcVT = DstVT = MVT::v8f64;
32920 return true;
32921 }
32922 if (isTargetShuffleEquivalent(
32923 Mask, {0, 0, 2, 2, 4, 4, 6, 6, 8, 8, 10, 10, 12, 12, 14, 14})) {
32924 Shuffle = X86ISD::MOVSLDUP;
32925 SrcVT = DstVT = MVT::v16f32;
32926 return true;
32927 }
32928 if (isTargetShuffleEquivalent(
32929 Mask, {1, 1, 3, 3, 5, 5, 7, 7, 9, 9, 11, 11, 13, 13, 15, 15})) {
32930 Shuffle = X86ISD::MOVSHDUP;
32931 SrcVT = DstVT = MVT::v16f32;
32932 return true;
32933 }
32934 }
32935
32936 return false;
32937}
32938
32939// Attempt to match a combined shuffle mask against supported unary immediate
32940// permute instructions.
32941// TODO: Investigate sharing more of this with shuffle lowering.
32942static bool matchUnaryPermuteShuffle(MVT MaskVT, ArrayRef<int> Mask,
32943 const APInt &Zeroable,
32944 bool AllowFloatDomain, bool AllowIntDomain,
32945 const X86Subtarget &Subtarget,
32946 unsigned &Shuffle, MVT &ShuffleVT,
32947 unsigned &PermuteImm) {
32948 unsigned NumMaskElts = Mask.size();
32949 unsigned InputSizeInBits = MaskVT.getSizeInBits();
32950 unsigned MaskScalarSizeInBits = InputSizeInBits / NumMaskElts;
32951 MVT MaskEltVT = MVT::getIntegerVT(MaskScalarSizeInBits);
32952
32953 bool ContainsZeros =
32954 llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; });
32955
32956 // Handle VPERMI/VPERMILPD vXi64/vXi64 patterns.
32957 if (!ContainsZeros && MaskScalarSizeInBits == 64) {
32958 // Check for lane crossing permutes.
32959 if (is128BitLaneCrossingShuffleMask(MaskEltVT, Mask)) {
32960 // PERMPD/PERMQ permutes within a 256-bit vector (AVX2+).
32961 if (Subtarget.hasAVX2() && MaskVT.is256BitVector()) {
32962 Shuffle = X86ISD::VPERMI;
32963 ShuffleVT = (AllowFloatDomain ? MVT::v4f64 : MVT::v4i64);
32964 PermuteImm = getV4X86ShuffleImm(Mask);
32965 return true;
32966 }
32967 if (Subtarget.hasAVX512() && MaskVT.is512BitVector()) {
32968 SmallVector<int, 4> RepeatedMask;
32969 if (is256BitLaneRepeatedShuffleMask(MVT::v8f64, Mask, RepeatedMask)) {
32970 Shuffle = X86ISD::VPERMI;
32971 ShuffleVT = (AllowFloatDomain ? MVT::v8f64 : MVT::v8i64);
32972 PermuteImm = getV4X86ShuffleImm(RepeatedMask);
32973 return true;
32974 }
32975 }
32976 } else if (AllowFloatDomain && Subtarget.hasAVX()) {
32977 // VPERMILPD can permute with a non-repeating shuffle.
32978 Shuffle = X86ISD::VPERMILPI;
32979 ShuffleVT = MVT::getVectorVT(MVT::f64, Mask.size());
32980 PermuteImm = 0;
32981 for (int i = 0, e = Mask.size(); i != e; ++i) {
32982 int M = Mask[i];
32983 if (M == SM_SentinelUndef)
32984 continue;
32985 assert(((M / 2) == (i / 2)) && "Out of range shuffle mask index")((((M / 2) == (i / 2)) && "Out of range shuffle mask index"
) ? static_cast<void> (0) : __assert_fail ("((M / 2) == (i / 2)) && \"Out of range shuffle mask index\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 32985, __PRETTY_FUNCTION__));
32986 PermuteImm |= (M & 1) << i;
32987 }
32988 return true;
32989 }
32990 }
32991
32992 // Handle PSHUFD/VPERMILPI vXi32/vXf32 repeated patterns.
32993 // AVX introduced the VPERMILPD/VPERMILPS float permutes, before then we
32994 // had to use 2-input SHUFPD/SHUFPS shuffles (not handled here).
32995 if ((MaskScalarSizeInBits == 64 || MaskScalarSizeInBits == 32) &&
32996 !ContainsZeros && (AllowIntDomain || Subtarget.hasAVX())) {
32997 SmallVector<int, 4> RepeatedMask;
32998 if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
32999 // Narrow the repeated mask to create 32-bit element permutes.
33000 SmallVector<int, 4> WordMask = RepeatedMask;
33001 if (MaskScalarSizeInBits == 64)
33002 scaleShuffleMask<int>(2, RepeatedMask, WordMask);
33003
33004 Shuffle = (AllowIntDomain ? X86ISD::PSHUFD : X86ISD::VPERMILPI);
33005 ShuffleVT = (AllowIntDomain ? MVT::i32 : MVT::f32);
33006 ShuffleVT = MVT::getVectorVT(ShuffleVT, InputSizeInBits / 32);
33007 PermuteImm = getV4X86ShuffleImm(WordMask);
33008 return true;
33009 }
33010 }
33011
33012 // Handle PSHUFLW/PSHUFHW vXi16 repeated patterns.
33013 if (!ContainsZeros && AllowIntDomain && MaskScalarSizeInBits == 16) {
33014 SmallVector<int, 4> RepeatedMask;
33015 if (is128BitLaneRepeatedShuffleMask(MaskEltVT, Mask, RepeatedMask)) {
33016 ArrayRef<int> LoMask(RepeatedMask.data() + 0, 4);
33017 ArrayRef<int> HiMask(RepeatedMask.data() + 4, 4);
33018
33019 // PSHUFLW: permute lower 4 elements only.
33020 if (isUndefOrInRange(LoMask, 0, 4) &&
33021 isSequentialOrUndefInRange(HiMask, 0, 4, 4)) {
33022 Shuffle = X86ISD::PSHUFLW;
33023 ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
33024 PermuteImm = getV4X86ShuffleImm(LoMask);
33025 return true;
33026 }
33027
33028 // PSHUFHW: permute upper 4 elements only.
33029 if (isUndefOrInRange(HiMask, 4, 8) &&
33030 isSequentialOrUndefInRange(LoMask, 0, 4, 0)) {
33031 // Offset the HiMask so that we can create the shuffle immediate.
33032 int OffsetHiMask[4];
33033 for (int i = 0; i != 4; ++i)
33034 OffsetHiMask[i] = (HiMask[i] < 0 ? HiMask[i] : HiMask[i] - 4);
33035
33036 Shuffle = X86ISD::PSHUFHW;
33037 ShuffleVT = MVT::getVectorVT(MVT::i16, InputSizeInBits / 16);
33038 PermuteImm = getV4X86ShuffleImm(OffsetHiMask);
33039 return true;
33040 }
33041 }
33042 }
33043
33044 // Attempt to match against byte/bit shifts.
33045 // FIXME: Add 512-bit support.
33046 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
33047 (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) {
33048 int ShiftAmt = matchShuffleAsShift(ShuffleVT, Shuffle, MaskScalarSizeInBits,
33049 Mask, 0, Zeroable, Subtarget);
33050 if (0 < ShiftAmt) {
33051 PermuteImm = (unsigned)ShiftAmt;
33052 return true;
33053 }
33054 }
33055
33056 return false;
33057}
33058
33059// Attempt to match a combined unary shuffle mask against supported binary
33060// shuffle instructions.
33061// TODO: Investigate sharing more of this with shuffle lowering.
33062static bool matchBinaryShuffle(MVT MaskVT, ArrayRef<int> Mask,
33063 bool AllowFloatDomain, bool AllowIntDomain,
33064 SDValue &V1, SDValue &V2, const SDLoc &DL,
33065 SelectionDAG &DAG, const X86Subtarget &Subtarget,
33066 unsigned &Shuffle, MVT &SrcVT, MVT &DstVT,
33067 bool IsUnary) {
33068 unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
33069
33070 if (MaskVT.is128BitVector()) {
33071 if (isTargetShuffleEquivalent(Mask, {0, 0}) && AllowFloatDomain) {
33072 V2 = V1;
33073 V1 = (SM_SentinelUndef == Mask[0] ? DAG.getUNDEF(MVT::v4f32) : V1);
33074 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKL : X86ISD::MOVLHPS;
33075 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
33076 return true;
33077 }
33078 if (isTargetShuffleEquivalent(Mask, {1, 1}) && AllowFloatDomain) {
33079 V2 = V1;
33080 Shuffle = Subtarget.hasSSE2() ? X86ISD::UNPCKH : X86ISD::MOVHLPS;
33081 SrcVT = DstVT = Subtarget.hasSSE2() ? MVT::v2f64 : MVT::v4f32;
33082 return true;
33083 }
33084 if (isTargetShuffleEquivalent(Mask, {0, 3}) && Subtarget.hasSSE2() &&
33085 (AllowFloatDomain || !Subtarget.hasSSE41())) {
33086 std::swap(V1, V2);
33087 Shuffle = X86ISD::MOVSD;
33088 SrcVT = DstVT = MVT::v2f64;
33089 return true;
33090 }
33091 if (isTargetShuffleEquivalent(Mask, {4, 1, 2, 3}) &&
33092 (AllowFloatDomain || !Subtarget.hasSSE41())) {
33093 Shuffle = X86ISD::MOVSS;
33094 SrcVT = DstVT = MVT::v4f32;
33095 return true;
33096 }
33097 }
33098
33099 // Attempt to match against either an unary or binary PACKSS/PACKUS shuffle.
33100 if (((MaskVT == MVT::v8i16 || MaskVT == MVT::v16i8) && Subtarget.hasSSE2()) ||
33101 ((MaskVT == MVT::v16i16 || MaskVT == MVT::v32i8) && Subtarget.hasInt256()) ||
33102 ((MaskVT == MVT::v32i16 || MaskVT == MVT::v64i8) && Subtarget.hasBWI())) {
33103 if (matchShuffleWithPACK(MaskVT, SrcVT, V1, V2, Shuffle, Mask, DAG,
33104 Subtarget)) {
33105 DstVT = MaskVT;
33106 return true;
33107 }
33108 }
33109
33110 // Attempt to match against either a unary or binary UNPCKL/UNPCKH shuffle.
33111 if ((MaskVT == MVT::v4f32 && Subtarget.hasSSE1()) ||
33112 (MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
33113 (MaskVT.is256BitVector() && 32 <= EltSizeInBits && Subtarget.hasAVX()) ||
33114 (MaskVT.is256BitVector() && Subtarget.hasAVX2()) ||
33115 (MaskVT.is512BitVector() && Subtarget.hasAVX512())) {
33116 if (matchShuffleWithUNPCK(MaskVT, V1, V2, Shuffle, IsUnary, Mask, DL, DAG,
33117 Subtarget)) {
33118 SrcVT = DstVT = MaskVT;
33119 if (MaskVT.is256BitVector() && !Subtarget.hasAVX2())
33120 SrcVT = DstVT = (32 == EltSizeInBits ? MVT::v8f32 : MVT::v4f64);
33121 return true;
33122 }
33123 }
33124
33125 return false;
33126}
33127
33128static bool matchBinaryPermuteShuffle(
33129 MVT MaskVT, ArrayRef<int> Mask, const APInt &Zeroable,
33130 bool AllowFloatDomain, bool AllowIntDomain, SDValue &V1, SDValue &V2,
33131 const SDLoc &DL, SelectionDAG &DAG, const X86Subtarget &Subtarget,
33132 unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) {
33133 unsigned NumMaskElts = Mask.size();
33134 unsigned EltSizeInBits = MaskVT.getScalarSizeInBits();
33135
33136 // Attempt to match against PALIGNR byte rotate.
33137 if (AllowIntDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) ||
33138 (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) {
33139 int ByteRotation = matchShuffleAsByteRotate(MaskVT, V1, V2, Mask);
33140 if (0 < ByteRotation) {
33141 Shuffle = X86ISD::PALIGNR;
33142 ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8);
33143 PermuteImm = ByteRotation;
33144 return true;
33145 }
33146 }
33147
33148 // Attempt to combine to X86ISD::BLENDI.
33149 if ((NumMaskElts <= 8 && ((Subtarget.hasSSE41() && MaskVT.is128BitVector()) ||
33150 (Subtarget.hasAVX() && MaskVT.is256BitVector()))) ||
33151 (MaskVT == MVT::v16i16 && Subtarget.hasAVX2())) {
33152 uint64_t BlendMask = 0;
33153 bool ForceV1Zero = false, ForceV2Zero = false;
33154 SmallVector<int, 8> TargetMask(Mask.begin(), Mask.end());
33155 if (matchShuffleAsBlend(V1, V2, TargetMask, Zeroable, ForceV1Zero,
33156 ForceV2Zero, BlendMask)) {
33157 if (MaskVT == MVT::v16i16) {
33158 // We can only use v16i16 PBLENDW if the lanes are repeated.
33159 SmallVector<int, 8> RepeatedMask;
33160 if (isRepeatedTargetShuffleMask(128, MaskVT, TargetMask,
33161 RepeatedMask)) {
33162 assert(RepeatedMask.size() == 8 &&((RepeatedMask.size() == 8 && "Repeated mask size doesn't match!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 8 && \"Repeated mask size doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33163, __PRETTY_FUNCTION__))
33163 "Repeated mask size doesn't match!")((RepeatedMask.size() == 8 && "Repeated mask size doesn't match!"
) ? static_cast<void> (0) : __assert_fail ("RepeatedMask.size() == 8 && \"Repeated mask size doesn't match!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33163, __PRETTY_FUNCTION__));
33164 PermuteImm = 0;
33165 for (int i = 0; i < 8; ++i)
33166 if (RepeatedMask[i] >= 8)
33167 PermuteImm |= 1 << i;
33168 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
33169 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
33170 Shuffle = X86ISD::BLENDI;
33171 ShuffleVT = MaskVT;
33172 return true;
33173 }
33174 } else {
33175 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
33176 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
33177 PermuteImm = (unsigned)BlendMask;
33178 Shuffle = X86ISD::BLENDI;
33179 ShuffleVT = MaskVT;
33180 return true;
33181 }
33182 }
33183 }
33184
33185 // Attempt to combine to INSERTPS, but only if it has elements that need to
33186 // be set to zero.
33187 if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
33188 MaskVT.is128BitVector() &&
33189 llvm::any_of(Mask, [](int M) { return M == SM_SentinelZero; }) &&
33190 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
33191 Shuffle = X86ISD::INSERTPS;
33192 ShuffleVT = MVT::v4f32;
33193 return true;
33194 }
33195
33196 // Attempt to combine to SHUFPD.
33197 if (AllowFloatDomain && EltSizeInBits == 64 &&
33198 ((MaskVT.is128BitVector() && Subtarget.hasSSE2()) ||
33199 (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
33200 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
33201 bool ForceV1Zero = false, ForceV2Zero = false;
33202 if (matchShuffleWithSHUFPD(MaskVT, V1, V2, ForceV1Zero, ForceV2Zero,
33203 PermuteImm, Mask, Zeroable)) {
33204 V1 = ForceV1Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V1;
33205 V2 = ForceV2Zero ? getZeroVector(MaskVT, Subtarget, DAG, DL) : V2;
33206 Shuffle = X86ISD::SHUFP;
33207 ShuffleVT = MVT::getVectorVT(MVT::f64, MaskVT.getSizeInBits() / 64);
33208 return true;
33209 }
33210 }
33211
33212 // Attempt to combine to SHUFPS.
33213 if (AllowFloatDomain && EltSizeInBits == 32 &&
33214 ((MaskVT.is128BitVector() && Subtarget.hasSSE1()) ||
33215 (MaskVT.is256BitVector() && Subtarget.hasAVX()) ||
33216 (MaskVT.is512BitVector() && Subtarget.hasAVX512()))) {
33217 SmallVector<int, 4> RepeatedMask;
33218 if (isRepeatedTargetShuffleMask(128, MaskVT, Mask, RepeatedMask)) {
33219 // Match each half of the repeated mask, to determine if its just
33220 // referencing one of the vectors, is zeroable or entirely undef.
33221 auto MatchHalf = [&](unsigned Offset, int &S0, int &S1) {
33222 int M0 = RepeatedMask[Offset];
33223 int M1 = RepeatedMask[Offset + 1];
33224
33225 if (isUndefInRange(RepeatedMask, Offset, 2)) {
33226 return DAG.getUNDEF(MaskVT);
33227 } else if (isUndefOrZeroInRange(RepeatedMask, Offset, 2)) {
33228 S0 = (SM_SentinelUndef == M0 ? -1 : 0);
33229 S1 = (SM_SentinelUndef == M1 ? -1 : 1);
33230 return getZeroVector(MaskVT, Subtarget, DAG, DL);
33231 } else if (isUndefOrInRange(M0, 0, 4) && isUndefOrInRange(M1, 0, 4)) {
33232 S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
33233 S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
33234 return V1;
33235 } else if (isUndefOrInRange(M0, 4, 8) && isUndefOrInRange(M1, 4, 8)) {
33236 S0 = (SM_SentinelUndef == M0 ? -1 : M0 & 3);
33237 S1 = (SM_SentinelUndef == M1 ? -1 : M1 & 3);
33238 return V2;
33239 }
33240
33241 return SDValue();
33242 };
33243
33244 int ShufMask[4] = {-1, -1, -1, -1};
33245 SDValue Lo = MatchHalf(0, ShufMask[0], ShufMask[1]);
33246 SDValue Hi = MatchHalf(2, ShufMask[2], ShufMask[3]);
33247
33248 if (Lo && Hi) {
33249 V1 = Lo;
33250 V2 = Hi;
33251 Shuffle = X86ISD::SHUFP;
33252 ShuffleVT = MVT::getVectorVT(MVT::f32, MaskVT.getSizeInBits() / 32);
33253 PermuteImm = getV4X86ShuffleImm(ShufMask);
33254 return true;
33255 }
33256 }
33257 }
33258
33259 // Attempt to combine to INSERTPS more generally if X86ISD::SHUFP failed.
33260 if (AllowFloatDomain && EltSizeInBits == 32 && Subtarget.hasSSE41() &&
33261 MaskVT.is128BitVector() &&
33262 matchShuffleAsInsertPS(V1, V2, PermuteImm, Zeroable, Mask, DAG)) {
33263 Shuffle = X86ISD::INSERTPS;
33264 ShuffleVT = MVT::v4f32;
33265 return true;
33266 }
33267
33268 return false;
33269}
33270
33271static SDValue combineX86ShuffleChainWithExtract(
33272 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
33273 bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG,
33274 const X86Subtarget &Subtarget);
33275
33276/// Combine an arbitrary chain of shuffles into a single instruction if
33277/// possible.
33278///
33279/// This is the leaf of the recursive combine below. When we have found some
33280/// chain of single-use x86 shuffle instructions and accumulated the combined
33281/// shuffle mask represented by them, this will try to pattern match that mask
33282/// into either a single instruction if there is a special purpose instruction
33283/// for this operation, or into a PSHUFB instruction which is a fully general
33284/// instruction but should only be used to replace chains over a certain depth.
33285static SDValue combineX86ShuffleChain(ArrayRef<SDValue> Inputs, SDValue Root,
33286 ArrayRef<int> BaseMask, int Depth,
33287 bool HasVariableMask,
33288 bool AllowVariableMask, SelectionDAG &DAG,
33289 const X86Subtarget &Subtarget) {
33290 assert(!BaseMask.empty() && "Cannot combine an empty shuffle mask!")((!BaseMask.empty() && "Cannot combine an empty shuffle mask!"
) ? static_cast<void> (0) : __assert_fail ("!BaseMask.empty() && \"Cannot combine an empty shuffle mask!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33290, __PRETTY_FUNCTION__));
33291 assert((Inputs.size() == 1 || Inputs.size() == 2) &&(((Inputs.size() == 1 || Inputs.size() == 2) && "Unexpected number of shuffle inputs!"
) ? static_cast<void> (0) : __assert_fail ("(Inputs.size() == 1 || Inputs.size() == 2) && \"Unexpected number of shuffle inputs!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33292, __PRETTY_FUNCTION__))
33292 "Unexpected number of shuffle inputs!")(((Inputs.size() == 1 || Inputs.size() == 2) && "Unexpected number of shuffle inputs!"
) ? static_cast<void> (0) : __assert_fail ("(Inputs.size() == 1 || Inputs.size() == 2) && \"Unexpected number of shuffle inputs!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33292, __PRETTY_FUNCTION__));
33293
33294 // Find the inputs that enter the chain. Note that multiple uses are OK
33295 // here, we're not going to remove the operands we find.
33296 bool UnaryShuffle = (Inputs.size() == 1);
33297 SDValue V1 = peekThroughBitcasts(Inputs[0]);
33298 SDValue V2 = (UnaryShuffle ? DAG.getUNDEF(V1.getValueType())
33299 : peekThroughBitcasts(Inputs[1]));
33300
33301 MVT VT1 = V1.getSimpleValueType();
33302 MVT VT2 = V2.getSimpleValueType();
33303 MVT RootVT = Root.getSimpleValueType();
33304 assert(VT1.getSizeInBits() == RootVT.getSizeInBits() &&((VT1.getSizeInBits() == RootVT.getSizeInBits() && VT2
.getSizeInBits() == RootVT.getSizeInBits() && "Vector size mismatch"
) ? static_cast<void> (0) : __assert_fail ("VT1.getSizeInBits() == RootVT.getSizeInBits() && VT2.getSizeInBits() == RootVT.getSizeInBits() && \"Vector size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33306, __PRETTY_FUNCTION__))
33305 VT2.getSizeInBits() == RootVT.getSizeInBits() &&((VT1.getSizeInBits() == RootVT.getSizeInBits() && VT2
.getSizeInBits() == RootVT.getSizeInBits() && "Vector size mismatch"
) ? static_cast<void> (0) : __assert_fail ("VT1.getSizeInBits() == RootVT.getSizeInBits() && VT2.getSizeInBits() == RootVT.getSizeInBits() && \"Vector size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33306, __PRETTY_FUNCTION__))
33306 "Vector size mismatch")((VT1.getSizeInBits() == RootVT.getSizeInBits() && VT2
.getSizeInBits() == RootVT.getSizeInBits() && "Vector size mismatch"
) ? static_cast<void> (0) : __assert_fail ("VT1.getSizeInBits() == RootVT.getSizeInBits() && VT2.getSizeInBits() == RootVT.getSizeInBits() && \"Vector size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33306, __PRETTY_FUNCTION__));
33307
33308 SDLoc DL(Root);
33309 SDValue Res;
33310
33311 unsigned NumBaseMaskElts = BaseMask.size();
33312 if (NumBaseMaskElts == 1) {
33313 assert(BaseMask[0] == 0 && "Invalid shuffle index found!")((BaseMask[0] == 0 && "Invalid shuffle index found!")
? static_cast<void> (0) : __assert_fail ("BaseMask[0] == 0 && \"Invalid shuffle index found!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33313, __PRETTY_FUNCTION__));
33314 return DAG.getBitcast(RootVT, V1);
33315 }
33316
33317 unsigned RootSizeInBits = RootVT.getSizeInBits();
33318 unsigned NumRootElts = RootVT.getVectorNumElements();
33319 unsigned BaseMaskEltSizeInBits = RootSizeInBits / NumBaseMaskElts;
33320 bool FloatDomain = VT1.isFloatingPoint() || VT2.isFloatingPoint() ||
33321 (RootVT.isFloatingPoint() && Depth >= 1) ||
33322 (RootVT.is256BitVector() && !Subtarget.hasAVX2());
33323
33324 // Don't combine if we are a AVX512/EVEX target and the mask element size
33325 // is different from the root element size - this would prevent writemasks
33326 // from being reused.
33327 // TODO - this currently prevents all lane shuffles from occurring.
33328 // TODO - check for writemasks usage instead of always preventing combining.
33329 // TODO - attempt to narrow Mask back to writemask size.
33330 bool IsEVEXShuffle =
33331 RootSizeInBits == 512 || (Subtarget.hasVLX() && RootSizeInBits >= 128);
33332
33333 // Attempt to match a subvector broadcast.
33334 // shuffle(insert_subvector(undef, sub, 0), undef, 0, 0, 0, 0)
33335 if (UnaryShuffle &&
33336 (BaseMaskEltSizeInBits == 128 || BaseMaskEltSizeInBits == 256)) {
33337 SmallVector<int, 64> BroadcastMask(NumBaseMaskElts, 0);
33338 if (isTargetShuffleEquivalent(BaseMask, BroadcastMask)) {
33339 SDValue Src = Inputs[0];
33340 if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
33341 Src.getOperand(0).isUndef() &&
33342 Src.getOperand(1).getValueSizeInBits() == BaseMaskEltSizeInBits &&
33343 MayFoldLoad(Src.getOperand(1)) && isNullConstant(Src.getOperand(2))) {
33344 return DAG.getBitcast(RootVT, DAG.getNode(X86ISD::SUBV_BROADCAST, DL,
33345 Src.getValueType(),
33346 Src.getOperand(1)));
33347 }
33348 }
33349 }
33350
33351 // TODO - handle 128/256-bit lane shuffles of 512-bit vectors.
33352
33353 // Handle 128-bit lane shuffles of 256-bit vectors.
33354 // If we have AVX2, prefer to use VPERMQ/VPERMPD for unary shuffles unless
33355 // we need to use the zeroing feature.
33356 // TODO - this should support binary shuffles.
33357 if (UnaryShuffle && RootVT.is256BitVector() && NumBaseMaskElts == 2 &&
33358 !(Subtarget.hasAVX2() && BaseMask[0] >= -1 && BaseMask[1] >= -1) &&
33359 !isSequentialOrUndefOrZeroInRange(BaseMask, 0, 2, 0)) {
33360 if (Depth == 0 && Root.getOpcode() == X86ISD::VPERM2X128)
33361 return SDValue(); // Nothing to do!
33362 MVT ShuffleVT = (FloatDomain ? MVT::v4f64 : MVT::v4i64);
33363 unsigned PermMask = 0;
33364 PermMask |= ((BaseMask[0] < 0 ? 0x8 : (BaseMask[0] & 1)) << 0);
33365 PermMask |= ((BaseMask[1] < 0 ? 0x8 : (BaseMask[1] & 1)) << 4);
33366
33367 Res = DAG.getBitcast(ShuffleVT, V1);
33368 Res = DAG.getNode(X86ISD::VPERM2X128, DL, ShuffleVT, Res,
33369 DAG.getUNDEF(ShuffleVT),
33370 DAG.getTargetConstant(PermMask, DL, MVT::i8));
33371 return DAG.getBitcast(RootVT, Res);
33372 }
33373
33374 // For masks that have been widened to 128-bit elements or more,
33375 // narrow back down to 64-bit elements.
33376 SmallVector<int, 64> Mask;
33377 if (BaseMaskEltSizeInBits > 64) {
33378 assert((BaseMaskEltSizeInBits % 64) == 0 && "Illegal mask size")(((BaseMaskEltSizeInBits % 64) == 0 && "Illegal mask size"
) ? static_cast<void> (0) : __assert_fail ("(BaseMaskEltSizeInBits % 64) == 0 && \"Illegal mask size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33378, __PRETTY_FUNCTION__));
33379 int MaskScale = BaseMaskEltSizeInBits / 64;
33380 scaleShuffleMask<int>(MaskScale, BaseMask, Mask);
33381 } else {
33382 Mask = SmallVector<int, 64>(BaseMask.begin(), BaseMask.end());
33383 }
33384
33385 unsigned NumMaskElts = Mask.size();
33386 unsigned MaskEltSizeInBits = RootSizeInBits / NumMaskElts;
33387
33388 // Determine the effective mask value type.
33389 FloatDomain &= (32 <= MaskEltSizeInBits);
33390 MVT MaskVT = FloatDomain ? MVT::getFloatingPointVT(MaskEltSizeInBits)
33391 : MVT::getIntegerVT(MaskEltSizeInBits);
33392 MaskVT = MVT::getVectorVT(MaskVT, NumMaskElts);
33393
33394 // Only allow legal mask types.
33395 if (!DAG.getTargetLoweringInfo().isTypeLegal(MaskVT))
33396 return SDValue();
33397
33398 // Attempt to match the mask against known shuffle patterns.
33399 MVT ShuffleSrcVT, ShuffleVT;
33400 unsigned Shuffle, PermuteImm;
33401
33402 // Which shuffle domains are permitted?
33403 // Permit domain crossing at higher combine depths.
33404 // TODO: Should we indicate which domain is preferred if both are allowed?
33405 bool AllowFloatDomain = FloatDomain || (Depth >= 3);
33406 bool AllowIntDomain = (!FloatDomain || (Depth >= 3)) && Subtarget.hasSSE2() &&
33407 (!MaskVT.is256BitVector() || Subtarget.hasAVX2());
33408
33409 // Determine zeroable mask elements.
33410 APInt KnownUndef, KnownZero;
33411 resolveZeroablesFromTargetShuffle(Mask, KnownUndef, KnownZero);
33412 APInt Zeroable = KnownUndef | KnownZero;
33413
33414 if (UnaryShuffle) {
33415 // If we are shuffling a X86ISD::VZEXT_LOAD then we can use the load
33416 // directly if we don't shuffle the lower element and we shuffle the upper
33417 // (zero) elements within themselves.
33418 if (V1.getOpcode() == X86ISD::VZEXT_LOAD &&
33419 (cast<MemIntrinsicSDNode>(V1)->getMemoryVT().getScalarSizeInBits() %
33420 MaskEltSizeInBits) == 0) {
33421 unsigned Scale =
33422 cast<MemIntrinsicSDNode>(V1)->getMemoryVT().getScalarSizeInBits() /
33423 MaskEltSizeInBits;
33424 ArrayRef<int> HiMask(Mask.data() + Scale, NumMaskElts - Scale);
33425 if (isSequentialOrUndefInRange(Mask, 0, Scale, 0) &&
33426 isUndefOrZeroOrInRange(HiMask, Scale, NumMaskElts)) {
33427 return DAG.getBitcast(RootVT, V1);
33428 }
33429 }
33430
33431 // Attempt to match against broadcast-from-vector.
33432 // Limit AVX1 to cases where we're loading+broadcasting a scalar element.
33433 if ((Subtarget.hasAVX2() || (Subtarget.hasAVX() && 32 <= MaskEltSizeInBits))
33434 && (!IsEVEXShuffle || NumRootElts == NumMaskElts)) {
33435 SmallVector<int, 64> BroadcastMask(NumMaskElts, 0);
33436 if (isTargetShuffleEquivalent(Mask, BroadcastMask)) {
33437 if (V1.getValueType() == MaskVT &&
33438 V1.getOpcode() == ISD::SCALAR_TO_VECTOR &&
33439 MayFoldLoad(V1.getOperand(0))) {
33440 if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
33441 return SDValue(); // Nothing to do!
33442 Res = V1.getOperand(0);
33443 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
33444 return DAG.getBitcast(RootVT, Res);
33445 }
33446 if (Subtarget.hasAVX2()) {
33447 if (Depth == 0 && Root.getOpcode() == X86ISD::VBROADCAST)
33448 return SDValue(); // Nothing to do!
33449 Res = DAG.getBitcast(MaskVT, V1);
33450 Res = DAG.getNode(X86ISD::VBROADCAST, DL, MaskVT, Res);
33451 return DAG.getBitcast(RootVT, Res);
33452 }
33453 }
33454 }
33455
33456 SDValue NewV1 = V1; // Save operand in case early exit happens.
33457 if (matchUnaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, NewV1,
33458 DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
33459 ShuffleVT) &&
33460 (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
33461 if (Depth == 0 && Root.getOpcode() == Shuffle)
33462 return SDValue(); // Nothing to do!
33463 Res = DAG.getBitcast(ShuffleSrcVT, NewV1);
33464 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res);
33465 return DAG.getBitcast(RootVT, Res);
33466 }
33467
33468 if (matchUnaryPermuteShuffle(MaskVT, Mask, Zeroable, AllowFloatDomain,
33469 AllowIntDomain, Subtarget, Shuffle, ShuffleVT,
33470 PermuteImm) &&
33471 (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
33472 if (Depth == 0 && Root.getOpcode() == Shuffle)
33473 return SDValue(); // Nothing to do!
33474 Res = DAG.getBitcast(ShuffleVT, V1);
33475 Res = DAG.getNode(Shuffle, DL, ShuffleVT, Res,
33476 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
33477 return DAG.getBitcast(RootVT, Res);
33478 }
33479 }
33480
33481 SDValue NewV1 = V1; // Save operands in case early exit happens.
33482 SDValue NewV2 = V2;
33483 if (matchBinaryShuffle(MaskVT, Mask, AllowFloatDomain, AllowIntDomain, NewV1,
33484 NewV2, DL, DAG, Subtarget, Shuffle, ShuffleSrcVT,
33485 ShuffleVT, UnaryShuffle) &&
33486 (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
33487 if (Depth == 0 && Root.getOpcode() == Shuffle)
33488 return SDValue(); // Nothing to do!
33489 NewV1 = DAG.getBitcast(ShuffleSrcVT, NewV1);
33490 NewV2 = DAG.getBitcast(ShuffleSrcVT, NewV2);
33491 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2);
33492 return DAG.getBitcast(RootVT, Res);
33493 }
33494
33495 NewV1 = V1; // Save operands in case early exit happens.
33496 NewV2 = V2;
33497 if (matchBinaryPermuteShuffle(
33498 MaskVT, Mask, Zeroable, AllowFloatDomain, AllowIntDomain, NewV1,
33499 NewV2, DL, DAG, Subtarget, Shuffle, ShuffleVT, PermuteImm) &&
33500 (!IsEVEXShuffle || (NumRootElts == ShuffleVT.getVectorNumElements()))) {
33501 if (Depth == 0 && Root.getOpcode() == Shuffle)
33502 return SDValue(); // Nothing to do!
33503 NewV1 = DAG.getBitcast(ShuffleVT, NewV1);
33504 NewV2 = DAG.getBitcast(ShuffleVT, NewV2);
33505 Res = DAG.getNode(Shuffle, DL, ShuffleVT, NewV1, NewV2,
33506 DAG.getTargetConstant(PermuteImm, DL, MVT::i8));
33507 return DAG.getBitcast(RootVT, Res);
33508 }
33509
33510 // Typically from here on, we need an integer version of MaskVT.
33511 MVT IntMaskVT = MVT::getIntegerVT(MaskEltSizeInBits);
33512 IntMaskVT = MVT::getVectorVT(IntMaskVT, NumMaskElts);
33513
33514 // Annoyingly, SSE4A instructions don't map into the above match helpers.
33515 if (Subtarget.hasSSE4A() && AllowIntDomain && RootSizeInBits == 128) {
33516 uint64_t BitLen, BitIdx;
33517 if (matchShuffleAsEXTRQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx,
33518 Zeroable)) {
33519 if (Depth == 0 && Root.getOpcode() == X86ISD::EXTRQI)
33520 return SDValue(); // Nothing to do!
33521 V1 = DAG.getBitcast(IntMaskVT, V1);
33522 Res = DAG.getNode(X86ISD::EXTRQI, DL, IntMaskVT, V1,
33523 DAG.getTargetConstant(BitLen, DL, MVT::i8),
33524 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
33525 return DAG.getBitcast(RootVT, Res);
33526 }
33527
33528 if (matchShuffleAsINSERTQ(IntMaskVT, V1, V2, Mask, BitLen, BitIdx)) {
33529 if (Depth == 0 && Root.getOpcode() == X86ISD::INSERTQI)
33530 return SDValue(); // Nothing to do!
33531 V1 = DAG.getBitcast(IntMaskVT, V1);
33532 V2 = DAG.getBitcast(IntMaskVT, V2);
33533 Res = DAG.getNode(X86ISD::INSERTQI, DL, IntMaskVT, V1, V2,
33534 DAG.getTargetConstant(BitLen, DL, MVT::i8),
33535 DAG.getTargetConstant(BitIdx, DL, MVT::i8));
33536 return DAG.getBitcast(RootVT, Res);
33537 }
33538 }
33539
33540 // Don't try to re-form single instruction chains under any circumstances now
33541 // that we've done encoding canonicalization for them.
33542 if (Depth < 1)
33543 return SDValue();
33544
33545 // Depth threshold above which we can efficiently use variable mask shuffles.
33546 int VariableShuffleDepth = Subtarget.hasFastVariableShuffle() ? 1 : 2;
33547 AllowVariableMask &= (Depth >= VariableShuffleDepth) || HasVariableMask;
33548
33549 bool MaskContainsZeros =
33550 any_of(Mask, [](int M) { return M == SM_SentinelZero; });
33551
33552 if (is128BitLaneCrossingShuffleMask(MaskVT, Mask)) {
33553 // If we have a single input lane-crossing shuffle then lower to VPERMV.
33554 if (UnaryShuffle && AllowVariableMask && !MaskContainsZeros &&
33555 ((Subtarget.hasAVX2() &&
33556 (MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||
33557 (Subtarget.hasAVX512() &&
33558 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
33559 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
33560 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) ||
33561 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) ||
33562 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) ||
33563 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) {
33564 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
33565 Res = DAG.getBitcast(MaskVT, V1);
33566 Res = DAG.getNode(X86ISD::VPERMV, DL, MaskVT, VPermMask, Res);
33567 return DAG.getBitcast(RootVT, Res);
33568 }
33569
33570 // Lower a unary+zero lane-crossing shuffle as VPERMV3 with a zero
33571 // vector as the second source.
33572 if (UnaryShuffle && AllowVariableMask &&
33573 ((Subtarget.hasAVX512() &&
33574 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
33575 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
33576 (Subtarget.hasVLX() &&
33577 (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
33578 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||
33579 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) ||
33580 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) ||
33581 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) ||
33582 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) {
33583 // Adjust shuffle mask - replace SM_SentinelZero with second source index.
33584 for (unsigned i = 0; i != NumMaskElts; ++i)
33585 if (Mask[i] == SM_SentinelZero)
33586 Mask[i] = NumMaskElts + i;
33587
33588 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
33589 Res = DAG.getBitcast(MaskVT, V1);
33590 SDValue Zero = getZeroVector(MaskVT, Subtarget, DAG, DL);
33591 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, Res, VPermMask, Zero);
33592 return DAG.getBitcast(RootVT, Res);
33593 }
33594
33595 // If that failed and either input is extracted then try to combine as a
33596 // shuffle with the larger type.
33597 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
33598 Inputs, Root, BaseMask, Depth, HasVariableMask, AllowVariableMask,
33599 DAG, Subtarget))
33600 return WideShuffle;
33601
33602 // If we have a dual input lane-crossing shuffle then lower to VPERMV3.
33603 if (AllowVariableMask && !MaskContainsZeros &&
33604 ((Subtarget.hasAVX512() &&
33605 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
33606 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
33607 (Subtarget.hasVLX() &&
33608 (MaskVT == MVT::v4f64 || MaskVT == MVT::v4i64 ||
33609 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||
33610 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) ||
33611 (Subtarget.hasBWI() && Subtarget.hasVLX() && MaskVT == MVT::v16i16) ||
33612 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) ||
33613 (Subtarget.hasVBMI() && Subtarget.hasVLX() && MaskVT == MVT::v32i8))) {
33614 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
33615 V1 = DAG.getBitcast(MaskVT, V1);
33616 V2 = DAG.getBitcast(MaskVT, V2);
33617 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2);
33618 return DAG.getBitcast(RootVT, Res);
33619 }
33620 return SDValue();
33621 }
33622
33623 // See if we can combine a single input shuffle with zeros to a bit-mask,
33624 // which is much simpler than any shuffle.
33625 if (UnaryShuffle && MaskContainsZeros && AllowVariableMask &&
33626 isSequentialOrUndefOrZeroInRange(Mask, 0, NumMaskElts, 0) &&
33627 DAG.getTargetLoweringInfo().isTypeLegal(MaskVT)) {
33628 APInt Zero = APInt::getNullValue(MaskEltSizeInBits);
33629 APInt AllOnes = APInt::getAllOnesValue(MaskEltSizeInBits);
33630 APInt UndefElts(NumMaskElts, 0);
33631 SmallVector<APInt, 64> EltBits(NumMaskElts, Zero);
33632 for (unsigned i = 0; i != NumMaskElts; ++i) {
33633 int M = Mask[i];
33634 if (M == SM_SentinelUndef) {
33635 UndefElts.setBit(i);
33636 continue;
33637 }
33638 if (M == SM_SentinelZero)
33639 continue;
33640 EltBits[i] = AllOnes;
33641 }
33642 SDValue BitMask = getConstVector(EltBits, UndefElts, MaskVT, DAG, DL);
33643 Res = DAG.getBitcast(MaskVT, V1);
33644 unsigned AndOpcode =
33645 FloatDomain ? unsigned(X86ISD::FAND) : unsigned(ISD::AND);
33646 Res = DAG.getNode(AndOpcode, DL, MaskVT, Res, BitMask);
33647 return DAG.getBitcast(RootVT, Res);
33648 }
33649
33650 // If we have a single input shuffle with different shuffle patterns in the
33651 // the 128-bit lanes use the variable mask to VPERMILPS.
33652 // TODO Combine other mask types at higher depths.
33653 if (UnaryShuffle && AllowVariableMask && !MaskContainsZeros &&
33654 ((MaskVT == MVT::v8f32 && Subtarget.hasAVX()) ||
33655 (MaskVT == MVT::v16f32 && Subtarget.hasAVX512()))) {
33656 SmallVector<SDValue, 16> VPermIdx;
33657 for (int M : Mask) {
33658 SDValue Idx =
33659 M < 0 ? DAG.getUNDEF(MVT::i32) : DAG.getConstant(M % 4, DL, MVT::i32);
33660 VPermIdx.push_back(Idx);
33661 }
33662 SDValue VPermMask = DAG.getBuildVector(IntMaskVT, DL, VPermIdx);
33663 Res = DAG.getBitcast(MaskVT, V1);
33664 Res = DAG.getNode(X86ISD::VPERMILPV, DL, MaskVT, Res, VPermMask);
33665 return DAG.getBitcast(RootVT, Res);
33666 }
33667
33668 // With XOP, binary shuffles of 128/256-bit floating point vectors can combine
33669 // to VPERMIL2PD/VPERMIL2PS.
33670 if (AllowVariableMask && Subtarget.hasXOP() &&
33671 (MaskVT == MVT::v2f64 || MaskVT == MVT::v4f64 || MaskVT == MVT::v4f32 ||
33672 MaskVT == MVT::v8f32)) {
33673 // VPERMIL2 Operation.
33674 // Bits[3] - Match Bit.
33675 // Bits[2:1] - (Per Lane) PD Shuffle Mask.
33676 // Bits[2:0] - (Per Lane) PS Shuffle Mask.
33677 unsigned NumLanes = MaskVT.getSizeInBits() / 128;
33678 unsigned NumEltsPerLane = NumMaskElts / NumLanes;
33679 SmallVector<int, 8> VPerm2Idx;
33680 unsigned M2ZImm = 0;
33681 for (int M : Mask) {
33682 if (M == SM_SentinelUndef) {
33683 VPerm2Idx.push_back(-1);
33684 continue;
33685 }
33686 if (M == SM_SentinelZero) {
33687 M2ZImm = 2;
33688 VPerm2Idx.push_back(8);
33689 continue;
33690 }
33691 int Index = (M % NumEltsPerLane) + ((M / NumMaskElts) * NumEltsPerLane);
33692 Index = (MaskVT.getScalarSizeInBits() == 64 ? Index << 1 : Index);
33693 VPerm2Idx.push_back(Index);
33694 }
33695 V1 = DAG.getBitcast(MaskVT, V1);
33696 V2 = DAG.getBitcast(MaskVT, V2);
33697 SDValue VPerm2MaskOp = getConstVector(VPerm2Idx, IntMaskVT, DAG, DL, true);
33698 Res = DAG.getNode(X86ISD::VPERMIL2, DL, MaskVT, V1, V2, VPerm2MaskOp,
33699 DAG.getTargetConstant(M2ZImm, DL, MVT::i8));
33700 return DAG.getBitcast(RootVT, Res);
33701 }
33702
33703 // If we have 3 or more shuffle instructions or a chain involving a variable
33704 // mask, we can replace them with a single PSHUFB instruction profitably.
33705 // Intel's manuals suggest only using PSHUFB if doing so replacing 5
33706 // instructions, but in practice PSHUFB tends to be *very* fast so we're
33707 // more aggressive.
33708 if (UnaryShuffle && AllowVariableMask &&
33709 ((RootVT.is128BitVector() && Subtarget.hasSSSE3()) ||
33710 (RootVT.is256BitVector() && Subtarget.hasAVX2()) ||
33711 (RootVT.is512BitVector() && Subtarget.hasBWI()))) {
33712 SmallVector<SDValue, 16> PSHUFBMask;
33713 int NumBytes = RootVT.getSizeInBits() / 8;
33714 int Ratio = NumBytes / NumMaskElts;
33715 for (int i = 0; i < NumBytes; ++i) {
33716 int M = Mask[i / Ratio];
33717 if (M == SM_SentinelUndef) {
33718 PSHUFBMask.push_back(DAG.getUNDEF(MVT::i8));
33719 continue;
33720 }
33721 if (M == SM_SentinelZero) {
33722 PSHUFBMask.push_back(DAG.getConstant(255, DL, MVT::i8));
33723 continue;
33724 }
33725 M = Ratio * M + i % Ratio;
33726 assert((M / 16) == (i / 16) && "Lane crossing detected")(((M / 16) == (i / 16) && "Lane crossing detected") ?
static_cast<void> (0) : __assert_fail ("(M / 16) == (i / 16) && \"Lane crossing detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33726, __PRETTY_FUNCTION__));
33727 PSHUFBMask.push_back(DAG.getConstant(M, DL, MVT::i8));
33728 }
33729 MVT ByteVT = MVT::getVectorVT(MVT::i8, NumBytes);
33730 Res = DAG.getBitcast(ByteVT, V1);
33731 SDValue PSHUFBMaskOp = DAG.getBuildVector(ByteVT, DL, PSHUFBMask);
33732 Res = DAG.getNode(X86ISD::PSHUFB, DL, ByteVT, Res, PSHUFBMaskOp);
33733 return DAG.getBitcast(RootVT, Res);
33734 }
33735
33736 // With XOP, if we have a 128-bit binary input shuffle we can always combine
33737 // to VPPERM. We match the depth requirement of PSHUFB - VPPERM is never
33738 // slower than PSHUFB on targets that support both.
33739 if (AllowVariableMask && RootVT.is128BitVector() && Subtarget.hasXOP()) {
33740 // VPPERM Mask Operation
33741 // Bits[4:0] - Byte Index (0 - 31)
33742 // Bits[7:5] - Permute Operation (0 - Source byte, 4 - ZERO)
33743 SmallVector<SDValue, 16> VPPERMMask;
33744 int NumBytes = 16;
33745 int Ratio = NumBytes / NumMaskElts;
33746 for (int i = 0; i < NumBytes; ++i) {
33747 int M = Mask[i / Ratio];
33748 if (M == SM_SentinelUndef) {
33749 VPPERMMask.push_back(DAG.getUNDEF(MVT::i8));
33750 continue;
33751 }
33752 if (M == SM_SentinelZero) {
33753 VPPERMMask.push_back(DAG.getConstant(128, DL, MVT::i8));
33754 continue;
33755 }
33756 M = Ratio * M + i % Ratio;
33757 VPPERMMask.push_back(DAG.getConstant(M, DL, MVT::i8));
33758 }
33759 MVT ByteVT = MVT::v16i8;
33760 V1 = DAG.getBitcast(ByteVT, V1);
33761 V2 = DAG.getBitcast(ByteVT, V2);
33762 SDValue VPPERMMaskOp = DAG.getBuildVector(ByteVT, DL, VPPERMMask);
33763 Res = DAG.getNode(X86ISD::VPPERM, DL, ByteVT, V1, V2, VPPERMMaskOp);
33764 return DAG.getBitcast(RootVT, Res);
33765 }
33766
33767 // If that failed and either input is extracted then try to combine as a
33768 // shuffle with the larger type.
33769 if (SDValue WideShuffle = combineX86ShuffleChainWithExtract(
33770 Inputs, Root, BaseMask, Depth, HasVariableMask, AllowVariableMask,
33771 DAG, Subtarget))
33772 return WideShuffle;
33773
33774 // If we have a dual input shuffle then lower to VPERMV3.
33775 if (!UnaryShuffle && AllowVariableMask && !MaskContainsZeros &&
33776 ((Subtarget.hasAVX512() &&
33777 (MaskVT == MVT::v8f64 || MaskVT == MVT::v8i64 ||
33778 MaskVT == MVT::v16f32 || MaskVT == MVT::v16i32)) ||
33779 (Subtarget.hasVLX() &&
33780 (MaskVT == MVT::v2f64 || MaskVT == MVT::v2i64 || MaskVT == MVT::v4f64 ||
33781 MaskVT == MVT::v4i64 || MaskVT == MVT::v4f32 || MaskVT == MVT::v4i32 ||
33782 MaskVT == MVT::v8f32 || MaskVT == MVT::v8i32)) ||
33783 (Subtarget.hasBWI() && MaskVT == MVT::v32i16) ||
33784 (Subtarget.hasBWI() && Subtarget.hasVLX() &&
33785 (MaskVT == MVT::v8i16 || MaskVT == MVT::v16i16)) ||
33786 (Subtarget.hasVBMI() && MaskVT == MVT::v64i8) ||
33787 (Subtarget.hasVBMI() && Subtarget.hasVLX() &&
33788 (MaskVT == MVT::v16i8 || MaskVT == MVT::v32i8)))) {
33789 SDValue VPermMask = getConstVector(Mask, IntMaskVT, DAG, DL, true);
33790 V1 = DAG.getBitcast(MaskVT, V1);
33791 V2 = DAG.getBitcast(MaskVT, V2);
33792 Res = DAG.getNode(X86ISD::VPERMV3, DL, MaskVT, V1, VPermMask, V2);
33793 return DAG.getBitcast(RootVT, Res);
33794 }
33795
33796 // Failed to find any combines.
33797 return SDValue();
33798}
33799
33800// Combine an arbitrary chain of shuffles + extract_subvectors into a single
33801// instruction if possible.
33802//
33803// Wrapper for combineX86ShuffleChain that extends the shuffle mask to a larger
33804// type size to attempt to combine:
33805// shuffle(extract_subvector(x,c1),extract_subvector(y,c2),m1)
33806// -->
33807// extract_subvector(shuffle(x,y,m2),0)
33808static SDValue combineX86ShuffleChainWithExtract(
33809 ArrayRef<SDValue> Inputs, SDValue Root, ArrayRef<int> BaseMask, int Depth,
33810 bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG,
33811 const X86Subtarget &Subtarget) {
33812 unsigned NumMaskElts = BaseMask.size();
33813 unsigned NumInputs = Inputs.size();
33814 if (NumInputs == 0)
33815 return SDValue();
33816
33817 SmallVector<SDValue, 4> WideInputs(Inputs.begin(), Inputs.end());
33818 SmallVector<unsigned, 4> Offsets(NumInputs, 0);
33819
33820 // Peek through subvectors.
33821 // TODO: Support inter-mixed EXTRACT_SUBVECTORs + BITCASTs?
33822 unsigned WideSizeInBits = WideInputs[0].getValueSizeInBits();
33823 for (unsigned i = 0; i != NumInputs; ++i) {
33824 SDValue &Src = WideInputs[i];
33825 unsigned &Offset = Offsets[i];
33826 Src = peekThroughBitcasts(Src);
33827 EVT BaseVT = Src.getValueType();
33828 while (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
33829 isa<ConstantSDNode>(Src.getOperand(1))) {
33830 Offset += Src.getConstantOperandVal(1);
33831 Src = Src.getOperand(0);
33832 }
33833 WideSizeInBits = std::max(WideSizeInBits,
33834 (unsigned)Src.getValueSizeInBits());
33835 assert((Offset % BaseVT.getVectorNumElements()) == 0 &&(((Offset % BaseVT.getVectorNumElements()) == 0 && "Unexpected subvector extraction"
) ? static_cast<void> (0) : __assert_fail ("(Offset % BaseVT.getVectorNumElements()) == 0 && \"Unexpected subvector extraction\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33836, __PRETTY_FUNCTION__))
33836 "Unexpected subvector extraction")(((Offset % BaseVT.getVectorNumElements()) == 0 && "Unexpected subvector extraction"
) ? static_cast<void> (0) : __assert_fail ("(Offset % BaseVT.getVectorNumElements()) == 0 && \"Unexpected subvector extraction\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33836, __PRETTY_FUNCTION__));
33837 Offset /= BaseVT.getVectorNumElements();
33838 Offset *= NumMaskElts;
33839 }
33840
33841 // Bail if we're always extracting from the lowest subvectors,
33842 // combineX86ShuffleChain should match this for the current width.
33843 if (llvm::all_of(Offsets, [](unsigned Offset) { return Offset == 0; }))
33844 return SDValue();
33845
33846 EVT RootVT = Root.getValueType();
33847 unsigned RootSizeInBits = RootVT.getSizeInBits();
33848 unsigned Scale = WideSizeInBits / RootSizeInBits;
33849 assert((WideSizeInBits % RootSizeInBits) == 0 &&(((WideSizeInBits % RootSizeInBits) == 0 && "Unexpected subvector extraction"
) ? static_cast<void> (0) : __assert_fail ("(WideSizeInBits % RootSizeInBits) == 0 && \"Unexpected subvector extraction\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33850, __PRETTY_FUNCTION__))
33850 "Unexpected subvector extraction")(((WideSizeInBits % RootSizeInBits) == 0 && "Unexpected subvector extraction"
) ? static_cast<void> (0) : __assert_fail ("(WideSizeInBits % RootSizeInBits) == 0 && \"Unexpected subvector extraction\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33850, __PRETTY_FUNCTION__));
33851
33852 // If the src vector types aren't the same, see if we can extend
33853 // them to match each other.
33854 // TODO: Support different scalar types?
33855 EVT WideSVT = WideInputs[0].getValueType().getScalarType();
33856 if (llvm::any_of(WideInputs, [&WideSVT, &DAG](SDValue Op) {
33857 return !DAG.getTargetLoweringInfo().isTypeLegal(Op.getValueType()) ||
33858 Op.getValueType().getScalarType() != WideSVT;
33859 }))
33860 return SDValue();
33861
33862 for (SDValue &NewInput : WideInputs) {
33863 assert((WideSizeInBits % NewInput.getValueSizeInBits()) == 0 &&(((WideSizeInBits % NewInput.getValueSizeInBits()) == 0 &&
"Shuffle vector size mismatch") ? static_cast<void> (0
) : __assert_fail ("(WideSizeInBits % NewInput.getValueSizeInBits()) == 0 && \"Shuffle vector size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33864, __PRETTY_FUNCTION__))
33864 "Shuffle vector size mismatch")(((WideSizeInBits % NewInput.getValueSizeInBits()) == 0 &&
"Shuffle vector size mismatch") ? static_cast<void> (0
) : __assert_fail ("(WideSizeInBits % NewInput.getValueSizeInBits()) == 0 && \"Shuffle vector size mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33864, __PRETTY_FUNCTION__));
33865 if (WideSizeInBits > NewInput.getValueSizeInBits())
33866 NewInput = widenSubVector(NewInput, false, Subtarget, DAG,
33867 SDLoc(NewInput), WideSizeInBits);
33868 assert(WideSizeInBits == NewInput.getValueSizeInBits() &&((WideSizeInBits == NewInput.getValueSizeInBits() && "Unexpected subvector extraction"
) ? static_cast<void> (0) : __assert_fail ("WideSizeInBits == NewInput.getValueSizeInBits() && \"Unexpected subvector extraction\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33869, __PRETTY_FUNCTION__))
33869 "Unexpected subvector extraction")((WideSizeInBits == NewInput.getValueSizeInBits() && "Unexpected subvector extraction"
) ? static_cast<void> (0) : __assert_fail ("WideSizeInBits == NewInput.getValueSizeInBits() && \"Unexpected subvector extraction\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33869, __PRETTY_FUNCTION__));
33870 }
33871
33872 // Create new mask for larger type.
33873 for (unsigned i = 1; i != NumInputs; ++i)
33874 Offsets[i] += i * Scale * NumMaskElts;
33875
33876 SmallVector<int, 64> WideMask(BaseMask.begin(), BaseMask.end());
33877 for (int &M : WideMask) {
33878 if (M < 0)
33879 continue;
33880 M = (M % NumMaskElts) + Offsets[M / NumMaskElts];
33881 }
33882 WideMask.append((Scale - 1) * NumMaskElts, SM_SentinelUndef);
33883
33884 // Remove unused/repeated shuffle source ops.
33885 resolveTargetShuffleInputsAndMask(WideInputs, WideMask);
33886 assert(!WideInputs.empty() && "Shuffle with no inputs detected")((!WideInputs.empty() && "Shuffle with no inputs detected"
) ? static_cast<void> (0) : __assert_fail ("!WideInputs.empty() && \"Shuffle with no inputs detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33886, __PRETTY_FUNCTION__));
33887
33888 if (WideInputs.size() > 2)
33889 return SDValue();
33890
33891 // Increase depth for every upper subvector we've peeked through.
33892 Depth += count_if(Offsets, [](unsigned Offset) { return Offset > 0; });
33893
33894 // Attempt to combine wider chain.
33895 // TODO: Can we use a better Root?
33896 SDValue WideRoot = WideInputs[0];
33897 if (SDValue WideShuffle = combineX86ShuffleChain(
33898 WideInputs, WideRoot, WideMask, Depth, HasVariableMask,
33899 AllowVariableMask, DAG, Subtarget)) {
33900 WideShuffle =
33901 extractSubVector(WideShuffle, 0, DAG, SDLoc(Root), RootSizeInBits);
33902 return DAG.getBitcast(RootVT, WideShuffle);
33903 }
33904 return SDValue();
33905}
33906
33907// Attempt to constant fold all of the constant source ops.
33908// Returns true if the entire shuffle is folded to a constant.
33909// TODO: Extend this to merge multiple constant Ops and update the mask.
33910static SDValue combineX86ShufflesConstants(ArrayRef<SDValue> Ops,
33911 ArrayRef<int> Mask, SDValue Root,
33912 bool HasVariableMask,
33913 SelectionDAG &DAG,
33914 const X86Subtarget &Subtarget) {
33915 MVT VT = Root.getSimpleValueType();
33916
33917 unsigned SizeInBits = VT.getSizeInBits();
33918 unsigned NumMaskElts = Mask.size();
33919 unsigned MaskSizeInBits = SizeInBits / NumMaskElts;
33920 unsigned NumOps = Ops.size();
33921
33922 // Extract constant bits from each source op.
33923 bool OneUseConstantOp = false;
33924 SmallVector<APInt, 16> UndefEltsOps(NumOps);
33925 SmallVector<SmallVector<APInt, 16>, 16> RawBitsOps(NumOps);
33926 for (unsigned i = 0; i != NumOps; ++i) {
33927 SDValue SrcOp = Ops[i];
33928 OneUseConstantOp |= SrcOp.hasOneUse();
33929 if (!getTargetConstantBitsFromNode(SrcOp, MaskSizeInBits, UndefEltsOps[i],
33930 RawBitsOps[i]))
33931 return SDValue();
33932 }
33933
33934 // Only fold if at least one of the constants is only used once or
33935 // the combined shuffle has included a variable mask shuffle, this
33936 // is to avoid constant pool bloat.
33937 if (!OneUseConstantOp && !HasVariableMask)
33938 return SDValue();
33939
33940 // Shuffle the constant bits according to the mask.
33941 APInt UndefElts(NumMaskElts, 0);
33942 APInt ZeroElts(NumMaskElts, 0);
33943 APInt ConstantElts(NumMaskElts, 0);
33944 SmallVector<APInt, 8> ConstantBitData(NumMaskElts,
33945 APInt::getNullValue(MaskSizeInBits));
33946 for (unsigned i = 0; i != NumMaskElts; ++i) {
33947 int M = Mask[i];
33948 if (M == SM_SentinelUndef) {
33949 UndefElts.setBit(i);
33950 continue;
33951 } else if (M == SM_SentinelZero) {
33952 ZeroElts.setBit(i);
33953 continue;
33954 }
33955 assert(0 <= M && M < (int)(NumMaskElts * NumOps))((0 <= M && M < (int)(NumMaskElts * NumOps)) ? static_cast
<void> (0) : __assert_fail ("0 <= M && M < (int)(NumMaskElts * NumOps)"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33955, __PRETTY_FUNCTION__));
33956
33957 unsigned SrcOpIdx = (unsigned)M / NumMaskElts;
33958 unsigned SrcMaskIdx = (unsigned)M % NumMaskElts;
33959
33960 auto &SrcUndefElts = UndefEltsOps[SrcOpIdx];
33961 if (SrcUndefElts[SrcMaskIdx]) {
33962 UndefElts.setBit(i);
33963 continue;
33964 }
33965
33966 auto &SrcEltBits = RawBitsOps[SrcOpIdx];
33967 APInt &Bits = SrcEltBits[SrcMaskIdx];
33968 if (!Bits) {
33969 ZeroElts.setBit(i);
33970 continue;
33971 }
33972
33973 ConstantElts.setBit(i);
33974 ConstantBitData[i] = Bits;
33975 }
33976 assert((UndefElts | ZeroElts | ConstantElts).isAllOnesValue())(((UndefElts | ZeroElts | ConstantElts).isAllOnesValue()) ? static_cast
<void> (0) : __assert_fail ("(UndefElts | ZeroElts | ConstantElts).isAllOnesValue()"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 33976, __PRETTY_FUNCTION__));
33977
33978 // Create the constant data.
33979 MVT MaskSVT;
33980 if (VT.isFloatingPoint() && (MaskSizeInBits == 32 || MaskSizeInBits == 64))
33981 MaskSVT = MVT::getFloatingPointVT(MaskSizeInBits);
33982 else
33983 MaskSVT = MVT::getIntegerVT(MaskSizeInBits);
33984
33985 MVT MaskVT = MVT::getVectorVT(MaskSVT, NumMaskElts);
33986
33987 SDLoc DL(Root);
33988 SDValue CstOp = getConstVector(ConstantBitData, UndefElts, MaskVT, DAG, DL);
33989 return DAG.getBitcast(VT, CstOp);
33990}
33991
33992/// Fully generic combining of x86 shuffle instructions.
33993///
33994/// This should be the last combine run over the x86 shuffle instructions. Once
33995/// they have been fully optimized, this will recursively consider all chains
33996/// of single-use shuffle instructions, build a generic model of the cumulative
33997/// shuffle operation, and check for simpler instructions which implement this
33998/// operation. We use this primarily for two purposes:
33999///
34000/// 1) Collapse generic shuffles to specialized single instructions when
34001/// equivalent. In most cases, this is just an encoding size win, but
34002/// sometimes we will collapse multiple generic shuffles into a single
34003/// special-purpose shuffle.
34004/// 2) Look for sequences of shuffle instructions with 3 or more total
34005/// instructions, and replace them with the slightly more expensive SSSE3
34006/// PSHUFB instruction if available. We do this as the last combining step
34007/// to ensure we avoid using PSHUFB if we can implement the shuffle with
34008/// a suitable short sequence of other instructions. The PSHUFB will either
34009/// use a register or have to read from memory and so is slightly (but only
34010/// slightly) more expensive than the other shuffle instructions.
34011///
34012/// Because this is inherently a quadratic operation (for each shuffle in
34013/// a chain, we recurse up the chain), the depth is limited to 8 instructions.
34014/// This should never be an issue in practice as the shuffle lowering doesn't
34015/// produce sequences of more than 8 instructions.
34016///
34017/// FIXME: We will currently miss some cases where the redundant shuffling
34018/// would simplify under the threshold for PSHUFB formation because of
34019/// combine-ordering. To fix this, we should do the redundant instruction
34020/// combining in this recursive walk.
34021static SDValue combineX86ShufflesRecursively(
34022 ArrayRef<SDValue> SrcOps, int SrcOpIndex, SDValue Root,
34023 ArrayRef<int> RootMask, ArrayRef<const SDNode *> SrcNodes, unsigned Depth,
34024 bool HasVariableMask, bool AllowVariableMask, SelectionDAG &DAG,
34025 const X86Subtarget &Subtarget) {
34026 assert(RootMask.size() > 0 &&((RootMask.size() > 0 && (RootMask.size() > 1 ||
(RootMask[0] == 0 && SrcOpIndex == 0)) && "Illegal shuffle root mask"
) ? static_cast<void> (0) : __assert_fail ("RootMask.size() > 0 && (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) && \"Illegal shuffle root mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34028, __PRETTY_FUNCTION__))
34027 (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) &&((RootMask.size() > 0 && (RootMask.size() > 1 ||
(RootMask[0] == 0 && SrcOpIndex == 0)) && "Illegal shuffle root mask"
) ? static_cast<void> (0) : __assert_fail ("RootMask.size() > 0 && (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) && \"Illegal shuffle root mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34028, __PRETTY_FUNCTION__))
34028 "Illegal shuffle root mask")((RootMask.size() > 0 && (RootMask.size() > 1 ||
(RootMask[0] == 0 && SrcOpIndex == 0)) && "Illegal shuffle root mask"
) ? static_cast<void> (0) : __assert_fail ("RootMask.size() > 0 && (RootMask.size() > 1 || (RootMask[0] == 0 && SrcOpIndex == 0)) && \"Illegal shuffle root mask\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34028, __PRETTY_FUNCTION__));
34029
34030 // Bound the depth of our recursive combine because this is ultimately
34031 // quadratic in nature.
34032 const unsigned MaxRecursionDepth = 8;
34033 if (Depth >= MaxRecursionDepth)
34034 return SDValue();
34035
34036 // Directly rip through bitcasts to find the underlying operand.
34037 SDValue Op = SrcOps[SrcOpIndex];
34038 Op = peekThroughOneUseBitcasts(Op);
34039
34040 MVT VT = Op.getSimpleValueType();
34041 if (!VT.isVector())
34042 return SDValue(); // Bail if we hit a non-vector.
34043
34044 assert(Root.getSimpleValueType().isVector() &&((Root.getSimpleValueType().isVector() && "Shuffles operate on vector types!"
) ? static_cast<void> (0) : __assert_fail ("Root.getSimpleValueType().isVector() && \"Shuffles operate on vector types!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34045, __PRETTY_FUNCTION__))
34045 "Shuffles operate on vector types!")((Root.getSimpleValueType().isVector() && "Shuffles operate on vector types!"
) ? static_cast<void> (0) : __assert_fail ("Root.getSimpleValueType().isVector() && \"Shuffles operate on vector types!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34045, __PRETTY_FUNCTION__));
34046 assert(VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits() &&((VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits
() && "Can only combine shuffles of the same vector register size."
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits() && \"Can only combine shuffles of the same vector register size.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34047, __PRETTY_FUNCTION__))
34047 "Can only combine shuffles of the same vector register size.")((VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits
() && "Can only combine shuffles of the same vector register size."
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() == Root.getSimpleValueType().getSizeInBits() && \"Can only combine shuffles of the same vector register size.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34047, __PRETTY_FUNCTION__));
34048
34049 // Extract target shuffle mask and resolve sentinels and inputs.
34050 // TODO - determine Op's demanded elts from RootMask.
34051 SmallVector<int, 64> OpMask;
34052 SmallVector<SDValue, 2> OpInputs;
34053 APInt OpUndef, OpZero;
34054 APInt OpDemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
34055 bool IsOpVariableMask = isTargetShuffleVariableMask(Op.getOpcode());
34056 if (!getTargetShuffleInputs(Op, OpDemandedElts, OpInputs, OpMask, OpUndef,
34057 OpZero, DAG, Depth, false))
34058 return SDValue();
34059
34060 SmallVector<int, 64> Mask;
34061 SmallVector<SDValue, 16> Ops;
34062
34063 // We don't need to merge masks if the root is empty.
34064 bool EmptyRoot = (Depth == 0) && (RootMask.size() == 1);
34065 if (EmptyRoot) {
34066 // Only resolve zeros if it will remove an input, otherwise we might end
34067 // up in an infinite loop.
34068 bool ResolveKnownZeros = true;
34069 if (!OpZero.isNullValue()) {
34070 APInt UsedInputs = APInt::getNullValue(OpInputs.size());
34071 for (int i = 0, e = OpMask.size(); i != e; ++i) {
34072 int M = OpMask[i];
34073 if (OpUndef[i] || OpZero[i] || isUndefOrZero(M))
34074 continue;
34075 UsedInputs.setBit(M / OpMask.size());
34076 if (UsedInputs.isAllOnesValue()) {
34077 ResolveKnownZeros = false;
34078 break;
34079 }
34080 }
34081 }
34082 resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero,
34083 ResolveKnownZeros);
34084
34085 Mask = OpMask;
34086 Ops.append(OpInputs.begin(), OpInputs.end());
34087 } else {
34088 resolveTargetShuffleFromZeroables(OpMask, OpUndef, OpZero);
34089
34090 // Add the inputs to the Ops list, avoiding duplicates.
34091 Ops.append(SrcOps.begin(), SrcOps.end());
34092
34093 auto AddOp = [&Ops](SDValue Input, int InsertionPoint) -> int {
34094 // Attempt to find an existing match.
34095 SDValue InputBC = peekThroughBitcasts(Input);
34096 for (int i = 0, e = Ops.size(); i < e; ++i)
34097 if (InputBC == peekThroughBitcasts(Ops[i]))
34098 return i;
34099 // Match failed - should we replace an existing Op?
34100 if (InsertionPoint >= 0) {
34101 Ops[InsertionPoint] = Input;
34102 return InsertionPoint;
34103 }
34104 // Add to the end of the Ops list.
34105 Ops.push_back(Input);
34106 return Ops.size() - 1;
34107 };
34108
34109 SmallVector<int, 2> OpInputIdx;
34110 for (SDValue OpInput : OpInputs)
34111 OpInputIdx.push_back(
34112 AddOp(OpInput, OpInputIdx.empty() ? SrcOpIndex : -1));
34113
34114 assert(((RootMask.size() > OpMask.size() &&((((RootMask.size() > OpMask.size() && RootMask.size
() % OpMask.size() == 0) || (OpMask.size() > RootMask.size
() && OpMask.size() % RootMask.size() == 0) || OpMask
.size() == RootMask.size()) && "The smaller number of elements must divide the larger."
) ? static_cast<void> (0) : __assert_fail ("((RootMask.size() > OpMask.size() && RootMask.size() % OpMask.size() == 0) || (OpMask.size() > RootMask.size() && OpMask.size() % RootMask.size() == 0) || OpMask.size() == RootMask.size()) && \"The smaller number of elements must divide the larger.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34119, __PRETTY_FUNCTION__))
34115 RootMask.size() % OpMask.size() == 0) ||((((RootMask.size() > OpMask.size() && RootMask.size
() % OpMask.size() == 0) || (OpMask.size() > RootMask.size
() && OpMask.size() % RootMask.size() == 0) || OpMask
.size() == RootMask.size()) && "The smaller number of elements must divide the larger."
) ? static_cast<void> (0) : __assert_fail ("((RootMask.size() > OpMask.size() && RootMask.size() % OpMask.size() == 0) || (OpMask.size() > RootMask.size() && OpMask.size() % RootMask.size() == 0) || OpMask.size() == RootMask.size()) && \"The smaller number of elements must divide the larger.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34119, __PRETTY_FUNCTION__))
34116 (OpMask.size() > RootMask.size() &&((((RootMask.size() > OpMask.size() && RootMask.size
() % OpMask.size() == 0) || (OpMask.size() > RootMask.size
() && OpMask.size() % RootMask.size() == 0) || OpMask
.size() == RootMask.size()) && "The smaller number of elements must divide the larger."
) ? static_cast<void> (0) : __assert_fail ("((RootMask.size() > OpMask.size() && RootMask.size() % OpMask.size() == 0) || (OpMask.size() > RootMask.size() && OpMask.size() % RootMask.size() == 0) || OpMask.size() == RootMask.size()) && \"The smaller number of elements must divide the larger.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34119, __PRETTY_FUNCTION__))
34117 OpMask.size() % RootMask.size() == 0) ||((((RootMask.size() > OpMask.size() && RootMask.size
() % OpMask.size() == 0) || (OpMask.size() > RootMask.size
() && OpMask.size() % RootMask.size() == 0) || OpMask
.size() == RootMask.size()) && "The smaller number of elements must divide the larger."
) ? static_cast<void> (0) : __assert_fail ("((RootMask.size() > OpMask.size() && RootMask.size() % OpMask.size() == 0) || (OpMask.size() > RootMask.size() && OpMask.size() % RootMask.size() == 0) || OpMask.size() == RootMask.size()) && \"The smaller number of elements must divide the larger.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34119, __PRETTY_FUNCTION__))
34118 OpMask.size() == RootMask.size()) &&((((RootMask.size() > OpMask.size() && RootMask.size
() % OpMask.size() == 0) || (OpMask.size() > RootMask.size
() && OpMask.size() % RootMask.size() == 0) || OpMask
.size() == RootMask.size()) && "The smaller number of elements must divide the larger."
) ? static_cast<void> (0) : __assert_fail ("((RootMask.size() > OpMask.size() && RootMask.size() % OpMask.size() == 0) || (OpMask.size() > RootMask.size() && OpMask.size() % RootMask.size() == 0) || OpMask.size() == RootMask.size()) && \"The smaller number of elements must divide the larger.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34119, __PRETTY_FUNCTION__))
34119 "The smaller number of elements must divide the larger.")((((RootMask.size() > OpMask.size() && RootMask.size
() % OpMask.size() == 0) || (OpMask.size() > RootMask.size
() && OpMask.size() % RootMask.size() == 0) || OpMask
.size() == RootMask.size()) && "The smaller number of elements must divide the larger."
) ? static_cast<void> (0) : __assert_fail ("((RootMask.size() > OpMask.size() && RootMask.size() % OpMask.size() == 0) || (OpMask.size() > RootMask.size() && OpMask.size() % RootMask.size() == 0) || OpMask.size() == RootMask.size()) && \"The smaller number of elements must divide the larger.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34119, __PRETTY_FUNCTION__));
34120
34121 // This function can be performance-critical, so we rely on the power-of-2
34122 // knowledge that we have about the mask sizes to replace div/rem ops with
34123 // bit-masks and shifts.
34124 assert(isPowerOf2_32(RootMask.size()) &&((isPowerOf2_32(RootMask.size()) && "Non-power-of-2 shuffle mask sizes"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(RootMask.size()) && \"Non-power-of-2 shuffle mask sizes\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34125, __PRETTY_FUNCTION__))
34125 "Non-power-of-2 shuffle mask sizes")((isPowerOf2_32(RootMask.size()) && "Non-power-of-2 shuffle mask sizes"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(RootMask.size()) && \"Non-power-of-2 shuffle mask sizes\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34125, __PRETTY_FUNCTION__));
34126 assert(isPowerOf2_32(OpMask.size()) && "Non-power-of-2 shuffle mask sizes")((isPowerOf2_32(OpMask.size()) && "Non-power-of-2 shuffle mask sizes"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(OpMask.size()) && \"Non-power-of-2 shuffle mask sizes\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34126, __PRETTY_FUNCTION__));
34127 unsigned RootMaskSizeLog2 = countTrailingZeros(RootMask.size());
34128 unsigned OpMaskSizeLog2 = countTrailingZeros(OpMask.size());
34129
34130 unsigned MaskWidth = std::max<unsigned>(OpMask.size(), RootMask.size());
34131 unsigned RootRatio =
34132 std::max<unsigned>(1, OpMask.size() >> RootMaskSizeLog2);
34133 unsigned OpRatio = std::max<unsigned>(1, RootMask.size() >> OpMaskSizeLog2);
34134 assert((RootRatio == 1 || OpRatio == 1) &&(((RootRatio == 1 || OpRatio == 1) && "Must not have a ratio for both incoming and op masks!"
) ? static_cast<void> (0) : __assert_fail ("(RootRatio == 1 || OpRatio == 1) && \"Must not have a ratio for both incoming and op masks!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34135, __PRETTY_FUNCTION__))
34135 "Must not have a ratio for both incoming and op masks!")(((RootRatio == 1 || OpRatio == 1) && "Must not have a ratio for both incoming and op masks!"
) ? static_cast<void> (0) : __assert_fail ("(RootRatio == 1 || OpRatio == 1) && \"Must not have a ratio for both incoming and op masks!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34135, __PRETTY_FUNCTION__));
34136
34137 assert(isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes")((isPowerOf2_32(MaskWidth) && "Non-power-of-2 shuffle mask sizes"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(MaskWidth) && \"Non-power-of-2 shuffle mask sizes\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34137, __PRETTY_FUNCTION__));
34138 assert(isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes")((isPowerOf2_32(RootRatio) && "Non-power-of-2 shuffle mask sizes"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(RootRatio) && \"Non-power-of-2 shuffle mask sizes\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34138, __PRETTY_FUNCTION__));
34139 assert(isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes")((isPowerOf2_32(OpRatio) && "Non-power-of-2 shuffle mask sizes"
) ? static_cast<void> (0) : __assert_fail ("isPowerOf2_32(OpRatio) && \"Non-power-of-2 shuffle mask sizes\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34139, __PRETTY_FUNCTION__));
34140 unsigned RootRatioLog2 = countTrailingZeros(RootRatio);
34141 unsigned OpRatioLog2 = countTrailingZeros(OpRatio);
34142
34143 Mask.resize(MaskWidth, SM_SentinelUndef);
34144
34145 // Merge this shuffle operation's mask into our accumulated mask. Note that
34146 // this shuffle's mask will be the first applied to the input, followed by
34147 // the root mask to get us all the way to the root value arrangement. The
34148 // reason for this order is that we are recursing up the operation chain.
34149 for (unsigned i = 0; i < MaskWidth; ++i) {
34150 unsigned RootIdx = i >> RootRatioLog2;
34151 if (RootMask[RootIdx] < 0) {
34152 // This is a zero or undef lane, we're done.
34153 Mask[i] = RootMask[RootIdx];
34154 continue;
34155 }
34156
34157 unsigned RootMaskedIdx =
34158 RootRatio == 1
34159 ? RootMask[RootIdx]
34160 : (RootMask[RootIdx] << RootRatioLog2) + (i & (RootRatio - 1));
34161
34162 // Just insert the scaled root mask value if it references an input other
34163 // than the SrcOp we're currently inserting.
34164 if ((RootMaskedIdx < (SrcOpIndex * MaskWidth)) ||
34165 (((SrcOpIndex + 1) * MaskWidth) <= RootMaskedIdx)) {
34166 Mask[i] = RootMaskedIdx;
34167 continue;
34168 }
34169
34170 RootMaskedIdx = RootMaskedIdx & (MaskWidth - 1);
34171 unsigned OpIdx = RootMaskedIdx >> OpRatioLog2;
34172 if (OpMask[OpIdx] < 0) {
34173 // The incoming lanes are zero or undef, it doesn't matter which ones we
34174 // are using.
34175 Mask[i] = OpMask[OpIdx];
34176 continue;
34177 }
34178
34179 // Ok, we have non-zero lanes, map them through to one of the Op's inputs.
34180 unsigned OpMaskedIdx = OpRatio == 1 ? OpMask[OpIdx]
34181 : (OpMask[OpIdx] << OpRatioLog2) +
34182 (RootMaskedIdx & (OpRatio - 1));
34183
34184 OpMaskedIdx = OpMaskedIdx & (MaskWidth - 1);
34185 int InputIdx = OpMask[OpIdx] / (int)OpMask.size();
34186 assert(0 <= OpInputIdx[InputIdx] && "Unknown target shuffle input")((0 <= OpInputIdx[InputIdx] && "Unknown target shuffle input"
) ? static_cast<void> (0) : __assert_fail ("0 <= OpInputIdx[InputIdx] && \"Unknown target shuffle input\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34186, __PRETTY_FUNCTION__));
34187 OpMaskedIdx += OpInputIdx[InputIdx] * MaskWidth;
34188
34189 Mask[i] = OpMaskedIdx;
34190 }
34191 }
34192
34193 // Remove unused/repeated shuffle source ops.
34194 resolveTargetShuffleInputsAndMask(Ops, Mask);
34195
34196 // Handle the all undef/zero cases early.
34197 if (all_of(Mask, [](int Idx) { return Idx == SM_SentinelUndef; }))
34198 return DAG.getUNDEF(Root.getValueType());
34199
34200 // TODO - should we handle the mixed zero/undef case as well? Just returning
34201 // a zero mask will lose information on undef elements possibly reducing
34202 // future combine possibilities.
34203 if (all_of(Mask, [](int Idx) { return Idx < 0; }))
34204 return getZeroVector(Root.getSimpleValueType(), Subtarget, DAG,
34205 SDLoc(Root));
34206
34207 assert(!Ops.empty() && "Shuffle with no inputs detected")((!Ops.empty() && "Shuffle with no inputs detected") ?
static_cast<void> (0) : __assert_fail ("!Ops.empty() && \"Shuffle with no inputs detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34207, __PRETTY_FUNCTION__));
34208 HasVariableMask |= IsOpVariableMask;
34209
34210 // Update the list of shuffle nodes that have been combined so far.
34211 SmallVector<const SDNode *, 16> CombinedNodes(SrcNodes.begin(),
34212 SrcNodes.end());
34213 CombinedNodes.push_back(Op.getNode());
34214
34215 // See if we can recurse into each shuffle source op (if it's a target
34216 // shuffle). The source op should only be generally combined if it either has
34217 // a single use (i.e. current Op) or all its users have already been combined,
34218 // if not then we can still combine but should prevent generation of variable
34219 // shuffles to avoid constant pool bloat.
34220 // Don't recurse if we already have more source ops than we can combine in
34221 // the remaining recursion depth.
34222 if (Ops.size() < (MaxRecursionDepth - Depth)) {
34223 for (int i = 0, e = Ops.size(); i < e; ++i) {
34224 // For empty roots, we need to resolve zeroable elements before combining
34225 // them with other shuffles.
34226 SmallVector<int, 64> ResolvedMask = Mask;
34227 if (EmptyRoot)
34228 resolveTargetShuffleFromZeroables(ResolvedMask, OpUndef, OpZero);
34229 bool AllowVar = false;
34230 if (Ops[i].getNode()->hasOneUse() ||
34231 SDNode::areOnlyUsersOf(CombinedNodes, Ops[i].getNode()))
34232 AllowVar = AllowVariableMask;
34233 if (SDValue Res = combineX86ShufflesRecursively(
34234 Ops, i, Root, ResolvedMask, CombinedNodes, Depth + 1,
34235 HasVariableMask, AllowVar, DAG, Subtarget))
34236 return Res;
34237 }
34238 }
34239
34240 // Attempt to constant fold all of the constant source ops.
34241 if (SDValue Cst = combineX86ShufflesConstants(
34242 Ops, Mask, Root, HasVariableMask, DAG, Subtarget))
34243 return Cst;
34244
34245 // We can only combine unary and binary shuffle mask cases.
34246 if (Ops.size() <= 2) {
34247 // Minor canonicalization of the accumulated shuffle mask to make it easier
34248 // to match below. All this does is detect masks with sequential pairs of
34249 // elements, and shrink them to the half-width mask. It does this in a loop
34250 // so it will reduce the size of the mask to the minimal width mask which
34251 // performs an equivalent shuffle.
34252 SmallVector<int, 64> WidenedMask;
34253 while (Mask.size() > 1 && canWidenShuffleElements(Mask, WidenedMask)) {
34254 Mask = std::move(WidenedMask);
34255 }
34256
34257 // Canonicalization of binary shuffle masks to improve pattern matching by
34258 // commuting the inputs.
34259 if (Ops.size() == 2 && canonicalizeShuffleMaskWithCommute(Mask)) {
34260 ShuffleVectorSDNode::commuteMask(Mask);
34261 std::swap(Ops[0], Ops[1]);
34262 }
34263
34264 // Finally, try to combine into a single shuffle instruction.
34265 return combineX86ShuffleChain(Ops, Root, Mask, Depth, HasVariableMask,
34266 AllowVariableMask, DAG, Subtarget);
34267 }
34268
34269 // If that failed and any input is extracted then try to combine as a
34270 // shuffle with the larger type.
34271 return combineX86ShuffleChainWithExtract(Ops, Root, Mask, Depth,
34272 HasVariableMask, AllowVariableMask,
34273 DAG, Subtarget);
34274}
34275
34276/// Helper entry wrapper to combineX86ShufflesRecursively.
34277static SDValue combineX86ShufflesRecursively(SDValue Op, SelectionDAG &DAG,
34278 const X86Subtarget &Subtarget) {
34279 return combineX86ShufflesRecursively({Op}, 0, Op, {0}, {}, /*Depth*/ 0,
34280 /*HasVarMask*/ false,
34281 /*AllowVarMask*/ true, DAG, Subtarget);
34282}
34283
34284/// Get the PSHUF-style mask from PSHUF node.
34285///
34286/// This is a very minor wrapper around getTargetShuffleMask to easy forming v4
34287/// PSHUF-style masks that can be reused with such instructions.
34288static SmallVector<int, 4> getPSHUFShuffleMask(SDValue N) {
34289 MVT VT = N.getSimpleValueType();
34290 SmallVector<int, 4> Mask;
34291 SmallVector<SDValue, 2> Ops;
34292 bool IsUnary;
34293 bool HaveMask =
34294 getTargetShuffleMask(N.getNode(), VT, false, Ops, Mask, IsUnary);
34295 (void)HaveMask;
34296 assert(HaveMask)((HaveMask) ? static_cast<void> (0) : __assert_fail ("HaveMask"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34296, __PRETTY_FUNCTION__));
34297
34298 // If we have more than 128-bits, only the low 128-bits of shuffle mask
34299 // matter. Check that the upper masks are repeats and remove them.
34300 if (VT.getSizeInBits() > 128) {
34301 int LaneElts = 128 / VT.getScalarSizeInBits();
34302#ifndef NDEBUG
34303 for (int i = 1, NumLanes = VT.getSizeInBits() / 128; i < NumLanes; ++i)
34304 for (int j = 0; j < LaneElts; ++j)
34305 assert(Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) &&((Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) &&
"Mask doesn't repeat in high 128-bit lanes!") ? static_cast<
void> (0) : __assert_fail ("Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) && \"Mask doesn't repeat in high 128-bit lanes!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34306, __PRETTY_FUNCTION__))
34306 "Mask doesn't repeat in high 128-bit lanes!")((Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) &&
"Mask doesn't repeat in high 128-bit lanes!") ? static_cast<
void> (0) : __assert_fail ("Mask[j] == Mask[i * LaneElts + j] - (LaneElts * i) && \"Mask doesn't repeat in high 128-bit lanes!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34306, __PRETTY_FUNCTION__));
34307#endif
34308 Mask.resize(LaneElts);
34309 }
34310
34311 switch (N.getOpcode()) {
34312 case X86ISD::PSHUFD:
34313 return Mask;
34314 case X86ISD::PSHUFLW:
34315 Mask.resize(4);
34316 return Mask;
34317 case X86ISD::PSHUFHW:
34318 Mask.erase(Mask.begin(), Mask.begin() + 4);
34319 for (int &M : Mask)
34320 M -= 4;
34321 return Mask;
34322 default:
34323 llvm_unreachable("No valid shuffle instruction found!")::llvm::llvm_unreachable_internal("No valid shuffle instruction found!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34323);
34324 }
34325}
34326
34327/// Search for a combinable shuffle across a chain ending in pshufd.
34328///
34329/// We walk up the chain and look for a combinable shuffle, skipping over
34330/// shuffles that we could hoist this shuffle's transformation past without
34331/// altering anything.
34332static SDValue
34333combineRedundantDWordShuffle(SDValue N, MutableArrayRef<int> Mask,
34334 SelectionDAG &DAG) {
34335 assert(N.getOpcode() == X86ISD::PSHUFD &&((N.getOpcode() == X86ISD::PSHUFD && "Called with something other than an x86 128-bit half shuffle!"
) ? static_cast<void> (0) : __assert_fail ("N.getOpcode() == X86ISD::PSHUFD && \"Called with something other than an x86 128-bit half shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34336, __PRETTY_FUNCTION__))
34336 "Called with something other than an x86 128-bit half shuffle!")((N.getOpcode() == X86ISD::PSHUFD && "Called with something other than an x86 128-bit half shuffle!"
) ? static_cast<void> (0) : __assert_fail ("N.getOpcode() == X86ISD::PSHUFD && \"Called with something other than an x86 128-bit half shuffle!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34336, __PRETTY_FUNCTION__));
34337 SDLoc DL(N);
34338
34339 // Walk up a single-use chain looking for a combinable shuffle. Keep a stack
34340 // of the shuffles in the chain so that we can form a fresh chain to replace
34341 // this one.
34342 SmallVector<SDValue, 8> Chain;
34343 SDValue V = N.getOperand(0);
34344 for (; V.hasOneUse(); V = V.getOperand(0)) {
34345 switch (V.getOpcode()) {
34346 default:
34347 return SDValue(); // Nothing combined!
34348
34349 case ISD::BITCAST:
34350 // Skip bitcasts as we always know the type for the target specific
34351 // instructions.
34352 continue;
34353
34354 case X86ISD::PSHUFD:
34355 // Found another dword shuffle.
34356 break;
34357
34358 case X86ISD::PSHUFLW:
34359 // Check that the low words (being shuffled) are the identity in the
34360 // dword shuffle, and the high words are self-contained.
34361 if (Mask[0] != 0 || Mask[1] != 1 ||
34362 !(Mask[2] >= 2 && Mask[2] < 4 && Mask[3] >= 2 && Mask[3] < 4))
34363 return SDValue();
34364
34365 Chain.push_back(V);
34366 continue;
34367
34368 case X86ISD::PSHUFHW:
34369 // Check that the high words (being shuffled) are the identity in the
34370 // dword shuffle, and the low words are self-contained.
34371 if (Mask[2] != 2 || Mask[3] != 3 ||
34372 !(Mask[0] >= 0 && Mask[0] < 2 && Mask[1] >= 0 && Mask[1] < 2))
34373 return SDValue();
34374
34375 Chain.push_back(V);
34376 continue;
34377
34378 case X86ISD::UNPCKL:
34379 case X86ISD::UNPCKH:
34380 // For either i8 -> i16 or i16 -> i32 unpacks, we can combine a dword
34381 // shuffle into a preceding word shuffle.
34382 if (V.getSimpleValueType().getVectorElementType() != MVT::i8 &&
34383 V.getSimpleValueType().getVectorElementType() != MVT::i16)
34384 return SDValue();
34385
34386 // Search for a half-shuffle which we can combine with.
34387 unsigned CombineOp =
34388 V.getOpcode() == X86ISD::UNPCKL ? X86ISD::PSHUFLW : X86ISD::PSHUFHW;
34389 if (V.getOperand(0) != V.getOperand(1) ||
34390 !V->isOnlyUserOf(V.getOperand(0).getNode()))
34391 return SDValue();
34392 Chain.push_back(V);
34393 V = V.getOperand(0);
34394 do {
34395 switch (V.getOpcode()) {
34396 default:
34397 return SDValue(); // Nothing to combine.
34398
34399 case X86ISD::PSHUFLW:
34400 case X86ISD::PSHUFHW:
34401 if (V.getOpcode() == CombineOp)
34402 break;
34403
34404 Chain.push_back(V);
34405
34406 LLVM_FALLTHROUGH[[gnu::fallthrough]];
34407 case ISD::BITCAST:
34408 V = V.getOperand(0);
34409 continue;
34410 }
34411 break;
34412 } while (V.hasOneUse());
34413 break;
34414 }
34415 // Break out of the loop if we break out of the switch.
34416 break;
34417 }
34418
34419 if (!V.hasOneUse())
34420 // We fell out of the loop without finding a viable combining instruction.
34421 return SDValue();
34422
34423 // Merge this node's mask and our incoming mask.
34424 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
34425 for (int &M : Mask)
34426 M = VMask[M];
34427 V = DAG.getNode(V.getOpcode(), DL, V.getValueType(), V.getOperand(0),
34428 getV4X86ShuffleImm8ForMask(Mask, DL, DAG));
34429
34430 // Rebuild the chain around this new shuffle.
34431 while (!Chain.empty()) {
34432 SDValue W = Chain.pop_back_val();
34433
34434 if (V.getValueType() != W.getOperand(0).getValueType())
34435 V = DAG.getBitcast(W.getOperand(0).getValueType(), V);
34436
34437 switch (W.getOpcode()) {
34438 default:
34439 llvm_unreachable("Only PSHUF and UNPCK instructions get here!")::llvm::llvm_unreachable_internal("Only PSHUF and UNPCK instructions get here!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34439);
34440
34441 case X86ISD::UNPCKL:
34442 case X86ISD::UNPCKH:
34443 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, V);
34444 break;
34445
34446 case X86ISD::PSHUFD:
34447 case X86ISD::PSHUFLW:
34448 case X86ISD::PSHUFHW:
34449 V = DAG.getNode(W.getOpcode(), DL, W.getValueType(), V, W.getOperand(1));
34450 break;
34451 }
34452 }
34453 if (V.getValueType() != N.getValueType())
34454 V = DAG.getBitcast(N.getValueType(), V);
34455
34456 // Return the new chain to replace N.
34457 return V;
34458}
34459
34460/// Try to combine x86 target specific shuffles.
34461static SDValue combineTargetShuffle(SDValue N, SelectionDAG &DAG,
34462 TargetLowering::DAGCombinerInfo &DCI,
34463 const X86Subtarget &Subtarget) {
34464 SDLoc DL(N);
34465 MVT VT = N.getSimpleValueType();
34466 SmallVector<int, 4> Mask;
34467 unsigned Opcode = N.getOpcode();
34468
34469 // Combine binary shuffle of 2 similar 'Horizontal' instructions into a
34470 // single instruction.
34471 if (VT.getScalarSizeInBits() == 64 &&
34472 (Opcode == X86ISD::MOVSD || Opcode == X86ISD::UNPCKH ||
34473 Opcode == X86ISD::UNPCKL)) {
34474 auto BC0 = peekThroughBitcasts(N.getOperand(0));
34475 auto BC1 = peekThroughBitcasts(N.getOperand(1));
34476 EVT VT0 = BC0.getValueType();
34477 EVT VT1 = BC1.getValueType();
34478 unsigned Opcode0 = BC0.getOpcode();
34479 unsigned Opcode1 = BC1.getOpcode();
34480 if (Opcode0 == Opcode1 && VT0 == VT1 &&
34481 (Opcode0 == X86ISD::FHADD || Opcode0 == X86ISD::HADD ||
34482 Opcode0 == X86ISD::FHSUB || Opcode0 == X86ISD::HSUB ||
34483 Opcode0 == X86ISD::PACKSS || Opcode0 == X86ISD::PACKUS)) {
34484 SDValue Lo, Hi;
34485 if (Opcode == X86ISD::MOVSD) {
34486 Lo = BC1.getOperand(0);
34487 Hi = BC0.getOperand(1);
34488 } else {
34489 Lo = BC0.getOperand(Opcode == X86ISD::UNPCKH ? 1 : 0);
34490 Hi = BC1.getOperand(Opcode == X86ISD::UNPCKH ? 1 : 0);
34491 }
34492 SDValue Horiz = DAG.getNode(Opcode0, DL, VT0, Lo, Hi);
34493 return DAG.getBitcast(VT, Horiz);
34494 }
34495 }
34496
34497 switch (Opcode) {
34498 case X86ISD::VBROADCAST: {
34499 SDValue Src = N.getOperand(0);
34500 SDValue BC = peekThroughBitcasts(Src);
34501 EVT SrcVT = Src.getValueType();
34502 EVT BCVT = BC.getValueType();
34503
34504 // If broadcasting from another shuffle, attempt to simplify it.
34505 // TODO - we really need a general SimplifyDemandedVectorElts mechanism.
34506 if (isTargetShuffle(BC.getOpcode()) &&
34507 VT.getScalarSizeInBits() % BCVT.getScalarSizeInBits() == 0) {
34508 unsigned Scale = VT.getScalarSizeInBits() / BCVT.getScalarSizeInBits();
34509 SmallVector<int, 16> DemandedMask(BCVT.getVectorNumElements(),
34510 SM_SentinelUndef);
34511 for (unsigned i = 0; i != Scale; ++i)
34512 DemandedMask[i] = i;
34513 if (SDValue Res = combineX86ShufflesRecursively(
34514 {BC}, 0, BC, DemandedMask, {}, /*Depth*/ 0,
34515 /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget))
34516 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
34517 DAG.getBitcast(SrcVT, Res));
34518 }
34519
34520 // broadcast(bitcast(src)) -> bitcast(broadcast(src))
34521 // 32-bit targets have to bitcast i64 to f64, so better to bitcast upward.
34522 if (Src.getOpcode() == ISD::BITCAST &&
34523 SrcVT.getScalarSizeInBits() == BCVT.getScalarSizeInBits()) {
34524 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), BCVT.getScalarType(),
34525 VT.getVectorNumElements());
34526 return DAG.getBitcast(VT, DAG.getNode(X86ISD::VBROADCAST, DL, NewVT, BC));
34527 }
34528
34529 // Reduce broadcast source vector to lowest 128-bits.
34530 if (SrcVT.getSizeInBits() > 128)
34531 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
34532 extract128BitVector(Src, 0, DAG, DL));
34533
34534 // broadcast(scalar_to_vector(x)) -> broadcast(x).
34535 if (Src.getOpcode() == ISD::SCALAR_TO_VECTOR)
34536 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Src.getOperand(0));
34537
34538 // Share broadcast with the longest vector and extract low subvector (free).
34539 for (SDNode *User : Src->uses())
34540 if (User != N.getNode() && User->getOpcode() == X86ISD::VBROADCAST &&
34541 User->getValueSizeInBits(0) > VT.getSizeInBits()) {
34542 return extractSubVector(SDValue(User, 0), 0, DAG, DL,
34543 VT.getSizeInBits());
34544 }
34545
34546 // vbroadcast(scalarload X) -> vbroadcast_load X
34547 // For float loads, extract other uses of the scalar from the broadcast.
34548 if (!SrcVT.isVector() && (Src.hasOneUse() || VT.isFloatingPoint()) &&
34549 ISD::isNormalLoad(Src.getNode())) {
34550 LoadSDNode *LN = cast<LoadSDNode>(Src);
34551 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
34552 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
34553 SDValue BcastLd =
34554 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, DL, Tys, Ops,
34555 LN->getMemoryVT(), LN->getMemOperand());
34556 // If the load value is used only by N, replace it via CombineTo N.
34557 bool NoReplaceExtract = Src.hasOneUse();
34558 DCI.CombineTo(N.getNode(), BcastLd);
34559 if (NoReplaceExtract) {
34560 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BcastLd.getValue(1));
34561 DCI.recursivelyDeleteUnusedNodes(LN);
34562 } else {
34563 SDValue Scl = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SrcVT, BcastLd,
34564 DAG.getIntPtrConstant(0, DL));
34565 DCI.CombineTo(LN, Scl, BcastLd.getValue(1));
34566 }
34567 return N; // Return N so it doesn't get rechecked!
34568 }
34569
34570 return SDValue();
34571 }
34572 case X86ISD::BLENDI: {
34573 SDValue N0 = N.getOperand(0);
34574 SDValue N1 = N.getOperand(1);
34575
34576 // blend(bitcast(x),bitcast(y)) -> bitcast(blend(x,y)) to narrower types.
34577 // TODO: Handle MVT::v16i16 repeated blend mask.
34578 if (N0.getOpcode() == ISD::BITCAST && N1.getOpcode() == ISD::BITCAST &&
34579 N0.getOperand(0).getValueType() == N1.getOperand(0).getValueType()) {
34580 MVT SrcVT = N0.getOperand(0).getSimpleValueType();
34581 if ((VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits()) == 0 &&
34582 SrcVT.getScalarSizeInBits() >= 32) {
34583 unsigned BlendMask = N.getConstantOperandVal(2);
34584 unsigned Size = VT.getVectorNumElements();
34585 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
34586 BlendMask = scaleVectorShuffleBlendMask(BlendMask, Size, Scale);
34587 return DAG.getBitcast(
34588 VT, DAG.getNode(X86ISD::BLENDI, DL, SrcVT, N0.getOperand(0),
34589 N1.getOperand(0),
34590 DAG.getTargetConstant(BlendMask, DL, MVT::i8)));
34591 }
34592 }
34593 return SDValue();
34594 }
34595 case X86ISD::VPERMI: {
34596 // vpermi(bitcast(x)) -> bitcast(vpermi(x)) for same number of elements.
34597 // TODO: Remove when we have preferred domains in combineX86ShuffleChain.
34598 SDValue N0 = N.getOperand(0);
34599 SDValue N1 = N.getOperand(1);
34600 unsigned EltSizeInBits = VT.getScalarSizeInBits();
34601 if (N0.getOpcode() == ISD::BITCAST &&
34602 N0.getOperand(0).getScalarValueSizeInBits() == EltSizeInBits) {
34603 SDValue Src = N0.getOperand(0);
34604 EVT SrcVT = Src.getValueType();
34605 SDValue Res = DAG.getNode(X86ISD::VPERMI, DL, SrcVT, Src, N1);
34606 return DAG.getBitcast(VT, Res);
34607 }
34608 return SDValue();
34609 }
34610 case X86ISD::PSHUFD:
34611 case X86ISD::PSHUFLW:
34612 case X86ISD::PSHUFHW:
34613 Mask = getPSHUFShuffleMask(N);
34614 assert(Mask.size() == 4)((Mask.size() == 4) ? static_cast<void> (0) : __assert_fail
("Mask.size() == 4", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34614, __PRETTY_FUNCTION__));
34615 break;
34616 case X86ISD::MOVSD:
34617 case X86ISD::MOVSS: {
34618 SDValue N0 = N.getOperand(0);
34619 SDValue N1 = N.getOperand(1);
34620
34621 // Canonicalize scalar FPOps:
34622 // MOVS*(N0, OP(N0, N1)) --> MOVS*(N0, SCALAR_TO_VECTOR(OP(N0[0], N1[0])))
34623 // If commutable, allow OP(N1[0], N0[0]).
34624 unsigned Opcode1 = N1.getOpcode();
34625 if (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL || Opcode1 == ISD::FSUB ||
34626 Opcode1 == ISD::FDIV) {
34627 SDValue N10 = N1.getOperand(0);
34628 SDValue N11 = N1.getOperand(1);
34629 if (N10 == N0 ||
34630 (N11 == N0 && (Opcode1 == ISD::FADD || Opcode1 == ISD::FMUL))) {
34631 if (N10 != N0)
34632 std::swap(N10, N11);
34633 MVT SVT = VT.getVectorElementType();
34634 SDValue ZeroIdx = DAG.getIntPtrConstant(0, DL);
34635 N10 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N10, ZeroIdx);
34636 N11 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, SVT, N11, ZeroIdx);
34637 SDValue Scl = DAG.getNode(Opcode1, DL, SVT, N10, N11);
34638 SDValue SclVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
34639 return DAG.getNode(Opcode, DL, VT, N0, SclVec);
34640 }
34641 }
34642
34643 return SDValue();
34644 }
34645 case X86ISD::INSERTPS: {
34646 assert(VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32")((VT == MVT::v4f32 && "INSERTPS ValueType must be MVT::v4f32"
) ? static_cast<void> (0) : __assert_fail ("VT == MVT::v4f32 && \"INSERTPS ValueType must be MVT::v4f32\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34646, __PRETTY_FUNCTION__));
34647 SDValue Op0 = N.getOperand(0);
34648 SDValue Op1 = N.getOperand(1);
34649 SDValue Op2 = N.getOperand(2);
34650 unsigned InsertPSMask = cast<ConstantSDNode>(Op2)->getZExtValue();
34651 unsigned SrcIdx = (InsertPSMask >> 6) & 0x3;
34652 unsigned DstIdx = (InsertPSMask >> 4) & 0x3;
34653 unsigned ZeroMask = InsertPSMask & 0xF;
34654
34655 // If we zero out all elements from Op0 then we don't need to reference it.
34656 if (((ZeroMask | (1u << DstIdx)) == 0xF) && !Op0.isUndef())
34657 return DAG.getNode(X86ISD::INSERTPS, DL, VT, DAG.getUNDEF(VT), Op1,
34658 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
34659
34660 // If we zero out the element from Op1 then we don't need to reference it.
34661 if ((ZeroMask & (1u << DstIdx)) && !Op1.isUndef())
34662 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
34663 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
34664
34665 // Attempt to merge insertps Op1 with an inner target shuffle node.
34666 SmallVector<int, 8> TargetMask1;
34667 SmallVector<SDValue, 2> Ops1;
34668 APInt KnownUndef1, KnownZero1;
34669 if (getTargetShuffleAndZeroables(Op1, TargetMask1, Ops1, KnownUndef1,
34670 KnownZero1)) {
34671 if (KnownUndef1[SrcIdx] || KnownZero1[SrcIdx]) {
34672 // Zero/UNDEF insertion - zero out element and remove dependency.
34673 InsertPSMask |= (1u << DstIdx);
34674 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, DAG.getUNDEF(VT),
34675 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
34676 }
34677 // Update insertps mask srcidx and reference the source input directly.
34678 int M = TargetMask1[SrcIdx];
34679 assert(0 <= M && M < 8 && "Shuffle index out of range")((0 <= M && M < 8 && "Shuffle index out of range"
) ? static_cast<void> (0) : __assert_fail ("0 <= M && M < 8 && \"Shuffle index out of range\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34679, __PRETTY_FUNCTION__));
34680 InsertPSMask = (InsertPSMask & 0x3f) | ((M & 0x3) << 6);
34681 Op1 = Ops1[M < 4 ? 0 : 1];
34682 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
34683 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
34684 }
34685
34686 // Attempt to merge insertps Op0 with an inner target shuffle node.
34687 SmallVector<int, 8> TargetMask0;
34688 SmallVector<SDValue, 2> Ops0;
34689 APInt KnownUndef0, KnownZero0;
34690 if (getTargetShuffleAndZeroables(Op0, TargetMask0, Ops0, KnownUndef0,
34691 KnownZero0)) {
34692 bool Updated = false;
34693 bool UseInput00 = false;
34694 bool UseInput01 = false;
34695 for (int i = 0; i != 4; ++i) {
34696 if ((InsertPSMask & (1u << i)) || (i == (int)DstIdx)) {
34697 // No change if element is already zero or the inserted element.
34698 continue;
34699 } else if (KnownUndef0[i] || KnownZero0[i]) {
34700 // If the target mask is undef/zero then we must zero the element.
34701 InsertPSMask |= (1u << i);
34702 Updated = true;
34703 continue;
34704 }
34705
34706 // The input vector element must be inline.
34707 int M = TargetMask0[i];
34708 if (M != i && M != (i + 4))
34709 return SDValue();
34710
34711 // Determine which inputs of the target shuffle we're using.
34712 UseInput00 |= (0 <= M && M < 4);
34713 UseInput01 |= (4 <= M);
34714 }
34715
34716 // If we're not using both inputs of the target shuffle then use the
34717 // referenced input directly.
34718 if (UseInput00 && !UseInput01) {
34719 Updated = true;
34720 Op0 = Ops0[0];
34721 } else if (!UseInput00 && UseInput01) {
34722 Updated = true;
34723 Op0 = Ops0[1];
34724 }
34725
34726 if (Updated)
34727 return DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0, Op1,
34728 DAG.getTargetConstant(InsertPSMask, DL, MVT::i8));
34729 }
34730
34731 // If we're inserting an element from a vbroadcast load, fold the
34732 // load into the X86insertps instruction. We need to convert the scalar
34733 // load to a vector and clear the source lane of the INSERTPS control.
34734 if (Op1.getOpcode() == X86ISD::VBROADCAST_LOAD && Op1.hasOneUse()) {
34735 auto *MemIntr = cast<MemIntrinsicSDNode>(Op1);
34736 if (MemIntr->getMemoryVT().getScalarSizeInBits() == 32) {
34737 SDValue Load = DAG.getLoad(MVT::f32, DL, MemIntr->getChain(),
34738 MemIntr->getBasePtr(),
34739 MemIntr->getMemOperand());
34740 SDValue Insert = DAG.getNode(X86ISD::INSERTPS, DL, VT, Op0,
34741 DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT,
34742 Load),
34743 DAG.getTargetConstant(InsertPSMask & 0x3f, DL, MVT::i8));
34744 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
34745 return Insert;
34746 }
34747 }
34748
34749 return SDValue();
34750 }
34751 default:
34752 return SDValue();
34753 }
34754
34755 // Nuke no-op shuffles that show up after combining.
34756 if (isNoopShuffleMask(Mask))
34757 return N.getOperand(0);
34758
34759 // Look for simplifications involving one or two shuffle instructions.
34760 SDValue V = N.getOperand(0);
34761 switch (N.getOpcode()) {
34762 default:
34763 break;
34764 case X86ISD::PSHUFLW:
34765 case X86ISD::PSHUFHW:
34766 assert(VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!")((VT.getVectorElementType() == MVT::i16 && "Bad word shuffle type!"
) ? static_cast<void> (0) : __assert_fail ("VT.getVectorElementType() == MVT::i16 && \"Bad word shuffle type!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34766, __PRETTY_FUNCTION__));
34767
34768 // See if this reduces to a PSHUFD which is no more expensive and can
34769 // combine with more operations. Note that it has to at least flip the
34770 // dwords as otherwise it would have been removed as a no-op.
34771 if (makeArrayRef(Mask).equals({2, 3, 0, 1})) {
34772 int DMask[] = {0, 1, 2, 3};
34773 int DOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 2;
34774 DMask[DOffset + 0] = DOffset + 1;
34775 DMask[DOffset + 1] = DOffset + 0;
34776 MVT DVT = MVT::getVectorVT(MVT::i32, VT.getVectorNumElements() / 2);
34777 V = DAG.getBitcast(DVT, V);
34778 V = DAG.getNode(X86ISD::PSHUFD, DL, DVT, V,
34779 getV4X86ShuffleImm8ForMask(DMask, DL, DAG));
34780 return DAG.getBitcast(VT, V);
34781 }
34782
34783 // Look for shuffle patterns which can be implemented as a single unpack.
34784 // FIXME: This doesn't handle the location of the PSHUFD generically, and
34785 // only works when we have a PSHUFD followed by two half-shuffles.
34786 if (Mask[0] == Mask[1] && Mask[2] == Mask[3] &&
34787 (V.getOpcode() == X86ISD::PSHUFLW ||
34788 V.getOpcode() == X86ISD::PSHUFHW) &&
34789 V.getOpcode() != N.getOpcode() &&
34790 V.hasOneUse()) {
34791 SDValue D = peekThroughOneUseBitcasts(V.getOperand(0));
34792 if (D.getOpcode() == X86ISD::PSHUFD && D.hasOneUse()) {
34793 SmallVector<int, 4> VMask = getPSHUFShuffleMask(V);
34794 SmallVector<int, 4> DMask = getPSHUFShuffleMask(D);
34795 int NOffset = N.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
34796 int VOffset = V.getOpcode() == X86ISD::PSHUFLW ? 0 : 4;
34797 int WordMask[8];
34798 for (int i = 0; i < 4; ++i) {
34799 WordMask[i + NOffset] = Mask[i] + NOffset;
34800 WordMask[i + VOffset] = VMask[i] + VOffset;
34801 }
34802 // Map the word mask through the DWord mask.
34803 int MappedMask[8];
34804 for (int i = 0; i < 8; ++i)
34805 MappedMask[i] = 2 * DMask[WordMask[i] / 2] + WordMask[i] % 2;
34806 if (makeArrayRef(MappedMask).equals({0, 0, 1, 1, 2, 2, 3, 3}) ||
34807 makeArrayRef(MappedMask).equals({4, 4, 5, 5, 6, 6, 7, 7})) {
34808 // We can replace all three shuffles with an unpack.
34809 V = DAG.getBitcast(VT, D.getOperand(0));
34810 return DAG.getNode(MappedMask[0] == 0 ? X86ISD::UNPCKL
34811 : X86ISD::UNPCKH,
34812 DL, VT, V, V);
34813 }
34814 }
34815 }
34816
34817 break;
34818
34819 case X86ISD::PSHUFD:
34820 if (SDValue NewN = combineRedundantDWordShuffle(N, Mask, DAG))
34821 return NewN;
34822
34823 break;
34824 }
34825
34826 return SDValue();
34827}
34828
34829/// Checks if the shuffle mask takes subsequent elements
34830/// alternately from two vectors.
34831/// For example <0, 5, 2, 7> or <8, 1, 10, 3, 12, 5, 14, 7> are both correct.
34832static bool isAddSubOrSubAddMask(ArrayRef<int> Mask, bool &Op0Even) {
34833
34834 int ParitySrc[2] = {-1, -1};
34835 unsigned Size = Mask.size();
34836 for (unsigned i = 0; i != Size; ++i) {
34837 int M = Mask[i];
34838 if (M < 0)
34839 continue;
34840
34841 // Make sure we are using the matching element from the input.
34842 if ((M % Size) != i)
34843 return false;
34844
34845 // Make sure we use the same input for all elements of the same parity.
34846 int Src = M / Size;
34847 if (ParitySrc[i % 2] >= 0 && ParitySrc[i % 2] != Src)
34848 return false;
34849 ParitySrc[i % 2] = Src;
34850 }
34851
34852 // Make sure each input is used.
34853 if (ParitySrc[0] < 0 || ParitySrc[1] < 0 || ParitySrc[0] == ParitySrc[1])
34854 return false;
34855
34856 Op0Even = ParitySrc[0] == 0;
34857 return true;
34858}
34859
34860/// Returns true iff the shuffle node \p N can be replaced with ADDSUB(SUBADD)
34861/// operation. If true is returned then the operands of ADDSUB(SUBADD) operation
34862/// are written to the parameters \p Opnd0 and \p Opnd1.
34863///
34864/// We combine shuffle to ADDSUB(SUBADD) directly on the abstract vector shuffle nodes
34865/// so it is easier to generically match. We also insert dummy vector shuffle
34866/// nodes for the operands which explicitly discard the lanes which are unused
34867/// by this operation to try to flow through the rest of the combiner
34868/// the fact that they're unused.
34869static bool isAddSubOrSubAdd(SDNode *N, const X86Subtarget &Subtarget,
34870 SelectionDAG &DAG, SDValue &Opnd0, SDValue &Opnd1,
34871 bool &IsSubAdd) {
34872
34873 EVT VT = N->getValueType(0);
34874 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
34875 if (!Subtarget.hasSSE3() || !TLI.isTypeLegal(VT) ||
34876 !VT.getSimpleVT().isFloatingPoint())
34877 return false;
34878
34879 // We only handle target-independent shuffles.
34880 // FIXME: It would be easy and harmless to use the target shuffle mask
34881 // extraction tool to support more.
34882 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
34883 return false;
34884
34885 SDValue V1 = N->getOperand(0);
34886 SDValue V2 = N->getOperand(1);
34887
34888 // Make sure we have an FADD and an FSUB.
34889 if ((V1.getOpcode() != ISD::FADD && V1.getOpcode() != ISD::FSUB) ||
34890 (V2.getOpcode() != ISD::FADD && V2.getOpcode() != ISD::FSUB) ||
34891 V1.getOpcode() == V2.getOpcode())
34892 return false;
34893
34894 // If there are other uses of these operations we can't fold them.
34895 if (!V1->hasOneUse() || !V2->hasOneUse())
34896 return false;
34897
34898 // Ensure that both operations have the same operands. Note that we can
34899 // commute the FADD operands.
34900 SDValue LHS, RHS;
34901 if (V1.getOpcode() == ISD::FSUB) {
34902 LHS = V1->getOperand(0); RHS = V1->getOperand(1);
34903 if ((V2->getOperand(0) != LHS || V2->getOperand(1) != RHS) &&
34904 (V2->getOperand(0) != RHS || V2->getOperand(1) != LHS))
34905 return false;
34906 } else {
34907 assert(V2.getOpcode() == ISD::FSUB && "Unexpected opcode")((V2.getOpcode() == ISD::FSUB && "Unexpected opcode")
? static_cast<void> (0) : __assert_fail ("V2.getOpcode() == ISD::FSUB && \"Unexpected opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 34907, __PRETTY_FUNCTION__));
34908 LHS = V2->getOperand(0); RHS = V2->getOperand(1);
34909 if ((V1->getOperand(0) != LHS || V1->getOperand(1) != RHS) &&
34910 (V1->getOperand(0) != RHS || V1->getOperand(1) != LHS))
34911 return false;
34912 }
34913
34914 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
34915 bool Op0Even;
34916 if (!isAddSubOrSubAddMask(Mask, Op0Even))
34917 return false;
34918
34919 // It's a subadd if the vector in the even parity is an FADD.
34920 IsSubAdd = Op0Even ? V1->getOpcode() == ISD::FADD
34921 : V2->getOpcode() == ISD::FADD;
34922
34923 Opnd0 = LHS;
34924 Opnd1 = RHS;
34925 return true;
34926}
34927
34928/// Combine shuffle of two fma nodes into FMAddSub or FMSubAdd.
34929static SDValue combineShuffleToFMAddSub(SDNode *N,
34930 const X86Subtarget &Subtarget,
34931 SelectionDAG &DAG) {
34932 // We only handle target-independent shuffles.
34933 // FIXME: It would be easy and harmless to use the target shuffle mask
34934 // extraction tool to support more.
34935 if (N->getOpcode() != ISD::VECTOR_SHUFFLE)
34936 return SDValue();
34937
34938 MVT VT = N->getSimpleValueType(0);
34939 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
34940 if (!Subtarget.hasAnyFMA() || !TLI.isTypeLegal(VT))
34941 return SDValue();
34942
34943 // We're trying to match (shuffle fma(a, b, c), X86Fmsub(a, b, c).
34944 SDValue Op0 = N->getOperand(0);
34945 SDValue Op1 = N->getOperand(1);
34946 SDValue FMAdd = Op0, FMSub = Op1;
34947 if (FMSub.getOpcode() != X86ISD::FMSUB)
34948 std::swap(FMAdd, FMSub);
34949
34950 if (FMAdd.getOpcode() != ISD::FMA || FMSub.getOpcode() != X86ISD::FMSUB ||
34951 FMAdd.getOperand(0) != FMSub.getOperand(0) || !FMAdd.hasOneUse() ||
34952 FMAdd.getOperand(1) != FMSub.getOperand(1) || !FMSub.hasOneUse() ||
34953 FMAdd.getOperand(2) != FMSub.getOperand(2))
34954 return SDValue();
34955
34956 // Check for correct shuffle mask.
34957 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
34958 bool Op0Even;
34959 if (!isAddSubOrSubAddMask(Mask, Op0Even))
34960 return SDValue();
34961
34962 // FMAddSub takes zeroth operand from FMSub node.
34963 SDLoc DL(N);
34964 bool IsSubAdd = Op0Even ? Op0 == FMAdd : Op1 == FMAdd;
34965 unsigned Opcode = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
34966 return DAG.getNode(Opcode, DL, VT, FMAdd.getOperand(0), FMAdd.getOperand(1),
34967 FMAdd.getOperand(2));
34968}
34969
34970/// Try to combine a shuffle into a target-specific add-sub or
34971/// mul-add-sub node.
34972static SDValue combineShuffleToAddSubOrFMAddSub(SDNode *N,
34973 const X86Subtarget &Subtarget,
34974 SelectionDAG &DAG) {
34975 if (SDValue V = combineShuffleToFMAddSub(N, Subtarget, DAG))
34976 return V;
34977
34978 SDValue Opnd0, Opnd1;
34979 bool IsSubAdd;
34980 if (!isAddSubOrSubAdd(N, Subtarget, DAG, Opnd0, Opnd1, IsSubAdd))
34981 return SDValue();
34982
34983 MVT VT = N->getSimpleValueType(0);
34984 SDLoc DL(N);
34985
34986 // Try to generate X86ISD::FMADDSUB node here.
34987 SDValue Opnd2;
34988 if (isFMAddSubOrFMSubAdd(Subtarget, DAG, Opnd0, Opnd1, Opnd2, 2)) {
34989 unsigned Opc = IsSubAdd ? X86ISD::FMSUBADD : X86ISD::FMADDSUB;
34990 return DAG.getNode(Opc, DL, VT, Opnd0, Opnd1, Opnd2);
34991 }
34992
34993 if (IsSubAdd)
34994 return SDValue();
34995
34996 // Do not generate X86ISD::ADDSUB node for 512-bit types even though
34997 // the ADDSUB idiom has been successfully recognized. There are no known
34998 // X86 targets with 512-bit ADDSUB instructions!
34999 if (VT.is512BitVector())
35000 return SDValue();
35001
35002 return DAG.getNode(X86ISD::ADDSUB, DL, VT, Opnd0, Opnd1);
35003}
35004
35005// We are looking for a shuffle where both sources are concatenated with undef
35006// and have a width that is half of the output's width. AVX2 has VPERMD/Q, so
35007// if we can express this as a single-source shuffle, that's preferable.
35008static SDValue combineShuffleOfConcatUndef(SDNode *N, SelectionDAG &DAG,
35009 const X86Subtarget &Subtarget) {
35010 if (!Subtarget.hasAVX2() || !isa<ShuffleVectorSDNode>(N))
35011 return SDValue();
35012
35013 EVT VT = N->getValueType(0);
35014
35015 // We only care about shuffles of 128/256-bit vectors of 32/64-bit values.
35016 if (!VT.is128BitVector() && !VT.is256BitVector())
35017 return SDValue();
35018
35019 if (VT.getVectorElementType() != MVT::i32 &&
35020 VT.getVectorElementType() != MVT::i64 &&
35021 VT.getVectorElementType() != MVT::f32 &&
35022 VT.getVectorElementType() != MVT::f64)
35023 return SDValue();
35024
35025 SDValue N0 = N->getOperand(0);
35026 SDValue N1 = N->getOperand(1);
35027
35028 // Check that both sources are concats with undef.
35029 if (N0.getOpcode() != ISD::CONCAT_VECTORS ||
35030 N1.getOpcode() != ISD::CONCAT_VECTORS || N0.getNumOperands() != 2 ||
35031 N1.getNumOperands() != 2 || !N0.getOperand(1).isUndef() ||
35032 !N1.getOperand(1).isUndef())
35033 return SDValue();
35034
35035 // Construct the new shuffle mask. Elements from the first source retain their
35036 // index, but elements from the second source no longer need to skip an undef.
35037 SmallVector<int, 8> Mask;
35038 int NumElts = VT.getVectorNumElements();
35039
35040 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(N);
35041 for (int Elt : SVOp->getMask())
35042 Mask.push_back(Elt < NumElts ? Elt : (Elt - NumElts / 2));
35043
35044 SDLoc DL(N);
35045 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, N0.getOperand(0),
35046 N1.getOperand(0));
35047 return DAG.getVectorShuffle(VT, DL, Concat, DAG.getUNDEF(VT), Mask);
35048}
35049
35050/// Eliminate a redundant shuffle of a horizontal math op.
35051static SDValue foldShuffleOfHorizOp(SDNode *N, SelectionDAG &DAG) {
35052 unsigned Opcode = N->getOpcode();
35053 if (Opcode != X86ISD::MOVDDUP && Opcode != X86ISD::VBROADCAST)
35054 if (Opcode != ISD::VECTOR_SHUFFLE || !N->getOperand(1).isUndef())
35055 return SDValue();
35056
35057 // For a broadcast, peek through an extract element of index 0 to find the
35058 // horizontal op: broadcast (ext_vec_elt HOp, 0)
35059 EVT VT = N->getValueType(0);
35060 if (Opcode == X86ISD::VBROADCAST) {
35061 SDValue SrcOp = N->getOperand(0);
35062 if (SrcOp.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
35063 SrcOp.getValueType() == MVT::f64 &&
35064 SrcOp.getOperand(0).getValueType() == VT &&
35065 isNullConstant(SrcOp.getOperand(1)))
35066 N = SrcOp.getNode();
35067 }
35068
35069 SDValue HOp = N->getOperand(0);
35070 if (HOp.getOpcode() != X86ISD::HADD && HOp.getOpcode() != X86ISD::FHADD &&
35071 HOp.getOpcode() != X86ISD::HSUB && HOp.getOpcode() != X86ISD::FHSUB)
35072 return SDValue();
35073
35074 // 128-bit horizontal math instructions are defined to operate on adjacent
35075 // lanes of each operand as:
35076 // v4X32: A[0] + A[1] , A[2] + A[3] , B[0] + B[1] , B[2] + B[3]
35077 // ...similarly for v2f64 and v8i16.
35078 if (!HOp.getOperand(0).isUndef() && !HOp.getOperand(1).isUndef() &&
35079 HOp.getOperand(0) != HOp.getOperand(1))
35080 return SDValue();
35081
35082 // The shuffle that we are eliminating may have allowed the horizontal op to
35083 // have an undemanded (undefined) operand. Duplicate the other (defined)
35084 // operand to ensure that the results are defined across all lanes without the
35085 // shuffle.
35086 auto updateHOp = [](SDValue HorizOp, SelectionDAG &DAG) {
35087 SDValue X;
35088 if (HorizOp.getOperand(0).isUndef()) {
35089 assert(!HorizOp.getOperand(1).isUndef() && "Not expecting foldable h-op")((!HorizOp.getOperand(1).isUndef() && "Not expecting foldable h-op"
) ? static_cast<void> (0) : __assert_fail ("!HorizOp.getOperand(1).isUndef() && \"Not expecting foldable h-op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35089, __PRETTY_FUNCTION__));
35090 X = HorizOp.getOperand(1);
35091 } else if (HorizOp.getOperand(1).isUndef()) {
35092 assert(!HorizOp.getOperand(0).isUndef() && "Not expecting foldable h-op")((!HorizOp.getOperand(0).isUndef() && "Not expecting foldable h-op"
) ? static_cast<void> (0) : __assert_fail ("!HorizOp.getOperand(0).isUndef() && \"Not expecting foldable h-op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35092, __PRETTY_FUNCTION__));
35093 X = HorizOp.getOperand(0);
35094 } else {
35095 return HorizOp;
35096 }
35097 return DAG.getNode(HorizOp.getOpcode(), SDLoc(HorizOp),
35098 HorizOp.getValueType(), X, X);
35099 };
35100
35101 // When the operands of a horizontal math op are identical, the low half of
35102 // the result is the same as the high half. If a target shuffle is also
35103 // replicating low and high halves (and without changing the type/length of
35104 // the vector), we don't need the shuffle.
35105 if (Opcode == X86ISD::MOVDDUP || Opcode == X86ISD::VBROADCAST) {
35106 if (HOp.getScalarValueSizeInBits() == 64 && HOp.getValueType() == VT) {
35107 // movddup (hadd X, X) --> hadd X, X
35108 // broadcast (extract_vec_elt (hadd X, X), 0) --> hadd X, X
35109 assert((HOp.getValueType() == MVT::v2f64 ||(((HOp.getValueType() == MVT::v2f64 || HOp.getValueType() == MVT
::v4f64) && "Unexpected type for h-op") ? static_cast
<void> (0) : __assert_fail ("(HOp.getValueType() == MVT::v2f64 || HOp.getValueType() == MVT::v4f64) && \"Unexpected type for h-op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35110, __PRETTY_FUNCTION__))
35110 HOp.getValueType() == MVT::v4f64) && "Unexpected type for h-op")(((HOp.getValueType() == MVT::v2f64 || HOp.getValueType() == MVT
::v4f64) && "Unexpected type for h-op") ? static_cast
<void> (0) : __assert_fail ("(HOp.getValueType() == MVT::v2f64 || HOp.getValueType() == MVT::v4f64) && \"Unexpected type for h-op\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35110, __PRETTY_FUNCTION__));
35111 return updateHOp(HOp, DAG);
35112 }
35113 return SDValue();
35114 }
35115
35116 // shuffle (hadd X, X), undef, [low half...high half] --> hadd X, X
35117 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(N)->getMask();
35118 // TODO: Other mask possibilities like {1,1} and {1,0} could be added here,
35119 // but this should be tied to whatever horizontal op matching and shuffle
35120 // canonicalization are producing.
35121 if (HOp.getValueSizeInBits() == 128 &&
35122 (isTargetShuffleEquivalent(Mask, {0, 0}) ||
35123 isTargetShuffleEquivalent(Mask, {0, 1, 0, 1}) ||
35124 isTargetShuffleEquivalent(Mask, {0, 1, 2, 3, 0, 1, 2, 3})))
35125 return updateHOp(HOp, DAG);
35126
35127 if (HOp.getValueSizeInBits() == 256 &&
35128 (isTargetShuffleEquivalent(Mask, {0, 0, 2, 2}) ||
35129 isTargetShuffleEquivalent(Mask, {0, 1, 0, 1, 4, 5, 4, 5}) ||
35130 isTargetShuffleEquivalent(
35131 Mask, {0, 1, 2, 3, 0, 1, 2, 3, 8, 9, 10, 11, 8, 9, 10, 11})))
35132 return updateHOp(HOp, DAG);
35133
35134 return SDValue();
35135}
35136
35137/// If we have a shuffle of AVX/AVX512 (256/512 bit) vectors that only uses the
35138/// low half of each source vector and does not set any high half elements in
35139/// the destination vector, narrow the shuffle to half its original size.
35140static SDValue narrowShuffle(ShuffleVectorSDNode *Shuf, SelectionDAG &DAG) {
35141 if (!Shuf->getValueType(0).isSimple())
35142 return SDValue();
35143 MVT VT = Shuf->getSimpleValueType(0);
35144 if (!VT.is256BitVector() && !VT.is512BitVector())
35145 return SDValue();
35146
35147 // See if we can ignore all of the high elements of the shuffle.
35148 ArrayRef<int> Mask = Shuf->getMask();
35149 if (!isUndefUpperHalf(Mask))
35150 return SDValue();
35151
35152 // Check if the shuffle mask accesses only the low half of each input vector
35153 // (half-index output is 0 or 2).
35154 int HalfIdx1, HalfIdx2;
35155 SmallVector<int, 8> HalfMask(Mask.size() / 2);
35156 if (!getHalfShuffleMask(Mask, HalfMask, HalfIdx1, HalfIdx2) ||
35157 (HalfIdx1 % 2 == 1) || (HalfIdx2 % 2 == 1))
35158 return SDValue();
35159
35160 // Create a half-width shuffle to replace the unnecessarily wide shuffle.
35161 // The trick is knowing that all of the insert/extract are actually free
35162 // subregister (zmm<->ymm or ymm<->xmm) ops. That leaves us with a shuffle
35163 // of narrow inputs into a narrow output, and that is always cheaper than
35164 // the wide shuffle that we started with.
35165 return getShuffleHalfVectors(SDLoc(Shuf), Shuf->getOperand(0),
35166 Shuf->getOperand(1), HalfMask, HalfIdx1,
35167 HalfIdx2, false, DAG, /*UseConcat*/true);
35168}
35169
35170static SDValue combineShuffle(SDNode *N, SelectionDAG &DAG,
35171 TargetLowering::DAGCombinerInfo &DCI,
35172 const X86Subtarget &Subtarget) {
35173 if (auto *Shuf = dyn_cast<ShuffleVectorSDNode>(N))
35174 if (SDValue V = narrowShuffle(Shuf, DAG))
35175 return V;
35176
35177 // If we have legalized the vector types, look for blends of FADD and FSUB
35178 // nodes that we can fuse into an ADDSUB, FMADDSUB, or FMSUBADD node.
35179 SDLoc dl(N);
35180 EVT VT = N->getValueType(0);
35181 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
35182 if (TLI.isTypeLegal(VT)) {
35183 if (SDValue AddSub = combineShuffleToAddSubOrFMAddSub(N, Subtarget, DAG))
35184 return AddSub;
35185
35186 if (SDValue HAddSub = foldShuffleOfHorizOp(N, DAG))
35187 return HAddSub;
35188 }
35189
35190 // Attempt to combine into a vector load/broadcast.
35191 if (SDValue LD = combineToConsecutiveLoads(VT, N, dl, DAG, Subtarget, true))
35192 return LD;
35193
35194 // For AVX2, we sometimes want to combine
35195 // (vector_shuffle <mask> (concat_vectors t1, undef)
35196 // (concat_vectors t2, undef))
35197 // Into:
35198 // (vector_shuffle <mask> (concat_vectors t1, t2), undef)
35199 // Since the latter can be efficiently lowered with VPERMD/VPERMQ
35200 if (SDValue ShufConcat = combineShuffleOfConcatUndef(N, DAG, Subtarget))
35201 return ShufConcat;
35202
35203 if (isTargetShuffle(N->getOpcode())) {
35204 SDValue Op(N, 0);
35205 if (SDValue Shuffle = combineTargetShuffle(Op, DAG, DCI, Subtarget))
35206 return Shuffle;
35207
35208 // Try recursively combining arbitrary sequences of x86 shuffle
35209 // instructions into higher-order shuffles. We do this after combining
35210 // specific PSHUF instruction sequences into their minimal form so that we
35211 // can evaluate how many specialized shuffle instructions are involved in
35212 // a particular chain.
35213 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
35214 return Res;
35215
35216 // Simplify source operands based on shuffle mask.
35217 // TODO - merge this into combineX86ShufflesRecursively.
35218 APInt KnownUndef, KnownZero;
35219 APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
35220 if (TLI.SimplifyDemandedVectorElts(Op, DemandedElts, KnownUndef, KnownZero, DCI))
35221 return SDValue(N, 0);
35222 }
35223
35224 // Look for a v2i64/v2f64 VZEXT_MOVL of a node that already produces zeros
35225 // in the upper 64 bits.
35226 // TODO: Can we generalize this using computeKnownBits.
35227 if (N->getOpcode() == X86ISD::VZEXT_MOVL &&
35228 (VT == MVT::v2f64 || VT == MVT::v2i64) &&
35229 N->getOperand(0).getOpcode() == ISD::BITCAST &&
35230 (N->getOperand(0).getOperand(0).getValueType() == MVT::v4f32 ||
35231 N->getOperand(0).getOperand(0).getValueType() == MVT::v4i32)) {
35232 SDValue In = N->getOperand(0).getOperand(0);
35233 switch (In.getOpcode()) {
35234 default:
35235 break;
35236 case X86ISD::CVTP2SI: case X86ISD::CVTP2UI:
35237 case X86ISD::MCVTP2SI: case X86ISD::MCVTP2UI:
35238 case X86ISD::CVTTP2SI: case X86ISD::CVTTP2UI:
35239 case X86ISD::MCVTTP2SI: case X86ISD::MCVTTP2UI:
35240 case X86ISD::CVTSI2P: case X86ISD::CVTUI2P:
35241 case X86ISD::MCVTSI2P: case X86ISD::MCVTUI2P:
35242 case X86ISD::VFPROUND: case X86ISD::VMFPROUND:
35243 if (In.getOperand(0).getValueType() == MVT::v2f64 ||
35244 In.getOperand(0).getValueType() == MVT::v2i64)
35245 return N->getOperand(0); // return the bitcast
35246 break;
35247 case X86ISD::STRICT_CVTTP2SI:
35248 case X86ISD::STRICT_CVTTP2UI:
35249 case X86ISD::STRICT_CVTSI2P:
35250 case X86ISD::STRICT_CVTUI2P:
35251 case X86ISD::STRICT_VFPROUND:
35252 if (In.getOperand(1).getValueType() == MVT::v2f64 ||
35253 In.getOperand(1).getValueType() == MVT::v2i64)
35254 return N->getOperand(0);
35255 break;
35256 }
35257 }
35258
35259 // Pull subvector inserts into undef through VZEXT_MOVL by making it an
35260 // insert into a zero vector. This helps get VZEXT_MOVL closer to
35261 // scalar_to_vectors where 256/512 are canonicalized to an insert and a
35262 // 128-bit scalar_to_vector. This reduces the number of isel patterns.
35263 if (N->getOpcode() == X86ISD::VZEXT_MOVL && !DCI.isBeforeLegalizeOps() &&
35264 N->getOperand(0).getOpcode() == ISD::INSERT_SUBVECTOR &&
35265 N->getOperand(0).hasOneUse() &&
35266 N->getOperand(0).getOperand(0).isUndef() &&
35267 isNullConstant(N->getOperand(0).getOperand(2))) {
35268 SDValue In = N->getOperand(0).getOperand(1);
35269 SDValue Movl = DAG.getNode(X86ISD::VZEXT_MOVL, dl, In.getValueType(), In);
35270 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, VT,
35271 getZeroVector(VT.getSimpleVT(), Subtarget, DAG, dl),
35272 Movl, N->getOperand(0).getOperand(2));
35273 }
35274
35275 // If this a vzmovl of a full vector load, replace it with a vzload, unless
35276 // the load is volatile.
35277 if (N->getOpcode() == X86ISD::VZEXT_MOVL && N->getOperand(0).hasOneUse() &&
35278 ISD::isNormalLoad(N->getOperand(0).getNode())) {
35279 LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(0));
35280 if (LN->isSimple()) {
35281 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
35282 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
35283 SDValue VZLoad =
35284 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops,
35285 VT.getVectorElementType(),
35286 LN->getPointerInfo(),
35287 LN->getAlignment(),
35288 MachineMemOperand::MOLoad);
35289 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
35290 return VZLoad;
35291 }
35292 }
35293
35294 return SDValue();
35295}
35296
35297bool X86TargetLowering::SimplifyDemandedVectorEltsForTargetNode(
35298 SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero,
35299 TargetLoweringOpt &TLO, unsigned Depth) const {
35300 int NumElts = DemandedElts.getBitWidth();
35301 unsigned Opc = Op.getOpcode();
35302 EVT VT = Op.getValueType();
35303
35304 // Handle special case opcodes.
35305 switch (Opc) {
35306 case X86ISD::PMULDQ:
35307 case X86ISD::PMULUDQ: {
35308 APInt LHSUndef, LHSZero;
35309 APInt RHSUndef, RHSZero;
35310 SDValue LHS = Op.getOperand(0);
35311 SDValue RHS = Op.getOperand(1);
35312 if (SimplifyDemandedVectorElts(LHS, DemandedElts, LHSUndef, LHSZero, TLO,
35313 Depth + 1))
35314 return true;
35315 if (SimplifyDemandedVectorElts(RHS, DemandedElts, RHSUndef, RHSZero, TLO,
35316 Depth + 1))
35317 return true;
35318 // Multiply by zero.
35319 KnownZero = LHSZero | RHSZero;
35320 break;
35321 }
35322 case X86ISD::VSHL:
35323 case X86ISD::VSRL:
35324 case X86ISD::VSRA: {
35325 // We only need the bottom 64-bits of the (128-bit) shift amount.
35326 SDValue Amt = Op.getOperand(1);
35327 MVT AmtVT = Amt.getSimpleValueType();
35328 assert(AmtVT.is128BitVector() && "Unexpected value type")((AmtVT.is128BitVector() && "Unexpected value type") ?
static_cast<void> (0) : __assert_fail ("AmtVT.is128BitVector() && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35328, __PRETTY_FUNCTION__));
35329
35330 // If we reuse the shift amount just for sse shift amounts then we know that
35331 // only the bottom 64-bits are only ever used.
35332 bool AssumeSingleUse = llvm::all_of(Amt->uses(), [&Amt](SDNode *Use) {
35333 unsigned UseOpc = Use->getOpcode();
35334 return (UseOpc == X86ISD::VSHL || UseOpc == X86ISD::VSRL ||
35335 UseOpc == X86ISD::VSRA) &&
35336 Use->getOperand(0) != Amt;
35337 });
35338
35339 APInt AmtUndef, AmtZero;
35340 unsigned NumAmtElts = AmtVT.getVectorNumElements();
35341 APInt AmtElts = APInt::getLowBitsSet(NumAmtElts, NumAmtElts / 2);
35342 if (SimplifyDemandedVectorElts(Amt, AmtElts, AmtUndef, AmtZero, TLO,
35343 Depth + 1, AssumeSingleUse))
35344 return true;
35345 LLVM_FALLTHROUGH[[gnu::fallthrough]];
35346 }
35347 case X86ISD::VSHLI:
35348 case X86ISD::VSRLI:
35349 case X86ISD::VSRAI: {
35350 SDValue Src = Op.getOperand(0);
35351 APInt SrcUndef;
35352 if (SimplifyDemandedVectorElts(Src, DemandedElts, SrcUndef, KnownZero, TLO,
35353 Depth + 1))
35354 return true;
35355 // TODO convert SrcUndef to KnownUndef.
35356 break;
35357 }
35358 case X86ISD::KSHIFTL: {
35359 SDValue Src = Op.getOperand(0);
35360 auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
35361 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount")((Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount"
) ? static_cast<void> (0) : __assert_fail ("Amt->getAPIntValue().ult(NumElts) && \"Out of range shift amount\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35361, __PRETTY_FUNCTION__));
35362 unsigned ShiftAmt = Amt->getZExtValue();
35363
35364 if (ShiftAmt == 0)
35365 return TLO.CombineTo(Op, Src);
35366
35367 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
35368 // single shift. We can do this if the bottom bits (which are shifted
35369 // out) are never demanded.
35370 if (Src.getOpcode() == X86ISD::KSHIFTR) {
35371 if (!DemandedElts.intersects(APInt::getLowBitsSet(NumElts, ShiftAmt))) {
35372 unsigned C1 = Src.getConstantOperandVal(1);
35373 unsigned NewOpc = X86ISD::KSHIFTL;
35374 int Diff = ShiftAmt - C1;
35375 if (Diff < 0) {
35376 Diff = -Diff;
35377 NewOpc = X86ISD::KSHIFTR;
35378 }
35379
35380 SDLoc dl(Op);
35381 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
35382 return TLO.CombineTo(
35383 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
35384 }
35385 }
35386
35387 APInt DemandedSrc = DemandedElts.lshr(ShiftAmt);
35388 if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
35389 Depth + 1))
35390 return true;
35391
35392 KnownUndef <<= ShiftAmt;
35393 KnownZero <<= ShiftAmt;
35394 KnownZero.setLowBits(ShiftAmt);
35395 break;
35396 }
35397 case X86ISD::KSHIFTR: {
35398 SDValue Src = Op.getOperand(0);
35399 auto *Amt = cast<ConstantSDNode>(Op.getOperand(1));
35400 assert(Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount")((Amt->getAPIntValue().ult(NumElts) && "Out of range shift amount"
) ? static_cast<void> (0) : __assert_fail ("Amt->getAPIntValue().ult(NumElts) && \"Out of range shift amount\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35400, __PRETTY_FUNCTION__));
35401 unsigned ShiftAmt = Amt->getZExtValue();
35402
35403 if (ShiftAmt == 0)
35404 return TLO.CombineTo(Op, Src);
35405
35406 // If this is ((X << C1) >>u ShAmt), see if we can simplify this into a
35407 // single shift. We can do this if the top bits (which are shifted
35408 // out) are never demanded.
35409 if (Src.getOpcode() == X86ISD::KSHIFTL) {
35410 if (!DemandedElts.intersects(APInt::getHighBitsSet(NumElts, ShiftAmt))) {
35411 unsigned C1 = Src.getConstantOperandVal(1);
35412 unsigned NewOpc = X86ISD::KSHIFTR;
35413 int Diff = ShiftAmt - C1;
35414 if (Diff < 0) {
35415 Diff = -Diff;
35416 NewOpc = X86ISD::KSHIFTL;
35417 }
35418
35419 SDLoc dl(Op);
35420 SDValue NewSA = TLO.DAG.getTargetConstant(Diff, dl, MVT::i8);
35421 return TLO.CombineTo(
35422 Op, TLO.DAG.getNode(NewOpc, dl, VT, Src.getOperand(0), NewSA));
35423 }
35424 }
35425
35426 APInt DemandedSrc = DemandedElts.shl(ShiftAmt);
35427 if (SimplifyDemandedVectorElts(Src, DemandedSrc, KnownUndef, KnownZero, TLO,
35428 Depth + 1))
35429 return true;
35430
35431 KnownUndef.lshrInPlace(ShiftAmt);
35432 KnownZero.lshrInPlace(ShiftAmt);
35433 KnownZero.setHighBits(ShiftAmt);
35434 break;
35435 }
35436 case X86ISD::CVTSI2P:
35437 case X86ISD::CVTUI2P: {
35438 SDValue Src = Op.getOperand(0);
35439 MVT SrcVT = Src.getSimpleValueType();
35440 APInt SrcUndef, SrcZero;
35441 APInt SrcElts = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
35442 if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
35443 Depth + 1))
35444 return true;
35445 break;
35446 }
35447 case X86ISD::PACKSS:
35448 case X86ISD::PACKUS: {
35449 SDValue N0 = Op.getOperand(0);
35450 SDValue N1 = Op.getOperand(1);
35451
35452 APInt DemandedLHS, DemandedRHS;
35453 getPackDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
35454
35455 APInt SrcUndef, SrcZero;
35456 if (SimplifyDemandedVectorElts(N0, DemandedLHS, SrcUndef, SrcZero, TLO,
35457 Depth + 1))
35458 return true;
35459 if (SimplifyDemandedVectorElts(N1, DemandedRHS, SrcUndef, SrcZero, TLO,
35460 Depth + 1))
35461 return true;
35462
35463 // Aggressively peek through ops to get at the demanded elts.
35464 // TODO - we should do this for all target/faux shuffles ops.
35465 if (!DemandedElts.isAllOnesValue()) {
35466 APInt DemandedSrcBits =
35467 APInt::getAllOnesValue(N0.getScalarValueSizeInBits());
35468 SDValue NewN0 = SimplifyMultipleUseDemandedBits(
35469 N0, DemandedSrcBits, DemandedLHS, TLO.DAG, Depth + 1);
35470 SDValue NewN1 = SimplifyMultipleUseDemandedBits(
35471 N1, DemandedSrcBits, DemandedRHS, TLO.DAG, Depth + 1);
35472 if (NewN0 || NewN1) {
35473 NewN0 = NewN0 ? NewN0 : N0;
35474 NewN1 = NewN1 ? NewN1 : N1;
35475 return TLO.CombineTo(Op,
35476 TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewN0, NewN1));
35477 }
35478 }
35479 break;
35480 }
35481 case X86ISD::HADD:
35482 case X86ISD::HSUB:
35483 case X86ISD::FHADD:
35484 case X86ISD::FHSUB: {
35485 APInt DemandedLHS, DemandedRHS;
35486 getHorizDemandedElts(VT, DemandedElts, DemandedLHS, DemandedRHS);
35487
35488 APInt LHSUndef, LHSZero;
35489 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedLHS, LHSUndef,
35490 LHSZero, TLO, Depth + 1))
35491 return true;
35492 APInt RHSUndef, RHSZero;
35493 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedRHS, RHSUndef,
35494 RHSZero, TLO, Depth + 1))
35495 return true;
35496 break;
35497 }
35498 case X86ISD::VTRUNC:
35499 case X86ISD::VTRUNCS:
35500 case X86ISD::VTRUNCUS: {
35501 SDValue Src = Op.getOperand(0);
35502 MVT SrcVT = Src.getSimpleValueType();
35503 APInt DemandedSrc = DemandedElts.zextOrTrunc(SrcVT.getVectorNumElements());
35504 APInt SrcUndef, SrcZero;
35505 if (SimplifyDemandedVectorElts(Src, DemandedSrc, SrcUndef, SrcZero, TLO,
35506 Depth + 1))
35507 return true;
35508 KnownZero = SrcZero.zextOrTrunc(NumElts);
35509 KnownUndef = SrcUndef.zextOrTrunc(NumElts);
35510 break;
35511 }
35512 case X86ISD::BLENDV: {
35513 APInt SelUndef, SelZero;
35514 if (SimplifyDemandedVectorElts(Op.getOperand(0), DemandedElts, SelUndef,
35515 SelZero, TLO, Depth + 1))
35516 return true;
35517
35518 // TODO: Use SelZero to adjust LHS/RHS DemandedElts.
35519 APInt LHSUndef, LHSZero;
35520 if (SimplifyDemandedVectorElts(Op.getOperand(1), DemandedElts, LHSUndef,
35521 LHSZero, TLO, Depth + 1))
35522 return true;
35523
35524 APInt RHSUndef, RHSZero;
35525 if (SimplifyDemandedVectorElts(Op.getOperand(2), DemandedElts, RHSUndef,
35526 RHSZero, TLO, Depth + 1))
35527 return true;
35528
35529 KnownZero = LHSZero & RHSZero;
35530 KnownUndef = LHSUndef & RHSUndef;
35531 break;
35532 }
35533 case X86ISD::VBROADCAST: {
35534 SDValue Src = Op.getOperand(0);
35535 MVT SrcVT = Src.getSimpleValueType();
35536 if (!SrcVT.isVector())
35537 return false;
35538 // Don't bother broadcasting if we just need the 0'th element.
35539 if (DemandedElts == 1) {
35540 if (Src.getValueType() != VT)
35541 Src = widenSubVector(VT.getSimpleVT(), Src, false, Subtarget, TLO.DAG,
35542 SDLoc(Op));
35543 return TLO.CombineTo(Op, Src);
35544 }
35545 APInt SrcUndef, SrcZero;
35546 APInt SrcElts = APInt::getOneBitSet(SrcVT.getVectorNumElements(), 0);
35547 if (SimplifyDemandedVectorElts(Src, SrcElts, SrcUndef, SrcZero, TLO,
35548 Depth + 1))
35549 return true;
35550 break;
35551 }
35552 case X86ISD::VPERMV: {
35553 SDValue Mask = Op.getOperand(0);
35554 APInt MaskUndef, MaskZero;
35555 if (SimplifyDemandedVectorElts(Mask, DemandedElts, MaskUndef, MaskZero, TLO,
35556 Depth + 1))
35557 return true;
35558 break;
35559 }
35560 case X86ISD::PSHUFB:
35561 case X86ISD::VPERMV3:
35562 case X86ISD::VPERMILPV: {
35563 SDValue Mask = Op.getOperand(1);
35564 APInt MaskUndef, MaskZero;
35565 if (SimplifyDemandedVectorElts(Mask, DemandedElts, MaskUndef, MaskZero, TLO,
35566 Depth + 1))
35567 return true;
35568 break;
35569 }
35570 case X86ISD::VPPERM:
35571 case X86ISD::VPERMIL2: {
35572 SDValue Mask = Op.getOperand(2);
35573 APInt MaskUndef, MaskZero;
35574 if (SimplifyDemandedVectorElts(Mask, DemandedElts, MaskUndef, MaskZero, TLO,
35575 Depth + 1))
35576 return true;
35577 break;
35578 }
35579 }
35580
35581 // For 256/512-bit ops that are 128/256-bit ops glued together, if we do not
35582 // demand any of the high elements, then narrow the op to 128/256-bits: e.g.
35583 // (op ymm0, ymm1) --> insert undef, (op xmm0, xmm1), 0
35584 if ((VT.is256BitVector() || VT.is512BitVector()) &&
35585 DemandedElts.lshr(NumElts / 2) == 0) {
35586 unsigned SizeInBits = VT.getSizeInBits();
35587 unsigned ExtSizeInBits = SizeInBits / 2;
35588
35589 // See if 512-bit ops only use the bottom 128-bits.
35590 if (VT.is512BitVector() && DemandedElts.lshr(NumElts / 4) == 0)
35591 ExtSizeInBits = SizeInBits / 4;
35592
35593 switch (Opc) {
35594 // Zero upper elements.
35595 case X86ISD::VZEXT_MOVL: {
35596 SDLoc DL(Op);
35597 SDValue Ext0 =
35598 extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
35599 SDValue ExtOp =
35600 TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0);
35601 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
35602 SDValue Insert =
35603 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
35604 return TLO.CombineTo(Op, Insert);
35605 }
35606 // Subvector broadcast.
35607 case X86ISD::SUBV_BROADCAST: {
35608 SDLoc DL(Op);
35609 SDValue Src = Op.getOperand(0);
35610 if (Src.getValueSizeInBits() > ExtSizeInBits)
35611 Src = extractSubVector(Src, 0, TLO.DAG, DL, ExtSizeInBits);
35612 else if (Src.getValueSizeInBits() < ExtSizeInBits) {
35613 MVT SrcSVT = Src.getSimpleValueType().getScalarType();
35614 MVT SrcVT =
35615 MVT::getVectorVT(SrcSVT, ExtSizeInBits / SrcSVT.getSizeInBits());
35616 Src = TLO.DAG.getNode(X86ISD::SUBV_BROADCAST, DL, SrcVT, Src);
35617 }
35618 return TLO.CombineTo(Op, insertSubVector(TLO.DAG.getUNDEF(VT), Src, 0,
35619 TLO.DAG, DL, ExtSizeInBits));
35620 }
35621 // Byte shifts by immediate.
35622 case X86ISD::VSHLDQ:
35623 case X86ISD::VSRLDQ:
35624 // Shift by uniform.
35625 case X86ISD::VSHL:
35626 case X86ISD::VSRL:
35627 case X86ISD::VSRA:
35628 // Shift by immediate.
35629 case X86ISD::VSHLI:
35630 case X86ISD::VSRLI:
35631 case X86ISD::VSRAI: {
35632 SDLoc DL(Op);
35633 SDValue Ext0 =
35634 extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
35635 SDValue ExtOp =
35636 TLO.DAG.getNode(Opc, DL, Ext0.getValueType(), Ext0, Op.getOperand(1));
35637 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
35638 SDValue Insert =
35639 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
35640 return TLO.CombineTo(Op, Insert);
35641 }
35642 case X86ISD::VPERMI: {
35643 // Simplify PERMPD/PERMQ to extract_subvector.
35644 // TODO: This should be done in shuffle combining.
35645 if (VT == MVT::v4f64 || VT == MVT::v4i64) {
35646 SmallVector<int, 4> Mask;
35647 DecodeVPERMMask(NumElts, Op.getConstantOperandVal(1), Mask);
35648 if (isUndefOrEqual(Mask[0], 2) && isUndefOrEqual(Mask[1], 3)) {
35649 SDLoc DL(Op);
35650 SDValue Ext = extractSubVector(Op.getOperand(0), 2, TLO.DAG, DL, 128);
35651 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
35652 SDValue Insert = insertSubVector(UndefVec, Ext, 0, TLO.DAG, DL, 128);
35653 return TLO.CombineTo(Op, Insert);
35654 }
35655 }
35656 break;
35657 }
35658 // Target Shuffles.
35659 case X86ISD::PSHUFB:
35660 case X86ISD::UNPCKL:
35661 case X86ISD::UNPCKH:
35662 // Saturated Packs.
35663 case X86ISD::PACKSS:
35664 case X86ISD::PACKUS:
35665 // Horizontal Ops.
35666 case X86ISD::HADD:
35667 case X86ISD::HSUB:
35668 case X86ISD::FHADD:
35669 case X86ISD::FHSUB: {
35670 SDLoc DL(Op);
35671 MVT ExtVT = VT.getSimpleVT();
35672 ExtVT = MVT::getVectorVT(ExtVT.getScalarType(),
35673 ExtSizeInBits / ExtVT.getScalarSizeInBits());
35674 SDValue Ext0 =
35675 extractSubVector(Op.getOperand(0), 0, TLO.DAG, DL, ExtSizeInBits);
35676 SDValue Ext1 =
35677 extractSubVector(Op.getOperand(1), 0, TLO.DAG, DL, ExtSizeInBits);
35678 SDValue ExtOp = TLO.DAG.getNode(Opc, DL, ExtVT, Ext0, Ext1);
35679 SDValue UndefVec = TLO.DAG.getUNDEF(VT);
35680 SDValue Insert =
35681 insertSubVector(UndefVec, ExtOp, 0, TLO.DAG, DL, ExtSizeInBits);
35682 return TLO.CombineTo(Op, Insert);
35683 }
35684 }
35685 }
35686
35687 // Get target/faux shuffle mask.
35688 APInt OpUndef, OpZero;
35689 SmallVector<int, 64> OpMask;
35690 SmallVector<SDValue, 2> OpInputs;
35691 if (!getTargetShuffleInputs(Op, DemandedElts, OpInputs, OpMask, OpUndef,
35692 OpZero, TLO.DAG, Depth, false))
35693 return false;
35694
35695 // Shuffle inputs must be the same size as the result.
35696 if (OpMask.size() != (unsigned)NumElts ||
35697 llvm::any_of(OpInputs, [VT](SDValue V) {
35698 return VT.getSizeInBits() != V.getValueSizeInBits() ||
35699 !V.getValueType().isVector();
35700 }))
35701 return false;
35702
35703 KnownZero = OpZero;
35704 KnownUndef = OpUndef;
35705
35706 // Check if shuffle mask can be simplified to undef/zero/identity.
35707 int NumSrcs = OpInputs.size();
35708 for (int i = 0; i != NumElts; ++i)
35709 if (!DemandedElts[i])
35710 OpMask[i] = SM_SentinelUndef;
35711
35712 if (isUndefInRange(OpMask, 0, NumElts)) {
35713 KnownUndef.setAllBits();
35714 return TLO.CombineTo(Op, TLO.DAG.getUNDEF(VT));
35715 }
35716 if (isUndefOrZeroInRange(OpMask, 0, NumElts)) {
35717 KnownZero.setAllBits();
35718 return TLO.CombineTo(
35719 Op, getZeroVector(VT.getSimpleVT(), Subtarget, TLO.DAG, SDLoc(Op)));
35720 }
35721 for (int Src = 0; Src != NumSrcs; ++Src)
35722 if (isSequentialOrUndefInRange(OpMask, 0, NumElts, Src * NumElts))
35723 return TLO.CombineTo(Op, TLO.DAG.getBitcast(VT, OpInputs[Src]));
35724
35725 // Attempt to simplify inputs.
35726 for (int Src = 0; Src != NumSrcs; ++Src) {
35727 // TODO: Support inputs of different types.
35728 if (OpInputs[Src].getValueType() != VT)
35729 continue;
35730
35731 int Lo = Src * NumElts;
35732 APInt SrcElts = APInt::getNullValue(NumElts);
35733 for (int i = 0; i != NumElts; ++i)
35734 if (DemandedElts[i]) {
35735 int M = OpMask[i] - Lo;
35736 if (0 <= M && M < NumElts)
35737 SrcElts.setBit(M);
35738 }
35739
35740 // TODO - Propagate input undef/zero elts.
35741 APInt SrcUndef, SrcZero;
35742 if (SimplifyDemandedVectorElts(OpInputs[Src], SrcElts, SrcUndef, SrcZero,
35743 TLO, Depth + 1))
35744 return true;
35745 }
35746
35747 // If we don't demand all elements, then attempt to combine to a simpler
35748 // shuffle.
35749 // TODO: Handle other depths, but first we need to handle the fact that
35750 // it might combine to the same shuffle.
35751 if (!DemandedElts.isAllOnesValue() && Depth == 0) {
35752 SmallVector<int, 64> DemandedMask(NumElts, SM_SentinelUndef);
35753 for (int i = 0; i != NumElts; ++i)
35754 if (DemandedElts[i])
35755 DemandedMask[i] = i;
35756
35757 SDValue NewShuffle = combineX86ShufflesRecursively(
35758 {Op}, 0, Op, DemandedMask, {}, Depth, /*HasVarMask*/ false,
35759 /*AllowVarMask*/ true, TLO.DAG, Subtarget);
35760 if (NewShuffle)
35761 return TLO.CombineTo(Op, NewShuffle);
35762 }
35763
35764 return false;
35765}
35766
35767bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
35768 SDValue Op, const APInt &OriginalDemandedBits,
35769 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
35770 unsigned Depth) const {
35771 EVT VT = Op.getValueType();
35772 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
35773 unsigned Opc = Op.getOpcode();
35774 switch(Opc) {
35775 case X86ISD::PMULDQ:
35776 case X86ISD::PMULUDQ: {
35777 // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
35778 KnownBits KnownOp;
35779 SDValue LHS = Op.getOperand(0);
35780 SDValue RHS = Op.getOperand(1);
35781 // FIXME: Can we bound this better?
35782 APInt DemandedMask = APInt::getLowBitsSet(64, 32);
35783 if (SimplifyDemandedBits(LHS, DemandedMask, OriginalDemandedElts, KnownOp,
35784 TLO, Depth + 1))
35785 return true;
35786 if (SimplifyDemandedBits(RHS, DemandedMask, OriginalDemandedElts, KnownOp,
35787 TLO, Depth + 1))
35788 return true;
35789
35790 // Aggressively peek through ops to get at the demanded low bits.
35791 SDValue DemandedLHS = SimplifyMultipleUseDemandedBits(
35792 LHS, DemandedMask, OriginalDemandedElts, TLO.DAG, Depth + 1);
35793 SDValue DemandedRHS = SimplifyMultipleUseDemandedBits(
35794 RHS, DemandedMask, OriginalDemandedElts, TLO.DAG, Depth + 1);
35795 if (DemandedLHS || DemandedRHS) {
35796 DemandedLHS = DemandedLHS ? DemandedLHS : LHS;
35797 DemandedRHS = DemandedRHS ? DemandedRHS : RHS;
35798 return TLO.CombineTo(
35799 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, DemandedLHS, DemandedRHS));
35800 }
35801 break;
35802 }
35803 case X86ISD::VSHLI: {
35804 SDValue Op0 = Op.getOperand(0);
35805
35806 unsigned ShAmt = Op.getConstantOperandVal(1);
35807 if (ShAmt >= BitWidth)
35808 break;
35809
35810 APInt DemandedMask = OriginalDemandedBits.lshr(ShAmt);
35811
35812 // If this is ((X >>u C1) << ShAmt), see if we can simplify this into a
35813 // single shift. We can do this if the bottom bits (which are shifted
35814 // out) are never demanded.
35815 if (Op0.getOpcode() == X86ISD::VSRLI &&
35816 OriginalDemandedBits.countTrailingZeros() >= ShAmt) {
35817 unsigned Shift2Amt = Op0.getConstantOperandVal(1);
35818 if (Shift2Amt < BitWidth) {
35819 int Diff = ShAmt - Shift2Amt;
35820 if (Diff == 0)
35821 return TLO.CombineTo(Op, Op0.getOperand(0));
35822
35823 unsigned NewOpc = Diff < 0 ? X86ISD::VSRLI : X86ISD::VSHLI;
35824 SDValue NewShift = TLO.DAG.getNode(
35825 NewOpc, SDLoc(Op), VT, Op0.getOperand(0),
35826 TLO.DAG.getTargetConstant(std::abs(Diff), SDLoc(Op), MVT::i8));
35827 return TLO.CombineTo(Op, NewShift);
35828 }
35829 }
35830
35831 if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
35832 TLO, Depth + 1))
35833 return true;
35834
35835 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35835, __PRETTY_FUNCTION__));
35836 Known.Zero <<= ShAmt;
35837 Known.One <<= ShAmt;
35838
35839 // Low bits known zero.
35840 Known.Zero.setLowBits(ShAmt);
35841 break;
35842 }
35843 case X86ISD::VSRLI: {
35844 unsigned ShAmt = Op.getConstantOperandVal(1);
35845 if (ShAmt >= BitWidth)
35846 break;
35847
35848 APInt DemandedMask = OriginalDemandedBits << ShAmt;
35849
35850 if (SimplifyDemandedBits(Op.getOperand(0), DemandedMask,
35851 OriginalDemandedElts, Known, TLO, Depth + 1))
35852 return true;
35853
35854 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35854, __PRETTY_FUNCTION__));
35855 Known.Zero.lshrInPlace(ShAmt);
35856 Known.One.lshrInPlace(ShAmt);
35857
35858 // High bits known zero.
35859 Known.Zero.setHighBits(ShAmt);
35860 break;
35861 }
35862 case X86ISD::VSRAI: {
35863 SDValue Op0 = Op.getOperand(0);
35864 SDValue Op1 = Op.getOperand(1);
35865
35866 unsigned ShAmt = cast<ConstantSDNode>(Op1)->getZExtValue();
35867 if (ShAmt >= BitWidth)
35868 break;
35869
35870 APInt DemandedMask = OriginalDemandedBits << ShAmt;
35871
35872 // If we just want the sign bit then we don't need to shift it.
35873 if (OriginalDemandedBits.isSignMask())
35874 return TLO.CombineTo(Op, Op0);
35875
35876 // fold (VSRAI (VSHLI X, C1), C1) --> X iff NumSignBits(X) > C1
35877 if (Op0.getOpcode() == X86ISD::VSHLI &&
35878 Op.getOperand(1) == Op0.getOperand(1)) {
35879 SDValue Op00 = Op0.getOperand(0);
35880 unsigned NumSignBits =
35881 TLO.DAG.ComputeNumSignBits(Op00, OriginalDemandedElts);
35882 if (ShAmt < NumSignBits)
35883 return TLO.CombineTo(Op, Op00);
35884 }
35885
35886 // If any of the demanded bits are produced by the sign extension, we also
35887 // demand the input sign bit.
35888 if (OriginalDemandedBits.countLeadingZeros() < ShAmt)
35889 DemandedMask.setSignBit();
35890
35891 if (SimplifyDemandedBits(Op0, DemandedMask, OriginalDemandedElts, Known,
35892 TLO, Depth + 1))
35893 return true;
35894
35895 assert(!Known.hasConflict() && "Bits known to be one AND zero?")((!Known.hasConflict() && "Bits known to be one AND zero?"
) ? static_cast<void> (0) : __assert_fail ("!Known.hasConflict() && \"Bits known to be one AND zero?\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 35895, __PRETTY_FUNCTION__));
35896 Known.Zero.lshrInPlace(ShAmt);
35897 Known.One.lshrInPlace(ShAmt);
35898
35899 // If the input sign bit is known to be zero, or if none of the top bits
35900 // are demanded, turn this into an unsigned shift right.
35901 if (Known.Zero[BitWidth - ShAmt - 1] ||
35902 OriginalDemandedBits.countLeadingZeros() >= ShAmt)
35903 return TLO.CombineTo(
35904 Op, TLO.DAG.getNode(X86ISD::VSRLI, SDLoc(Op), VT, Op0, Op1));
35905
35906 // High bits are known one.
35907 if (Known.One[BitWidth - ShAmt - 1])
35908 Known.One.setHighBits(ShAmt);
35909 break;
35910 }
35911 case X86ISD::PEXTRB:
35912 case X86ISD::PEXTRW: {
35913 SDValue Vec = Op.getOperand(0);
35914 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(1));
35915 MVT VecVT = Vec.getSimpleValueType();
35916 unsigned NumVecElts = VecVT.getVectorNumElements();
35917
35918 if (CIdx && CIdx->getAPIntValue().ult(NumVecElts)) {
35919 unsigned Idx = CIdx->getZExtValue();
35920 unsigned VecBitWidth = VecVT.getScalarSizeInBits();
35921
35922 // If we demand no bits from the vector then we must have demanded
35923 // bits from the implict zext - simplify to zero.
35924 APInt DemandedVecBits = OriginalDemandedBits.trunc(VecBitWidth);
35925 if (DemandedVecBits == 0)
35926 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
35927
35928 APInt KnownUndef, KnownZero;
35929 APInt DemandedVecElts = APInt::getOneBitSet(NumVecElts, Idx);
35930 if (SimplifyDemandedVectorElts(Vec, DemandedVecElts, KnownUndef,
35931 KnownZero, TLO, Depth + 1))
35932 return true;
35933
35934 KnownBits KnownVec;
35935 if (SimplifyDemandedBits(Vec, DemandedVecBits, DemandedVecElts,
35936 KnownVec, TLO, Depth + 1))
35937 return true;
35938
35939 if (SDValue V = SimplifyMultipleUseDemandedBits(
35940 Vec, DemandedVecBits, DemandedVecElts, TLO.DAG, Depth + 1))
35941 return TLO.CombineTo(
35942 Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, V, Op.getOperand(1)));
35943
35944 Known = KnownVec.zext(BitWidth, true);
35945 return false;
35946 }
35947 break;
35948 }
35949 case X86ISD::PINSRB:
35950 case X86ISD::PINSRW: {
35951 SDValue Vec = Op.getOperand(0);
35952 SDValue Scl = Op.getOperand(1);
35953 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
35954 MVT VecVT = Vec.getSimpleValueType();
35955
35956 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements())) {
35957 unsigned Idx = CIdx->getZExtValue();
35958 if (!OriginalDemandedElts[Idx])
35959 return TLO.CombineTo(Op, Vec);
35960
35961 KnownBits KnownVec;
35962 APInt DemandedVecElts(OriginalDemandedElts);
35963 DemandedVecElts.clearBit(Idx);
35964 if (SimplifyDemandedBits(Vec, OriginalDemandedBits, DemandedVecElts,
35965 KnownVec, TLO, Depth + 1))
35966 return true;
35967
35968 KnownBits KnownScl;
35969 unsigned NumSclBits = Scl.getScalarValueSizeInBits();
35970 APInt DemandedSclBits = OriginalDemandedBits.zext(NumSclBits);
35971 if (SimplifyDemandedBits(Scl, DemandedSclBits, KnownScl, TLO, Depth + 1))
35972 return true;
35973
35974 KnownScl = KnownScl.trunc(VecVT.getScalarSizeInBits());
35975 Known.One = KnownVec.One & KnownScl.One;
35976 Known.Zero = KnownVec.Zero & KnownScl.Zero;
35977 return false;
35978 }
35979 break;
35980 }
35981 case X86ISD::PACKSS:
35982 // PACKSS saturates to MIN/MAX integer values. So if we just want the
35983 // sign bit then we can just ask for the source operands sign bit.
35984 // TODO - add known bits handling.
35985 if (OriginalDemandedBits.isSignMask()) {
35986 APInt DemandedLHS, DemandedRHS;
35987 getPackDemandedElts(VT, OriginalDemandedElts, DemandedLHS, DemandedRHS);
35988
35989 KnownBits KnownLHS, KnownRHS;
35990 APInt SignMask = APInt::getSignMask(BitWidth * 2);
35991 if (SimplifyDemandedBits(Op.getOperand(0), SignMask, DemandedLHS,
35992 KnownLHS, TLO, Depth + 1))
35993 return true;
35994 if (SimplifyDemandedBits(Op.getOperand(1), SignMask, DemandedRHS,
35995 KnownRHS, TLO, Depth + 1))
35996 return true;
35997 }
35998 // TODO - add general PACKSS/PACKUS SimplifyDemandedBits support.
35999 break;
36000 case X86ISD::PCMPGT:
36001 // icmp sgt(0, R) == ashr(R, BitWidth-1).
36002 // iff we only need the sign bit then we can use R directly.
36003 if (OriginalDemandedBits.isSignMask() &&
36004 ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
36005 return TLO.CombineTo(Op, Op.getOperand(1));
36006 break;
36007 case X86ISD::MOVMSK: {
36008 SDValue Src = Op.getOperand(0);
36009 MVT SrcVT = Src.getSimpleValueType();
36010 unsigned SrcBits = SrcVT.getScalarSizeInBits();
36011 unsigned NumElts = SrcVT.getVectorNumElements();
36012
36013 // If we don't need the sign bits at all just return zero.
36014 if (OriginalDemandedBits.countTrailingZeros() >= NumElts)
36015 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
36016
36017 // Only demand the vector elements of the sign bits we need.
36018 APInt KnownUndef, KnownZero;
36019 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(NumElts);
36020 if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero,
36021 TLO, Depth + 1))
36022 return true;
36023
36024 Known.Zero = KnownZero.zextOrSelf(BitWidth);
36025 Known.Zero.setHighBits(BitWidth - NumElts);
36026
36027 // MOVMSK only uses the MSB from each vector element.
36028 KnownBits KnownSrc;
36029 if (SimplifyDemandedBits(Src, APInt::getSignMask(SrcBits), DemandedElts,
36030 KnownSrc, TLO, Depth + 1))
36031 return true;
36032
36033 if (KnownSrc.One[SrcBits - 1])
36034 Known.One.setLowBits(NumElts);
36035 else if (KnownSrc.Zero[SrcBits - 1])
36036 Known.Zero.setLowBits(NumElts);
36037 return false;
36038 }
36039 }
36040
36041 return TargetLowering::SimplifyDemandedBitsForTargetNode(
36042 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
36043}
36044
36045SDValue X86TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
36046 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
36047 SelectionDAG &DAG, unsigned Depth) const {
36048 int NumElts = DemandedElts.getBitWidth();
36049 unsigned Opc = Op.getOpcode();
36050 EVT VT = Op.getValueType();
36051
36052 switch (Opc) {
36053 case X86ISD::PINSRB:
36054 case X86ISD::PINSRW: {
36055 // If we don't demand the inserted element, return the base vector.
36056 SDValue Vec = Op.getOperand(0);
36057 auto *CIdx = dyn_cast<ConstantSDNode>(Op.getOperand(2));
36058 MVT VecVT = Vec.getSimpleValueType();
36059 if (CIdx && CIdx->getAPIntValue().ult(VecVT.getVectorNumElements()) &&
36060 !DemandedElts[CIdx->getZExtValue()])
36061 return Vec;
36062 break;
36063 }
36064 case X86ISD::PCMPGT:
36065 // icmp sgt(0, R) == ashr(R, BitWidth-1).
36066 // iff we only need the sign bit then we can use R directly.
36067 if (DemandedBits.isSignMask() &&
36068 ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
36069 return Op.getOperand(1);
36070 break;
36071 }
36072
36073 APInt ShuffleUndef, ShuffleZero;
36074 SmallVector<int, 16> ShuffleMask;
36075 SmallVector<SDValue, 2> ShuffleOps;
36076 if (getTargetShuffleInputs(Op, DemandedElts, ShuffleOps, ShuffleMask,
36077 ShuffleUndef, ShuffleZero, DAG, Depth, false)) {
36078 // If all the demanded elts are from one operand and are inline,
36079 // then we can use the operand directly.
36080 int NumOps = ShuffleOps.size();
36081 if (ShuffleMask.size() == (unsigned)NumElts &&
36082 llvm::all_of(ShuffleOps, [VT](SDValue V) {
36083 return VT.getSizeInBits() == V.getValueSizeInBits();
36084 })) {
36085
36086 if (DemandedElts.isSubsetOf(ShuffleUndef))
36087 return DAG.getUNDEF(VT);
36088 if (DemandedElts.isSubsetOf(ShuffleUndef | ShuffleZero))
36089 return getZeroVector(VT.getSimpleVT(), Subtarget, DAG, SDLoc(Op));
36090
36091 // Bitmask that indicates which ops have only been accessed 'inline'.
36092 APInt IdentityOp = APInt::getAllOnesValue(NumOps);
36093 for (int i = 0; i != NumElts; ++i) {
36094 int M = ShuffleMask[i];
36095 if (!DemandedElts[i] || ShuffleUndef[i])
36096 continue;
36097 int Op = M / NumElts;
36098 int Index = M % NumElts;
36099 if (M < 0 || Index != i) {
36100 IdentityOp.clearAllBits();
36101 break;
36102 }
36103 IdentityOp &= APInt::getOneBitSet(NumOps, Op);
36104 if (IdentityOp == 0)
36105 break;
36106 }
36107 assert((IdentityOp == 0 || IdentityOp.countPopulation() == 1) &&(((IdentityOp == 0 || IdentityOp.countPopulation() == 1) &&
"Multiple identity shuffles detected") ? static_cast<void
> (0) : __assert_fail ("(IdentityOp == 0 || IdentityOp.countPopulation() == 1) && \"Multiple identity shuffles detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36108, __PRETTY_FUNCTION__))
36108 "Multiple identity shuffles detected")(((IdentityOp == 0 || IdentityOp.countPopulation() == 1) &&
"Multiple identity shuffles detected") ? static_cast<void
> (0) : __assert_fail ("(IdentityOp == 0 || IdentityOp.countPopulation() == 1) && \"Multiple identity shuffles detected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36108, __PRETTY_FUNCTION__));
36109
36110 if (IdentityOp != 0)
36111 return DAG.getBitcast(VT, ShuffleOps[IdentityOp.countTrailingZeros()]);
36112 }
36113 }
36114
36115 return TargetLowering::SimplifyMultipleUseDemandedBitsForTargetNode(
36116 Op, DemandedBits, DemandedElts, DAG, Depth);
36117}
36118
36119// Helper to peek through bitops/setcc to determine size of source vector.
36120// Allows combineBitcastvxi1 to determine what size vector generated a <X x i1>.
36121static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size) {
36122 switch (Src.getOpcode()) {
36123 case ISD::SETCC:
36124 return Src.getOperand(0).getValueSizeInBits() == Size;
36125 case ISD::AND:
36126 case ISD::XOR:
36127 case ISD::OR:
36128 return checkBitcastSrcVectorSize(Src.getOperand(0), Size) &&
36129 checkBitcastSrcVectorSize(Src.getOperand(1), Size);
36130 }
36131 return false;
36132}
36133
36134// Helper to push sign extension of vXi1 SETCC result through bitops.
36135static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT,
36136 SDValue Src, const SDLoc &DL) {
36137 switch (Src.getOpcode()) {
36138 case ISD::SETCC:
36139 return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
36140 case ISD::AND:
36141 case ISD::XOR:
36142 case ISD::OR:
36143 return DAG.getNode(
36144 Src.getOpcode(), DL, SExtVT,
36145 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
36146 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
36147 }
36148 llvm_unreachable("Unexpected node type for vXi1 sign extension")::llvm::llvm_unreachable_internal("Unexpected node type for vXi1 sign extension"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36148);
36149}
36150
36151// Try to match patterns such as
36152// (i16 bitcast (v16i1 x))
36153// ->
36154// (i16 movmsk (16i8 sext (v16i1 x)))
36155// before the illegal vector is scalarized on subtargets that don't have legal
36156// vxi1 types.
36157static SDValue combineBitcastvxi1(SelectionDAG &DAG, EVT VT, SDValue Src,
36158 const SDLoc &DL,
36159 const X86Subtarget &Subtarget) {
36160 EVT SrcVT = Src.getValueType();
36161 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
36162 return SDValue();
36163
36164 // If the input is a truncate from v16i8 or v32i8 go ahead and use a
36165 // movmskb even with avx512. This will be better than truncating to vXi1 and
36166 // using a kmov. This can especially help KNL if the input is a v16i8/v32i8
36167 // vpcmpeqb/vpcmpgtb.
36168 bool IsTruncated = Src.getOpcode() == ISD::TRUNCATE && Src.hasOneUse() &&
36169 (Src.getOperand(0).getValueType() == MVT::v16i8 ||
36170 Src.getOperand(0).getValueType() == MVT::v32i8 ||
36171 Src.getOperand(0).getValueType() == MVT::v64i8);
36172
36173 // With AVX512 vxi1 types are legal and we prefer using k-regs.
36174 // MOVMSK is supported in SSE2 or later.
36175 if (!Subtarget.hasSSE2() || (Subtarget.hasAVX512() && !IsTruncated))
36176 return SDValue();
36177
36178 // There are MOVMSK flavors for types v16i8, v32i8, v4f32, v8f32, v4f64 and
36179 // v8f64. So all legal 128-bit and 256-bit vectors are covered except for
36180 // v8i16 and v16i16.
36181 // For these two cases, we can shuffle the upper element bytes to a
36182 // consecutive sequence at the start of the vector and treat the results as
36183 // v16i8 or v32i8, and for v16i8 this is the preferable solution. However,
36184 // for v16i16 this is not the case, because the shuffle is expensive, so we
36185 // avoid sign-extending to this type entirely.
36186 // For example, t0 := (v8i16 sext(v8i1 x)) needs to be shuffled as:
36187 // (v16i8 shuffle <0,2,4,6,8,10,12,14,u,u,...,u> (v16i8 bitcast t0), undef)
36188 MVT SExtVT;
36189 bool PropagateSExt = false;
36190 switch (SrcVT.getSimpleVT().SimpleTy) {
36191 default:
36192 return SDValue();
36193 case MVT::v2i1:
36194 SExtVT = MVT::v2i64;
36195 break;
36196 case MVT::v4i1:
36197 SExtVT = MVT::v4i32;
36198 // For cases such as (i4 bitcast (v4i1 setcc v4i64 v1, v2))
36199 // sign-extend to a 256-bit operation to avoid truncation.
36200 if (Subtarget.hasAVX() && checkBitcastSrcVectorSize(Src, 256)) {
36201 SExtVT = MVT::v4i64;
36202 PropagateSExt = true;
36203 }
36204 break;
36205 case MVT::v8i1:
36206 SExtVT = MVT::v8i16;
36207 // For cases such as (i8 bitcast (v8i1 setcc v8i32 v1, v2)),
36208 // sign-extend to a 256-bit operation to match the compare.
36209 // If the setcc operand is 128-bit, prefer sign-extending to 128-bit over
36210 // 256-bit because the shuffle is cheaper than sign extending the result of
36211 // the compare.
36212 if (Subtarget.hasAVX() && (checkBitcastSrcVectorSize(Src, 256) ||
36213 checkBitcastSrcVectorSize(Src, 512))) {
36214 SExtVT = MVT::v8i32;
36215 PropagateSExt = true;
36216 }
36217 break;
36218 case MVT::v16i1:
36219 SExtVT = MVT::v16i8;
36220 // For the case (i16 bitcast (v16i1 setcc v16i16 v1, v2)),
36221 // it is not profitable to sign-extend to 256-bit because this will
36222 // require an extra cross-lane shuffle which is more expensive than
36223 // truncating the result of the compare to 128-bits.
36224 break;
36225 case MVT::v32i1:
36226 SExtVT = MVT::v32i8;
36227 break;
36228 case MVT::v64i1:
36229 // If we have AVX512F, but not AVX512BW and the input is truncated from
36230 // v64i8 checked earlier. Then split the input and make two pmovmskbs.
36231 if (Subtarget.hasAVX512() && !Subtarget.hasBWI()) {
36232 SExtVT = MVT::v64i8;
36233 break;
36234 }
36235 return SDValue();
36236 };
36237
36238 SDValue V = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
36239 : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
36240
36241 if (SExtVT == MVT::v16i8 || SExtVT == MVT::v32i8 || SExtVT == MVT::v64i8) {
36242 V = getPMOVMSKB(DL, V, DAG, Subtarget);
36243 } else {
36244 if (SExtVT == MVT::v8i16)
36245 V = DAG.getNode(X86ISD::PACKSS, DL, MVT::v16i8, V,
36246 DAG.getUNDEF(MVT::v8i16));
36247 V = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, V);
36248 }
36249
36250 EVT IntVT =
36251 EVT::getIntegerVT(*DAG.getContext(), SrcVT.getVectorNumElements());
36252 V = DAG.getZExtOrTrunc(V, DL, IntVT);
36253 return DAG.getBitcast(VT, V);
36254}
36255
36256// Convert a vXi1 constant build vector to the same width scalar integer.
36257static SDValue combinevXi1ConstantToInteger(SDValue Op, SelectionDAG &DAG) {
36258 EVT SrcVT = Op.getValueType();
36259 assert(SrcVT.getVectorElementType() == MVT::i1 &&((SrcVT.getVectorElementType() == MVT::i1 && "Expected a vXi1 vector"
) ? static_cast<void> (0) : __assert_fail ("SrcVT.getVectorElementType() == MVT::i1 && \"Expected a vXi1 vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36260, __PRETTY_FUNCTION__))
36260 "Expected a vXi1 vector")((SrcVT.getVectorElementType() == MVT::i1 && "Expected a vXi1 vector"
) ? static_cast<void> (0) : __assert_fail ("SrcVT.getVectorElementType() == MVT::i1 && \"Expected a vXi1 vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36260, __PRETTY_FUNCTION__));
36261 assert(ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&((ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
"Expected a constant build vector") ? static_cast<void>
(0) : __assert_fail ("ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && \"Expected a constant build vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36262, __PRETTY_FUNCTION__))
36262 "Expected a constant build vector")((ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) &&
"Expected a constant build vector") ? static_cast<void>
(0) : __assert_fail ("ISD::isBuildVectorOfConstantSDNodes(Op.getNode()) && \"Expected a constant build vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36262, __PRETTY_FUNCTION__));
36263
36264 APInt Imm(SrcVT.getVectorNumElements(), 0);
36265 for (unsigned Idx = 0, e = Op.getNumOperands(); Idx < e; ++Idx) {
36266 SDValue In = Op.getOperand(Idx);
36267 if (!In.isUndef() && (cast<ConstantSDNode>(In)->getZExtValue() & 0x1))
36268 Imm.setBit(Idx);
36269 }
36270 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), Imm.getBitWidth());
36271 return DAG.getConstant(Imm, SDLoc(Op), IntVT);
36272}
36273
36274static SDValue combineCastedMaskArithmetic(SDNode *N, SelectionDAG &DAG,
36275 TargetLowering::DAGCombinerInfo &DCI,
36276 const X86Subtarget &Subtarget) {
36277 assert(N->getOpcode() == ISD::BITCAST && "Expected a bitcast")((N->getOpcode() == ISD::BITCAST && "Expected a bitcast"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::BITCAST && \"Expected a bitcast\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36277, __PRETTY_FUNCTION__));
36278
36279 if (!DCI.isBeforeLegalizeOps())
36280 return SDValue();
36281
36282 // Only do this if we have k-registers.
36283 if (!Subtarget.hasAVX512())
36284 return SDValue();
36285
36286 EVT DstVT = N->getValueType(0);
36287 SDValue Op = N->getOperand(0);
36288 EVT SrcVT = Op.getValueType();
36289
36290 if (!Op.hasOneUse())
36291 return SDValue();
36292
36293 // Look for logic ops.
36294 if (Op.getOpcode() != ISD::AND &&
36295 Op.getOpcode() != ISD::OR &&
36296 Op.getOpcode() != ISD::XOR)
36297 return SDValue();
36298
36299 // Make sure we have a bitcast between mask registers and a scalar type.
36300 if (!(SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
36301 DstVT.isScalarInteger()) &&
36302 !(DstVT.isVector() && DstVT.getVectorElementType() == MVT::i1 &&
36303 SrcVT.isScalarInteger()))
36304 return SDValue();
36305
36306 SDValue LHS = Op.getOperand(0);
36307 SDValue RHS = Op.getOperand(1);
36308
36309 if (LHS.hasOneUse() && LHS.getOpcode() == ISD::BITCAST &&
36310 LHS.getOperand(0).getValueType() == DstVT)
36311 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT, LHS.getOperand(0),
36312 DAG.getBitcast(DstVT, RHS));
36313
36314 if (RHS.hasOneUse() && RHS.getOpcode() == ISD::BITCAST &&
36315 RHS.getOperand(0).getValueType() == DstVT)
36316 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
36317 DAG.getBitcast(DstVT, LHS), RHS.getOperand(0));
36318
36319 // If the RHS is a vXi1 build vector, this is a good reason to flip too.
36320 // Most of these have to move a constant from the scalar domain anyway.
36321 if (ISD::isBuildVectorOfConstantSDNodes(RHS.getNode())) {
36322 RHS = combinevXi1ConstantToInteger(RHS, DAG);
36323 return DAG.getNode(Op.getOpcode(), SDLoc(N), DstVT,
36324 DAG.getBitcast(DstVT, LHS), RHS);
36325 }
36326
36327 return SDValue();
36328}
36329
36330static SDValue createMMXBuildVector(BuildVectorSDNode *BV, SelectionDAG &DAG,
36331 const X86Subtarget &Subtarget) {
36332 SDLoc DL(BV);
36333 unsigned NumElts = BV->getNumOperands();
36334 SDValue Splat = BV->getSplatValue();
36335
36336 // Build MMX element from integer GPR or SSE float values.
36337 auto CreateMMXElement = [&](SDValue V) {
36338 if (V.isUndef())
36339 return DAG.getUNDEF(MVT::x86mmx);
36340 if (V.getValueType().isFloatingPoint()) {
36341 if (Subtarget.hasSSE1() && !isa<ConstantFPSDNode>(V)) {
36342 V = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4f32, V);
36343 V = DAG.getBitcast(MVT::v2i64, V);
36344 return DAG.getNode(X86ISD::MOVDQ2Q, DL, MVT::x86mmx, V);
36345 }
36346 V = DAG.getBitcast(MVT::i32, V);
36347 } else {
36348 V = DAG.getAnyExtOrTrunc(V, DL, MVT::i32);
36349 }
36350 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, MVT::x86mmx, V);
36351 };
36352
36353 // Convert build vector ops to MMX data in the bottom elements.
36354 SmallVector<SDValue, 8> Ops;
36355
36356 // Broadcast - use (PUNPCKL+)PSHUFW to broadcast single element.
36357 if (Splat) {
36358 if (Splat.isUndef())
36359 return DAG.getUNDEF(MVT::x86mmx);
36360
36361 Splat = CreateMMXElement(Splat);
36362
36363 if (Subtarget.hasSSE1()) {
36364 // Unpack v8i8 to splat i8 elements to lowest 16-bits.
36365 if (NumElts == 8)
36366 Splat = DAG.getNode(
36367 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
36368 DAG.getConstant(Intrinsic::x86_mmx_punpcklbw, DL, MVT::i32), Splat,
36369 Splat);
36370
36371 // Use PSHUFW to repeat 16-bit elements.
36372 unsigned ShufMask = (NumElts > 2 ? 0 : 0x44);
36373 return DAG.getNode(
36374 ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx,
36375 DAG.getTargetConstant(Intrinsic::x86_sse_pshuf_w, DL, MVT::i32),
36376 Splat, DAG.getTargetConstant(ShufMask, DL, MVT::i8));
36377 }
36378 Ops.append(NumElts, Splat);
36379 } else {
36380 for (unsigned i = 0; i != NumElts; ++i)
36381 Ops.push_back(CreateMMXElement(BV->getOperand(i)));
36382 }
36383
36384 // Use tree of PUNPCKLs to build up general MMX vector.
36385 while (Ops.size() > 1) {
36386 unsigned NumOps = Ops.size();
36387 unsigned IntrinOp =
36388 (NumOps == 2 ? Intrinsic::x86_mmx_punpckldq
36389 : (NumOps == 4 ? Intrinsic::x86_mmx_punpcklwd
36390 : Intrinsic::x86_mmx_punpcklbw));
36391 SDValue Intrin = DAG.getConstant(IntrinOp, DL, MVT::i32);
36392 for (unsigned i = 0; i != NumOps; i += 2)
36393 Ops[i / 2] = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::x86mmx, Intrin,
36394 Ops[i], Ops[i + 1]);
36395 Ops.resize(NumOps / 2);
36396 }
36397
36398 return Ops[0];
36399}
36400
36401static SDValue combineBitcast(SDNode *N, SelectionDAG &DAG,
36402 TargetLowering::DAGCombinerInfo &DCI,
36403 const X86Subtarget &Subtarget) {
36404 SDValue N0 = N->getOperand(0);
36405 EVT VT = N->getValueType(0);
36406 EVT SrcVT = N0.getValueType();
36407
36408 // Try to match patterns such as
36409 // (i16 bitcast (v16i1 x))
36410 // ->
36411 // (i16 movmsk (16i8 sext (v16i1 x)))
36412 // before the setcc result is scalarized on subtargets that don't have legal
36413 // vxi1 types.
36414 if (DCI.isBeforeLegalize()) {
36415 SDLoc dl(N);
36416 if (SDValue V = combineBitcastvxi1(DAG, VT, N0, dl, Subtarget))
36417 return V;
36418
36419 // Recognize the IR pattern for the movmsk intrinsic under SSE1 befoer type
36420 // legalization destroys the v4i32 type.
36421 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && SrcVT == MVT::v4i1 &&
36422 VT.isScalarInteger() && N0.getOpcode() == ISD::SETCC &&
36423 N0.getOperand(0).getValueType() == MVT::v4i32 &&
36424 ISD::isBuildVectorAllZeros(N0.getOperand(1).getNode()) &&
36425 cast<CondCodeSDNode>(N0.getOperand(2))->get() == ISD::SETLT) {
36426 SDValue N00 = N0.getOperand(0);
36427 // Only do this if we can avoid scalarizing the input.
36428 if (ISD::isNormalLoad(N00.getNode()) ||
36429 (N00.getOpcode() == ISD::BITCAST &&
36430 N00.getOperand(0).getValueType() == MVT::v4f32)) {
36431 SDValue V = DAG.getNode(X86ISD::MOVMSK, dl, MVT::i32,
36432 DAG.getBitcast(MVT::v4f32, N00));
36433 return DAG.getZExtOrTrunc(V, dl, VT);
36434 }
36435 }
36436
36437 // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
36438 // type, widen both sides to avoid a trip through memory.
36439 if ((VT == MVT::v4i1 || VT == MVT::v2i1) && SrcVT.isScalarInteger() &&
36440 Subtarget.hasAVX512()) {
36441 N0 = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::i8, N0);
36442 N0 = DAG.getBitcast(MVT::v8i1, N0);
36443 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, dl, VT, N0,
36444 DAG.getIntPtrConstant(0, dl));
36445 }
36446
36447 // If this is a bitcast between a MVT::v4i1/v2i1 and an illegal integer
36448 // type, widen both sides to avoid a trip through memory.
36449 if ((SrcVT == MVT::v4i1 || SrcVT == MVT::v2i1) && VT.isScalarInteger() &&
36450 Subtarget.hasAVX512()) {
36451 // Use zeros for the widening if we already have some zeroes. This can
36452 // allow SimplifyDemandedBits to remove scalar ANDs that may be down
36453 // stream of this.
36454 // FIXME: It might make sense to detect a concat_vectors with a mix of
36455 // zeroes and undef and turn it into insert_subvector for i1 vectors as
36456 // a separate combine. What we can't do is canonicalize the operands of
36457 // such a concat or we'll get into a loop with SimplifyDemandedBits.
36458 if (N0.getOpcode() == ISD::CONCAT_VECTORS) {
36459 SDValue LastOp = N0.getOperand(N0.getNumOperands() - 1);
36460 if (ISD::isBuildVectorAllZeros(LastOp.getNode())) {
36461 SrcVT = LastOp.getValueType();
36462 unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
36463 SmallVector<SDValue, 4> Ops(N0->op_begin(), N0->op_end());
36464 Ops.resize(NumConcats, DAG.getConstant(0, dl, SrcVT));
36465 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
36466 N0 = DAG.getBitcast(MVT::i8, N0);
36467 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
36468 }
36469 }
36470
36471 unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
36472 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(SrcVT));
36473 Ops[0] = N0;
36474 N0 = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
36475 N0 = DAG.getBitcast(MVT::i8, N0);
36476 return DAG.getNode(ISD::TRUNCATE, dl, VT, N0);
36477 }
36478 }
36479
36480 // Look for (i8 (bitcast (v8i1 (extract_subvector (v16i1 X), 0)))) and
36481 // replace with (i8 (trunc (i16 (bitcast (v16i1 X))))). This can occur
36482 // due to insert_subvector legalization on KNL. By promoting the copy to i16
36483 // we can help with known bits propagation from the vXi1 domain to the
36484 // scalar domain.
36485 if (VT == MVT::i8 && SrcVT == MVT::v8i1 && Subtarget.hasAVX512() &&
36486 !Subtarget.hasDQI() && N0.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
36487 N0.getOperand(0).getValueType() == MVT::v16i1 &&
36488 isNullConstant(N0.getOperand(1)))
36489 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT,
36490 DAG.getBitcast(MVT::i16, N0.getOperand(0)));
36491
36492 // Combine (bitcast (vbroadcast_load)) -> (vbroadcast_load). The memory VT
36493 // determines // the number of bits loaded. Remaining bits are zero.
36494 if (N0.getOpcode() == X86ISD::VBROADCAST_LOAD && N0.hasOneUse() &&
36495 VT.getScalarSizeInBits() == SrcVT.getScalarSizeInBits()) {
36496 auto *BCast = cast<MemIntrinsicSDNode>(N0);
36497 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
36498 SDValue Ops[] = { BCast->getChain(), BCast->getBasePtr() };
36499 SDValue ResNode =
36500 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
36501 VT.getVectorElementType(),
36502 BCast->getMemOperand());
36503 DAG.ReplaceAllUsesOfValueWith(SDValue(BCast, 1), ResNode.getValue(1));
36504 return ResNode;
36505 }
36506
36507 // Since MMX types are special and don't usually play with other vector types,
36508 // it's better to handle them early to be sure we emit efficient code by
36509 // avoiding store-load conversions.
36510 if (VT == MVT::x86mmx) {
36511 // Detect MMX constant vectors.
36512 APInt UndefElts;
36513 SmallVector<APInt, 1> EltBits;
36514 if (getTargetConstantBitsFromNode(N0, 64, UndefElts, EltBits)) {
36515 SDLoc DL(N0);
36516 // Handle zero-extension of i32 with MOVD.
36517 if (EltBits[0].countLeadingZeros() >= 32)
36518 return DAG.getNode(X86ISD::MMX_MOVW2D, DL, VT,
36519 DAG.getConstant(EltBits[0].trunc(32), DL, MVT::i32));
36520 // Else, bitcast to a double.
36521 // TODO - investigate supporting sext 32-bit immediates on x86_64.
36522 APFloat F64(APFloat::IEEEdouble(), EltBits[0]);
36523 return DAG.getBitcast(VT, DAG.getConstantFP(F64, DL, MVT::f64));
36524 }
36525
36526 // Detect bitcasts to x86mmx low word.
36527 if (N0.getOpcode() == ISD::BUILD_VECTOR &&
36528 (SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 || SrcVT == MVT::v8i8) &&
36529 N0.getOperand(0).getValueType() == SrcVT.getScalarType()) {
36530 bool LowUndef = true, AllUndefOrZero = true;
36531 for (unsigned i = 1, e = SrcVT.getVectorNumElements(); i != e; ++i) {
36532 SDValue Op = N0.getOperand(i);
36533 LowUndef &= Op.isUndef() || (i >= e/2);
36534 AllUndefOrZero &= (Op.isUndef() || isNullConstant(Op));
36535 }
36536 if (AllUndefOrZero) {
36537 SDValue N00 = N0.getOperand(0);
36538 SDLoc dl(N00);
36539 N00 = LowUndef ? DAG.getAnyExtOrTrunc(N00, dl, MVT::i32)
36540 : DAG.getZExtOrTrunc(N00, dl, MVT::i32);
36541 return DAG.getNode(X86ISD::MMX_MOVW2D, dl, VT, N00);
36542 }
36543 }
36544
36545 // Detect bitcasts of 64-bit build vectors and convert to a
36546 // MMX UNPCK/PSHUFW which takes MMX type inputs with the value in the
36547 // lowest element.
36548 if (N0.getOpcode() == ISD::BUILD_VECTOR &&
36549 (SrcVT == MVT::v2f32 || SrcVT == MVT::v2i32 || SrcVT == MVT::v4i16 ||
36550 SrcVT == MVT::v8i8))
36551 return createMMXBuildVector(cast<BuildVectorSDNode>(N0), DAG, Subtarget);
36552
36553 // Detect bitcasts between element or subvector extraction to x86mmx.
36554 if ((N0.getOpcode() == ISD::EXTRACT_VECTOR_ELT ||
36555 N0.getOpcode() == ISD::EXTRACT_SUBVECTOR) &&
36556 isNullConstant(N0.getOperand(1))) {
36557 SDValue N00 = N0.getOperand(0);
36558 if (N00.getValueType().is128BitVector())
36559 return DAG.getNode(X86ISD::MOVDQ2Q, SDLoc(N00), VT,
36560 DAG.getBitcast(MVT::v2i64, N00));
36561 }
36562
36563 // Detect bitcasts from FP_TO_SINT to x86mmx.
36564 if (SrcVT == MVT::v2i32 && N0.getOpcode() == ISD::FP_TO_SINT) {
36565 SDLoc DL(N0);
36566 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v4i32, N0,
36567 DAG.getUNDEF(MVT::v2i32));
36568 return DAG.getNode(X86ISD::MOVDQ2Q, DL, VT,
36569 DAG.getBitcast(MVT::v2i64, Res));
36570 }
36571 }
36572
36573 // Try to remove a bitcast of constant vXi1 vector. We have to legalize
36574 // most of these to scalar anyway.
36575 if (Subtarget.hasAVX512() && VT.isScalarInteger() &&
36576 SrcVT.isVector() && SrcVT.getVectorElementType() == MVT::i1 &&
36577 ISD::isBuildVectorOfConstantSDNodes(N0.getNode())) {
36578 return combinevXi1ConstantToInteger(N0, DAG);
36579 }
36580
36581 if (Subtarget.hasAVX512() && SrcVT.isScalarInteger() &&
36582 VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
36583 isa<ConstantSDNode>(N0)) {
36584 auto *C = cast<ConstantSDNode>(N0);
36585 if (C->isAllOnesValue())
36586 return DAG.getConstant(1, SDLoc(N0), VT);
36587 if (C->isNullValue())
36588 return DAG.getConstant(0, SDLoc(N0), VT);
36589 }
36590
36591 // Try to remove bitcasts from input and output of mask arithmetic to
36592 // remove GPR<->K-register crossings.
36593 if (SDValue V = combineCastedMaskArithmetic(N, DAG, DCI, Subtarget))
36594 return V;
36595
36596 // Convert a bitcasted integer logic operation that has one bitcasted
36597 // floating-point operand into a floating-point logic operation. This may
36598 // create a load of a constant, but that is cheaper than materializing the
36599 // constant in an integer register and transferring it to an SSE register or
36600 // transferring the SSE operand to integer register and back.
36601 unsigned FPOpcode;
36602 switch (N0.getOpcode()) {
36603 case ISD::AND: FPOpcode = X86ISD::FAND; break;
36604 case ISD::OR: FPOpcode = X86ISD::FOR; break;
36605 case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
36606 default: return SDValue();
36607 }
36608
36609 if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
36610 (Subtarget.hasSSE2() && VT == MVT::f64)))
36611 return SDValue();
36612
36613 SDValue LogicOp0 = N0.getOperand(0);
36614 SDValue LogicOp1 = N0.getOperand(1);
36615 SDLoc DL0(N0);
36616
36617 // bitcast(logic(bitcast(X), Y)) --> logic'(X, bitcast(Y))
36618 if (N0.hasOneUse() && LogicOp0.getOpcode() == ISD::BITCAST &&
36619 LogicOp0.hasOneUse() && LogicOp0.getOperand(0).getValueType() == VT &&
36620 !isa<ConstantSDNode>(LogicOp0.getOperand(0))) {
36621 SDValue CastedOp1 = DAG.getBitcast(VT, LogicOp1);
36622 return DAG.getNode(FPOpcode, DL0, VT, LogicOp0.getOperand(0), CastedOp1);
36623 }
36624 // bitcast(logic(X, bitcast(Y))) --> logic'(bitcast(X), Y)
36625 if (N0.hasOneUse() && LogicOp1.getOpcode() == ISD::BITCAST &&
36626 LogicOp1.hasOneUse() && LogicOp1.getOperand(0).getValueType() == VT &&
36627 !isa<ConstantSDNode>(LogicOp1.getOperand(0))) {
36628 SDValue CastedOp0 = DAG.getBitcast(VT, LogicOp0);
36629 return DAG.getNode(FPOpcode, DL0, VT, LogicOp1.getOperand(0), CastedOp0);
36630 }
36631
36632 return SDValue();
36633}
36634
36635// Given a ABS node, detect the following pattern:
36636// (ABS (SUB (ZERO_EXTEND a), (ZERO_EXTEND b))).
36637// This is useful as it is the input into a SAD pattern.
36638static bool detectZextAbsDiff(const SDValue &Abs, SDValue &Op0, SDValue &Op1) {
36639 SDValue AbsOp1 = Abs->getOperand(0);
36640 if (AbsOp1.getOpcode() != ISD::SUB)
36641 return false;
36642
36643 Op0 = AbsOp1.getOperand(0);
36644 Op1 = AbsOp1.getOperand(1);
36645
36646 // Check if the operands of the sub are zero-extended from vectors of i8.
36647 if (Op0.getOpcode() != ISD::ZERO_EXTEND ||
36648 Op0.getOperand(0).getValueType().getVectorElementType() != MVT::i8 ||
36649 Op1.getOpcode() != ISD::ZERO_EXTEND ||
36650 Op1.getOperand(0).getValueType().getVectorElementType() != MVT::i8)
36651 return false;
36652
36653 return true;
36654}
36655
36656// Given two zexts of <k x i8> to <k x i32>, create a PSADBW of the inputs
36657// to these zexts.
36658static SDValue createPSADBW(SelectionDAG &DAG, const SDValue &Zext0,
36659 const SDValue &Zext1, const SDLoc &DL,
36660 const X86Subtarget &Subtarget) {
36661 // Find the appropriate width for the PSADBW.
36662 EVT InVT = Zext0.getOperand(0).getValueType();
36663 unsigned RegSize = std::max(128u, (unsigned)InVT.getSizeInBits());
36664
36665 // "Zero-extend" the i8 vectors. This is not a per-element zext, rather we
36666 // fill in the missing vector elements with 0.
36667 unsigned NumConcat = RegSize / InVT.getSizeInBits();
36668 SmallVector<SDValue, 16> Ops(NumConcat, DAG.getConstant(0, DL, InVT));
36669 Ops[0] = Zext0.getOperand(0);
36670 MVT ExtendedVT = MVT::getVectorVT(MVT::i8, RegSize / 8);
36671 SDValue SadOp0 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
36672 Ops[0] = Zext1.getOperand(0);
36673 SDValue SadOp1 = DAG.getNode(ISD::CONCAT_VECTORS, DL, ExtendedVT, Ops);
36674
36675 // Actually build the SAD, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
36676 auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
36677 ArrayRef<SDValue> Ops) {
36678 MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
36679 return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops);
36680 };
36681 MVT SadVT = MVT::getVectorVT(MVT::i64, RegSize / 64);
36682 return SplitOpsAndApply(DAG, Subtarget, DL, SadVT, { SadOp0, SadOp1 },
36683 PSADBWBuilder);
36684}
36685
36686// Attempt to replace an min/max v8i16/v16i8 horizontal reduction with
36687// PHMINPOSUW.
36688static SDValue combineHorizontalMinMaxResult(SDNode *Extract, SelectionDAG &DAG,
36689 const X86Subtarget &Subtarget) {
36690 // Bail without SSE41.
36691 if (!Subtarget.hasSSE41())
36692 return SDValue();
36693
36694 EVT ExtractVT = Extract->getValueType(0);
36695 if (ExtractVT != MVT::i16 && ExtractVT != MVT::i8)
36696 return SDValue();
36697
36698 // Check for SMAX/SMIN/UMAX/UMIN horizontal reduction patterns.
36699 ISD::NodeType BinOp;
36700 SDValue Src = DAG.matchBinOpReduction(
36701 Extract, BinOp, {ISD::SMAX, ISD::SMIN, ISD::UMAX, ISD::UMIN}, true);
36702 if (!Src)
36703 return SDValue();
36704
36705 EVT SrcVT = Src.getValueType();
36706 EVT SrcSVT = SrcVT.getScalarType();
36707 if (SrcSVT != ExtractVT || (SrcVT.getSizeInBits() % 128) != 0)
36708 return SDValue();
36709
36710 SDLoc DL(Extract);
36711 SDValue MinPos = Src;
36712
36713 // First, reduce the source down to 128-bit, applying BinOp to lo/hi.
36714 while (SrcVT.getSizeInBits() > 128) {
36715 unsigned NumElts = SrcVT.getVectorNumElements();
36716 unsigned NumSubElts = NumElts / 2;
36717 SrcVT = EVT::getVectorVT(*DAG.getContext(), SrcSVT, NumSubElts);
36718 unsigned SubSizeInBits = SrcVT.getSizeInBits();
36719 SDValue Lo = extractSubVector(MinPos, 0, DAG, DL, SubSizeInBits);
36720 SDValue Hi = extractSubVector(MinPos, NumSubElts, DAG, DL, SubSizeInBits);
36721 MinPos = DAG.getNode(BinOp, DL, SrcVT, Lo, Hi);
36722 }
36723 assert(((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) ||((((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || (
SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
"Unexpected value type") ? static_cast<void> (0) : __assert_fail
("((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36725, __PRETTY_FUNCTION__))
36724 (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&((((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || (
SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
"Unexpected value type") ? static_cast<void> (0) : __assert_fail
("((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36725, __PRETTY_FUNCTION__))
36725 "Unexpected value type")((((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || (
SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) &&
"Unexpected value type") ? static_cast<void> (0) : __assert_fail
("((SrcVT == MVT::v8i16 && ExtractVT == MVT::i16) || (SrcVT == MVT::v16i8 && ExtractVT == MVT::i8)) && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36725, __PRETTY_FUNCTION__));
36726
36727 // PHMINPOSUW applies to UMIN(v8i16), for SMIN/SMAX/UMAX we must apply a mask
36728 // to flip the value accordingly.
36729 SDValue Mask;
36730 unsigned MaskEltsBits = ExtractVT.getSizeInBits();
36731 if (BinOp == ISD::SMAX)
36732 Mask = DAG.getConstant(APInt::getSignedMaxValue(MaskEltsBits), DL, SrcVT);
36733 else if (BinOp == ISD::SMIN)
36734 Mask = DAG.getConstant(APInt::getSignedMinValue(MaskEltsBits), DL, SrcVT);
36735 else if (BinOp == ISD::UMAX)
36736 Mask = DAG.getConstant(APInt::getAllOnesValue(MaskEltsBits), DL, SrcVT);
36737
36738 if (Mask)
36739 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
36740
36741 // For v16i8 cases we need to perform UMIN on pairs of byte elements,
36742 // shuffling each upper element down and insert zeros. This means that the
36743 // v16i8 UMIN will leave the upper element as zero, performing zero-extension
36744 // ready for the PHMINPOS.
36745 if (ExtractVT == MVT::i8) {
36746 SDValue Upper = DAG.getVectorShuffle(
36747 SrcVT, DL, MinPos, DAG.getConstant(0, DL, MVT::v16i8),
36748 {1, 16, 3, 16, 5, 16, 7, 16, 9, 16, 11, 16, 13, 16, 15, 16});
36749 MinPos = DAG.getNode(ISD::UMIN, DL, SrcVT, MinPos, Upper);
36750 }
36751
36752 // Perform the PHMINPOS on a v8i16 vector,
36753 MinPos = DAG.getBitcast(MVT::v8i16, MinPos);
36754 MinPos = DAG.getNode(X86ISD::PHMINPOS, DL, MVT::v8i16, MinPos);
36755 MinPos = DAG.getBitcast(SrcVT, MinPos);
36756
36757 if (Mask)
36758 MinPos = DAG.getNode(ISD::XOR, DL, SrcVT, Mask, MinPos);
36759
36760 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, MinPos,
36761 DAG.getIntPtrConstant(0, DL));
36762}
36763
36764// Attempt to replace an all_of/any_of/parity style horizontal reduction with a MOVMSK.
36765static SDValue combineHorizontalPredicateResult(SDNode *Extract,
36766 SelectionDAG &DAG,
36767 const X86Subtarget &Subtarget) {
36768 // Bail without SSE2.
36769 if (!Subtarget.hasSSE2())
36770 return SDValue();
36771
36772 EVT ExtractVT = Extract->getValueType(0);
36773 unsigned BitWidth = ExtractVT.getSizeInBits();
36774 if (ExtractVT != MVT::i64 && ExtractVT != MVT::i32 && ExtractVT != MVT::i16 &&
36775 ExtractVT != MVT::i8 && ExtractVT != MVT::i1)
36776 return SDValue();
36777
36778 // Check for OR(any_of)/AND(all_of)/XOR(parity) horizontal reduction patterns.
36779 ISD::NodeType BinOp;
36780 SDValue Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::OR, ISD::AND});
36781 if (!Match && ExtractVT == MVT::i1)
36782 Match = DAG.matchBinOpReduction(Extract, BinOp, {ISD::XOR});
36783 if (!Match)
36784 return SDValue();
36785
36786 // EXTRACT_VECTOR_ELT can require implicit extension of the vector element
36787 // which we can't support here for now.
36788 if (Match.getScalarValueSizeInBits() != BitWidth)
36789 return SDValue();
36790
36791 SDValue Movmsk;
36792 SDLoc DL(Extract);
36793 EVT MatchVT = Match.getValueType();
36794 unsigned NumElts = MatchVT.getVectorNumElements();
36795 unsigned MaxElts = Subtarget.hasInt256() ? 32 : 16;
36796 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
36797
36798 if (ExtractVT == MVT::i1) {
36799 // Special case for (pre-legalization) vXi1 reductions.
36800 if (NumElts > 64 || !isPowerOf2_32(NumElts))
36801 return SDValue();
36802 if (TLI.isTypeLegal(MatchVT)) {
36803 // If this is a legal AVX512 predicate type then we can just bitcast.
36804 EVT MovmskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
36805 Movmsk = DAG.getBitcast(MovmskVT, Match);
36806 } else {
36807 // Use combineBitcastvxi1 to create the MOVMSK.
36808 while (NumElts > MaxElts) {
36809 SDValue Lo, Hi;
36810 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
36811 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
36812 NumElts /= 2;
36813 }
36814 EVT MovmskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
36815 Movmsk = combineBitcastvxi1(DAG, MovmskVT, Match, DL, Subtarget);
36816 }
36817 if (!Movmsk)
36818 return SDValue();
36819 Movmsk = DAG.getZExtOrTrunc(Movmsk, DL, NumElts > 32 ? MVT::i64 : MVT::i32);
36820 } else {
36821 // Bail with AVX512VL (which uses predicate registers).
36822 if (Subtarget.hasVLX())
36823 return SDValue();
36824
36825 unsigned MatchSizeInBits = Match.getValueSizeInBits();
36826 if (!(MatchSizeInBits == 128 ||
36827 (MatchSizeInBits == 256 && Subtarget.hasAVX())))
36828 return SDValue();
36829
36830 // Make sure this isn't a vector of 1 element. The perf win from using
36831 // MOVMSK diminishes with less elements in the reduction, but it is
36832 // generally better to get the comparison over to the GPRs as soon as
36833 // possible to reduce the number of vector ops.
36834 if (Match.getValueType().getVectorNumElements() < 2)
36835 return SDValue();
36836
36837 // Check that we are extracting a reduction of all sign bits.
36838 if (DAG.ComputeNumSignBits(Match) != BitWidth)
36839 return SDValue();
36840
36841 if (MatchSizeInBits == 256 && BitWidth < 32 && !Subtarget.hasInt256()) {
36842 SDValue Lo, Hi;
36843 std::tie(Lo, Hi) = DAG.SplitVector(Match, DL);
36844 Match = DAG.getNode(BinOp, DL, Lo.getValueType(), Lo, Hi);
36845 MatchSizeInBits = Match.getValueSizeInBits();
36846 }
36847
36848 // For 32/64 bit comparisons use MOVMSKPS/MOVMSKPD, else PMOVMSKB.
36849 MVT MaskSrcVT;
36850 if (64 == BitWidth || 32 == BitWidth)
36851 MaskSrcVT = MVT::getVectorVT(MVT::getFloatingPointVT(BitWidth),
36852 MatchSizeInBits / BitWidth);
36853 else
36854 MaskSrcVT = MVT::getVectorVT(MVT::i8, MatchSizeInBits / 8);
36855
36856 SDValue BitcastLogicOp = DAG.getBitcast(MaskSrcVT, Match);
36857 Movmsk = getPMOVMSKB(DL, BitcastLogicOp, DAG, Subtarget);
36858 NumElts = MaskSrcVT.getVectorNumElements();
36859 }
36860 assert((NumElts <= 32 || NumElts == 64) &&(((NumElts <= 32 || NumElts == 64) && "Not expecting more than 64 elements"
) ? static_cast<void> (0) : __assert_fail ("(NumElts <= 32 || NumElts == 64) && \"Not expecting more than 64 elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36861, __PRETTY_FUNCTION__))
36861 "Not expecting more than 64 elements")(((NumElts <= 32 || NumElts == 64) && "Not expecting more than 64 elements"
) ? static_cast<void> (0) : __assert_fail ("(NumElts <= 32 || NumElts == 64) && \"Not expecting more than 64 elements\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 36861, __PRETTY_FUNCTION__));
36862
36863 MVT CmpVT = NumElts == 64 ? MVT::i64 : MVT::i32;
36864 if (BinOp == ISD::XOR) {
36865 // parity -> (AND (CTPOP(MOVMSK X)), 1)
36866 SDValue Mask = DAG.getConstant(1, DL, CmpVT);
36867 SDValue Result = DAG.getNode(ISD::CTPOP, DL, CmpVT, Movmsk);
36868 Result = DAG.getNode(ISD::AND, DL, CmpVT, Result, Mask);
36869 return DAG.getZExtOrTrunc(Result, DL, ExtractVT);
36870 }
36871
36872 SDValue CmpC;
36873 ISD::CondCode CondCode;
36874 if (BinOp == ISD::OR) {
36875 // any_of -> MOVMSK != 0
36876 CmpC = DAG.getConstant(0, DL, CmpVT);
36877 CondCode = ISD::CondCode::SETNE;
36878 } else {
36879 // all_of -> MOVMSK == ((1 << NumElts) - 1)
36880 CmpC = DAG.getConstant(APInt::getLowBitsSet(CmpVT.getSizeInBits(), NumElts),
36881 DL, CmpVT);
36882 CondCode = ISD::CondCode::SETEQ;
36883 }
36884
36885 // The setcc produces an i8 of 0/1, so extend that to the result width and
36886 // negate to get the final 0/-1 mask value.
36887 EVT SetccVT =
36888 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), CmpVT);
36889 SDValue Setcc = DAG.getSetCC(DL, SetccVT, Movmsk, CmpC, CondCode);
36890 SDValue Zext = DAG.getZExtOrTrunc(Setcc, DL, ExtractVT);
36891 SDValue Zero = DAG.getConstant(0, DL, ExtractVT);
36892 return DAG.getNode(ISD::SUB, DL, ExtractVT, Zero, Zext);
36893}
36894
36895static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
36896 const X86Subtarget &Subtarget) {
36897 // PSADBW is only supported on SSE2 and up.
36898 if (!Subtarget.hasSSE2())
36899 return SDValue();
36900
36901 // Verify the type we're extracting from is any integer type above i16.
36902 EVT VT = Extract->getOperand(0).getValueType();
36903 if (!VT.isSimple() || !(VT.getVectorElementType().getSizeInBits() > 16))
36904 return SDValue();
36905
36906 unsigned RegSize = 128;
36907 if (Subtarget.useBWIRegs())
36908 RegSize = 512;
36909 else if (Subtarget.hasAVX())
36910 RegSize = 256;
36911
36912 // We handle upto v16i* for SSE2 / v32i* for AVX / v64i* for AVX512.
36913 // TODO: We should be able to handle larger vectors by splitting them before
36914 // feeding them into several SADs, and then reducing over those.
36915 if (RegSize / VT.getVectorNumElements() < 8)
36916 return SDValue();
36917
36918 // Match shuffle + add pyramid.
36919 ISD::NodeType BinOp;
36920 SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
36921
36922 // The operand is expected to be zero extended from i8
36923 // (verified in detectZextAbsDiff).
36924 // In order to convert to i64 and above, additional any/zero/sign
36925 // extend is expected.
36926 // The zero extend from 32 bit has no mathematical effect on the result.
36927 // Also the sign extend is basically zero extend
36928 // (extends the sign bit which is zero).
36929 // So it is correct to skip the sign/zero extend instruction.
36930 if (Root && (Root.getOpcode() == ISD::SIGN_EXTEND ||
36931 Root.getOpcode() == ISD::ZERO_EXTEND ||
36932 Root.getOpcode() == ISD::ANY_EXTEND))
36933 Root = Root.getOperand(0);
36934
36935 // If there was a match, we want Root to be a select that is the root of an
36936 // abs-diff pattern.
36937 if (!Root || Root.getOpcode() != ISD::ABS)
36938 return SDValue();
36939
36940 // Check whether we have an abs-diff pattern feeding into the select.
36941 SDValue Zext0, Zext1;
36942 if (!detectZextAbsDiff(Root, Zext0, Zext1))
36943 return SDValue();
36944
36945 // Create the SAD instruction.
36946 SDLoc DL(Extract);
36947 SDValue SAD = createPSADBW(DAG, Zext0, Zext1, DL, Subtarget);
36948
36949 // If the original vector was wider than 8 elements, sum over the results
36950 // in the SAD vector.
36951 unsigned Stages = Log2_32(VT.getVectorNumElements());
36952 MVT SadVT = SAD.getSimpleValueType();
36953 if (Stages > 3) {
36954 unsigned SadElems = SadVT.getVectorNumElements();
36955
36956 for(unsigned i = Stages - 3; i > 0; --i) {
36957 SmallVector<int, 16> Mask(SadElems, -1);
36958 for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
36959 Mask[j] = MaskEnd + j;
36960
36961 SDValue Shuffle =
36962 DAG.getVectorShuffle(SadVT, DL, SAD, DAG.getUNDEF(SadVT), Mask);
36963 SAD = DAG.getNode(ISD::ADD, DL, SadVT, SAD, Shuffle);
36964 }
36965 }
36966
36967 MVT Type = Extract->getSimpleValueType(0);
36968 unsigned TypeSizeInBits = Type.getSizeInBits();
36969 // Return the lowest TypeSizeInBits bits.
36970 MVT ResVT = MVT::getVectorVT(Type, SadVT.getSizeInBits() / TypeSizeInBits);
36971 SAD = DAG.getBitcast(ResVT, SAD);
36972 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, Type, SAD,
36973 Extract->getOperand(1));
36974}
36975
36976// Attempt to peek through a target shuffle and extract the scalar from the
36977// source.
36978static SDValue combineExtractWithShuffle(SDNode *N, SelectionDAG &DAG,
36979 TargetLowering::DAGCombinerInfo &DCI,
36980 const X86Subtarget &Subtarget) {
36981 if (DCI.isBeforeLegalizeOps())
36982 return SDValue();
36983
36984 SDLoc dl(N);
36985 SDValue Src = N->getOperand(0);
36986 SDValue Idx = N->getOperand(1);
36987
36988 EVT VT = N->getValueType(0);
36989 EVT SrcVT = Src.getValueType();
36990 EVT SrcSVT = SrcVT.getVectorElementType();
36991 unsigned NumSrcElts = SrcVT.getVectorNumElements();
36992
36993 // Don't attempt this for boolean mask vectors or unknown extraction indices.
36994 if (SrcSVT == MVT::i1 || !isa<ConstantSDNode>(Idx))
36995 return SDValue();
36996
36997 SDValue SrcBC = peekThroughBitcasts(Src);
36998
36999 // Handle extract(broadcast(scalar_value)), it doesn't matter what index is.
37000 if (X86ISD::VBROADCAST == SrcBC.getOpcode()) {
37001 SDValue SrcOp = SrcBC.getOperand(0);
37002 if (SrcOp.getValueSizeInBits() == VT.getSizeInBits())
37003 return DAG.getBitcast(VT, SrcOp);
37004 }
37005
37006 // If we're extracting a single element from a broadcast load and there are
37007 // no other users, just create a single load.
37008 if (SrcBC.getOpcode() == X86ISD::VBROADCAST_LOAD && SrcBC.hasOneUse()) {
37009 auto *MemIntr = cast<MemIntrinsicSDNode>(SrcBC);
37010 unsigned SrcBCWidth = SrcBC.getScalarValueSizeInBits();
37011 if (MemIntr->getMemoryVT().getSizeInBits() == SrcBCWidth &&
37012 VT.getSizeInBits() == SrcBCWidth) {
37013 SDValue Load = DAG.getLoad(VT, dl, MemIntr->getChain(),
37014 MemIntr->getBasePtr(),
37015 MemIntr->getPointerInfo(),
37016 MemIntr->getAlignment(),
37017 MemIntr->getMemOperand()->getFlags());
37018 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), Load.getValue(1));
37019 return Load;
37020 }
37021 }
37022
37023 // Handle extract(truncate(x)) for 0'th index.
37024 // TODO: Treat this as a faux shuffle?
37025 // TODO: When can we use this for general indices?
37026 if (ISD::TRUNCATE == Src.getOpcode() && SrcVT.is128BitVector() &&
37027 isNullConstant(Idx)) {
37028 Src = extract128BitVector(Src.getOperand(0), 0, DAG, dl);
37029 Src = DAG.getBitcast(SrcVT, Src);
37030 return DAG.getNode(N->getOpcode(), dl, VT, Src, Idx);
37031 }
37032
37033 // Resolve the target shuffle inputs and mask.
37034 SmallVector<int, 16> Mask;
37035 SmallVector<SDValue, 2> Ops;
37036 if (!getTargetShuffleInputs(SrcBC, Ops, Mask, DAG))
37037 return SDValue();
37038
37039 // Attempt to narrow/widen the shuffle mask to the correct size.
37040 if (Mask.size() != NumSrcElts) {
37041 if ((NumSrcElts % Mask.size()) == 0) {
37042 SmallVector<int, 16> ScaledMask;
37043 int Scale = NumSrcElts / Mask.size();
37044 scaleShuffleMask<int>(Scale, Mask, ScaledMask);
37045 Mask = std::move(ScaledMask);
37046 } else if ((Mask.size() % NumSrcElts) == 0) {
37047 // Simplify Mask based on demanded element.
37048 int ExtractIdx = (int)N->getConstantOperandVal(1);
37049 int Scale = Mask.size() / NumSrcElts;
37050 int Lo = Scale * ExtractIdx;
37051 int Hi = Scale * (ExtractIdx + 1);
37052 for (int i = 0, e = (int)Mask.size(); i != e; ++i)
37053 if (i < Lo || Hi <= i)
37054 Mask[i] = SM_SentinelUndef;
37055
37056 SmallVector<int, 16> WidenedMask;
37057 while (Mask.size() > NumSrcElts &&
37058 canWidenShuffleElements(Mask, WidenedMask))
37059 Mask = std::move(WidenedMask);
37060 // TODO - investigate support for wider shuffle masks with known upper
37061 // undef/zero elements for implicit zero-extension.
37062 }
37063 }
37064
37065 // Check if narrowing/widening failed.
37066 if (Mask.size() != NumSrcElts)
37067 return SDValue();
37068
37069 int SrcIdx = Mask[N->getConstantOperandVal(1)];
37070
37071 // If the shuffle source element is undef/zero then we can just accept it.
37072 if (SrcIdx == SM_SentinelUndef)
37073 return DAG.getUNDEF(VT);
37074
37075 if (SrcIdx == SM_SentinelZero)
37076 return VT.isFloatingPoint() ? DAG.getConstantFP(0.0, dl, VT)
37077 : DAG.getConstant(0, dl, VT);
37078
37079 SDValue SrcOp = Ops[SrcIdx / Mask.size()];
37080 SrcIdx = SrcIdx % Mask.size();
37081
37082 // We can only extract other elements from 128-bit vectors and in certain
37083 // circumstances, depending on SSE-level.
37084 // TODO: Investigate using extract_subvector for larger vectors.
37085 // TODO: Investigate float/double extraction if it will be just stored.
37086 if ((SrcVT == MVT::v4i32 || SrcVT == MVT::v2i64) &&
37087 ((SrcIdx == 0 && Subtarget.hasSSE2()) || Subtarget.hasSSE41())) {
37088 assert(SrcSVT == VT && "Unexpected extraction type")((SrcSVT == VT && "Unexpected extraction type") ? static_cast
<void> (0) : __assert_fail ("SrcSVT == VT && \"Unexpected extraction type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37088, __PRETTY_FUNCTION__));
37089 SrcOp = DAG.getBitcast(SrcVT, SrcOp);
37090 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, SrcSVT, SrcOp,
37091 DAG.getIntPtrConstant(SrcIdx, dl));
37092 }
37093
37094 if ((SrcVT == MVT::v8i16 && Subtarget.hasSSE2()) ||
37095 (SrcVT == MVT::v16i8 && Subtarget.hasSSE41())) {
37096 assert(VT.getSizeInBits() >= SrcSVT.getSizeInBits() &&((VT.getSizeInBits() >= SrcSVT.getSizeInBits() && "Unexpected extraction type"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() >= SrcSVT.getSizeInBits() && \"Unexpected extraction type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37097, __PRETTY_FUNCTION__))
37097 "Unexpected extraction type")((VT.getSizeInBits() >= SrcSVT.getSizeInBits() && "Unexpected extraction type"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() >= SrcSVT.getSizeInBits() && \"Unexpected extraction type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37097, __PRETTY_FUNCTION__));
37098 unsigned OpCode = (SrcVT == MVT::v8i16 ? X86ISD::PEXTRW : X86ISD::PEXTRB);
37099 SrcOp = DAG.getBitcast(SrcVT, SrcOp);
37100 SDValue ExtOp = DAG.getNode(OpCode, dl, MVT::i32, SrcOp,
37101 DAG.getIntPtrConstant(SrcIdx, dl));
37102 return DAG.getZExtOrTrunc(ExtOp, dl, VT);
37103 }
37104
37105 return SDValue();
37106}
37107
37108/// Extracting a scalar FP value from vector element 0 is free, so extract each
37109/// operand first, then perform the math as a scalar op.
37110static SDValue scalarizeExtEltFP(SDNode *ExtElt, SelectionDAG &DAG) {
37111 assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Expected extract")((ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
"Expected extract") ? static_cast<void> (0) : __assert_fail
("ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && \"Expected extract\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37111, __PRETTY_FUNCTION__));
37112 SDValue Vec = ExtElt->getOperand(0);
37113 SDValue Index = ExtElt->getOperand(1);
37114 EVT VT = ExtElt->getValueType(0);
37115 EVT VecVT = Vec.getValueType();
37116
37117 // TODO: If this is a unary/expensive/expand op, allow extraction from a
37118 // non-zero element because the shuffle+scalar op will be cheaper?
37119 if (!Vec.hasOneUse() || !isNullConstant(Index) || VecVT.getScalarType() != VT)
37120 return SDValue();
37121
37122 // Vector FP compares don't fit the pattern of FP math ops (propagate, not
37123 // extract, the condition code), so deal with those as a special-case.
37124 if (Vec.getOpcode() == ISD::SETCC && VT == MVT::i1) {
37125 EVT OpVT = Vec.getOperand(0).getValueType().getScalarType();
37126 if (OpVT != MVT::f32 && OpVT != MVT::f64)
37127 return SDValue();
37128
37129 // extract (setcc X, Y, CC), 0 --> setcc (extract X, 0), (extract Y, 0), CC
37130 SDLoc DL(ExtElt);
37131 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
37132 Vec.getOperand(0), Index);
37133 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT,
37134 Vec.getOperand(1), Index);
37135 return DAG.getNode(Vec.getOpcode(), DL, VT, Ext0, Ext1, Vec.getOperand(2));
37136 }
37137
37138 if (VT != MVT::f32 && VT != MVT::f64)
37139 return SDValue();
37140
37141 // Vector FP selects don't fit the pattern of FP math ops (because the
37142 // condition has a different type and we have to change the opcode), so deal
37143 // with those here.
37144 // FIXME: This is restricted to pre type legalization by ensuring the setcc
37145 // has i1 elements. If we loosen this we need to convert vector bool to a
37146 // scalar bool.
37147 if (Vec.getOpcode() == ISD::VSELECT &&
37148 Vec.getOperand(0).getOpcode() == ISD::SETCC &&
37149 Vec.getOperand(0).getValueType().getScalarType() == MVT::i1 &&
37150 Vec.getOperand(0).getOperand(0).getValueType() == VecVT) {
37151 // ext (sel Cond, X, Y), 0 --> sel (ext Cond, 0), (ext X, 0), (ext Y, 0)
37152 SDLoc DL(ExtElt);
37153 SDValue Ext0 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL,
37154 Vec.getOperand(0).getValueType().getScalarType(),
37155 Vec.getOperand(0), Index);
37156 SDValue Ext1 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
37157 Vec.getOperand(1), Index);
37158 SDValue Ext2 = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT,
37159 Vec.getOperand(2), Index);
37160 return DAG.getNode(ISD::SELECT, DL, VT, Ext0, Ext1, Ext2);
37161 }
37162
37163 // TODO: This switch could include FNEG and the x86-specific FP logic ops
37164 // (FAND, FANDN, FOR, FXOR). But that may require enhancements to avoid
37165 // missed load folding and fma+fneg combining.
37166 switch (Vec.getOpcode()) {
37167 case ISD::FMA: // Begin 3 operands
37168 case ISD::FMAD:
37169 case ISD::FADD: // Begin 2 operands
37170 case ISD::FSUB:
37171 case ISD::FMUL:
37172 case ISD::FDIV:
37173 case ISD::FREM:
37174 case ISD::FCOPYSIGN:
37175 case ISD::FMINNUM:
37176 case ISD::FMAXNUM:
37177 case ISD::FMINNUM_IEEE:
37178 case ISD::FMAXNUM_IEEE:
37179 case ISD::FMAXIMUM:
37180 case ISD::FMINIMUM:
37181 case X86ISD::FMAX:
37182 case X86ISD::FMIN:
37183 case ISD::FABS: // Begin 1 operand
37184 case ISD::FSQRT:
37185 case ISD::FRINT:
37186 case ISD::FCEIL:
37187 case ISD::FTRUNC:
37188 case ISD::FNEARBYINT:
37189 case ISD::FROUND:
37190 case ISD::FFLOOR:
37191 case X86ISD::FRCP:
37192 case X86ISD::FRSQRT: {
37193 // extract (fp X, Y, ...), 0 --> fp (extract X, 0), (extract Y, 0), ...
37194 SDLoc DL(ExtElt);
37195 SmallVector<SDValue, 4> ExtOps;
37196 for (SDValue Op : Vec->ops())
37197 ExtOps.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Op, Index));
37198 return DAG.getNode(Vec.getOpcode(), DL, VT, ExtOps);
37199 }
37200 default:
37201 return SDValue();
37202 }
37203 llvm_unreachable("All opcodes should return within switch")::llvm::llvm_unreachable_internal("All opcodes should return within switch"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37203);
37204}
37205
37206/// Try to convert a vector reduction sequence composed of binops and shuffles
37207/// into horizontal ops.
37208static SDValue combineReductionToHorizontal(SDNode *ExtElt, SelectionDAG &DAG,
37209 const X86Subtarget &Subtarget) {
37210 assert(ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && "Unexpected caller")((ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
"Unexpected caller") ? static_cast<void> (0) : __assert_fail
("ExtElt->getOpcode() == ISD::EXTRACT_VECTOR_ELT && \"Unexpected caller\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37210, __PRETTY_FUNCTION__));
37211
37212 // We need at least SSE2 to anything here.
37213 if (!Subtarget.hasSSE2())
37214 return SDValue();
37215
37216 ISD::NodeType Opc;
37217 SDValue Rdx =
37218 DAG.matchBinOpReduction(ExtElt, Opc, {ISD::ADD, ISD::FADD}, true);
37219 if (!Rdx)
37220 return SDValue();
37221
37222 SDValue Index = ExtElt->getOperand(1);
37223 assert(isNullConstant(Index) &&((isNullConstant(Index) && "Reduction doesn't end in an extract from index 0"
) ? static_cast<void> (0) : __assert_fail ("isNullConstant(Index) && \"Reduction doesn't end in an extract from index 0\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37224, __PRETTY_FUNCTION__))
37224 "Reduction doesn't end in an extract from index 0")((isNullConstant(Index) && "Reduction doesn't end in an extract from index 0"
) ? static_cast<void> (0) : __assert_fail ("isNullConstant(Index) && \"Reduction doesn't end in an extract from index 0\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37224, __PRETTY_FUNCTION__));
37225
37226 EVT VT = ExtElt->getValueType(0);
37227 EVT VecVT = Rdx.getValueType();
37228 if (VecVT.getScalarType() != VT)
37229 return SDValue();
37230
37231 SDLoc DL(ExtElt);
37232
37233 // vXi8 reduction - sub 128-bit vector.
37234 if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
37235 if (VecVT == MVT::v4i8) {
37236 // Pad with zero.
37237 if (Subtarget.hasSSE41()) {
37238 Rdx = DAG.getBitcast(MVT::i32, Rdx);
37239 Rdx = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
37240 DAG.getConstant(0, DL, MVT::v4i32), Rdx,
37241 DAG.getIntPtrConstant(0, DL));
37242 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
37243 } else {
37244 Rdx = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, Rdx,
37245 DAG.getConstant(0, DL, VecVT));
37246 }
37247 }
37248 if (Rdx.getValueType() == MVT::v8i8) {
37249 // Pad with undef.
37250 Rdx = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, Rdx,
37251 DAG.getUNDEF(MVT::v8i8));
37252 }
37253 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
37254 DAG.getConstant(0, DL, MVT::v16i8));
37255 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
37256 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
37257 }
37258
37259 // Must be a >=128-bit vector with pow2 elements.
37260 if ((VecVT.getSizeInBits() % 128) != 0 ||
37261 !isPowerOf2_32(VecVT.getVectorNumElements()))
37262 return SDValue();
37263
37264 // vXi8 reduction - sum lo/hi halves then use PSADBW.
37265 if (VT == MVT::i8) {
37266 while (Rdx.getValueSizeInBits() > 128) {
37267 unsigned HalfSize = VecVT.getSizeInBits() / 2;
37268 unsigned HalfElts = VecVT.getVectorNumElements() / 2;
37269 SDValue Lo = extractSubVector(Rdx, 0, DAG, DL, HalfSize);
37270 SDValue Hi = extractSubVector(Rdx, HalfElts, DAG, DL, HalfSize);
37271 Rdx = DAG.getNode(ISD::ADD, DL, Lo.getValueType(), Lo, Hi);
37272 VecVT = Rdx.getValueType();
37273 }
37274 assert(VecVT == MVT::v16i8 && "v16i8 reduction expected")((VecVT == MVT::v16i8 && "v16i8 reduction expected") ?
static_cast<void> (0) : __assert_fail ("VecVT == MVT::v16i8 && \"v16i8 reduction expected\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37274, __PRETTY_FUNCTION__));
37275
37276 SDValue Hi = DAG.getVectorShuffle(
37277 MVT::v16i8, DL, Rdx, Rdx,
37278 {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
37279 Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
37280 Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
37281 getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
37282 Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
37283 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
37284 }
37285
37286 // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
37287 bool OptForSize = DAG.getMachineFunction().getFunction().hasOptSize();
37288 if (!Subtarget.hasFastHorizontalOps() && !OptForSize)
37289 return SDValue();
37290
37291 unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
37292
37293 // 256-bit horizontal instructions operate on 128-bit chunks rather than
37294 // across the whole vector, so we need an extract + hop preliminary stage.
37295 // This is the only step where the operands of the hop are not the same value.
37296 // TODO: We could extend this to handle 512-bit or even longer vectors.
37297 if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) ||
37298 ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {
37299 unsigned NumElts = VecVT.getVectorNumElements();
37300 SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
37301 SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
37302 Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
37303 VecVT = Rdx.getValueType();
37304 }
37305 if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&
37306 !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
37307 return SDValue();
37308
37309 // extract (add (shuf X), X), 0 --> extract (hadd X, X), 0
37310 unsigned ReductionSteps = Log2_32(VecVT.getVectorNumElements());
37311 for (unsigned i = 0; i != ReductionSteps; ++i)
37312 Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
37313
37314 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
37315}
37316
37317/// Detect vector gather/scatter index generation and convert it from being a
37318/// bunch of shuffles and extracts into a somewhat faster sequence.
37319/// For i686, the best sequence is apparently storing the value and loading
37320/// scalars back, while for x64 we should use 64-bit extracts and shifts.
37321static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
37322 TargetLowering::DAGCombinerInfo &DCI,
37323 const X86Subtarget &Subtarget) {
37324 if (SDValue NewOp = combineExtractWithShuffle(N, DAG, DCI, Subtarget))
37325 return NewOp;
37326
37327 SDValue InputVector = N->getOperand(0);
37328 SDValue EltIdx = N->getOperand(1);
37329 auto *CIdx = dyn_cast<ConstantSDNode>(EltIdx);
37330
37331 EVT SrcVT = InputVector.getValueType();
37332 EVT VT = N->getValueType(0);
37333 SDLoc dl(InputVector);
37334 bool IsPextr = N->getOpcode() != ISD::EXTRACT_VECTOR_ELT;
37335 unsigned NumSrcElts = SrcVT.getVectorNumElements();
37336
37337 if (CIdx && CIdx->getAPIntValue().uge(NumSrcElts))
37338 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
37339
37340 // Integer Constant Folding.
37341 if (CIdx && VT.isInteger()) {
37342 APInt UndefVecElts;
37343 SmallVector<APInt, 16> EltBits;
37344 unsigned VecEltBitWidth = SrcVT.getScalarSizeInBits();
37345 if (getTargetConstantBitsFromNode(InputVector, VecEltBitWidth, UndefVecElts,
37346 EltBits, true, false)) {
37347 uint64_t Idx = CIdx->getZExtValue();
37348 if (UndefVecElts[Idx])
37349 return IsPextr ? DAG.getConstant(0, dl, VT) : DAG.getUNDEF(VT);
37350 return DAG.getConstant(EltBits[Idx].zextOrSelf(VT.getScalarSizeInBits()),
37351 dl, VT);
37352 }
37353 }
37354
37355 if (IsPextr) {
37356 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
37357 if (TLI.SimplifyDemandedBits(
37358 SDValue(N, 0), APInt::getAllOnesValue(VT.getSizeInBits()), DCI))
37359 return SDValue(N, 0);
37360
37361 // PEXTR*(PINSR*(v, s, c), c) -> s (with implicit zext handling).
37362 if ((InputVector.getOpcode() == X86ISD::PINSRB ||
37363 InputVector.getOpcode() == X86ISD::PINSRW) &&
37364 InputVector.getOperand(2) == EltIdx) {
37365 assert(SrcVT == InputVector.getOperand(0).getValueType() &&((SrcVT == InputVector.getOperand(0).getValueType() &&
"Vector type mismatch") ? static_cast<void> (0) : __assert_fail
("SrcVT == InputVector.getOperand(0).getValueType() && \"Vector type mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37366, __PRETTY_FUNCTION__))
37366 "Vector type mismatch")((SrcVT == InputVector.getOperand(0).getValueType() &&
"Vector type mismatch") ? static_cast<void> (0) : __assert_fail
("SrcVT == InputVector.getOperand(0).getValueType() && \"Vector type mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37366, __PRETTY_FUNCTION__));
37367 SDValue Scl = InputVector.getOperand(1);
37368 Scl = DAG.getNode(ISD::TRUNCATE, dl, SrcVT.getScalarType(), Scl);
37369 return DAG.getZExtOrTrunc(Scl, dl, VT);
37370 }
37371
37372 // TODO - Remove this once we can handle the implicit zero-extension of
37373 // X86ISD::PEXTRW/X86ISD::PEXTRB in combineHorizontalPredicateResult and
37374 // combineBasicSADPattern.
37375 return SDValue();
37376 }
37377
37378 // Detect mmx extraction of all bits as a i64. It works better as a bitcast.
37379 if (InputVector.getOpcode() == ISD::BITCAST && InputVector.hasOneUse() &&
37380 VT == MVT::i64 && SrcVT == MVT::v1i64 && isNullConstant(EltIdx)) {
37381 SDValue MMXSrc = InputVector.getOperand(0);
37382
37383 // The bitcast source is a direct mmx result.
37384 if (MMXSrc.getValueType() == MVT::x86mmx)
37385 return DAG.getBitcast(VT, InputVector);
37386 }
37387
37388 // Detect mmx to i32 conversion through a v2i32 elt extract.
37389 if (InputVector.getOpcode() == ISD::BITCAST && InputVector.hasOneUse() &&
37390 VT == MVT::i32 && SrcVT == MVT::v2i32 && isNullConstant(EltIdx)) {
37391 SDValue MMXSrc = InputVector.getOperand(0);
37392
37393 // The bitcast source is a direct mmx result.
37394 if (MMXSrc.getValueType() == MVT::x86mmx)
37395 return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32, MMXSrc);
37396 }
37397
37398 // Check whether this extract is the root of a sum of absolute differences
37399 // pattern. This has to be done here because we really want it to happen
37400 // pre-legalization,
37401 if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
37402 return SAD;
37403
37404 // Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
37405 if (SDValue Cmp = combineHorizontalPredicateResult(N, DAG, Subtarget))
37406 return Cmp;
37407
37408 // Attempt to replace min/max v8i16/v16i8 reductions with PHMINPOSUW.
37409 if (SDValue MinMax = combineHorizontalMinMaxResult(N, DAG, Subtarget))
37410 return MinMax;
37411
37412 if (SDValue V = combineReductionToHorizontal(N, DAG, Subtarget))
37413 return V;
37414
37415 if (SDValue V = scalarizeExtEltFP(N, DAG))
37416 return V;
37417
37418 // Attempt to extract a i1 element by using MOVMSK to extract the signbits
37419 // and then testing the relevant element.
37420 if (CIdx && SrcVT.getScalarType() == MVT::i1) {
37421 SmallVector<SDNode *, 16> BoolExtracts;
37422 auto IsBoolExtract = [&BoolExtracts](SDNode *Use) {
37423 if (Use->getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
37424 isa<ConstantSDNode>(Use->getOperand(1)) &&
37425 Use->getValueType(0) == MVT::i1) {
37426 BoolExtracts.push_back(Use);
37427 return true;
37428 }
37429 return false;
37430 };
37431 if (all_of(InputVector->uses(), IsBoolExtract) &&
37432 BoolExtracts.size() > 1) {
37433 EVT BCVT = EVT::getIntegerVT(*DAG.getContext(), NumSrcElts);
37434 if (SDValue BC =
37435 combineBitcastvxi1(DAG, BCVT, InputVector, dl, Subtarget)) {
37436 for (SDNode *Use : BoolExtracts) {
37437 // extractelement vXi1 X, MaskIdx --> ((movmsk X) & Mask) == Mask
37438 unsigned MaskIdx = Use->getConstantOperandVal(1);
37439 APInt MaskBit = APInt::getOneBitSet(NumSrcElts, MaskIdx);
37440 SDValue Mask = DAG.getConstant(MaskBit, dl, BCVT);
37441 SDValue Res = DAG.getNode(ISD::AND, dl, BCVT, BC, Mask);
37442 Res = DAG.getSetCC(dl, MVT::i1, Res, Mask, ISD::SETEQ);
37443 DCI.CombineTo(Use, Res);
37444 }
37445 return SDValue(N, 0);
37446 }
37447 }
37448 }
37449
37450 return SDValue();
37451}
37452
37453/// If a vector select has an operand that is -1 or 0, try to simplify the
37454/// select to a bitwise logic operation.
37455/// TODO: Move to DAGCombiner, possibly using TargetLowering::hasAndNot()?
37456static SDValue
37457combineVSelectWithAllOnesOrZeros(SDNode *N, SelectionDAG &DAG,
37458 TargetLowering::DAGCombinerInfo &DCI,
37459 const X86Subtarget &Subtarget) {
37460 SDValue Cond = N->getOperand(0);
37461 SDValue LHS = N->getOperand(1);
37462 SDValue RHS = N->getOperand(2);
37463 EVT VT = LHS.getValueType();
37464 EVT CondVT = Cond.getValueType();
37465 SDLoc DL(N);
37466 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
37467
37468 if (N->getOpcode() != ISD::VSELECT)
37469 return SDValue();
37470
37471 assert(CondVT.isVector() && "Vector select expects a vector selector!")((CondVT.isVector() && "Vector select expects a vector selector!"
) ? static_cast<void> (0) : __assert_fail ("CondVT.isVector() && \"Vector select expects a vector selector!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37471, __PRETTY_FUNCTION__));
37472
37473 // Check if the first operand is all zeros and Cond type is vXi1.
37474 // This situation only applies to avx512.
37475 // TODO: Use isNullOrNullSplat() to distinguish constants with undefs?
37476 // TODO: Can we assert that both operands are not zeros (because that should
37477 // get simplified at node creation time)?
37478 bool TValIsAllZeros = ISD::isBuildVectorAllZeros(LHS.getNode());
37479 bool FValIsAllZeros = ISD::isBuildVectorAllZeros(RHS.getNode());
37480
37481 // If both inputs are 0/undef, create a complete zero vector.
37482 // FIXME: As noted above this should be handled by DAGCombiner/getNode.
37483 if (TValIsAllZeros && FValIsAllZeros) {
37484 if (VT.isFloatingPoint())
37485 return DAG.getConstantFP(0.0, DL, VT);
37486 return DAG.getConstant(0, DL, VT);
37487 }
37488
37489 if (TValIsAllZeros && !FValIsAllZeros && Subtarget.hasAVX512() &&
37490 Cond.hasOneUse() && CondVT.getVectorElementType() == MVT::i1) {
37491 // Invert the cond to not(cond) : xor(op,allones)=not(op)
37492 SDValue CondNew = DAG.getNOT(DL, Cond, CondVT);
37493 // Vselect cond, op1, op2 = Vselect not(cond), op2, op1
37494 return DAG.getSelect(DL, VT, CondNew, RHS, LHS);
37495 }
37496
37497 // To use the condition operand as a bitwise mask, it must have elements that
37498 // are the same size as the select elements. Ie, the condition operand must
37499 // have already been promoted from the IR select condition type <N x i1>.
37500 // Don't check if the types themselves are equal because that excludes
37501 // vector floating-point selects.
37502 if (CondVT.getScalarSizeInBits() != VT.getScalarSizeInBits())
37503 return SDValue();
37504
37505 // Try to invert the condition if true value is not all 1s and false value is
37506 // not all 0s. Only do this if the condition has one use.
37507 bool TValIsAllOnes = ISD::isBuildVectorAllOnes(LHS.getNode());
37508 if (!TValIsAllOnes && !FValIsAllZeros && Cond.hasOneUse() &&
37509 // Check if the selector will be produced by CMPP*/PCMP*.
37510 Cond.getOpcode() == ISD::SETCC &&
37511 // Check if SETCC has already been promoted.
37512 TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT) ==
37513 CondVT) {
37514 bool FValIsAllOnes = ISD::isBuildVectorAllOnes(RHS.getNode());
37515
37516 if (TValIsAllZeros || FValIsAllOnes) {
37517 SDValue CC = Cond.getOperand(2);
37518 ISD::CondCode NewCC = ISD::getSetCCInverse(
37519 cast<CondCodeSDNode>(CC)->get(), Cond.getOperand(0).getValueType());
37520 Cond = DAG.getSetCC(DL, CondVT, Cond.getOperand(0), Cond.getOperand(1),
37521 NewCC);
37522 std::swap(LHS, RHS);
37523 TValIsAllOnes = FValIsAllOnes;
37524 FValIsAllZeros = TValIsAllZeros;
37525 }
37526 }
37527
37528 // Cond value must be 'sign splat' to be converted to a logical op.
37529 if (DAG.ComputeNumSignBits(Cond) != CondVT.getScalarSizeInBits())
37530 return SDValue();
37531
37532 // vselect Cond, 111..., 000... -> Cond
37533 if (TValIsAllOnes && FValIsAllZeros)
37534 return DAG.getBitcast(VT, Cond);
37535
37536 if (!DCI.isBeforeLegalize() && !TLI.isTypeLegal(CondVT))
37537 return SDValue();
37538
37539 // vselect Cond, 111..., X -> or Cond, X
37540 if (TValIsAllOnes) {
37541 SDValue CastRHS = DAG.getBitcast(CondVT, RHS);
37542 SDValue Or = DAG.getNode(ISD::OR, DL, CondVT, Cond, CastRHS);
37543 return DAG.getBitcast(VT, Or);
37544 }
37545
37546 // vselect Cond, X, 000... -> and Cond, X
37547 if (FValIsAllZeros) {
37548 SDValue CastLHS = DAG.getBitcast(CondVT, LHS);
37549 SDValue And = DAG.getNode(ISD::AND, DL, CondVT, Cond, CastLHS);
37550 return DAG.getBitcast(VT, And);
37551 }
37552
37553 // vselect Cond, 000..., X -> andn Cond, X
37554 if (TValIsAllZeros) {
37555 MVT AndNVT = MVT::getVectorVT(MVT::i64, CondVT.getSizeInBits() / 64);
37556 SDValue CastCond = DAG.getBitcast(AndNVT, Cond);
37557 SDValue CastRHS = DAG.getBitcast(AndNVT, RHS);
37558 SDValue AndN = DAG.getNode(X86ISD::ANDNP, DL, AndNVT, CastCond, CastRHS);
37559 return DAG.getBitcast(VT, AndN);
37560 }
37561
37562 return SDValue();
37563}
37564
37565/// If both arms of a vector select are concatenated vectors, split the select,
37566/// and concatenate the result to eliminate a wide (256-bit) vector instruction:
37567/// vselect Cond, (concat T0, T1), (concat F0, F1) -->
37568/// concat (vselect (split Cond), T0, F0), (vselect (split Cond), T1, F1)
37569static SDValue narrowVectorSelect(SDNode *N, SelectionDAG &DAG,
37570 const X86Subtarget &Subtarget) {
37571 unsigned Opcode = N->getOpcode();
37572 if (Opcode != X86ISD::BLENDV && Opcode != ISD::VSELECT)
37573 return SDValue();
37574
37575 // TODO: Split 512-bit vectors too?
37576 EVT VT = N->getValueType(0);
37577 if (!VT.is256BitVector())
37578 return SDValue();
37579
37580 // TODO: Split as long as any 2 of the 3 operands are concatenated?
37581 SDValue Cond = N->getOperand(0);
37582 SDValue TVal = N->getOperand(1);
37583 SDValue FVal = N->getOperand(2);
37584 SmallVector<SDValue, 4> CatOpsT, CatOpsF;
37585 if (!TVal.hasOneUse() || !FVal.hasOneUse() ||
37586 !collectConcatOps(TVal.getNode(), CatOpsT) ||
37587 !collectConcatOps(FVal.getNode(), CatOpsF))
37588 return SDValue();
37589
37590 auto makeBlend = [Opcode](SelectionDAG &DAG, const SDLoc &DL,
37591 ArrayRef<SDValue> Ops) {
37592 return DAG.getNode(Opcode, DL, Ops[1].getValueType(), Ops);
37593 };
37594 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { Cond, TVal, FVal },
37595 makeBlend, /*CheckBWI*/ false);
37596}
37597
37598static SDValue combineSelectOfTwoConstants(SDNode *N, SelectionDAG &DAG) {
37599 SDValue Cond = N->getOperand(0);
37600 SDValue LHS = N->getOperand(1);
37601 SDValue RHS = N->getOperand(2);
37602 SDLoc DL(N);
37603
37604 auto *TrueC = dyn_cast<ConstantSDNode>(LHS);
37605 auto *FalseC = dyn_cast<ConstantSDNode>(RHS);
37606 if (!TrueC || !FalseC)
37607 return SDValue();
37608
37609 // Don't do this for crazy integer types.
37610 EVT VT = N->getValueType(0);
37611 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
37612 return SDValue();
37613
37614 // We're going to use the condition bit in math or logic ops. We could allow
37615 // this with a wider condition value (post-legalization it becomes an i8),
37616 // but if nothing is creating selects that late, it doesn't matter.
37617 if (Cond.getValueType() != MVT::i1)
37618 return SDValue();
37619
37620 // A power-of-2 multiply is just a shift. LEA also cheaply handles multiply by
37621 // 3, 5, or 9 with i32/i64, so those get transformed too.
37622 // TODO: For constants that overflow or do not differ by power-of-2 or small
37623 // multiplier, convert to 'and' + 'add'.
37624 const APInt &TrueVal = TrueC->getAPIntValue();
37625 const APInt &FalseVal = FalseC->getAPIntValue();
37626 bool OV;
37627 APInt Diff = TrueVal.ssub_ov(FalseVal, OV);
37628 if (OV)
37629 return SDValue();
37630
37631 APInt AbsDiff = Diff.abs();
37632 if (AbsDiff.isPowerOf2() ||
37633 ((VT == MVT::i32 || VT == MVT::i64) &&
37634 (AbsDiff == 3 || AbsDiff == 5 || AbsDiff == 9))) {
37635
37636 // We need a positive multiplier constant for shift/LEA codegen. The 'not'
37637 // of the condition can usually be folded into a compare predicate, but even
37638 // without that, the sequence should be cheaper than a CMOV alternative.
37639 if (TrueVal.slt(FalseVal)) {
37640 Cond = DAG.getNOT(DL, Cond, MVT::i1);
37641 std::swap(TrueC, FalseC);
37642 }
37643
37644 // select Cond, TC, FC --> (zext(Cond) * (TC - FC)) + FC
37645 SDValue R = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Cond);
37646
37647 // Multiply condition by the difference if non-one.
37648 if (!AbsDiff.isOneValue())
37649 R = DAG.getNode(ISD::MUL, DL, VT, R, DAG.getConstant(AbsDiff, DL, VT));
37650
37651 // Add the base if non-zero.
37652 if (!FalseC->isNullValue())
37653 R = DAG.getNode(ISD::ADD, DL, VT, R, SDValue(FalseC, 0));
37654
37655 return R;
37656 }
37657
37658 return SDValue();
37659}
37660
37661/// If this is a *dynamic* select (non-constant condition) and we can match
37662/// this node with one of the variable blend instructions, restructure the
37663/// condition so that blends can use the high (sign) bit of each element.
37664/// This function will also call SimplifyDemandedBits on already created
37665/// BLENDV to perform additional simplifications.
37666static SDValue combineVSelectToBLENDV(SDNode *N, SelectionDAG &DAG,
37667 TargetLowering::DAGCombinerInfo &DCI,
37668 const X86Subtarget &Subtarget) {
37669 SDValue Cond = N->getOperand(0);
37670 if ((N->getOpcode() != ISD::VSELECT &&
37671 N->getOpcode() != X86ISD::BLENDV) ||
37672 ISD::isBuildVectorOfConstantSDNodes(Cond.getNode()))
37673 return SDValue();
37674
37675 // Don't optimize before the condition has been transformed to a legal type
37676 // and don't ever optimize vector selects that map to AVX512 mask-registers.
37677 unsigned BitWidth = Cond.getScalarValueSizeInBits();
37678 if (BitWidth < 8 || BitWidth > 64)
37679 return SDValue();
37680
37681 // We can only handle the cases where VSELECT is directly legal on the
37682 // subtarget. We custom lower VSELECT nodes with constant conditions and
37683 // this makes it hard to see whether a dynamic VSELECT will correctly
37684 // lower, so we both check the operation's status and explicitly handle the
37685 // cases where a *dynamic* blend will fail even though a constant-condition
37686 // blend could be custom lowered.
37687 // FIXME: We should find a better way to handle this class of problems.
37688 // Potentially, we should combine constant-condition vselect nodes
37689 // pre-legalization into shuffles and not mark as many types as custom
37690 // lowered.
37691 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
37692 EVT VT = N->getValueType(0);
37693 if (!TLI.isOperationLegalOrCustom(ISD::VSELECT, VT))
37694 return SDValue();
37695 // FIXME: We don't support i16-element blends currently. We could and
37696 // should support them by making *all* the bits in the condition be set
37697 // rather than just the high bit and using an i8-element blend.
37698 if (VT.getVectorElementType() == MVT::i16)
37699 return SDValue();
37700 // Dynamic blending was only available from SSE4.1 onward.
37701 if (VT.is128BitVector() && !Subtarget.hasSSE41())
37702 return SDValue();
37703 // Byte blends are only available in AVX2
37704 if (VT == MVT::v32i8 && !Subtarget.hasAVX2())
37705 return SDValue();
37706 // There are no 512-bit blend instructions that use sign bits.
37707 if (VT.is512BitVector())
37708 return SDValue();
37709
37710 auto OnlyUsedAsSelectCond = [](SDValue Cond) {
37711 for (SDNode::use_iterator UI = Cond->use_begin(), UE = Cond->use_end();
37712 UI != UE; ++UI)
37713 if ((UI->getOpcode() != ISD::VSELECT &&
37714 UI->getOpcode() != X86ISD::BLENDV) ||
37715 UI.getOperandNo() != 0)
37716 return false;
37717
37718 return true;
37719 };
37720
37721 if (OnlyUsedAsSelectCond(Cond)) {
37722 APInt DemandedMask(APInt::getSignMask(BitWidth));
37723 KnownBits Known;
37724 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
37725 !DCI.isBeforeLegalizeOps());
37726 if (!TLI.SimplifyDemandedBits(Cond, DemandedMask, Known, TLO, 0, true))
37727 return SDValue();
37728
37729 // If we changed the computation somewhere in the DAG, this change will
37730 // affect all users of Cond. Update all the nodes so that we do not use
37731 // the generic VSELECT anymore. Otherwise, we may perform wrong
37732 // optimizations as we messed with the actual expectation for the vector
37733 // boolean values.
37734 for (SDNode *U : Cond->uses()) {
37735 if (U->getOpcode() == X86ISD::BLENDV)
37736 continue;
37737
37738 SDValue SB = DAG.getNode(X86ISD::BLENDV, SDLoc(U), U->getValueType(0),
37739 Cond, U->getOperand(1), U->getOperand(2));
37740 DAG.ReplaceAllUsesOfValueWith(SDValue(U, 0), SB);
37741 DCI.AddToWorklist(U);
37742 }
37743 DCI.CommitTargetLoweringOpt(TLO);
37744 return SDValue(N, 0);
37745 }
37746
37747 // Otherwise we can still at least try to simplify multiple use bits.
37748 APInt DemandedMask(APInt::getSignMask(BitWidth));
37749 APInt DemandedElts(APInt::getAllOnesValue(VT.getVectorNumElements()));
37750 KnownBits Known;
37751 TargetLowering::TargetLoweringOpt TLO(DAG, !DCI.isBeforeLegalize(),
37752 !DCI.isBeforeLegalizeOps());
37753 if (SDValue V = TLI.SimplifyMultipleUseDemandedBits(Cond, DemandedMask,
37754 DemandedElts, DAG, 0))
37755 return DAG.getNode(X86ISD::BLENDV, SDLoc(N), N->getValueType(0),
37756 V, N->getOperand(1), N->getOperand(2));
37757
37758 return SDValue();
37759}
37760
37761// Try to match:
37762// (or (and (M, (sub 0, X)), (pandn M, X)))
37763// which is a special case of:
37764// (select M, (sub 0, X), X)
37765// Per:
37766// http://graphics.stanford.edu/~seander/bithacks.html#ConditionalNegate
37767// We know that, if fNegate is 0 or 1:
37768// (fNegate ? -v : v) == ((v ^ -fNegate) + fNegate)
37769//
37770// Here, we have a mask, M (all 1s or 0), and, similarly, we know that:
37771// ((M & 1) ? -X : X) == ((X ^ -(M & 1)) + (M & 1))
37772// ( M ? -X : X) == ((X ^ M ) + (M & 1))
37773// This lets us transform our vselect to:
37774// (add (xor X, M), (and M, 1))
37775// And further to:
37776// (sub (xor X, M), M)
37777static SDValue combineLogicBlendIntoConditionalNegate(
37778 EVT VT, SDValue Mask, SDValue X, SDValue Y, const SDLoc &DL,
37779 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
37780 EVT MaskVT = Mask.getValueType();
37781 assert(MaskVT.isInteger() &&((MaskVT.isInteger() && DAG.ComputeNumSignBits(Mask) ==
MaskVT.getScalarSizeInBits() && "Mask must be zero/all-bits"
) ? static_cast<void> (0) : __assert_fail ("MaskVT.isInteger() && DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() && \"Mask must be zero/all-bits\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37783, __PRETTY_FUNCTION__))
37782 DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() &&((MaskVT.isInteger() && DAG.ComputeNumSignBits(Mask) ==
MaskVT.getScalarSizeInBits() && "Mask must be zero/all-bits"
) ? static_cast<void> (0) : __assert_fail ("MaskVT.isInteger() && DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() && \"Mask must be zero/all-bits\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37783, __PRETTY_FUNCTION__))
37783 "Mask must be zero/all-bits")((MaskVT.isInteger() && DAG.ComputeNumSignBits(Mask) ==
MaskVT.getScalarSizeInBits() && "Mask must be zero/all-bits"
) ? static_cast<void> (0) : __assert_fail ("MaskVT.isInteger() && DAG.ComputeNumSignBits(Mask) == MaskVT.getScalarSizeInBits() && \"Mask must be zero/all-bits\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 37783, __PRETTY_FUNCTION__));
37784
37785 if (X.getValueType() != MaskVT || Y.getValueType() != MaskVT)
37786 return SDValue();
37787 if (!DAG.getTargetLoweringInfo().isOperationLegal(ISD::SUB, MaskVT))
37788 return SDValue();
37789
37790 auto IsNegV = [](SDNode *N, SDValue V) {
37791 return N->getOpcode() == ISD::SUB && N->getOperand(1) == V &&
37792 ISD::isBuildVectorAllZeros(N->getOperand(0).getNode());
37793 };
37794
37795 SDValue V;
37796 if (IsNegV(Y.getNode(), X))
37797 V = X;
37798 else if (IsNegV(X.getNode(), Y))
37799 V = Y;
37800 else
37801 return SDValue();
37802
37803 SDValue SubOp1 = DAG.getNode(ISD::XOR, DL, MaskVT, V, Mask);
37804 SDValue SubOp2 = Mask;
37805
37806 // If the negate was on the false side of the select, then
37807 // the operands of the SUB need to be swapped. PR 27251.
37808 // This is because the pattern being matched above is
37809 // (vselect M, (sub (0, X), X) -> (sub (xor X, M), M)
37810 // but if the pattern matched was
37811 // (vselect M, X, (sub (0, X))), that is really negation of the pattern
37812 // above, -(vselect M, (sub 0, X), X), and therefore the replacement
37813 // pattern also needs to be a negation of the replacement pattern above.
37814 // And -(sub X, Y) is just sub (Y, X), so swapping the operands of the
37815 // sub accomplishes the negation of the replacement pattern.
37816 if (V == Y)
37817 std::swap(SubOp1, SubOp2);
37818
37819 SDValue Res = DAG.getNode(ISD::SUB, DL, MaskVT, SubOp1, SubOp2);
37820 return DAG.getBitcast(VT, Res);
37821}
37822
37823/// Do target-specific dag combines on SELECT and VSELECT nodes.
37824static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
37825 TargetLowering::DAGCombinerInfo &DCI,
37826 const X86Subtarget &Subtarget) {
37827 SDLoc DL(N);
37828 SDValue Cond = N->getOperand(0);
37829 SDValue LHS = N->getOperand(1);
37830 SDValue RHS = N->getOperand(2);
37831
37832 // Try simplification again because we use this function to optimize
37833 // BLENDV nodes that are not handled by the generic combiner.
37834 if (SDValue V = DAG.simplifySelect(Cond, LHS, RHS))
37835 return V;
37836
37837 EVT VT = LHS.getValueType();
37838 EVT CondVT = Cond.getValueType();
37839 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
37840 bool CondConstantVector = ISD::isBuildVectorOfConstantSDNodes(Cond.getNode());
37841
37842 // Attempt to combine (select M, (sub 0, X), X) -> (sub (xor X, M), M).
37843 // Limit this to cases of non-constant masks that createShuffleMaskFromVSELECT
37844 // can't catch, plus vXi8 cases where we'd likely end up with BLENDV.
37845 if (CondVT.isVector() && CondVT.isInteger() &&
37846 CondVT.getScalarSizeInBits() == VT.getScalarSizeInBits() &&
37847 (!CondConstantVector || CondVT.getScalarType() == MVT::i8) &&
37848 DAG.ComputeNumSignBits(Cond) == CondVT.getScalarSizeInBits())
37849 if (SDValue V = combineLogicBlendIntoConditionalNegate(VT, Cond, RHS, LHS,
37850 DL, DAG, Subtarget))
37851 return V;
37852
37853 // Convert vselects with constant condition into shuffles.
37854 if (CondConstantVector && DCI.isBeforeLegalizeOps()) {
37855 SmallVector<int, 64> Mask;
37856 if (createShuffleMaskFromVSELECT(Mask, Cond))
37857 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
37858 }
37859
37860 // If we have SSE[12] support, try to form min/max nodes. SSE min/max
37861 // instructions match the semantics of the common C idiom x<y?x:y but not
37862 // x<=y?x:y, because of how they handle negative zero (which can be
37863 // ignored in unsafe-math mode).
37864 // We also try to create v2f32 min/max nodes, which we later widen to v4f32.
37865 if (Cond.getOpcode() == ISD::SETCC && VT.isFloatingPoint() &&
37866 VT != MVT::f80 && VT != MVT::f128 &&
37867 (TLI.isTypeLegal(VT) || VT == MVT::v2f32) &&
37868 (Subtarget.hasSSE2() ||
37869 (Subtarget.hasSSE1() && VT.getScalarType() == MVT::f32))) {
37870 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
37871
37872 unsigned Opcode = 0;
37873 // Check for x CC y ? x : y.
37874 if (DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
37875 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
37876 switch (CC) {
37877 default: break;
37878 case ISD::SETULT:
37879 // Converting this to a min would handle NaNs incorrectly, and swapping
37880 // the operands would cause it to handle comparisons between positive
37881 // and negative zero incorrectly.
37882 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
37883 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
37884 !(DAG.isKnownNeverZeroFloat(LHS) ||
37885 DAG.isKnownNeverZeroFloat(RHS)))
37886 break;
37887 std::swap(LHS, RHS);
37888 }
37889 Opcode = X86ISD::FMIN;
37890 break;
37891 case ISD::SETOLE:
37892 // Converting this to a min would handle comparisons between positive
37893 // and negative zero incorrectly.
37894 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
37895 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
37896 break;
37897 Opcode = X86ISD::FMIN;
37898 break;
37899 case ISD::SETULE:
37900 // Converting this to a min would handle both negative zeros and NaNs
37901 // incorrectly, but we can swap the operands to fix both.
37902 std::swap(LHS, RHS);
37903 LLVM_FALLTHROUGH[[gnu::fallthrough]];
37904 case ISD::SETOLT:
37905 case ISD::SETLT:
37906 case ISD::SETLE:
37907 Opcode = X86ISD::FMIN;
37908 break;
37909
37910 case ISD::SETOGE:
37911 // Converting this to a max would handle comparisons between positive
37912 // and negative zero incorrectly.
37913 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
37914 !DAG.isKnownNeverZeroFloat(LHS) && !DAG.isKnownNeverZeroFloat(RHS))
37915 break;
37916 Opcode = X86ISD::FMAX;
37917 break;
37918 case ISD::SETUGT:
37919 // Converting this to a max would handle NaNs incorrectly, and swapping
37920 // the operands would cause it to handle comparisons between positive
37921 // and negative zero incorrectly.
37922 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS)) {
37923 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
37924 !(DAG.isKnownNeverZeroFloat(LHS) ||
37925 DAG.isKnownNeverZeroFloat(RHS)))
37926 break;
37927 std::swap(LHS, RHS);
37928 }
37929 Opcode = X86ISD::FMAX;
37930 break;
37931 case ISD::SETUGE:
37932 // Converting this to a max would handle both negative zeros and NaNs
37933 // incorrectly, but we can swap the operands to fix both.
37934 std::swap(LHS, RHS);
37935 LLVM_FALLTHROUGH[[gnu::fallthrough]];
37936 case ISD::SETOGT:
37937 case ISD::SETGT:
37938 case ISD::SETGE:
37939 Opcode = X86ISD::FMAX;
37940 break;
37941 }
37942 // Check for x CC y ? y : x -- a min/max with reversed arms.
37943 } else if (DAG.isEqualTo(LHS, Cond.getOperand(1)) &&
37944 DAG.isEqualTo(RHS, Cond.getOperand(0))) {
37945 switch (CC) {
37946 default: break;
37947 case ISD::SETOGE:
37948 // Converting this to a min would handle comparisons between positive
37949 // and negative zero incorrectly, and swapping the operands would
37950 // cause it to handle NaNs incorrectly.
37951 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
37952 !(DAG.isKnownNeverZeroFloat(LHS) ||
37953 DAG.isKnownNeverZeroFloat(RHS))) {
37954 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
37955 break;
37956 std::swap(LHS, RHS);
37957 }
37958 Opcode = X86ISD::FMIN;
37959 break;
37960 case ISD::SETUGT:
37961 // Converting this to a min would handle NaNs incorrectly.
37962 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
37963 break;
37964 Opcode = X86ISD::FMIN;
37965 break;
37966 case ISD::SETUGE:
37967 // Converting this to a min would handle both negative zeros and NaNs
37968 // incorrectly, but we can swap the operands to fix both.
37969 std::swap(LHS, RHS);
37970 LLVM_FALLTHROUGH[[gnu::fallthrough]];
37971 case ISD::SETOGT:
37972 case ISD::SETGT:
37973 case ISD::SETGE:
37974 Opcode = X86ISD::FMIN;
37975 break;
37976
37977 case ISD::SETULT:
37978 // Converting this to a max would handle NaNs incorrectly.
37979 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
37980 break;
37981 Opcode = X86ISD::FMAX;
37982 break;
37983 case ISD::SETOLE:
37984 // Converting this to a max would handle comparisons between positive
37985 // and negative zero incorrectly, and swapping the operands would
37986 // cause it to handle NaNs incorrectly.
37987 if (!DAG.getTarget().Options.NoSignedZerosFPMath &&
37988 !DAG.isKnownNeverZeroFloat(LHS) &&
37989 !DAG.isKnownNeverZeroFloat(RHS)) {
37990 if (!DAG.isKnownNeverNaN(LHS) || !DAG.isKnownNeverNaN(RHS))
37991 break;
37992 std::swap(LHS, RHS);
37993 }
37994 Opcode = X86ISD::FMAX;
37995 break;
37996 case ISD::SETULE:
37997 // Converting this to a max would handle both negative zeros and NaNs
37998 // incorrectly, but we can swap the operands to fix both.
37999 std::swap(LHS, RHS);
38000 LLVM_FALLTHROUGH[[gnu::fallthrough]];
38001 case ISD::SETOLT:
38002 case ISD::SETLT:
38003 case ISD::SETLE:
38004 Opcode = X86ISD::FMAX;
38005 break;
38006 }
38007 }
38008
38009 if (Opcode)
38010 return DAG.getNode(Opcode, DL, N->getValueType(0), LHS, RHS);
38011 }
38012
38013 // Some mask scalar intrinsics rely on checking if only one bit is set
38014 // and implement it in C code like this:
38015 // A[0] = (U & 1) ? A[0] : W[0];
38016 // This creates some redundant instructions that break pattern matching.
38017 // fold (select (setcc (and (X, 1), 0, seteq), Y, Z)) -> select(and(X, 1),Z,Y)
38018 if (Subtarget.hasAVX512() && N->getOpcode() == ISD::SELECT &&
38019 Cond.getOpcode() == ISD::SETCC && (VT == MVT::f32 || VT == MVT::f64)) {
38020 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
38021 SDValue AndNode = Cond.getOperand(0);
38022 if (AndNode.getOpcode() == ISD::AND && CC == ISD::SETEQ &&
38023 isNullConstant(Cond.getOperand(1)) &&
38024 isOneConstant(AndNode.getOperand(1))) {
38025 // LHS and RHS swapped due to
38026 // setcc outputting 1 when AND resulted in 0 and vice versa.
38027 AndNode = DAG.getZExtOrTrunc(AndNode, DL, MVT::i8);
38028 return DAG.getNode(ISD::SELECT, DL, VT, AndNode, RHS, LHS);
38029 }
38030 }
38031
38032 // v16i8 (select v16i1, v16i8, v16i8) does not have a proper
38033 // lowering on KNL. In this case we convert it to
38034 // v16i8 (select v16i8, v16i8, v16i8) and use AVX instruction.
38035 // The same situation all vectors of i8 and i16 without BWI.
38036 // Make sure we extend these even before type legalization gets a chance to
38037 // split wide vectors.
38038 // Since SKX these selects have a proper lowering.
38039 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && CondVT.isVector() &&
38040 CondVT.getVectorElementType() == MVT::i1 &&
38041 (VT.getVectorElementType() == MVT::i8 ||
38042 VT.getVectorElementType() == MVT::i16)) {
38043 Cond = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Cond);
38044 return DAG.getNode(N->getOpcode(), DL, VT, Cond, LHS, RHS);
38045 }
38046
38047 // AVX512 - Extend select with zero to merge with target shuffle.
38048 // select(mask, extract_subvector(shuffle(x)), zero) -->
38049 // extract_subvector(select(insert_subvector(mask), shuffle(x), zero))
38050 // TODO - support non target shuffles as well.
38051 if (Subtarget.hasAVX512() && CondVT.isVector() &&
38052 CondVT.getVectorElementType() == MVT::i1) {
38053 auto SelectableOp = [&TLI](SDValue Op) {
38054 return Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
38055 isTargetShuffle(Op.getOperand(0).getOpcode()) &&
38056 isNullConstant(Op.getOperand(1)) &&
38057 TLI.isTypeLegal(Op.getOperand(0).getValueType()) &&
38058 Op.hasOneUse() && Op.getOperand(0).hasOneUse();
38059 };
38060
38061 bool SelectableLHS = SelectableOp(LHS);
38062 bool SelectableRHS = SelectableOp(RHS);
38063 bool ZeroLHS = ISD::isBuildVectorAllZeros(LHS.getNode());
38064 bool ZeroRHS = ISD::isBuildVectorAllZeros(RHS.getNode());
38065
38066 if ((SelectableLHS && ZeroRHS) || (SelectableRHS && ZeroLHS)) {
38067 EVT SrcVT = SelectableLHS ? LHS.getOperand(0).getValueType()
38068 : RHS.getOperand(0).getValueType();
38069 unsigned NumSrcElts = SrcVT.getVectorNumElements();
38070 EVT SrcCondVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumSrcElts);
38071 LHS = insertSubVector(DAG.getUNDEF(SrcVT), LHS, 0, DAG, DL,
38072 VT.getSizeInBits());
38073 RHS = insertSubVector(DAG.getUNDEF(SrcVT), RHS, 0, DAG, DL,
38074 VT.getSizeInBits());
38075 Cond = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, SrcCondVT,
38076 DAG.getUNDEF(SrcCondVT), Cond,
38077 DAG.getIntPtrConstant(0, DL));
38078 SDValue Res = DAG.getSelect(DL, SrcVT, Cond, LHS, RHS);
38079 return extractSubVector(Res, 0, DAG, DL, VT.getSizeInBits());
38080 }
38081 }
38082
38083 if (SDValue V = combineSelectOfTwoConstants(N, DAG))
38084 return V;
38085
38086 // Canonicalize max and min:
38087 // (x > y) ? x : y -> (x >= y) ? x : y
38088 // (x < y) ? x : y -> (x <= y) ? x : y
38089 // This allows use of COND_S / COND_NS (see TranslateX86CC) which eliminates
38090 // the need for an extra compare
38091 // against zero. e.g.
38092 // (x - y) > 0 : (x - y) ? 0 -> (x - y) >= 0 : (x - y) ? 0
38093 // subl %esi, %edi
38094 // testl %edi, %edi
38095 // movl $0, %eax
38096 // cmovgl %edi, %eax
38097 // =>
38098 // xorl %eax, %eax
38099 // subl %esi, $edi
38100 // cmovsl %eax, %edi
38101 if (N->getOpcode() == ISD::SELECT && Cond.getOpcode() == ISD::SETCC &&
38102 Cond.hasOneUse() &&
38103 DAG.isEqualTo(LHS, Cond.getOperand(0)) &&
38104 DAG.isEqualTo(RHS, Cond.getOperand(1))) {
38105 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
38106 switch (CC) {
38107 default: break;
38108 case ISD::SETLT:
38109 case ISD::SETGT: {
38110 ISD::CondCode NewCC = (CC == ISD::SETLT) ? ISD::SETLE : ISD::SETGE;
38111 Cond = DAG.getSetCC(SDLoc(Cond), Cond.getValueType(),
38112 Cond.getOperand(0), Cond.getOperand(1), NewCC);
38113 return DAG.getSelect(DL, VT, Cond, LHS, RHS);
38114 }
38115 }
38116 }
38117
38118 // Match VSELECTs into subs with unsigned saturation.
38119 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
38120 // psubus is available in SSE2 for i8 and i16 vectors.
38121 Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 &&
38122 isPowerOf2_32(VT.getVectorNumElements()) &&
38123 (VT.getVectorElementType() == MVT::i8 ||
38124 VT.getVectorElementType() == MVT::i16)) {
38125 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
38126
38127 // Check if one of the arms of the VSELECT is a zero vector. If it's on the
38128 // left side invert the predicate to simplify logic below.
38129 SDValue Other;
38130 if (ISD::isBuildVectorAllZeros(LHS.getNode())) {
38131 Other = RHS;
38132 CC = ISD::getSetCCInverse(CC, VT.getVectorElementType());
38133 } else if (ISD::isBuildVectorAllZeros(RHS.getNode())) {
38134 Other = LHS;
38135 }
38136
38137 if (Other.getNode() && Other->getNumOperands() == 2 &&
38138 Other->getOperand(0) == Cond.getOperand(0)) {
38139 SDValue OpLHS = Other->getOperand(0), OpRHS = Other->getOperand(1);
38140 SDValue CondRHS = Cond->getOperand(1);
38141
38142 // Look for a general sub with unsigned saturation first.
38143 // x >= y ? x-y : 0 --> subus x, y
38144 // x > y ? x-y : 0 --> subus x, y
38145 if ((CC == ISD::SETUGE || CC == ISD::SETUGT) &&
38146 Other->getOpcode() == ISD::SUB && OpRHS == CondRHS)
38147 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
38148
38149 if (auto *OpRHSBV = dyn_cast<BuildVectorSDNode>(OpRHS)) {
38150 if (isa<BuildVectorSDNode>(CondRHS)) {
38151 // If the RHS is a constant we have to reverse the const
38152 // canonicalization.
38153 // x > C-1 ? x+-C : 0 --> subus x, C
38154 auto MatchUSUBSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
38155 return (!Op && !Cond) ||
38156 (Op && Cond &&
38157 Cond->getAPIntValue() == (-Op->getAPIntValue() - 1));
38158 };
38159 if (CC == ISD::SETUGT && Other->getOpcode() == ISD::ADD &&
38160 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUSUBSAT,
38161 /*AllowUndefs*/ true)) {
38162 OpRHS = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
38163 OpRHS);
38164 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
38165 }
38166
38167 // Another special case: If C was a sign bit, the sub has been
38168 // canonicalized into a xor.
38169 // FIXME: Would it be better to use computeKnownBits to determine
38170 // whether it's safe to decanonicalize the xor?
38171 // x s< 0 ? x^C : 0 --> subus x, C
38172 if (auto *OpRHSConst = OpRHSBV->getConstantSplatNode()) {
38173 if (CC == ISD::SETLT && Other.getOpcode() == ISD::XOR &&
38174 ISD::isBuildVectorAllZeros(CondRHS.getNode()) &&
38175 OpRHSConst->getAPIntValue().isSignMask()) {
38176 // Note that we have to rebuild the RHS constant here to ensure we
38177 // don't rely on particular values of undef lanes.
38178 OpRHS = DAG.getConstant(OpRHSConst->getAPIntValue(), DL, VT);
38179 return DAG.getNode(ISD::USUBSAT, DL, VT, OpLHS, OpRHS);
38180 }
38181 }
38182 }
38183 }
38184 }
38185 }
38186
38187 // Match VSELECTs into add with unsigned saturation.
38188 if (N->getOpcode() == ISD::VSELECT && Cond.getOpcode() == ISD::SETCC &&
38189 // paddus is available in SSE2 for i8 and i16 vectors.
38190 Subtarget.hasSSE2() && VT.getVectorNumElements() >= 2 &&
38191 isPowerOf2_32(VT.getVectorNumElements()) &&
38192 (VT.getVectorElementType() == MVT::i8 ||
38193 VT.getVectorElementType() == MVT::i16)) {
38194 ISD::CondCode CC = cast<CondCodeSDNode>(Cond.getOperand(2))->get();
38195
38196 SDValue CondLHS = Cond->getOperand(0);
38197 SDValue CondRHS = Cond->getOperand(1);
38198
38199 // Check if one of the arms of the VSELECT is vector with all bits set.
38200 // If it's on the left side invert the predicate to simplify logic below.
38201 SDValue Other;
38202 if (ISD::isBuildVectorAllOnes(LHS.getNode())) {
38203 Other = RHS;
38204 CC = ISD::getSetCCInverse(CC, VT.getVectorElementType());
38205 } else if (ISD::isBuildVectorAllOnes(RHS.getNode())) {
38206 Other = LHS;
38207 }
38208
38209 if (Other.getNode() && Other.getOpcode() == ISD::ADD) {
38210 SDValue OpLHS = Other.getOperand(0), OpRHS = Other.getOperand(1);
38211
38212 // Canonicalize condition operands.
38213 if (CC == ISD::SETUGE) {
38214 std::swap(CondLHS, CondRHS);
38215 CC = ISD::SETULE;
38216 }
38217
38218 // We can test against either of the addition operands.
38219 // x <= x+y ? x+y : ~0 --> addus x, y
38220 // x+y >= x ? x+y : ~0 --> addus x, y
38221 if (CC == ISD::SETULE && Other == CondRHS &&
38222 (OpLHS == CondLHS || OpRHS == CondLHS))
38223 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
38224
38225 if (isa<BuildVectorSDNode>(OpRHS) && isa<BuildVectorSDNode>(CondRHS) &&
38226 CondLHS == OpLHS) {
38227 // If the RHS is a constant we have to reverse the const
38228 // canonicalization.
38229 // x > ~C ? x+C : ~0 --> addus x, C
38230 auto MatchUADDSAT = [](ConstantSDNode *Op, ConstantSDNode *Cond) {
38231 return Cond->getAPIntValue() == ~Op->getAPIntValue();
38232 };
38233 if (CC == ISD::SETULE &&
38234 ISD::matchBinaryPredicate(OpRHS, CondRHS, MatchUADDSAT))
38235 return DAG.getNode(ISD::UADDSAT, DL, VT, OpLHS, OpRHS);
38236 }
38237 }
38238 }
38239
38240 // Early exit check
38241 if (!TLI.isTypeLegal(VT))
38242 return SDValue();
38243
38244 if (SDValue V = combineVSelectWithAllOnesOrZeros(N, DAG, DCI, Subtarget))
38245 return V;
38246
38247 if (SDValue V = combineVSelectToBLENDV(N, DAG, DCI, Subtarget))
38248 return V;
38249
38250 if (SDValue V = narrowVectorSelect(N, DAG, Subtarget))
38251 return V;
38252
38253 // select(~Cond, X, Y) -> select(Cond, Y, X)
38254 if (CondVT.getScalarType() != MVT::i1)
38255 if (SDValue CondNot = IsNOT(Cond, DAG))
38256 return DAG.getNode(N->getOpcode(), DL, VT,
38257 DAG.getBitcast(CondVT, CondNot), RHS, LHS);
38258
38259 // Custom action for SELECT MMX
38260 if (VT == MVT::x86mmx) {
38261 LHS = DAG.getBitcast(MVT::i64, LHS);
38262 RHS = DAG.getBitcast(MVT::i64, RHS);
38263 SDValue newSelect = DAG.getNode(ISD::SELECT, DL, MVT::i64, Cond, LHS, RHS);
38264 return DAG.getBitcast(VT, newSelect);
38265 }
38266
38267 return SDValue();
38268}
38269
38270/// Combine:
38271/// (brcond/cmov/setcc .., (cmp (atomic_load_add x, 1), 0), COND_S)
38272/// to:
38273/// (brcond/cmov/setcc .., (LADD x, 1), COND_LE)
38274/// i.e., reusing the EFLAGS produced by the LOCKed instruction.
38275/// Note that this is only legal for some op/cc combinations.
38276static SDValue combineSetCCAtomicArith(SDValue Cmp, X86::CondCode &CC,
38277 SelectionDAG &DAG,
38278 const X86Subtarget &Subtarget) {
38279 // This combine only operates on CMP-like nodes.
38280 if (!(Cmp.getOpcode() == X86ISD::CMP ||
38281 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
38282 return SDValue();
38283
38284 // Can't replace the cmp if it has more uses than the one we're looking at.
38285 // FIXME: We would like to be able to handle this, but would need to make sure
38286 // all uses were updated.
38287 if (!Cmp.hasOneUse())
38288 return SDValue();
38289
38290 // This only applies to variations of the common case:
38291 // (icmp slt x, 0) -> (icmp sle (add x, 1), 0)
38292 // (icmp sge x, 0) -> (icmp sgt (add x, 1), 0)
38293 // (icmp sle x, 0) -> (icmp slt (sub x, 1), 0)
38294 // (icmp sgt x, 0) -> (icmp sge (sub x, 1), 0)
38295 // Using the proper condcodes (see below), overflow is checked for.
38296
38297 // FIXME: We can generalize both constraints:
38298 // - XOR/OR/AND (if they were made to survive AtomicExpand)
38299 // - LHS != 1
38300 // if the result is compared.
38301
38302 SDValue CmpLHS = Cmp.getOperand(0);
38303 SDValue CmpRHS = Cmp.getOperand(1);
38304
38305 if (!CmpLHS.hasOneUse())
38306 return SDValue();
38307
38308 unsigned Opc = CmpLHS.getOpcode();
38309 if (Opc != ISD::ATOMIC_LOAD_ADD && Opc != ISD::ATOMIC_LOAD_SUB)
38310 return SDValue();
38311
38312 SDValue OpRHS = CmpLHS.getOperand(2);
38313 auto *OpRHSC = dyn_cast<ConstantSDNode>(OpRHS);
38314 if (!OpRHSC)
38315 return SDValue();
38316
38317 APInt Addend = OpRHSC->getAPIntValue();
38318 if (Opc == ISD::ATOMIC_LOAD_SUB)
38319 Addend = -Addend;
38320
38321 auto *CmpRHSC = dyn_cast<ConstantSDNode>(CmpRHS);
38322 if (!CmpRHSC)
38323 return SDValue();
38324
38325 APInt Comparison = CmpRHSC->getAPIntValue();
38326
38327 // If the addend is the negation of the comparison value, then we can do
38328 // a full comparison by emitting the atomic arithmetic as a locked sub.
38329 if (Comparison == -Addend) {
38330 // The CC is fine, but we need to rewrite the LHS of the comparison as an
38331 // atomic sub.
38332 auto *AN = cast<AtomicSDNode>(CmpLHS.getNode());
38333 auto AtomicSub = DAG.getAtomic(
38334 ISD::ATOMIC_LOAD_SUB, SDLoc(CmpLHS), CmpLHS.getValueType(),
38335 /*Chain*/ CmpLHS.getOperand(0), /*LHS*/ CmpLHS.getOperand(1),
38336 /*RHS*/ DAG.getConstant(-Addend, SDLoc(CmpRHS), CmpRHS.getValueType()),
38337 AN->getMemOperand());
38338 auto LockOp = lowerAtomicArithWithLOCK(AtomicSub, DAG, Subtarget);
38339 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0),
38340 DAG.getUNDEF(CmpLHS.getValueType()));
38341 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
38342 return LockOp;
38343 }
38344
38345 // We can handle comparisons with zero in a number of cases by manipulating
38346 // the CC used.
38347 if (!Comparison.isNullValue())
38348 return SDValue();
38349
38350 if (CC == X86::COND_S && Addend == 1)
38351 CC = X86::COND_LE;
38352 else if (CC == X86::COND_NS && Addend == 1)
38353 CC = X86::COND_G;
38354 else if (CC == X86::COND_G && Addend == -1)
38355 CC = X86::COND_GE;
38356 else if (CC == X86::COND_LE && Addend == -1)
38357 CC = X86::COND_L;
38358 else
38359 return SDValue();
38360
38361 SDValue LockOp = lowerAtomicArithWithLOCK(CmpLHS, DAG, Subtarget);
38362 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(0),
38363 DAG.getUNDEF(CmpLHS.getValueType()));
38364 DAG.ReplaceAllUsesOfValueWith(CmpLHS.getValue(1), LockOp.getValue(1));
38365 return LockOp;
38366}
38367
38368// Check whether a boolean test is testing a boolean value generated by
38369// X86ISD::SETCC. If so, return the operand of that SETCC and proper condition
38370// code.
38371//
38372// Simplify the following patterns:
38373// (Op (CMP (SETCC Cond EFLAGS) 1) EQ) or
38374// (Op (CMP (SETCC Cond EFLAGS) 0) NEQ)
38375// to (Op EFLAGS Cond)
38376//
38377// (Op (CMP (SETCC Cond EFLAGS) 0) EQ) or
38378// (Op (CMP (SETCC Cond EFLAGS) 1) NEQ)
38379// to (Op EFLAGS !Cond)
38380//
38381// where Op could be BRCOND or CMOV.
38382//
38383static SDValue checkBoolTestSetCCCombine(SDValue Cmp, X86::CondCode &CC) {
38384 // This combine only operates on CMP-like nodes.
38385 if (!(Cmp.getOpcode() == X86ISD::CMP ||
38386 (Cmp.getOpcode() == X86ISD::SUB && !Cmp->hasAnyUseOfValue(0))))
38387 return SDValue();
38388
38389 // Quit if not used as a boolean value.
38390 if (CC != X86::COND_E && CC != X86::COND_NE)
38391 return SDValue();
38392
38393 // Check CMP operands. One of them should be 0 or 1 and the other should be
38394 // an SetCC or extended from it.
38395 SDValue Op1 = Cmp.getOperand(0);
38396 SDValue Op2 = Cmp.getOperand(1);
38397
38398 SDValue SetCC;
38399 const ConstantSDNode* C = nullptr;
38400 bool needOppositeCond = (CC == X86::COND_E);
38401 bool checkAgainstTrue = false; // Is it a comparison against 1?
38402
38403 if ((C = dyn_cast<ConstantSDNode>(Op1)))
38404 SetCC = Op2;
38405 else if ((C = dyn_cast<ConstantSDNode>(Op2)))
38406 SetCC = Op1;
38407 else // Quit if all operands are not constants.
38408 return SDValue();
38409
38410 if (C->getZExtValue() == 1) {
38411 needOppositeCond = !needOppositeCond;
38412 checkAgainstTrue = true;
38413 } else if (C->getZExtValue() != 0)
38414 // Quit if the constant is neither 0 or 1.
38415 return SDValue();
38416
38417 bool truncatedToBoolWithAnd = false;
38418 // Skip (zext $x), (trunc $x), or (and $x, 1) node.
38419 while (SetCC.getOpcode() == ISD::ZERO_EXTEND ||
38420 SetCC.getOpcode() == ISD::TRUNCATE ||
38421 SetCC.getOpcode() == ISD::AND) {
38422 if (SetCC.getOpcode() == ISD::AND) {
38423 int OpIdx = -1;
38424 if (isOneConstant(SetCC.getOperand(0)))
38425 OpIdx = 1;
38426 if (isOneConstant(SetCC.getOperand(1)))
38427 OpIdx = 0;
38428 if (OpIdx < 0)
38429 break;
38430 SetCC = SetCC.getOperand(OpIdx);
38431 truncatedToBoolWithAnd = true;
38432 } else
38433 SetCC = SetCC.getOperand(0);
38434 }
38435
38436 switch (SetCC.getOpcode()) {
38437 case X86ISD::SETCC_CARRY:
38438 // Since SETCC_CARRY gives output based on R = CF ? ~0 : 0, it's unsafe to
38439 // simplify it if the result of SETCC_CARRY is not canonicalized to 0 or 1,
38440 // i.e. it's a comparison against true but the result of SETCC_CARRY is not
38441 // truncated to i1 using 'and'.
38442 if (checkAgainstTrue && !truncatedToBoolWithAnd)
38443 break;
38444 assert(X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B &&((X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B
&& "Invalid use of SETCC_CARRY!") ? static_cast<void
> (0) : __assert_fail ("X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B && \"Invalid use of SETCC_CARRY!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 38445, __PRETTY_FUNCTION__))
38445 "Invalid use of SETCC_CARRY!")((X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B
&& "Invalid use of SETCC_CARRY!") ? static_cast<void
> (0) : __assert_fail ("X86::CondCode(SetCC.getConstantOperandVal(0)) == X86::COND_B && \"Invalid use of SETCC_CARRY!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 38445, __PRETTY_FUNCTION__));
38446 LLVM_FALLTHROUGH[[gnu::fallthrough]];
38447 case X86ISD::SETCC:
38448 // Set the condition code or opposite one if necessary.
38449 CC = X86::CondCode(SetCC.getConstantOperandVal(0));
38450 if (needOppositeCond)
38451 CC = X86::GetOppositeBranchCondition(CC);
38452 return SetCC.getOperand(1);
38453 case X86ISD::CMOV: {
38454 // Check whether false/true value has canonical one, i.e. 0 or 1.
38455 ConstantSDNode *FVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(0));
38456 ConstantSDNode *TVal = dyn_cast<ConstantSDNode>(SetCC.getOperand(1));
38457 // Quit if true value is not a constant.
38458 if (!TVal)
38459 return SDValue();
38460 // Quit if false value is not a constant.
38461 if (!FVal) {
38462 SDValue Op = SetCC.getOperand(0);
38463 // Skip 'zext' or 'trunc' node.
38464 if (Op.getOpcode() == ISD::ZERO_EXTEND ||
38465 Op.getOpcode() == ISD::TRUNCATE)
38466 Op = Op.getOperand(0);
38467 // A special case for rdrand/rdseed, where 0 is set if false cond is
38468 // found.
38469 if ((Op.getOpcode() != X86ISD::RDRAND &&
38470 Op.getOpcode() != X86ISD::RDSEED) || Op.getResNo() != 0)
38471 return SDValue();
38472 }
38473 // Quit if false value is not the constant 0 or 1.
38474 bool FValIsFalse = true;
38475 if (FVal && FVal->getZExtValue() != 0) {
38476 if (FVal->getZExtValue() != 1)
38477 return SDValue();
38478 // If FVal is 1, opposite cond is needed.
38479 needOppositeCond = !needOppositeCond;
38480 FValIsFalse = false;
38481 }
38482 // Quit if TVal is not the constant opposite of FVal.
38483 if (FValIsFalse && TVal->getZExtValue() != 1)
38484 return SDValue();
38485 if (!FValIsFalse && TVal->getZExtValue() != 0)
38486 return SDValue();
38487 CC = X86::CondCode(SetCC.getConstantOperandVal(2));
38488 if (needOppositeCond)
38489 CC = X86::GetOppositeBranchCondition(CC);
38490 return SetCC.getOperand(3);
38491 }
38492 }
38493
38494 return SDValue();
38495}
38496
38497/// Check whether Cond is an AND/OR of SETCCs off of the same EFLAGS.
38498/// Match:
38499/// (X86or (X86setcc) (X86setcc))
38500/// (X86cmp (and (X86setcc) (X86setcc)), 0)
38501static bool checkBoolTestAndOrSetCCCombine(SDValue Cond, X86::CondCode &CC0,
38502 X86::CondCode &CC1, SDValue &Flags,
38503 bool &isAnd) {
38504 if (Cond->getOpcode() == X86ISD::CMP) {
38505 if (!isNullConstant(Cond->getOperand(1)))
38506 return false;
38507
38508 Cond = Cond->getOperand(0);
38509 }
38510
38511 isAnd = false;
38512
38513 SDValue SetCC0, SetCC1;
38514 switch (Cond->getOpcode()) {
38515 default: return false;
38516 case ISD::AND:
38517 case X86ISD::AND:
38518 isAnd = true;
38519 LLVM_FALLTHROUGH[[gnu::fallthrough]];
38520 case ISD::OR:
38521 case X86ISD::OR:
38522 SetCC0 = Cond->getOperand(0);
38523 SetCC1 = Cond->getOperand(1);
38524 break;
38525 };
38526
38527 // Make sure we have SETCC nodes, using the same flags value.
38528 if (SetCC0.getOpcode() != X86ISD::SETCC ||
38529 SetCC1.getOpcode() != X86ISD::SETCC ||
38530 SetCC0->getOperand(1) != SetCC1->getOperand(1))
38531 return false;
38532
38533 CC0 = (X86::CondCode)SetCC0->getConstantOperandVal(0);
38534 CC1 = (X86::CondCode)SetCC1->getConstantOperandVal(0);
38535 Flags = SetCC0->getOperand(1);
38536 return true;
38537}
38538
38539// When legalizing carry, we create carries via add X, -1
38540// If that comes from an actual carry, via setcc, we use the
38541// carry directly.
38542static SDValue combineCarryThroughADD(SDValue EFLAGS, SelectionDAG &DAG) {
38543 if (EFLAGS.getOpcode() == X86ISD::ADD) {
38544 if (isAllOnesConstant(EFLAGS.getOperand(1))) {
38545 SDValue Carry = EFLAGS.getOperand(0);
38546 while (Carry.getOpcode() == ISD::TRUNCATE ||
38547 Carry.getOpcode() == ISD::ZERO_EXTEND ||
38548 Carry.getOpcode() == ISD::SIGN_EXTEND ||
38549 Carry.getOpcode() == ISD::ANY_EXTEND ||
38550 (Carry.getOpcode() == ISD::AND &&
38551 isOneConstant(Carry.getOperand(1))))
38552 Carry = Carry.getOperand(0);
38553 if (Carry.getOpcode() == X86ISD::SETCC ||
38554 Carry.getOpcode() == X86ISD::SETCC_CARRY) {
38555 // TODO: Merge this code with equivalent in combineAddOrSubToADCOrSBB?
38556 uint64_t CarryCC = Carry.getConstantOperandVal(0);
38557 SDValue CarryOp1 = Carry.getOperand(1);
38558 if (CarryCC == X86::COND_B)
38559 return CarryOp1;
38560 if (CarryCC == X86::COND_A) {
38561 // Try to convert COND_A into COND_B in an attempt to facilitate
38562 // materializing "setb reg".
38563 //
38564 // Do not flip "e > c", where "c" is a constant, because Cmp
38565 // instruction cannot take an immediate as its first operand.
38566 //
38567 if (CarryOp1.getOpcode() == X86ISD::SUB &&
38568 CarryOp1.getNode()->hasOneUse() &&
38569 CarryOp1.getValueType().isInteger() &&
38570 !isa<ConstantSDNode>(CarryOp1.getOperand(1))) {
38571 SDValue SubCommute =
38572 DAG.getNode(X86ISD::SUB, SDLoc(CarryOp1), CarryOp1->getVTList(),
38573 CarryOp1.getOperand(1), CarryOp1.getOperand(0));
38574 return SDValue(SubCommute.getNode(), CarryOp1.getResNo());
38575 }
38576 }
38577 // If this is a check of the z flag of an add with 1, switch to the
38578 // C flag.
38579 if (CarryCC == X86::COND_E &&
38580 CarryOp1.getOpcode() == X86ISD::ADD &&
38581 isOneConstant(CarryOp1.getOperand(1)))
38582 return CarryOp1;
38583 }
38584 }
38585 }
38586
38587 return SDValue();
38588}
38589
38590/// Optimize an EFLAGS definition used according to the condition code \p CC
38591/// into a simpler EFLAGS value, potentially returning a new \p CC and replacing
38592/// uses of chain values.
38593static SDValue combineSetCCEFLAGS(SDValue EFLAGS, X86::CondCode &CC,
38594 SelectionDAG &DAG,
38595 const X86Subtarget &Subtarget) {
38596 if (CC == X86::COND_B)
38597 if (SDValue Flags = combineCarryThroughADD(EFLAGS, DAG))
38598 return Flags;
38599
38600 if (SDValue R = checkBoolTestSetCCCombine(EFLAGS, CC))
38601 return R;
38602 return combineSetCCAtomicArith(EFLAGS, CC, DAG, Subtarget);
38603}
38604
38605/// Optimize X86ISD::CMOV [LHS, RHS, CONDCODE (e.g. X86::COND_NE), CONDVAL]
38606static SDValue combineCMov(SDNode *N, SelectionDAG &DAG,
38607 TargetLowering::DAGCombinerInfo &DCI,
38608 const X86Subtarget &Subtarget) {
38609 SDLoc DL(N);
38610
38611 SDValue FalseOp = N->getOperand(0);
38612 SDValue TrueOp = N->getOperand(1);
38613 X86::CondCode CC = (X86::CondCode)N->getConstantOperandVal(2);
38614 SDValue Cond = N->getOperand(3);
38615
38616 // cmov X, X, ?, ? --> X
38617 if (TrueOp == FalseOp)
38618 return TrueOp;
38619
38620 // Try to simplify the EFLAGS and condition code operands.
38621 // We can't always do this as FCMOV only supports a subset of X86 cond.
38622 if (SDValue Flags = combineSetCCEFLAGS(Cond, CC, DAG, Subtarget)) {
38623 if (FalseOp.getValueType() != MVT::f80 || hasFPCMov(CC)) {
38624 SDValue Ops[] = {FalseOp, TrueOp, DAG.getTargetConstant(CC, DL, MVT::i8),
38625 Flags};
38626 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
38627 }
38628 }
38629
38630 // If this is a select between two integer constants, try to do some
38631 // optimizations. Note that the operands are ordered the opposite of SELECT
38632 // operands.
38633 if (ConstantSDNode *TrueC = dyn_cast<ConstantSDNode>(TrueOp)) {
38634 if (ConstantSDNode *FalseC = dyn_cast<ConstantSDNode>(FalseOp)) {
38635 // Canonicalize the TrueC/FalseC values so that TrueC (the true value) is
38636 // larger than FalseC (the false value).
38637 if (TrueC->getAPIntValue().ult(FalseC->getAPIntValue())) {
38638 CC = X86::GetOppositeBranchCondition(CC);
38639 std::swap(TrueC, FalseC);
38640 std::swap(TrueOp, FalseOp);
38641 }
38642
38643 // Optimize C ? 8 : 0 -> zext(setcc(C)) << 3. Likewise for any pow2/0.
38644 // This is efficient for any integer data type (including i8/i16) and
38645 // shift amount.
38646 if (FalseC->getAPIntValue() == 0 && TrueC->getAPIntValue().isPowerOf2()) {
38647 Cond = getSETCC(CC, Cond, DL, DAG);
38648
38649 // Zero extend the condition if needed.
38650 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, TrueC->getValueType(0), Cond);
38651
38652 unsigned ShAmt = TrueC->getAPIntValue().logBase2();
38653 Cond = DAG.getNode(ISD::SHL, DL, Cond.getValueType(), Cond,
38654 DAG.getConstant(ShAmt, DL, MVT::i8));
38655 return Cond;
38656 }
38657
38658 // Optimize Cond ? cst+1 : cst -> zext(setcc(C)+cst. This is efficient
38659 // for any integer data type, including i8/i16.
38660 if (FalseC->getAPIntValue()+1 == TrueC->getAPIntValue()) {
38661 Cond = getSETCC(CC, Cond, DL, DAG);
38662
38663 // Zero extend the condition if needed.
38664 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL,
38665 FalseC->getValueType(0), Cond);
38666 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
38667 SDValue(FalseC, 0));
38668 return Cond;
38669 }
38670
38671 // Optimize cases that will turn into an LEA instruction. This requires
38672 // an i32 or i64 and an efficient multiplier (1, 2, 3, 4, 5, 8, 9).
38673 if (N->getValueType(0) == MVT::i32 || N->getValueType(0) == MVT::i64) {
38674 APInt Diff = TrueC->getAPIntValue() - FalseC->getAPIntValue();
38675 assert(Diff.getBitWidth() == N->getValueType(0).getSizeInBits() &&((Diff.getBitWidth() == N->getValueType(0).getSizeInBits()
&& "Implicit constant truncation") ? static_cast<
void> (0) : __assert_fail ("Diff.getBitWidth() == N->getValueType(0).getSizeInBits() && \"Implicit constant truncation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 38676, __PRETTY_FUNCTION__))
38676 "Implicit constant truncation")((Diff.getBitWidth() == N->getValueType(0).getSizeInBits()
&& "Implicit constant truncation") ? static_cast<
void> (0) : __assert_fail ("Diff.getBitWidth() == N->getValueType(0).getSizeInBits() && \"Implicit constant truncation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 38676, __PRETTY_FUNCTION__));
38677
38678 bool isFastMultiplier = false;
38679 if (Diff.ult(10)) {
38680 switch (Diff.getZExtValue()) {
38681 default: break;
38682 case 1: // result = add base, cond
38683 case 2: // result = lea base( , cond*2)
38684 case 3: // result = lea base(cond, cond*2)
38685 case 4: // result = lea base( , cond*4)
38686 case 5: // result = lea base(cond, cond*4)
38687 case 8: // result = lea base( , cond*8)
38688 case 9: // result = lea base(cond, cond*8)
38689 isFastMultiplier = true;
38690 break;
38691 }
38692 }
38693
38694 if (isFastMultiplier) {
38695 Cond = getSETCC(CC, Cond, DL ,DAG);
38696 // Zero extend the condition if needed.
38697 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, FalseC->getValueType(0),
38698 Cond);
38699 // Scale the condition by the difference.
38700 if (Diff != 1)
38701 Cond = DAG.getNode(ISD::MUL, DL, Cond.getValueType(), Cond,
38702 DAG.getConstant(Diff, DL, Cond.getValueType()));
38703
38704 // Add the base if non-zero.
38705 if (FalseC->getAPIntValue() != 0)
38706 Cond = DAG.getNode(ISD::ADD, DL, Cond.getValueType(), Cond,
38707 SDValue(FalseC, 0));
38708 return Cond;
38709 }
38710 }
38711 }
38712 }
38713
38714 // Handle these cases:
38715 // (select (x != c), e, c) -> select (x != c), e, x),
38716 // (select (x == c), c, e) -> select (x == c), x, e)
38717 // where the c is an integer constant, and the "select" is the combination
38718 // of CMOV and CMP.
38719 //
38720 // The rationale for this change is that the conditional-move from a constant
38721 // needs two instructions, however, conditional-move from a register needs
38722 // only one instruction.
38723 //
38724 // CAVEAT: By replacing a constant with a symbolic value, it may obscure
38725 // some instruction-combining opportunities. This opt needs to be
38726 // postponed as late as possible.
38727 //
38728 if (!DCI.isBeforeLegalize() && !DCI.isBeforeLegalizeOps()) {
38729 // the DCI.xxxx conditions are provided to postpone the optimization as
38730 // late as possible.
38731
38732 ConstantSDNode *CmpAgainst = nullptr;
38733 if ((Cond.getOpcode() == X86ISD::CMP || Cond.getOpcode() == X86ISD::SUB) &&
38734 (CmpAgainst = dyn_cast<ConstantSDNode>(Cond.getOperand(1))) &&
38735 !isa<ConstantSDNode>(Cond.getOperand(0))) {
38736
38737 if (CC == X86::COND_NE &&
38738 CmpAgainst == dyn_cast<ConstantSDNode>(FalseOp)) {
38739 CC = X86::GetOppositeBranchCondition(CC);
38740 std::swap(TrueOp, FalseOp);
38741 }
38742
38743 if (CC == X86::COND_E &&
38744 CmpAgainst == dyn_cast<ConstantSDNode>(TrueOp)) {
38745 SDValue Ops[] = {FalseOp, Cond.getOperand(0),
38746 DAG.getTargetConstant(CC, DL, MVT::i8), Cond};
38747 return DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
38748 }
38749 }
38750 }
38751
38752 // Fold and/or of setcc's to double CMOV:
38753 // (CMOV F, T, ((cc1 | cc2) != 0)) -> (CMOV (CMOV F, T, cc1), T, cc2)
38754 // (CMOV F, T, ((cc1 & cc2) != 0)) -> (CMOV (CMOV T, F, !cc1), F, !cc2)
38755 //
38756 // This combine lets us generate:
38757 // cmovcc1 (jcc1 if we don't have CMOV)
38758 // cmovcc2 (same)
38759 // instead of:
38760 // setcc1
38761 // setcc2
38762 // and/or
38763 // cmovne (jne if we don't have CMOV)
38764 // When we can't use the CMOV instruction, it might increase branch
38765 // mispredicts.
38766 // When we can use CMOV, or when there is no mispredict, this improves
38767 // throughput and reduces register pressure.
38768 //
38769 if (CC == X86::COND_NE) {
38770 SDValue Flags;
38771 X86::CondCode CC0, CC1;
38772 bool isAndSetCC;
38773 if (checkBoolTestAndOrSetCCCombine(Cond, CC0, CC1, Flags, isAndSetCC)) {
38774 if (isAndSetCC) {
38775 std::swap(FalseOp, TrueOp);
38776 CC0 = X86::GetOppositeBranchCondition(CC0);
38777 CC1 = X86::GetOppositeBranchCondition(CC1);
38778 }
38779
38780 SDValue LOps[] = {FalseOp, TrueOp,
38781 DAG.getTargetConstant(CC0, DL, MVT::i8), Flags};
38782 SDValue LCMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), LOps);
38783 SDValue Ops[] = {LCMOV, TrueOp, DAG.getTargetConstant(CC1, DL, MVT::i8),
38784 Flags};
38785 SDValue CMOV = DAG.getNode(X86ISD::CMOV, DL, N->getValueType(0), Ops);
38786 return CMOV;
38787 }
38788 }
38789
38790 // Fold (CMOV C1, (ADD (CTTZ X), C2), (X != 0)) ->
38791 // (ADD (CMOV C1-C2, (CTTZ X), (X != 0)), C2)
38792 // Or (CMOV (ADD (CTTZ X), C2), C1, (X == 0)) ->
38793 // (ADD (CMOV (CTTZ X), C1-C2, (X == 0)), C2)
38794 if ((CC == X86::COND_NE || CC == X86::COND_E) &&
38795 Cond.getOpcode() == X86ISD::CMP && isNullConstant(Cond.getOperand(1))) {
38796 SDValue Add = TrueOp;
38797 SDValue Const = FalseOp;
38798 // Canonicalize the condition code for easier matching and output.
38799 if (CC == X86::COND_E)
38800 std::swap(Add, Const);
38801
38802 // We might have replaced the constant in the cmov with the LHS of the
38803 // compare. If so change it to the RHS of the compare.
38804 if (Const == Cond.getOperand(0))
38805 Const = Cond.getOperand(1);
38806
38807 // Ok, now make sure that Add is (add (cttz X), C2) and Const is a constant.
38808 if (isa<ConstantSDNode>(Const) && Add.getOpcode() == ISD::ADD &&
38809 Add.hasOneUse() && isa<ConstantSDNode>(Add.getOperand(1)) &&
38810 (Add.getOperand(0).getOpcode() == ISD::CTTZ_ZERO_UNDEF ||
38811 Add.getOperand(0).getOpcode() == ISD::CTTZ) &&
38812 Add.getOperand(0).getOperand(0) == Cond.getOperand(0)) {
38813 EVT VT = N->getValueType(0);
38814 // This should constant fold.
38815 SDValue Diff = DAG.getNode(ISD::SUB, DL, VT, Const, Add.getOperand(1));
38816 SDValue CMov =
38817 DAG.getNode(X86ISD::CMOV, DL, VT, Diff, Add.getOperand(0),
38818 DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8), Cond);
38819 return DAG.getNode(ISD::ADD, DL, VT, CMov, Add.getOperand(1));
38820 }
38821 }
38822
38823 return SDValue();
38824}
38825
38826/// Different mul shrinking modes.
38827enum class ShrinkMode { MULS8, MULU8, MULS16, MULU16 };
38828
38829static bool canReduceVMulWidth(SDNode *N, SelectionDAG &DAG, ShrinkMode &Mode) {
38830 EVT VT = N->getOperand(0).getValueType();
38831 if (VT.getScalarSizeInBits() != 32)
38832 return false;
38833
38834 assert(N->getNumOperands() == 2 && "NumOperands of Mul are 2")((N->getNumOperands() == 2 && "NumOperands of Mul are 2"
) ? static_cast<void> (0) : __assert_fail ("N->getNumOperands() == 2 && \"NumOperands of Mul are 2\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 38834, __PRETTY_FUNCTION__));
38835 unsigned SignBits[2] = {1, 1};
38836 bool IsPositive[2] = {false, false};
38837 for (unsigned i = 0; i < 2; i++) {
38838 SDValue Opd = N->getOperand(i);
38839
38840 SignBits[i] = DAG.ComputeNumSignBits(Opd);
38841 IsPositive[i] = DAG.SignBitIsZero(Opd);
38842 }
38843
38844 bool AllPositive = IsPositive[0] && IsPositive[1];
38845 unsigned MinSignBits = std::min(SignBits[0], SignBits[1]);
38846 // When ranges are from -128 ~ 127, use MULS8 mode.
38847 if (MinSignBits >= 25)
38848 Mode = ShrinkMode::MULS8;
38849 // When ranges are from 0 ~ 255, use MULU8 mode.
38850 else if (AllPositive && MinSignBits >= 24)
38851 Mode = ShrinkMode::MULU8;
38852 // When ranges are from -32768 ~ 32767, use MULS16 mode.
38853 else if (MinSignBits >= 17)
38854 Mode = ShrinkMode::MULS16;
38855 // When ranges are from 0 ~ 65535, use MULU16 mode.
38856 else if (AllPositive && MinSignBits >= 16)
38857 Mode = ShrinkMode::MULU16;
38858 else
38859 return false;
38860 return true;
38861}
38862
38863/// When the operands of vector mul are extended from smaller size values,
38864/// like i8 and i16, the type of mul may be shrinked to generate more
38865/// efficient code. Two typical patterns are handled:
38866/// Pattern1:
38867/// %2 = sext/zext <N x i8> %1 to <N x i32>
38868/// %4 = sext/zext <N x i8> %3 to <N x i32>
38869// or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
38870/// %5 = mul <N x i32> %2, %4
38871///
38872/// Pattern2:
38873/// %2 = zext/sext <N x i16> %1 to <N x i32>
38874/// %4 = zext/sext <N x i16> %3 to <N x i32>
38875/// or %4 = build_vector <N x i32> %C1, ..., %CN (%C1..%CN are constants)
38876/// %5 = mul <N x i32> %2, %4
38877///
38878/// There are four mul shrinking modes:
38879/// If %2 == sext32(trunc8(%2)), i.e., the scalar value range of %2 is
38880/// -128 to 128, and the scalar value range of %4 is also -128 to 128,
38881/// generate pmullw+sext32 for it (MULS8 mode).
38882/// If %2 == zext32(trunc8(%2)), i.e., the scalar value range of %2 is
38883/// 0 to 255, and the scalar value range of %4 is also 0 to 255,
38884/// generate pmullw+zext32 for it (MULU8 mode).
38885/// If %2 == sext32(trunc16(%2)), i.e., the scalar value range of %2 is
38886/// -32768 to 32767, and the scalar value range of %4 is also -32768 to 32767,
38887/// generate pmullw+pmulhw for it (MULS16 mode).
38888/// If %2 == zext32(trunc16(%2)), i.e., the scalar value range of %2 is
38889/// 0 to 65535, and the scalar value range of %4 is also 0 to 65535,
38890/// generate pmullw+pmulhuw for it (MULU16 mode).
38891static SDValue reduceVMULWidth(SDNode *N, SelectionDAG &DAG,
38892 const X86Subtarget &Subtarget) {
38893 // Check for legality
38894 // pmullw/pmulhw are not supported by SSE.
38895 if (!Subtarget.hasSSE2())
38896 return SDValue();
38897
38898 // Check for profitability
38899 // pmulld is supported since SSE41. It is better to use pmulld
38900 // instead of pmullw+pmulhw, except for subtargets where pmulld is slower than
38901 // the expansion.
38902 bool OptForMinSize = DAG.getMachineFunction().getFunction().hasMinSize();
38903 if (Subtarget.hasSSE41() && (OptForMinSize || !Subtarget.isPMULLDSlow()))
38904 return SDValue();
38905
38906 ShrinkMode Mode;
38907 if (!canReduceVMulWidth(N, DAG, Mode))
38908 return SDValue();
38909
38910 SDLoc DL(N);
38911 SDValue N0 = N->getOperand(0);
38912 SDValue N1 = N->getOperand(1);
38913 EVT VT = N->getOperand(0).getValueType();
38914 unsigned NumElts = VT.getVectorNumElements();
38915 if ((NumElts % 2) != 0)
38916 return SDValue();
38917
38918 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts);
38919
38920 // Shrink the operands of mul.
38921 SDValue NewN0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N0);
38922 SDValue NewN1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, N1);
38923
38924 // Generate the lower part of mul: pmullw. For MULU8/MULS8, only the
38925 // lower part is needed.
38926 SDValue MulLo = DAG.getNode(ISD::MUL, DL, ReducedVT, NewN0, NewN1);
38927 if (Mode == ShrinkMode::MULU8 || Mode == ShrinkMode::MULS8)
38928 return DAG.getNode((Mode == ShrinkMode::MULU8) ? ISD::ZERO_EXTEND
38929 : ISD::SIGN_EXTEND,
38930 DL, VT, MulLo);
38931
38932 MVT ResVT = MVT::getVectorVT(MVT::i32, NumElts / 2);
38933 // Generate the higher part of mul: pmulhw/pmulhuw. For MULU16/MULS16,
38934 // the higher part is also needed.
38935 SDValue MulHi =
38936 DAG.getNode(Mode == ShrinkMode::MULS16 ? ISD::MULHS : ISD::MULHU, DL,
38937 ReducedVT, NewN0, NewN1);
38938
38939 // Repack the lower part and higher part result of mul into a wider
38940 // result.
38941 // Generate shuffle functioning as punpcklwd.
38942 SmallVector<int, 16> ShuffleMask(NumElts);
38943 for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
38944 ShuffleMask[2 * i] = i;
38945 ShuffleMask[2 * i + 1] = i + NumElts;
38946 }
38947 SDValue ResLo =
38948 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
38949 ResLo = DAG.getBitcast(ResVT, ResLo);
38950 // Generate shuffle functioning as punpckhwd.
38951 for (unsigned i = 0, e = NumElts / 2; i < e; i++) {
38952 ShuffleMask[2 * i] = i + NumElts / 2;
38953 ShuffleMask[2 * i + 1] = i + NumElts * 3 / 2;
38954 }
38955 SDValue ResHi =
38956 DAG.getVectorShuffle(ReducedVT, DL, MulLo, MulHi, ShuffleMask);
38957 ResHi = DAG.getBitcast(ResVT, ResHi);
38958 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, ResLo, ResHi);
38959}
38960
38961static SDValue combineMulSpecial(uint64_t MulAmt, SDNode *N, SelectionDAG &DAG,
38962 EVT VT, const SDLoc &DL) {
38963
38964 auto combineMulShlAddOrSub = [&](int Mult, int Shift, bool isAdd) {
38965 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
38966 DAG.getConstant(Mult, DL, VT));
38967 Result = DAG.getNode(ISD::SHL, DL, VT, Result,
38968 DAG.getConstant(Shift, DL, MVT::i8));
38969 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
38970 N->getOperand(0));
38971 return Result;
38972 };
38973
38974 auto combineMulMulAddOrSub = [&](int Mul1, int Mul2, bool isAdd) {
38975 SDValue Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
38976 DAG.getConstant(Mul1, DL, VT));
38977 Result = DAG.getNode(X86ISD::MUL_IMM, DL, VT, Result,
38978 DAG.getConstant(Mul2, DL, VT));
38979 Result = DAG.getNode(isAdd ? ISD::ADD : ISD::SUB, DL, VT, Result,
38980 N->getOperand(0));
38981 return Result;
38982 };
38983
38984 switch (MulAmt) {
38985 default:
38986 break;
38987 case 11:
38988 // mul x, 11 => add ((shl (mul x, 5), 1), x)
38989 return combineMulShlAddOrSub(5, 1, /*isAdd*/ true);
38990 case 21:
38991 // mul x, 21 => add ((shl (mul x, 5), 2), x)
38992 return combineMulShlAddOrSub(5, 2, /*isAdd*/ true);
38993 case 41:
38994 // mul x, 41 => add ((shl (mul x, 5), 3), x)
38995 return combineMulShlAddOrSub(5, 3, /*isAdd*/ true);
38996 case 22:
38997 // mul x, 22 => add (add ((shl (mul x, 5), 2), x), x)
38998 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
38999 combineMulShlAddOrSub(5, 2, /*isAdd*/ true));
39000 case 19:
39001 // mul x, 19 => add ((shl (mul x, 9), 1), x)
39002 return combineMulShlAddOrSub(9, 1, /*isAdd*/ true);
39003 case 37:
39004 // mul x, 37 => add ((shl (mul x, 9), 2), x)
39005 return combineMulShlAddOrSub(9, 2, /*isAdd*/ true);
39006 case 73:
39007 // mul x, 73 => add ((shl (mul x, 9), 3), x)
39008 return combineMulShlAddOrSub(9, 3, /*isAdd*/ true);
39009 case 13:
39010 // mul x, 13 => add ((shl (mul x, 3), 2), x)
39011 return combineMulShlAddOrSub(3, 2, /*isAdd*/ true);
39012 case 23:
39013 // mul x, 23 => sub ((shl (mul x, 3), 3), x)
39014 return combineMulShlAddOrSub(3, 3, /*isAdd*/ false);
39015 case 26:
39016 // mul x, 26 => add ((mul (mul x, 5), 5), x)
39017 return combineMulMulAddOrSub(5, 5, /*isAdd*/ true);
39018 case 28:
39019 // mul x, 28 => add ((mul (mul x, 9), 3), x)
39020 return combineMulMulAddOrSub(9, 3, /*isAdd*/ true);
39021 case 29:
39022 // mul x, 29 => add (add ((mul (mul x, 9), 3), x), x)
39023 return DAG.getNode(ISD::ADD, DL, VT, N->getOperand(0),
39024 combineMulMulAddOrSub(9, 3, /*isAdd*/ true));
39025 }
39026
39027 // Another trick. If this is a power 2 + 2/4/8, we can use a shift followed
39028 // by a single LEA.
39029 // First check if this a sum of two power of 2s because that's easy. Then
39030 // count how many zeros are up to the first bit.
39031 // TODO: We can do this even without LEA at a cost of two shifts and an add.
39032 if (isPowerOf2_64(MulAmt & (MulAmt - 1))) {
39033 unsigned ScaleShift = countTrailingZeros(MulAmt);
39034 if (ScaleShift >= 1 && ScaleShift < 4) {
39035 unsigned ShiftAmt = Log2_64((MulAmt & (MulAmt - 1)));
39036 SDValue Shift1 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
39037 DAG.getConstant(ShiftAmt, DL, MVT::i8));
39038 SDValue Shift2 = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
39039 DAG.getConstant(ScaleShift, DL, MVT::i8));
39040 return DAG.getNode(ISD::ADD, DL, VT, Shift1, Shift2);
39041 }
39042 }
39043
39044 return SDValue();
39045}
39046
39047// If the upper 17 bits of each element are zero then we can use PMADDWD,
39048// which is always at least as quick as PMULLD, except on KNL.
39049static SDValue combineMulToPMADDWD(SDNode *N, SelectionDAG &DAG,
39050 const X86Subtarget &Subtarget) {
39051 if (!Subtarget.hasSSE2())
39052 return SDValue();
39053
39054 if (Subtarget.isPMADDWDSlow())
39055 return SDValue();
39056
39057 EVT VT = N->getValueType(0);
39058
39059 // Only support vXi32 vectors.
39060 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32)
39061 return SDValue();
39062
39063 // Make sure the vXi16 type is legal. This covers the AVX512 without BWI case.
39064 // Also allow v2i32 if it will be widened.
39065 MVT WVT = MVT::getVectorVT(MVT::i16, 2 * VT.getVectorNumElements());
39066 if (VT != MVT::v2i32 && !DAG.getTargetLoweringInfo().isTypeLegal(WVT))
39067 return SDValue();
39068
39069 SDValue N0 = N->getOperand(0);
39070 SDValue N1 = N->getOperand(1);
39071
39072 // If we are zero extending two steps without SSE4.1, its better to reduce
39073 // the vmul width instead.
39074 if (!Subtarget.hasSSE41() &&
39075 (N0.getOpcode() == ISD::ZERO_EXTEND &&
39076 N0.getOperand(0).getScalarValueSizeInBits() <= 8) &&
39077 (N1.getOpcode() == ISD::ZERO_EXTEND &&
39078 N1.getOperand(0).getScalarValueSizeInBits() <= 8))
39079 return SDValue();
39080
39081 APInt Mask17 = APInt::getHighBitsSet(32, 17);
39082 if (!DAG.MaskedValueIsZero(N1, Mask17) ||
39083 !DAG.MaskedValueIsZero(N0, Mask17))
39084 return SDValue();
39085
39086 // Use SplitOpsAndApply to handle AVX splitting.
39087 auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
39088 ArrayRef<SDValue> Ops) {
39089 MVT OpVT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
39090 return DAG.getNode(X86ISD::VPMADDWD, DL, OpVT, Ops);
39091 };
39092 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT,
39093 { DAG.getBitcast(WVT, N0), DAG.getBitcast(WVT, N1) },
39094 PMADDWDBuilder);
39095}
39096
39097static SDValue combineMulToPMULDQ(SDNode *N, SelectionDAG &DAG,
39098 const X86Subtarget &Subtarget) {
39099 if (!Subtarget.hasSSE2())
39100 return SDValue();
39101
39102 EVT VT = N->getValueType(0);
39103
39104 // Only support vXi64 vectors.
39105 if (!VT.isVector() || VT.getVectorElementType() != MVT::i64 ||
39106 VT.getVectorNumElements() < 2 ||
39107 !isPowerOf2_32(VT.getVectorNumElements()))
39108 return SDValue();
39109
39110 SDValue N0 = N->getOperand(0);
39111 SDValue N1 = N->getOperand(1);
39112
39113 // MULDQ returns the 64-bit result of the signed multiplication of the lower
39114 // 32-bits. We can lower with this if the sign bits stretch that far.
39115 if (Subtarget.hasSSE41() && DAG.ComputeNumSignBits(N0) > 32 &&
39116 DAG.ComputeNumSignBits(N1) > 32) {
39117 auto PMULDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
39118 ArrayRef<SDValue> Ops) {
39119 return DAG.getNode(X86ISD::PMULDQ, DL, Ops[0].getValueType(), Ops);
39120 };
39121 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },
39122 PMULDQBuilder, /*CheckBWI*/false);
39123 }
39124
39125 // If the upper bits are zero we can use a single pmuludq.
39126 APInt Mask = APInt::getHighBitsSet(64, 32);
39127 if (DAG.MaskedValueIsZero(N0, Mask) && DAG.MaskedValueIsZero(N1, Mask)) {
39128 auto PMULUDQBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
39129 ArrayRef<SDValue> Ops) {
39130 return DAG.getNode(X86ISD::PMULUDQ, DL, Ops[0].getValueType(), Ops);
39131 };
39132 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, { N0, N1 },
39133 PMULUDQBuilder, /*CheckBWI*/false);
39134 }
39135
39136 return SDValue();
39137}
39138
39139/// Optimize a single multiply with constant into two operations in order to
39140/// implement it with two cheaper instructions, e.g. LEA + SHL, LEA + LEA.
39141static SDValue combineMul(SDNode *N, SelectionDAG &DAG,
39142 TargetLowering::DAGCombinerInfo &DCI,
39143 const X86Subtarget &Subtarget) {
39144 EVT VT = N->getValueType(0);
39145
39146 if (SDValue V = combineMulToPMADDWD(N, DAG, Subtarget))
39147 return V;
39148
39149 if (SDValue V = combineMulToPMULDQ(N, DAG, Subtarget))
39150 return V;
39151
39152 if (DCI.isBeforeLegalize() && VT.isVector())
39153 return reduceVMULWidth(N, DAG, Subtarget);
39154
39155 if (!MulConstantOptimization)
39156 return SDValue();
39157 // An imul is usually smaller than the alternative sequence.
39158 if (DAG.getMachineFunction().getFunction().hasMinSize())
39159 return SDValue();
39160
39161 if (DCI.isBeforeLegalize() || DCI.isCalledByLegalizer())
39162 return SDValue();
39163
39164 if (VT != MVT::i64 && VT != MVT::i32)
39165 return SDValue();
39166
39167 ConstantSDNode *C = dyn_cast<ConstantSDNode>(N->getOperand(1));
39168 if (!C)
39169 return SDValue();
39170 if (isPowerOf2_64(C->getZExtValue()))
39171 return SDValue();
39172
39173 int64_t SignMulAmt = C->getSExtValue();
39174 assert(SignMulAmt != INT64_MIN && "Int min should have been handled!")((SignMulAmt != (-9223372036854775807L -1) && "Int min should have been handled!"
) ? static_cast<void> (0) : __assert_fail ("SignMulAmt != INT64_MIN && \"Int min should have been handled!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39174, __PRETTY_FUNCTION__));
39175 uint64_t AbsMulAmt = SignMulAmt < 0 ? -SignMulAmt : SignMulAmt;
39176
39177 SDLoc DL(N);
39178 if (AbsMulAmt == 3 || AbsMulAmt == 5 || AbsMulAmt == 9) {
39179 SDValue NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
39180 DAG.getConstant(AbsMulAmt, DL, VT));
39181 if (SignMulAmt < 0)
39182 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
39183 NewMul);
39184
39185 return NewMul;
39186 }
39187
39188 uint64_t MulAmt1 = 0;
39189 uint64_t MulAmt2 = 0;
39190 if ((AbsMulAmt % 9) == 0) {
39191 MulAmt1 = 9;
39192 MulAmt2 = AbsMulAmt / 9;
39193 } else if ((AbsMulAmt % 5) == 0) {
39194 MulAmt1 = 5;
39195 MulAmt2 = AbsMulAmt / 5;
39196 } else if ((AbsMulAmt % 3) == 0) {
39197 MulAmt1 = 3;
39198 MulAmt2 = AbsMulAmt / 3;
39199 }
39200
39201 SDValue NewMul;
39202 // For negative multiply amounts, only allow MulAmt2 to be a power of 2.
39203 if (MulAmt2 &&
39204 (isPowerOf2_64(MulAmt2) ||
39205 (SignMulAmt >= 0 && (MulAmt2 == 3 || MulAmt2 == 5 || MulAmt2 == 9)))) {
39206
39207 if (isPowerOf2_64(MulAmt2) &&
39208 !(SignMulAmt >= 0 && N->hasOneUse() &&
39209 N->use_begin()->getOpcode() == ISD::ADD))
39210 // If second multiplifer is pow2, issue it first. We want the multiply by
39211 // 3, 5, or 9 to be folded into the addressing mode unless the lone use
39212 // is an add. Only do this for positive multiply amounts since the
39213 // negate would prevent it from being used as an address mode anyway.
39214 std::swap(MulAmt1, MulAmt2);
39215
39216 if (isPowerOf2_64(MulAmt1))
39217 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
39218 DAG.getConstant(Log2_64(MulAmt1), DL, MVT::i8));
39219 else
39220 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, N->getOperand(0),
39221 DAG.getConstant(MulAmt1, DL, VT));
39222
39223 if (isPowerOf2_64(MulAmt2))
39224 NewMul = DAG.getNode(ISD::SHL, DL, VT, NewMul,
39225 DAG.getConstant(Log2_64(MulAmt2), DL, MVT::i8));
39226 else
39227 NewMul = DAG.getNode(X86ISD::MUL_IMM, DL, VT, NewMul,
39228 DAG.getConstant(MulAmt2, DL, VT));
39229
39230 // Negate the result.
39231 if (SignMulAmt < 0)
39232 NewMul = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT),
39233 NewMul);
39234 } else if (!Subtarget.slowLEA())
39235 NewMul = combineMulSpecial(C->getZExtValue(), N, DAG, VT, DL);
39236
39237 if (!NewMul) {
39238 assert(C->getZExtValue() != 0 &&((C->getZExtValue() != 0 && C->getZExtValue() !=
(VT == MVT::i64 ? (18446744073709551615UL) : (4294967295U)) &&
"Both cases that could cause potential overflows should have "
"already been handled.") ? static_cast<void> (0) : __assert_fail
("C->getZExtValue() != 0 && C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) && \"Both cases that could cause potential overflows should have \" \"already been handled.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39241, __PRETTY_FUNCTION__))
39239 C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) &&((C->getZExtValue() != 0 && C->getZExtValue() !=
(VT == MVT::i64 ? (18446744073709551615UL) : (4294967295U)) &&
"Both cases that could cause potential overflows should have "
"already been handled.") ? static_cast<void> (0) : __assert_fail
("C->getZExtValue() != 0 && C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) && \"Both cases that could cause potential overflows should have \" \"already been handled.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39241, __PRETTY_FUNCTION__))
39240 "Both cases that could cause potential overflows should have "((C->getZExtValue() != 0 && C->getZExtValue() !=
(VT == MVT::i64 ? (18446744073709551615UL) : (4294967295U)) &&
"Both cases that could cause potential overflows should have "
"already been handled.") ? static_cast<void> (0) : __assert_fail
("C->getZExtValue() != 0 && C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) && \"Both cases that could cause potential overflows should have \" \"already been handled.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39241, __PRETTY_FUNCTION__))
39241 "already been handled.")((C->getZExtValue() != 0 && C->getZExtValue() !=
(VT == MVT::i64 ? (18446744073709551615UL) : (4294967295U)) &&
"Both cases that could cause potential overflows should have "
"already been handled.") ? static_cast<void> (0) : __assert_fail
("C->getZExtValue() != 0 && C->getZExtValue() != (VT == MVT::i64 ? UINT64_MAX : UINT32_MAX) && \"Both cases that could cause potential overflows should have \" \"already been handled.\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39241, __PRETTY_FUNCTION__));
39242 if (isPowerOf2_64(AbsMulAmt - 1)) {
39243 // (mul x, 2^N + 1) => (add (shl x, N), x)
39244 NewMul = DAG.getNode(
39245 ISD::ADD, DL, VT, N->getOperand(0),
39246 DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
39247 DAG.getConstant(Log2_64(AbsMulAmt - 1), DL,
39248 MVT::i8)));
39249 // To negate, subtract the number from zero
39250 if (SignMulAmt < 0)
39251 NewMul = DAG.getNode(ISD::SUB, DL, VT,
39252 DAG.getConstant(0, DL, VT), NewMul);
39253 } else if (isPowerOf2_64(AbsMulAmt + 1)) {
39254 // (mul x, 2^N - 1) => (sub (shl x, N), x)
39255 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
39256 DAG.getConstant(Log2_64(AbsMulAmt + 1),
39257 DL, MVT::i8));
39258 // To negate, reverse the operands of the subtract.
39259 if (SignMulAmt < 0)
39260 NewMul = DAG.getNode(ISD::SUB, DL, VT, N->getOperand(0), NewMul);
39261 else
39262 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
39263 } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt - 2)) {
39264 // (mul x, 2^N + 2) => (add (add (shl x, N), x), x)
39265 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
39266 DAG.getConstant(Log2_64(AbsMulAmt - 2),
39267 DL, MVT::i8));
39268 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0));
39269 NewMul = DAG.getNode(ISD::ADD, DL, VT, NewMul, N->getOperand(0));
39270 } else if (SignMulAmt >= 0 && isPowerOf2_64(AbsMulAmt + 2)) {
39271 // (mul x, 2^N - 2) => (sub (sub (shl x, N), x), x)
39272 NewMul = DAG.getNode(ISD::SHL, DL, VT, N->getOperand(0),
39273 DAG.getConstant(Log2_64(AbsMulAmt + 2),
39274 DL, MVT::i8));
39275 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
39276 NewMul = DAG.getNode(ISD::SUB, DL, VT, NewMul, N->getOperand(0));
39277 }
39278 }
39279
39280 return NewMul;
39281}
39282
39283static SDValue combineShiftLeft(SDNode *N, SelectionDAG &DAG) {
39284 SDValue N0 = N->getOperand(0);
39285 SDValue N1 = N->getOperand(1);
39286 ConstantSDNode *N1C = dyn_cast<ConstantSDNode>(N1);
39287 EVT VT = N0.getValueType();
39288
39289 // fold (shl (and (setcc_c), c1), c2) -> (and setcc_c, (c1 << c2))
39290 // since the result of setcc_c is all zero's or all ones.
39291 if (VT.isInteger() && !VT.isVector() &&
39292 N1C && N0.getOpcode() == ISD::AND &&
39293 N0.getOperand(1).getOpcode() == ISD::Constant) {
39294 SDValue N00 = N0.getOperand(0);
39295 APInt Mask = N0.getConstantOperandAPInt(1);
39296 Mask <<= N1C->getAPIntValue();
39297 bool MaskOK = false;
39298 // We can handle cases concerning bit-widening nodes containing setcc_c if
39299 // we carefully interrogate the mask to make sure we are semantics
39300 // preserving.
39301 // The transform is not safe if the result of C1 << C2 exceeds the bitwidth
39302 // of the underlying setcc_c operation if the setcc_c was zero extended.
39303 // Consider the following example:
39304 // zext(setcc_c) -> i32 0x0000FFFF
39305 // c1 -> i32 0x0000FFFF
39306 // c2 -> i32 0x00000001
39307 // (shl (and (setcc_c), c1), c2) -> i32 0x0001FFFE
39308 // (and setcc_c, (c1 << c2)) -> i32 0x0000FFFE
39309 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
39310 MaskOK = true;
39311 } else if (N00.getOpcode() == ISD::SIGN_EXTEND &&
39312 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
39313 MaskOK = true;
39314 } else if ((N00.getOpcode() == ISD::ZERO_EXTEND ||
39315 N00.getOpcode() == ISD::ANY_EXTEND) &&
39316 N00.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY) {
39317 MaskOK = Mask.isIntN(N00.getOperand(0).getValueSizeInBits());
39318 }
39319 if (MaskOK && Mask != 0) {
39320 SDLoc DL(N);
39321 return DAG.getNode(ISD::AND, DL, VT, N00, DAG.getConstant(Mask, DL, VT));
39322 }
39323 }
39324
39325 // Hardware support for vector shifts is sparse which makes us scalarize the
39326 // vector operations in many cases. Also, on sandybridge ADD is faster than
39327 // shl.
39328 // (shl V, 1) -> add V,V
39329 if (auto *N1BV = dyn_cast<BuildVectorSDNode>(N1))
39330 if (auto *N1SplatC = N1BV->getConstantSplatNode()) {
39331 assert(N0.getValueType().isVector() && "Invalid vector shift type")((N0.getValueType().isVector() && "Invalid vector shift type"
) ? static_cast<void> (0) : __assert_fail ("N0.getValueType().isVector() && \"Invalid vector shift type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39331, __PRETTY_FUNCTION__));
39332 // We shift all of the values by one. In many cases we do not have
39333 // hardware support for this operation. This is better expressed as an ADD
39334 // of two values.
39335 if (N1SplatC->isOne())
39336 return DAG.getNode(ISD::ADD, SDLoc(N), VT, N0, N0);
39337 }
39338
39339 return SDValue();
39340}
39341
39342static SDValue combineShiftRightArithmetic(SDNode *N, SelectionDAG &DAG) {
39343 SDValue N0 = N->getOperand(0);
39344 SDValue N1 = N->getOperand(1);
39345 EVT VT = N0.getValueType();
39346 unsigned Size = VT.getSizeInBits();
39347
39348 // fold (ashr (shl, a, [56,48,32,24,16]), SarConst)
39349 // into (shl, (sext (a), [56,48,32,24,16] - SarConst)) or
39350 // into (lshr, (sext (a), SarConst - [56,48,32,24,16]))
39351 // depending on sign of (SarConst - [56,48,32,24,16])
39352
39353 // sexts in X86 are MOVs. The MOVs have the same code size
39354 // as above SHIFTs (only SHIFT on 1 has lower code size).
39355 // However the MOVs have 2 advantages to a SHIFT:
39356 // 1. MOVs can write to a register that differs from source
39357 // 2. MOVs accept memory operands
39358
39359 if (VT.isVector() || N1.getOpcode() != ISD::Constant ||
39360 N0.getOpcode() != ISD::SHL || !N0.hasOneUse() ||
39361 N0.getOperand(1).getOpcode() != ISD::Constant)
39362 return SDValue();
39363
39364 SDValue N00 = N0.getOperand(0);
39365 SDValue N01 = N0.getOperand(1);
39366 APInt ShlConst = (cast<ConstantSDNode>(N01))->getAPIntValue();
39367 APInt SarConst = (cast<ConstantSDNode>(N1))->getAPIntValue();
39368 EVT CVT = N1.getValueType();
39369
39370 if (SarConst.isNegative())
39371 return SDValue();
39372
39373 for (MVT SVT : { MVT::i8, MVT::i16, MVT::i32 }) {
39374 unsigned ShiftSize = SVT.getSizeInBits();
39375 // skipping types without corresponding sext/zext and
39376 // ShlConst that is not one of [56,48,32,24,16]
39377 if (ShiftSize >= Size || ShlConst != Size - ShiftSize)
39378 continue;
39379 SDLoc DL(N);
39380 SDValue NN =
39381 DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, N00, DAG.getValueType(SVT));
39382 SarConst = SarConst - (Size - ShiftSize);
39383 if (SarConst == 0)
39384 return NN;
39385 else if (SarConst.isNegative())
39386 return DAG.getNode(ISD::SHL, DL, VT, NN,
39387 DAG.getConstant(-SarConst, DL, CVT));
39388 else
39389 return DAG.getNode(ISD::SRA, DL, VT, NN,
39390 DAG.getConstant(SarConst, DL, CVT));
39391 }
39392 return SDValue();
39393}
39394
39395static SDValue combineShiftRightLogical(SDNode *N, SelectionDAG &DAG,
39396 TargetLowering::DAGCombinerInfo &DCI) {
39397 SDValue N0 = N->getOperand(0);
39398 SDValue N1 = N->getOperand(1);
39399 EVT VT = N0.getValueType();
39400
39401 // Only do this on the last DAG combine as it can interfere with other
39402 // combines.
39403 if (!DCI.isAfterLegalizeDAG())
39404 return SDValue();
39405
39406 // Try to improve a sequence of srl (and X, C1), C2 by inverting the order.
39407 // TODO: This is a generic DAG combine that became an x86-only combine to
39408 // avoid shortcomings in other folds such as bswap, bit-test ('bt'), and
39409 // and-not ('andn').
39410 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse())
39411 return SDValue();
39412
39413 auto *ShiftC = dyn_cast<ConstantSDNode>(N1);
39414 auto *AndC = dyn_cast<ConstantSDNode>(N0.getOperand(1));
39415 if (!ShiftC || !AndC)
39416 return SDValue();
39417
39418 // If we can shrink the constant mask below 8-bits or 32-bits, then this
39419 // transform should reduce code size. It may also enable secondary transforms
39420 // from improved known-bits analysis or instruction selection.
39421 APInt MaskVal = AndC->getAPIntValue();
39422
39423 // If this can be matched by a zero extend, don't optimize.
39424 if (MaskVal.isMask()) {
39425 unsigned TO = MaskVal.countTrailingOnes();
39426 if (TO >= 8 && isPowerOf2_32(TO))
39427 return SDValue();
39428 }
39429
39430 APInt NewMaskVal = MaskVal.lshr(ShiftC->getAPIntValue());
39431 unsigned OldMaskSize = MaskVal.getMinSignedBits();
39432 unsigned NewMaskSize = NewMaskVal.getMinSignedBits();
39433 if ((OldMaskSize > 8 && NewMaskSize <= 8) ||
39434 (OldMaskSize > 32 && NewMaskSize <= 32)) {
39435 // srl (and X, AndC), ShiftC --> and (srl X, ShiftC), (AndC >> ShiftC)
39436 SDLoc DL(N);
39437 SDValue NewMask = DAG.getConstant(NewMaskVal, DL, VT);
39438 SDValue NewShift = DAG.getNode(ISD::SRL, DL, VT, N0.getOperand(0), N1);
39439 return DAG.getNode(ISD::AND, DL, VT, NewShift, NewMask);
39440 }
39441 return SDValue();
39442}
39443
39444static SDValue combineVectorPack(SDNode *N, SelectionDAG &DAG,
39445 TargetLowering::DAGCombinerInfo &DCI,
39446 const X86Subtarget &Subtarget) {
39447 unsigned Opcode = N->getOpcode();
39448 assert((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&(((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
"Unexpected shift opcode") ? static_cast<void> (0) : __assert_fail
("(X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39449, __PRETTY_FUNCTION__))
39449 "Unexpected shift opcode")(((X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) &&
"Unexpected shift opcode") ? static_cast<void> (0) : __assert_fail
("(X86ISD::PACKSS == Opcode || X86ISD::PACKUS == Opcode) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39449, __PRETTY_FUNCTION__));
39450
39451 EVT VT = N->getValueType(0);
39452 SDValue N0 = N->getOperand(0);
39453 SDValue N1 = N->getOperand(1);
39454 unsigned DstBitsPerElt = VT.getScalarSizeInBits();
39455 unsigned SrcBitsPerElt = 2 * DstBitsPerElt;
39456 assert(N0.getScalarValueSizeInBits() == SrcBitsPerElt &&((N0.getScalarValueSizeInBits() == SrcBitsPerElt && N1
.getScalarValueSizeInBits() == SrcBitsPerElt && "Unexpected PACKSS/PACKUS input type"
) ? static_cast<void> (0) : __assert_fail ("N0.getScalarValueSizeInBits() == SrcBitsPerElt && N1.getScalarValueSizeInBits() == SrcBitsPerElt && \"Unexpected PACKSS/PACKUS input type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39458, __PRETTY_FUNCTION__))
39457 N1.getScalarValueSizeInBits() == SrcBitsPerElt &&((N0.getScalarValueSizeInBits() == SrcBitsPerElt && N1
.getScalarValueSizeInBits() == SrcBitsPerElt && "Unexpected PACKSS/PACKUS input type"
) ? static_cast<void> (0) : __assert_fail ("N0.getScalarValueSizeInBits() == SrcBitsPerElt && N1.getScalarValueSizeInBits() == SrcBitsPerElt && \"Unexpected PACKSS/PACKUS input type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39458, __PRETTY_FUNCTION__))
39458 "Unexpected PACKSS/PACKUS input type")((N0.getScalarValueSizeInBits() == SrcBitsPerElt && N1
.getScalarValueSizeInBits() == SrcBitsPerElt && "Unexpected PACKSS/PACKUS input type"
) ? static_cast<void> (0) : __assert_fail ("N0.getScalarValueSizeInBits() == SrcBitsPerElt && N1.getScalarValueSizeInBits() == SrcBitsPerElt && \"Unexpected PACKSS/PACKUS input type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39458, __PRETTY_FUNCTION__));
39459
39460 bool IsSigned = (X86ISD::PACKSS == Opcode);
39461
39462 // Constant Folding.
39463 APInt UndefElts0, UndefElts1;
39464 SmallVector<APInt, 32> EltBits0, EltBits1;
39465 if ((N0.isUndef() || N->isOnlyUserOf(N0.getNode())) &&
39466 (N1.isUndef() || N->isOnlyUserOf(N1.getNode())) &&
39467 getTargetConstantBitsFromNode(N0, SrcBitsPerElt, UndefElts0, EltBits0) &&
39468 getTargetConstantBitsFromNode(N1, SrcBitsPerElt, UndefElts1, EltBits1)) {
39469 unsigned NumLanes = VT.getSizeInBits() / 128;
39470 unsigned NumDstElts = VT.getVectorNumElements();
39471 unsigned NumSrcElts = NumDstElts / 2;
39472 unsigned NumDstEltsPerLane = NumDstElts / NumLanes;
39473 unsigned NumSrcEltsPerLane = NumSrcElts / NumLanes;
39474
39475 APInt Undefs(NumDstElts, 0);
39476 SmallVector<APInt, 32> Bits(NumDstElts, APInt::getNullValue(DstBitsPerElt));
39477 for (unsigned Lane = 0; Lane != NumLanes; ++Lane) {
39478 for (unsigned Elt = 0; Elt != NumDstEltsPerLane; ++Elt) {
39479 unsigned SrcIdx = Lane * NumSrcEltsPerLane + Elt % NumSrcEltsPerLane;
39480 auto &UndefElts = (Elt >= NumSrcEltsPerLane ? UndefElts1 : UndefElts0);
39481 auto &EltBits = (Elt >= NumSrcEltsPerLane ? EltBits1 : EltBits0);
39482
39483 if (UndefElts[SrcIdx]) {
39484 Undefs.setBit(Lane * NumDstEltsPerLane + Elt);
39485 continue;
39486 }
39487
39488 APInt &Val = EltBits[SrcIdx];
39489 if (IsSigned) {
39490 // PACKSS: Truncate signed value with signed saturation.
39491 // Source values less than dst minint are saturated to minint.
39492 // Source values greater than dst maxint are saturated to maxint.
39493 if (Val.isSignedIntN(DstBitsPerElt))
39494 Val = Val.trunc(DstBitsPerElt);
39495 else if (Val.isNegative())
39496 Val = APInt::getSignedMinValue(DstBitsPerElt);
39497 else
39498 Val = APInt::getSignedMaxValue(DstBitsPerElt);
39499 } else {
39500 // PACKUS: Truncate signed value with unsigned saturation.
39501 // Source values less than zero are saturated to zero.
39502 // Source values greater than dst maxuint are saturated to maxuint.
39503 if (Val.isIntN(DstBitsPerElt))
39504 Val = Val.trunc(DstBitsPerElt);
39505 else if (Val.isNegative())
39506 Val = APInt::getNullValue(DstBitsPerElt);
39507 else
39508 Val = APInt::getAllOnesValue(DstBitsPerElt);
39509 }
39510 Bits[Lane * NumDstEltsPerLane + Elt] = Val;
39511 }
39512 }
39513
39514 return getConstVector(Bits, Undefs, VT.getSimpleVT(), DAG, SDLoc(N));
39515 }
39516
39517 // Try to combine a PACKUSWB/PACKSSWB implemented truncate with a regular
39518 // truncate to create a larger truncate.
39519 if (Subtarget.hasAVX512() &&
39520 N0.getOpcode() == ISD::TRUNCATE && N1.isUndef() && VT == MVT::v16i8 &&
39521 N0.getOperand(0).getValueType() == MVT::v8i32) {
39522 if ((IsSigned && DAG.ComputeNumSignBits(N0) > 8) ||
39523 (!IsSigned &&
39524 DAG.MaskedValueIsZero(N0, APInt::getHighBitsSet(16, 8)))) {
39525 if (Subtarget.hasVLX())
39526 return DAG.getNode(X86ISD::VTRUNC, SDLoc(N), VT, N0.getOperand(0));
39527
39528 // Widen input to v16i32 so we can truncate that.
39529 SDLoc dl(N);
39530 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v16i32,
39531 N0.getOperand(0), DAG.getUNDEF(MVT::v8i32));
39532 return DAG.getNode(ISD::TRUNCATE, SDLoc(N), VT, Concat);
39533 }
39534 }
39535
39536 // Attempt to combine as shuffle.
39537 SDValue Op(N, 0);
39538 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
39539 return Res;
39540
39541 return SDValue();
39542}
39543
39544static SDValue combineVectorShiftVar(SDNode *N, SelectionDAG &DAG,
39545 TargetLowering::DAGCombinerInfo &DCI,
39546 const X86Subtarget &Subtarget) {
39547 assert((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() ||(((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->
getOpcode() || X86ISD::VSRL == N->getOpcode()) && "Unexpected shift opcode"
) ? static_cast<void> (0) : __assert_fail ("(X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() || X86ISD::VSRL == N->getOpcode()) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39549, __PRETTY_FUNCTION__))
39548 X86ISD::VSRL == N->getOpcode()) &&(((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->
getOpcode() || X86ISD::VSRL == N->getOpcode()) && "Unexpected shift opcode"
) ? static_cast<void> (0) : __assert_fail ("(X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() || X86ISD::VSRL == N->getOpcode()) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39549, __PRETTY_FUNCTION__))
39549 "Unexpected shift opcode")(((X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->
getOpcode() || X86ISD::VSRL == N->getOpcode()) && "Unexpected shift opcode"
) ? static_cast<void> (0) : __assert_fail ("(X86ISD::VSHL == N->getOpcode() || X86ISD::VSRA == N->getOpcode() || X86ISD::VSRL == N->getOpcode()) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39549, __PRETTY_FUNCTION__));
39550 EVT VT = N->getValueType(0);
39551 SDValue N0 = N->getOperand(0);
39552 SDValue N1 = N->getOperand(1);
39553
39554 // Shift zero -> zero.
39555 if (ISD::isBuildVectorAllZeros(N0.getNode()))
39556 return DAG.getConstant(0, SDLoc(N), VT);
39557
39558 // Detect constant shift amounts.
39559 APInt UndefElts;
39560 SmallVector<APInt, 32> EltBits;
39561 if (getTargetConstantBitsFromNode(N1, 64, UndefElts, EltBits, true, false)) {
39562 unsigned X86Opc = getTargetVShiftUniformOpcode(N->getOpcode(), false);
39563 return getTargetVShiftByConstNode(X86Opc, SDLoc(N), VT.getSimpleVT(), N0,
39564 EltBits[0].getZExtValue(), DAG);
39565 }
39566
39567 APInt KnownUndef, KnownZero;
39568 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
39569 APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
39570 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, KnownUndef,
39571 KnownZero, DCI))
39572 return SDValue(N, 0);
39573
39574 return SDValue();
39575}
39576
39577static SDValue combineVectorShiftImm(SDNode *N, SelectionDAG &DAG,
39578 TargetLowering::DAGCombinerInfo &DCI,
39579 const X86Subtarget &Subtarget) {
39580 unsigned Opcode = N->getOpcode();
39581 assert((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode ||(((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode || X86ISD
::VSRLI == Opcode) && "Unexpected shift opcode") ? static_cast
<void> (0) : __assert_fail ("(X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode || X86ISD::VSRLI == Opcode) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39583, __PRETTY_FUNCTION__))
39582 X86ISD::VSRLI == Opcode) &&(((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode || X86ISD
::VSRLI == Opcode) && "Unexpected shift opcode") ? static_cast
<void> (0) : __assert_fail ("(X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode || X86ISD::VSRLI == Opcode) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39583, __PRETTY_FUNCTION__))
39583 "Unexpected shift opcode")(((X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode || X86ISD
::VSRLI == Opcode) && "Unexpected shift opcode") ? static_cast
<void> (0) : __assert_fail ("(X86ISD::VSHLI == Opcode || X86ISD::VSRAI == Opcode || X86ISD::VSRLI == Opcode) && \"Unexpected shift opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39583, __PRETTY_FUNCTION__));
39584 bool LogicalShift = X86ISD::VSHLI == Opcode || X86ISD::VSRLI == Opcode;
39585 EVT VT = N->getValueType(0);
39586 SDValue N0 = N->getOperand(0);
39587 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
39588 assert(VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 &&((VT == N0.getValueType() && (NumBitsPerElt % 8) == 0
&& "Unexpected value type") ? static_cast<void>
(0) : __assert_fail ("VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39589, __PRETTY_FUNCTION__))
39589 "Unexpected value type")((VT == N0.getValueType() && (NumBitsPerElt % 8) == 0
&& "Unexpected value type") ? static_cast<void>
(0) : __assert_fail ("VT == N0.getValueType() && (NumBitsPerElt % 8) == 0 && \"Unexpected value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39589, __PRETTY_FUNCTION__));
39590 assert(N->getOperand(1).getValueType() == MVT::i8 &&((N->getOperand(1).getValueType() == MVT::i8 && "Unexpected shift amount type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == MVT::i8 && \"Unexpected shift amount type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39591, __PRETTY_FUNCTION__))
39591 "Unexpected shift amount type")((N->getOperand(1).getValueType() == MVT::i8 && "Unexpected shift amount type"
) ? static_cast<void> (0) : __assert_fail ("N->getOperand(1).getValueType() == MVT::i8 && \"Unexpected shift amount type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39591, __PRETTY_FUNCTION__));
39592
39593 // Out of range logical bit shifts are guaranteed to be zero.
39594 // Out of range arithmetic bit shifts splat the sign bit.
39595 unsigned ShiftVal = N->getConstantOperandVal(1);
39596 if (ShiftVal >= NumBitsPerElt) {
39597 if (LogicalShift)
39598 return DAG.getConstant(0, SDLoc(N), VT);
39599 else
39600 ShiftVal = NumBitsPerElt - 1;
39601 }
39602
39603 // Shift N0 by zero -> N0.
39604 if (!ShiftVal)
39605 return N0;
39606
39607 // Shift zero -> zero.
39608 if (ISD::isBuildVectorAllZeros(N0.getNode()))
39609 return DAG.getConstant(0, SDLoc(N), VT);
39610
39611 // Fold (VSRAI (VSRAI X, C1), C2) --> (VSRAI X, (C1 + C2)) with (C1 + C2)
39612 // clamped to (NumBitsPerElt - 1).
39613 if (Opcode == X86ISD::VSRAI && N0.getOpcode() == X86ISD::VSRAI) {
39614 unsigned ShiftVal2 = cast<ConstantSDNode>(N0.getOperand(1))->getZExtValue();
39615 unsigned NewShiftVal = ShiftVal + ShiftVal2;
39616 if (NewShiftVal >= NumBitsPerElt)
39617 NewShiftVal = NumBitsPerElt - 1;
39618 return DAG.getNode(X86ISD::VSRAI, SDLoc(N), VT, N0.getOperand(0),
39619 DAG.getTargetConstant(NewShiftVal, SDLoc(N), MVT::i8));
39620 }
39621
39622 // We can decode 'whole byte' logical bit shifts as shuffles.
39623 if (LogicalShift && (ShiftVal % 8) == 0) {
39624 SDValue Op(N, 0);
39625 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
39626 return Res;
39627 }
39628
39629 // Constant Folding.
39630 APInt UndefElts;
39631 SmallVector<APInt, 32> EltBits;
39632 if (N->isOnlyUserOf(N0.getNode()) &&
39633 getTargetConstantBitsFromNode(N0, NumBitsPerElt, UndefElts, EltBits)) {
39634 assert(EltBits.size() == VT.getVectorNumElements() &&((EltBits.size() == VT.getVectorNumElements() && "Unexpected shift value type"
) ? static_cast<void> (0) : __assert_fail ("EltBits.size() == VT.getVectorNumElements() && \"Unexpected shift value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39635, __PRETTY_FUNCTION__))
39635 "Unexpected shift value type")((EltBits.size() == VT.getVectorNumElements() && "Unexpected shift value type"
) ? static_cast<void> (0) : __assert_fail ("EltBits.size() == VT.getVectorNumElements() && \"Unexpected shift value type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39635, __PRETTY_FUNCTION__));
39636 for (APInt &Elt : EltBits) {
39637 if (X86ISD::VSHLI == Opcode)
39638 Elt <<= ShiftVal;
39639 else if (X86ISD::VSRAI == Opcode)
39640 Elt.ashrInPlace(ShiftVal);
39641 else
39642 Elt.lshrInPlace(ShiftVal);
39643 }
39644 return getConstVector(EltBits, UndefElts, VT.getSimpleVT(), DAG, SDLoc(N));
39645 }
39646
39647 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
39648 if (TLI.SimplifyDemandedBits(SDValue(N, 0),
39649 APInt::getAllOnesValue(NumBitsPerElt), DCI))
39650 return SDValue(N, 0);
39651
39652 return SDValue();
39653}
39654
39655static SDValue combineVectorInsert(SDNode *N, SelectionDAG &DAG,
39656 TargetLowering::DAGCombinerInfo &DCI,
39657 const X86Subtarget &Subtarget) {
39658 EVT VT = N->getValueType(0);
39659 assert(((N->getOpcode() == X86ISD::PINSRB && VT == MVT::v16i8) ||((((N->getOpcode() == X86ISD::PINSRB && VT == MVT::
v16i8) || (N->getOpcode() == X86ISD::PINSRW && VT ==
MVT::v8i16)) && "Unexpected vector insertion") ? static_cast
<void> (0) : __assert_fail ("((N->getOpcode() == X86ISD::PINSRB && VT == MVT::v16i8) || (N->getOpcode() == X86ISD::PINSRW && VT == MVT::v8i16)) && \"Unexpected vector insertion\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39661, __PRETTY_FUNCTION__))
39660 (N->getOpcode() == X86ISD::PINSRW && VT == MVT::v8i16)) &&((((N->getOpcode() == X86ISD::PINSRB && VT == MVT::
v16i8) || (N->getOpcode() == X86ISD::PINSRW && VT ==
MVT::v8i16)) && "Unexpected vector insertion") ? static_cast
<void> (0) : __assert_fail ("((N->getOpcode() == X86ISD::PINSRB && VT == MVT::v16i8) || (N->getOpcode() == X86ISD::PINSRW && VT == MVT::v8i16)) && \"Unexpected vector insertion\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39661, __PRETTY_FUNCTION__))
39661 "Unexpected vector insertion")((((N->getOpcode() == X86ISD::PINSRB && VT == MVT::
v16i8) || (N->getOpcode() == X86ISD::PINSRW && VT ==
MVT::v8i16)) && "Unexpected vector insertion") ? static_cast
<void> (0) : __assert_fail ("((N->getOpcode() == X86ISD::PINSRB && VT == MVT::v16i8) || (N->getOpcode() == X86ISD::PINSRW && VT == MVT::v8i16)) && \"Unexpected vector insertion\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39661, __PRETTY_FUNCTION__));
39662
39663 unsigned NumBitsPerElt = VT.getScalarSizeInBits();
39664 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
39665 if (TLI.SimplifyDemandedBits(SDValue(N, 0),
39666 APInt::getAllOnesValue(NumBitsPerElt), DCI))
39667 return SDValue(N, 0);
39668
39669 // Attempt to combine PINSRB/PINSRW patterns to a shuffle.
39670 SDValue Op(N, 0);
39671 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
39672 return Res;
39673
39674 return SDValue();
39675}
39676
39677/// Recognize the distinctive (AND (setcc ...) (setcc ..)) where both setccs
39678/// reference the same FP CMP, and rewrite for CMPEQSS and friends. Likewise for
39679/// OR -> CMPNEQSS.
39680static SDValue combineCompareEqual(SDNode *N, SelectionDAG &DAG,
39681 TargetLowering::DAGCombinerInfo &DCI,
39682 const X86Subtarget &Subtarget) {
39683 unsigned opcode;
39684
39685 // SSE1 supports CMP{eq|ne}SS, and SSE2 added CMP{eq|ne}SD, but
39686 // we're requiring SSE2 for both.
39687 if (Subtarget.hasSSE2() && isAndOrOfSetCCs(SDValue(N, 0U), opcode)) {
39688 SDValue N0 = N->getOperand(0);
39689 SDValue N1 = N->getOperand(1);
39690 SDValue CMP0 = N0.getOperand(1);
39691 SDValue CMP1 = N1.getOperand(1);
39692 SDLoc DL(N);
39693
39694 // The SETCCs should both refer to the same CMP.
39695 if (CMP0.getOpcode() != X86ISD::CMP || CMP0 != CMP1)
39696 return SDValue();
39697
39698 SDValue CMP00 = CMP0->getOperand(0);
39699 SDValue CMP01 = CMP0->getOperand(1);
39700 EVT VT = CMP00.getValueType();
39701
39702 if (VT == MVT::f32 || VT == MVT::f64) {
39703 bool ExpectingFlags = false;
39704 // Check for any users that want flags:
39705 for (SDNode::use_iterator UI = N->use_begin(), UE = N->use_end();
39706 !ExpectingFlags && UI != UE; ++UI)
39707 switch (UI->getOpcode()) {
39708 default:
39709 case ISD::BR_CC:
39710 case ISD::BRCOND:
39711 case ISD::SELECT:
39712 ExpectingFlags = true;
39713 break;
39714 case ISD::CopyToReg:
39715 case ISD::SIGN_EXTEND:
39716 case ISD::ZERO_EXTEND:
39717 case ISD::ANY_EXTEND:
39718 break;
39719 }
39720
39721 if (!ExpectingFlags) {
39722 enum X86::CondCode cc0 = (enum X86::CondCode)N0.getConstantOperandVal(0);
39723 enum X86::CondCode cc1 = (enum X86::CondCode)N1.getConstantOperandVal(0);
39724
39725 if (cc1 == X86::COND_E || cc1 == X86::COND_NE) {
39726 X86::CondCode tmp = cc0;
39727 cc0 = cc1;
39728 cc1 = tmp;
39729 }
39730
39731 if ((cc0 == X86::COND_E && cc1 == X86::COND_NP) ||
39732 (cc0 == X86::COND_NE && cc1 == X86::COND_P)) {
39733 // FIXME: need symbolic constants for these magic numbers.
39734 // See X86ATTInstPrinter.cpp:printSSECC().
39735 unsigned x86cc = (cc0 == X86::COND_E) ? 0 : 4;
39736 if (Subtarget.hasAVX512()) {
39737 SDValue FSetCC =
39738 DAG.getNode(X86ISD::FSETCCM, DL, MVT::v1i1, CMP00, CMP01,
39739 DAG.getTargetConstant(x86cc, DL, MVT::i8));
39740 // Need to fill with zeros to ensure the bitcast will produce zeroes
39741 // for the upper bits. An EXTRACT_ELEMENT here wouldn't guarantee that.
39742 SDValue Ins = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, MVT::v16i1,
39743 DAG.getConstant(0, DL, MVT::v16i1),
39744 FSetCC, DAG.getIntPtrConstant(0, DL));
39745 return DAG.getZExtOrTrunc(DAG.getBitcast(MVT::i16, Ins), DL,
39746 N->getSimpleValueType(0));
39747 }
39748 SDValue OnesOrZeroesF =
39749 DAG.getNode(X86ISD::FSETCC, DL, CMP00.getValueType(), CMP00,
39750 CMP01, DAG.getTargetConstant(x86cc, DL, MVT::i8));
39751
39752 bool is64BitFP = (CMP00.getValueType() == MVT::f64);
39753 MVT IntVT = is64BitFP ? MVT::i64 : MVT::i32;
39754
39755 if (is64BitFP && !Subtarget.is64Bit()) {
39756 // On a 32-bit target, we cannot bitcast the 64-bit float to a
39757 // 64-bit integer, since that's not a legal type. Since
39758 // OnesOrZeroesF is all ones of all zeroes, we don't need all the
39759 // bits, but can do this little dance to extract the lowest 32 bits
39760 // and work with those going forward.
39761 SDValue Vector64 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64,
39762 OnesOrZeroesF);
39763 SDValue Vector32 = DAG.getBitcast(MVT::v4f32, Vector64);
39764 OnesOrZeroesF = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f32,
39765 Vector32, DAG.getIntPtrConstant(0, DL));
39766 IntVT = MVT::i32;
39767 }
39768
39769 SDValue OnesOrZeroesI = DAG.getBitcast(IntVT, OnesOrZeroesF);
39770 SDValue ANDed = DAG.getNode(ISD::AND, DL, IntVT, OnesOrZeroesI,
39771 DAG.getConstant(1, DL, IntVT));
39772 SDValue OneBitOfTruth = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
39773 ANDed);
39774 return OneBitOfTruth;
39775 }
39776 }
39777 }
39778 }
39779 return SDValue();
39780}
39781
39782/// Try to fold: (and (xor X, -1), Y) -> (andnp X, Y).
39783static SDValue combineANDXORWithAllOnesIntoANDNP(SDNode *N, SelectionDAG &DAG) {
39784 assert(N->getOpcode() == ISD::AND)((N->getOpcode() == ISD::AND) ? static_cast<void> (0
) : __assert_fail ("N->getOpcode() == ISD::AND", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39784, __PRETTY_FUNCTION__));
39785
39786 MVT VT = N->getSimpleValueType(0);
39787 if (!VT.is128BitVector() && !VT.is256BitVector() && !VT.is512BitVector())
39788 return SDValue();
39789
39790 SDValue X, Y;
39791 SDValue N0 = N->getOperand(0);
39792 SDValue N1 = N->getOperand(1);
39793
39794 if (SDValue Not = IsNOT(N0, DAG)) {
39795 X = Not;
39796 Y = N1;
39797 } else if (SDValue Not = IsNOT(N1, DAG)) {
39798 X = Not;
39799 Y = N0;
39800 } else
39801 return SDValue();
39802
39803 X = DAG.getBitcast(VT, X);
39804 Y = DAG.getBitcast(VT, Y);
39805 return DAG.getNode(X86ISD::ANDNP, SDLoc(N), VT, X, Y);
39806}
39807
39808// On AVX/AVX2 the type v8i1 is legalized to v8i16, which is an XMM sized
39809// register. In most cases we actually compare or select YMM-sized registers
39810// and mixing the two types creates horrible code. This method optimizes
39811// some of the transition sequences.
39812// Even with AVX-512 this is still useful for removing casts around logical
39813// operations on vXi1 mask types.
39814static SDValue PromoteMaskArithmetic(SDNode *N, SelectionDAG &DAG,
39815 const X86Subtarget &Subtarget) {
39816 EVT VT = N->getValueType(0);
39817 assert(VT.isVector() && "Expected vector type")((VT.isVector() && "Expected vector type") ? static_cast
<void> (0) : __assert_fail ("VT.isVector() && \"Expected vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39817, __PRETTY_FUNCTION__));
39818
39819 assert((N->getOpcode() == ISD::ANY_EXTEND ||(((N->getOpcode() == ISD::ANY_EXTEND || N->getOpcode() ==
ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) &&
"Invalid Node") ? static_cast<void> (0) : __assert_fail
("(N->getOpcode() == ISD::ANY_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) && \"Invalid Node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39821, __PRETTY_FUNCTION__))
39820 N->getOpcode() == ISD::ZERO_EXTEND ||(((N->getOpcode() == ISD::ANY_EXTEND || N->getOpcode() ==
ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) &&
"Invalid Node") ? static_cast<void> (0) : __assert_fail
("(N->getOpcode() == ISD::ANY_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) && \"Invalid Node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39821, __PRETTY_FUNCTION__))
39821 N->getOpcode() == ISD::SIGN_EXTEND) && "Invalid Node")(((N->getOpcode() == ISD::ANY_EXTEND || N->getOpcode() ==
ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) &&
"Invalid Node") ? static_cast<void> (0) : __assert_fail
("(N->getOpcode() == ISD::ANY_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND || N->getOpcode() == ISD::SIGN_EXTEND) && \"Invalid Node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39821, __PRETTY_FUNCTION__));
39822
39823 SDValue Narrow = N->getOperand(0);
39824 EVT NarrowVT = Narrow.getValueType();
39825
39826 if (Narrow->getOpcode() != ISD::XOR &&
39827 Narrow->getOpcode() != ISD::AND &&
39828 Narrow->getOpcode() != ISD::OR)
39829 return SDValue();
39830
39831 SDValue N0 = Narrow->getOperand(0);
39832 SDValue N1 = Narrow->getOperand(1);
39833 SDLoc DL(Narrow);
39834
39835 // The Left side has to be a trunc.
39836 if (N0.getOpcode() != ISD::TRUNCATE)
39837 return SDValue();
39838
39839 // The type of the truncated inputs.
39840 if (N0.getOperand(0).getValueType() != VT)
39841 return SDValue();
39842
39843 // The right side has to be a 'trunc' or a constant vector.
39844 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
39845 N1.getOperand(0).getValueType() == VT;
39846 if (!RHSTrunc &&
39847 !ISD::isBuildVectorOfConstantSDNodes(N1.getNode()))
39848 return SDValue();
39849
39850 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
39851
39852 if (!TLI.isOperationLegalOrPromote(Narrow->getOpcode(), VT))
39853 return SDValue();
39854
39855 // Set N0 and N1 to hold the inputs to the new wide operation.
39856 N0 = N0.getOperand(0);
39857 if (RHSTrunc)
39858 N1 = N1.getOperand(0);
39859 else
39860 N1 = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N1);
39861
39862 // Generate the wide operation.
39863 SDValue Op = DAG.getNode(Narrow->getOpcode(), DL, VT, N0, N1);
39864 unsigned Opcode = N->getOpcode();
39865 switch (Opcode) {
39866 default: llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39866);
39867 case ISD::ANY_EXTEND:
39868 return Op;
39869 case ISD::ZERO_EXTEND:
39870 return DAG.getZeroExtendInReg(Op, DL, NarrowVT.getScalarType());
39871 case ISD::SIGN_EXTEND:
39872 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT,
39873 Op, DAG.getValueType(NarrowVT));
39874 }
39875}
39876
39877/// If both input operands of a logic op are being cast from floating point
39878/// types, try to convert this into a floating point logic node to avoid
39879/// unnecessary moves from SSE to integer registers.
39880static SDValue convertIntLogicToFPLogic(SDNode *N, SelectionDAG &DAG,
39881 const X86Subtarget &Subtarget) {
39882 EVT VT = N->getValueType(0);
39883 SDValue N0 = N->getOperand(0);
39884 SDValue N1 = N->getOperand(1);
39885 SDLoc DL(N);
39886
39887 if (N0.getOpcode() != ISD::BITCAST || N1.getOpcode() != ISD::BITCAST)
39888 return SDValue();
39889
39890 SDValue N00 = N0.getOperand(0);
39891 SDValue N10 = N1.getOperand(0);
39892 EVT N00Type = N00.getValueType();
39893 EVT N10Type = N10.getValueType();
39894
39895 // Ensure that both types are the same and are legal scalar fp types.
39896 if (N00Type != N10Type ||
39897 !((Subtarget.hasSSE1() && N00Type == MVT::f32) ||
39898 (Subtarget.hasSSE2() && N00Type == MVT::f64)))
39899 return SDValue();
39900
39901 unsigned FPOpcode;
39902 switch (N->getOpcode()) {
39903 default: llvm_unreachable("Unexpected input node for FP logic conversion")::llvm::llvm_unreachable_internal("Unexpected input node for FP logic conversion"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 39903);
39904 case ISD::AND: FPOpcode = X86ISD::FAND; break;
39905 case ISD::OR: FPOpcode = X86ISD::FOR; break;
39906 case ISD::XOR: FPOpcode = X86ISD::FXOR; break;
39907 }
39908
39909 SDValue FPLogic = DAG.getNode(FPOpcode, DL, N00Type, N00, N10);
39910 return DAG.getBitcast(VT, FPLogic);
39911}
39912
39913/// If this is a zero/all-bits result that is bitwise-anded with a low bits
39914/// mask. (Mask == 1 for the x86 lowering of a SETCC + ZEXT), replace the 'and'
39915/// with a shift-right to eliminate loading the vector constant mask value.
39916static SDValue combineAndMaskToShift(SDNode *N, SelectionDAG &DAG,
39917 const X86Subtarget &Subtarget) {
39918 SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
39919 SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
39920 EVT VT0 = Op0.getValueType();
39921 EVT VT1 = Op1.getValueType();
39922
39923 if (VT0 != VT1 || !VT0.isSimple() || !VT0.isInteger())
39924 return SDValue();
39925
39926 APInt SplatVal;
39927 if (!ISD::isConstantSplatVector(Op1.getNode(), SplatVal) ||
39928 !SplatVal.isMask())
39929 return SDValue();
39930
39931 // Don't prevent creation of ANDN.
39932 if (isBitwiseNot(Op0))
39933 return SDValue();
39934
39935 if (!SupportedVectorShiftWithImm(VT0.getSimpleVT(), Subtarget, ISD::SRL))
39936 return SDValue();
39937
39938 unsigned EltBitWidth = VT0.getScalarSizeInBits();
39939 if (EltBitWidth != DAG.ComputeNumSignBits(Op0))
39940 return SDValue();
39941
39942 SDLoc DL(N);
39943 unsigned ShiftVal = SplatVal.countTrailingOnes();
39944 SDValue ShAmt = DAG.getTargetConstant(EltBitWidth - ShiftVal, DL, MVT::i8);
39945 SDValue Shift = DAG.getNode(X86ISD::VSRLI, DL, VT0, Op0, ShAmt);
39946 return DAG.getBitcast(N->getValueType(0), Shift);
39947}
39948
39949// Get the index node from the lowered DAG of a GEP IR instruction with one
39950// indexing dimension.
39951static SDValue getIndexFromUnindexedLoad(LoadSDNode *Ld) {
39952 if (Ld->isIndexed())
39953 return SDValue();
39954
39955 SDValue Base = Ld->getBasePtr();
39956
39957 if (Base.getOpcode() != ISD::ADD)
39958 return SDValue();
39959
39960 SDValue ShiftedIndex = Base.getOperand(0);
39961
39962 if (ShiftedIndex.getOpcode() != ISD::SHL)
39963 return SDValue();
39964
39965 return ShiftedIndex.getOperand(0);
39966
39967}
39968
39969static bool hasBZHI(const X86Subtarget &Subtarget, MVT VT) {
39970 if (Subtarget.hasBMI2() && VT.isScalarInteger()) {
39971 switch (VT.getSizeInBits()) {
39972 default: return false;
39973 case 64: return Subtarget.is64Bit() ? true : false;
39974 case 32: return true;
39975 }
39976 }
39977 return false;
39978}
39979
39980// This function recognizes cases where X86 bzhi instruction can replace and
39981// 'and-load' sequence.
39982// In case of loading integer value from an array of constants which is defined
39983// as follows:
39984//
39985// int array[SIZE] = {0x0, 0x1, 0x3, 0x7, 0xF ..., 2^(SIZE-1) - 1}
39986//
39987// then applying a bitwise and on the result with another input.
39988// It's equivalent to performing bzhi (zero high bits) on the input, with the
39989// same index of the load.
39990static SDValue combineAndLoadToBZHI(SDNode *Node, SelectionDAG &DAG,
39991 const X86Subtarget &Subtarget) {
39992 MVT VT = Node->getSimpleValueType(0);
39993 SDLoc dl(Node);
39994
39995 // Check if subtarget has BZHI instruction for the node's type
39996 if (!hasBZHI(Subtarget, VT))
39997 return SDValue();
39998
39999 // Try matching the pattern for both operands.
40000 for (unsigned i = 0; i < 2; i++) {
40001 SDValue N = Node->getOperand(i);
40002 LoadSDNode *Ld = dyn_cast<LoadSDNode>(N.getNode());
40003
40004 // continue if the operand is not a load instruction
40005 if (!Ld)
40006 return SDValue();
40007
40008 const Value *MemOp = Ld->getMemOperand()->getValue();
40009
40010 if (!MemOp)
40011 return SDValue();
40012
40013 if (const GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(MemOp)) {
40014 if (GlobalVariable *GV = dyn_cast<GlobalVariable>(GEP->getOperand(0))) {
40015 if (GV->isConstant() && GV->hasDefinitiveInitializer()) {
40016
40017 Constant *Init = GV->getInitializer();
40018 Type *Ty = Init->getType();
40019 if (!isa<ConstantDataArray>(Init) ||
40020 !Ty->getArrayElementType()->isIntegerTy() ||
40021 Ty->getArrayElementType()->getScalarSizeInBits() !=
40022 VT.getSizeInBits() ||
40023 Ty->getArrayNumElements() >
40024 Ty->getArrayElementType()->getScalarSizeInBits())
40025 continue;
40026
40027 // Check if the array's constant elements are suitable to our case.
40028 uint64_t ArrayElementCount = Init->getType()->getArrayNumElements();
40029 bool ConstantsMatch = true;
40030 for (uint64_t j = 0; j < ArrayElementCount; j++) {
40031 ConstantInt *Elem =
40032 dyn_cast<ConstantInt>(Init->getAggregateElement(j));
40033 if (Elem->getZExtValue() != (((uint64_t)1 << j) - 1)) {
40034 ConstantsMatch = false;
40035 break;
40036 }
40037 }
40038 if (!ConstantsMatch)
40039 continue;
40040
40041 // Do the transformation (For 32-bit type):
40042 // -> (and (load arr[idx]), inp)
40043 // <- (and (srl 0xFFFFFFFF, (sub 32, idx)))
40044 // that will be replaced with one bzhi instruction.
40045 SDValue Inp = (i == 0) ? Node->getOperand(1) : Node->getOperand(0);
40046 SDValue SizeC = DAG.getConstant(VT.getSizeInBits(), dl, MVT::i32);
40047
40048 // Get the Node which indexes into the array.
40049 SDValue Index = getIndexFromUnindexedLoad(Ld);
40050 if (!Index)
40051 return SDValue();
40052 Index = DAG.getZExtOrTrunc(Index, dl, MVT::i32);
40053
40054 SDValue Sub = DAG.getNode(ISD::SUB, dl, MVT::i32, SizeC, Index);
40055 Sub = DAG.getNode(ISD::TRUNCATE, dl, MVT::i8, Sub);
40056
40057 SDValue AllOnes = DAG.getAllOnesConstant(dl, VT);
40058 SDValue LShr = DAG.getNode(ISD::SRL, dl, VT, AllOnes, Sub);
40059
40060 return DAG.getNode(ISD::AND, dl, VT, Inp, LShr);
40061 }
40062 }
40063 }
40064 }
40065 return SDValue();
40066}
40067
40068// Look for (and (ctpop X), 1) which is the IR form of __builtin_parity.
40069// Turn it into series of XORs and a setnp.
40070static SDValue combineParity(SDNode *N, SelectionDAG &DAG,
40071 const X86Subtarget &Subtarget) {
40072 EVT VT = N->getValueType(0);
40073
40074 // We only support 64-bit and 32-bit. 64-bit requires special handling
40075 // unless the 64-bit popcnt instruction is legal.
40076 if (VT != MVT::i32 && VT != MVT::i64)
40077 return SDValue();
40078
40079 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40080 if (TLI.isTypeLegal(VT) && TLI.isOperationLegal(ISD::CTPOP, VT))
40081 return SDValue();
40082
40083 SDValue N0 = N->getOperand(0);
40084 SDValue N1 = N->getOperand(1);
40085
40086 // LHS needs to be a single use CTPOP.
40087 if (N0.getOpcode() != ISD::CTPOP || !N0.hasOneUse())
40088 return SDValue();
40089
40090 // RHS needs to be 1.
40091 if (!isOneConstant(N1))
40092 return SDValue();
40093
40094 SDLoc DL(N);
40095 SDValue X = N0.getOperand(0);
40096
40097 // If this is 64-bit, its always best to xor the two 32-bit pieces together
40098 // even if we have popcnt.
40099 if (VT == MVT::i64) {
40100 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32,
40101 DAG.getNode(ISD::SRL, DL, VT, X,
40102 DAG.getConstant(32, DL, MVT::i8)));
40103 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, X);
40104 X = DAG.getNode(ISD::XOR, DL, MVT::i32, Lo, Hi);
40105 // Generate a 32-bit parity idiom. This will bring us back here if we need
40106 // to expand it too.
40107 SDValue Parity = DAG.getNode(ISD::AND, DL, MVT::i32,
40108 DAG.getNode(ISD::CTPOP, DL, MVT::i32, X),
40109 DAG.getConstant(1, DL, MVT::i32));
40110 return DAG.getNode(ISD::ZERO_EXTEND, DL, VT, Parity);
40111 }
40112 assert(VT == MVT::i32 && "Unexpected VT!")((VT == MVT::i32 && "Unexpected VT!") ? static_cast<
void> (0) : __assert_fail ("VT == MVT::i32 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40112, __PRETTY_FUNCTION__));
40113
40114 // Xor the high and low 16-bits together using a 32-bit operation.
40115 SDValue Hi16 = DAG.getNode(ISD::SRL, DL, VT, X,
40116 DAG.getConstant(16, DL, MVT::i8));
40117 X = DAG.getNode(ISD::XOR, DL, VT, X, Hi16);
40118
40119 // Finally xor the low 2 bytes together and use a 8-bit flag setting xor.
40120 // This should allow an h-reg to be used to save a shift.
40121 // FIXME: We only get an h-reg in 32-bit mode.
40122 SDValue Hi = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8,
40123 DAG.getNode(ISD::SRL, DL, VT, X,
40124 DAG.getConstant(8, DL, MVT::i8)));
40125 SDValue Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i8, X);
40126 SDVTList VTs = DAG.getVTList(MVT::i8, MVT::i32);
40127 SDValue Flags = DAG.getNode(X86ISD::XOR, DL, VTs, Lo, Hi).getValue(1);
40128
40129 // Copy the inverse of the parity flag into a register with setcc.
40130 SDValue Setnp = getSETCC(X86::COND_NP, Flags, DL, DAG);
40131 // Zero extend to original type.
40132 return DAG.getNode(ISD::ZERO_EXTEND, DL, N->getValueType(0), Setnp);
40133}
40134
40135
40136// Look for (and (bitcast (vXi1 (concat_vectors (vYi1 setcc), undef,))), C)
40137// Where C is a mask containing the same number of bits as the setcc and
40138// where the setcc will freely 0 upper bits of k-register. We can replace the
40139// undef in the concat with 0s and remove the AND. This mainly helps with
40140// v2i1/v4i1 setcc being casted to scalar.
40141static SDValue combineScalarAndWithMaskSetcc(SDNode *N, SelectionDAG &DAG,
40142 const X86Subtarget &Subtarget) {
40143 assert(N->getOpcode() == ISD::AND && "Unexpected opcode!")((N->getOpcode() == ISD::AND && "Unexpected opcode!"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::AND && \"Unexpected opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40143, __PRETTY_FUNCTION__));
40144
40145 EVT VT = N->getValueType(0);
40146
40147 // Make sure this is an AND with constant. We will check the value of the
40148 // constant later.
40149 if (!isa<ConstantSDNode>(N->getOperand(1)))
40150 return SDValue();
40151
40152 // This is implied by the ConstantSDNode.
40153 assert(!VT.isVector() && "Expected scalar VT!")((!VT.isVector() && "Expected scalar VT!") ? static_cast
<void> (0) : __assert_fail ("!VT.isVector() && \"Expected scalar VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40153, __PRETTY_FUNCTION__));
40154
40155 if (N->getOperand(0).getOpcode() != ISD::BITCAST ||
40156 !N->getOperand(0).hasOneUse() ||
40157 !N->getOperand(0).getOperand(0).hasOneUse())
40158 return SDValue();
40159
40160 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40161 SDValue Src = N->getOperand(0).getOperand(0);
40162 EVT SrcVT = Src.getValueType();
40163 if (!SrcVT.isVector() || SrcVT.getVectorElementType() != MVT::i1 ||
40164 !TLI.isTypeLegal(SrcVT))
40165 return SDValue();
40166
40167 if (Src.getOpcode() != ISD::CONCAT_VECTORS)
40168 return SDValue();
40169
40170 // We only care about the first subvector of the concat, we expect the
40171 // other subvectors to be ignored due to the AND if we make the change.
40172 SDValue SubVec = Src.getOperand(0);
40173 EVT SubVecVT = SubVec.getValueType();
40174
40175 // First subvector should be a setcc with a legal result type. The RHS of the
40176 // AND should be a mask with this many bits.
40177 if (SubVec.getOpcode() != ISD::SETCC || !TLI.isTypeLegal(SubVecVT) ||
40178 !N->getConstantOperandAPInt(1).isMask(SubVecVT.getVectorNumElements()))
40179 return SDValue();
40180
40181 EVT SetccVT = SubVec.getOperand(0).getValueType();
40182 if (!TLI.isTypeLegal(SetccVT) ||
40183 !(Subtarget.hasVLX() || SetccVT.is512BitVector()))
40184 return SDValue();
40185
40186 if (!(Subtarget.hasBWI() || SetccVT.getScalarSizeInBits() >= 32))
40187 return SDValue();
40188
40189 // We passed all the checks. Rebuild the concat_vectors with zeroes
40190 // and cast it back to VT.
40191 SDLoc dl(N);
40192 SmallVector<SDValue, 4> Ops(Src.getNumOperands(),
40193 DAG.getConstant(0, dl, SubVecVT));
40194 Ops[0] = SubVec;
40195 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, dl, SrcVT,
40196 Ops);
40197 return DAG.getBitcast(VT, Concat);
40198}
40199
40200static SDValue combineAnd(SDNode *N, SelectionDAG &DAG,
40201 TargetLowering::DAGCombinerInfo &DCI,
40202 const X86Subtarget &Subtarget) {
40203 EVT VT = N->getValueType(0);
40204
40205 // If this is SSE1 only convert to FAND to avoid scalarization.
40206 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
40207 return DAG.getBitcast(
40208 MVT::v4i32, DAG.getNode(X86ISD::FAND, SDLoc(N), MVT::v4f32,
40209 DAG.getBitcast(MVT::v4f32, N->getOperand(0)),
40210 DAG.getBitcast(MVT::v4f32, N->getOperand(1))));
40211 }
40212
40213 // Use a 32-bit and+zext if upper bits known zero.
40214 if (VT == MVT::i64 && Subtarget.is64Bit() &&
40215 !isa<ConstantSDNode>(N->getOperand(1))) {
40216 APInt HiMask = APInt::getHighBitsSet(64, 32);
40217 if (DAG.MaskedValueIsZero(N->getOperand(1), HiMask) ||
40218 DAG.MaskedValueIsZero(N->getOperand(0), HiMask)) {
40219 SDLoc dl(N);
40220 SDValue LHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(0));
40221 SDValue RHS = DAG.getNode(ISD::TRUNCATE, dl, MVT::i32, N->getOperand(1));
40222 return DAG.getNode(ISD::ZERO_EXTEND, dl, MVT::i64,
40223 DAG.getNode(ISD::AND, dl, MVT::i32, LHS, RHS));
40224 }
40225 }
40226
40227 // This must be done before legalization has expanded the ctpop.
40228 if (SDValue V = combineParity(N, DAG, Subtarget))
40229 return V;
40230
40231 // Match all-of bool scalar reductions into a bitcast/movmsk + cmp.
40232 // TODO: Support multiple SrcOps.
40233 if (VT == MVT::i1) {
40234 SmallVector<SDValue, 2> SrcOps;
40235 if (matchScalarReduction(SDValue(N, 0), ISD::AND, SrcOps) &&
40236 SrcOps.size() == 1) {
40237 SDLoc dl(N);
40238 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40239 unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
40240 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
40241 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
40242 if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
40243 Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
40244 if (Mask) {
40245 APInt AllBits = APInt::getAllOnesValue(NumElts);
40246 return DAG.getSetCC(dl, MVT::i1, Mask,
40247 DAG.getConstant(AllBits, dl, MaskVT), ISD::SETEQ);
40248 }
40249 }
40250 }
40251
40252 if (SDValue V = combineScalarAndWithMaskSetcc(N, DAG, Subtarget))
40253 return V;
40254
40255 if (DCI.isBeforeLegalizeOps())
40256 return SDValue();
40257
40258 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
40259 return R;
40260
40261 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
40262 return FPLogic;
40263
40264 if (SDValue R = combineANDXORWithAllOnesIntoANDNP(N, DAG))
40265 return R;
40266
40267 if (SDValue ShiftRight = combineAndMaskToShift(N, DAG, Subtarget))
40268 return ShiftRight;
40269
40270 if (SDValue R = combineAndLoadToBZHI(N, DAG, Subtarget))
40271 return R;
40272
40273 // Attempt to recursively combine a bitmask AND with shuffles.
40274 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
40275 SDValue Op(N, 0);
40276 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
40277 return Res;
40278 }
40279
40280 // Attempt to combine a scalar bitmask AND with an extracted shuffle.
40281 if ((VT.getScalarSizeInBits() % 8) == 0 &&
40282 N->getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
40283 isa<ConstantSDNode>(N->getOperand(0).getOperand(1))) {
40284 SDValue BitMask = N->getOperand(1);
40285 SDValue SrcVec = N->getOperand(0).getOperand(0);
40286 EVT SrcVecVT = SrcVec.getValueType();
40287
40288 // Check that the constant bitmask masks whole bytes.
40289 APInt UndefElts;
40290 SmallVector<APInt, 64> EltBits;
40291 if (VT == SrcVecVT.getScalarType() &&
40292 N->getOperand(0)->isOnlyUserOf(SrcVec.getNode()) &&
40293 getTargetConstantBitsFromNode(BitMask, 8, UndefElts, EltBits) &&
40294 llvm::all_of(EltBits, [](APInt M) {
40295 return M.isNullValue() || M.isAllOnesValue();
40296 })) {
40297 unsigned NumElts = SrcVecVT.getVectorNumElements();
40298 unsigned Scale = SrcVecVT.getScalarSizeInBits() / 8;
40299 unsigned Idx = N->getOperand(0).getConstantOperandVal(1);
40300
40301 // Create a root shuffle mask from the byte mask and the extracted index.
40302 SmallVector<int, 16> ShuffleMask(NumElts * Scale, SM_SentinelUndef);
40303 for (unsigned i = 0; i != Scale; ++i) {
40304 if (UndefElts[i])
40305 continue;
40306 int VecIdx = Scale * Idx + i;
40307 ShuffleMask[VecIdx] =
40308 EltBits[i].isNullValue() ? SM_SentinelZero : VecIdx;
40309 }
40310
40311 if (SDValue Shuffle = combineX86ShufflesRecursively(
40312 {SrcVec}, 0, SrcVec, ShuffleMask, {}, /*Depth*/ 1,
40313 /*HasVarMask*/ false, /*AllowVarMask*/ true, DAG, Subtarget))
40314 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(N), VT, Shuffle,
40315 N->getOperand(0).getOperand(1));
40316 }
40317 }
40318
40319 return SDValue();
40320}
40321
40322// Canonicalize OR(AND(X,C),AND(Y,~C)) -> OR(AND(X,C),ANDNP(C,Y))
40323static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG,
40324 const X86Subtarget &Subtarget) {
40325 assert(N->getOpcode() == ISD::OR && "Unexpected Opcode")((N->getOpcode() == ISD::OR && "Unexpected Opcode"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Unexpected Opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40325, __PRETTY_FUNCTION__));
40326
40327 MVT VT = N->getSimpleValueType(0);
40328 if (!VT.isVector() || (VT.getScalarSizeInBits() % 8) != 0)
40329 return SDValue();
40330
40331 SDValue N0 = peekThroughBitcasts(N->getOperand(0));
40332 SDValue N1 = peekThroughBitcasts(N->getOperand(1));
40333 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != ISD::AND)
40334 return SDValue();
40335
40336 // On XOP we'll lower to PCMOV so accept one use. With AVX512, we can use
40337 // VPTERNLOG. Otherwise only do this if either mask has multiple uses already.
40338 bool UseVPTERNLOG = (Subtarget.hasAVX512() && VT.is512BitVector()) ||
40339 Subtarget.hasVLX();
40340 if (!(Subtarget.hasXOP() || UseVPTERNLOG ||
40341 !N0.getOperand(1).hasOneUse() || !N1.getOperand(1).hasOneUse()))
40342 return SDValue();
40343
40344 // Attempt to extract constant byte masks.
40345 APInt UndefElts0, UndefElts1;
40346 SmallVector<APInt, 32> EltBits0, EltBits1;
40347 if (!getTargetConstantBitsFromNode(N0.getOperand(1), 8, UndefElts0, EltBits0,
40348 false, false))
40349 return SDValue();
40350 if (!getTargetConstantBitsFromNode(N1.getOperand(1), 8, UndefElts1, EltBits1,
40351 false, false))
40352 return SDValue();
40353
40354 for (unsigned i = 0, e = EltBits0.size(); i != e; ++i) {
40355 // TODO - add UNDEF elts support.
40356 if (UndefElts0[i] || UndefElts1[i])
40357 return SDValue();
40358 if (EltBits0[i] != ~EltBits1[i])
40359 return SDValue();
40360 }
40361
40362 SDLoc DL(N);
40363 SDValue X = N->getOperand(0);
40364 SDValue Y =
40365 DAG.getNode(X86ISD::ANDNP, DL, VT, DAG.getBitcast(VT, N0.getOperand(1)),
40366 DAG.getBitcast(VT, N1.getOperand(0)));
40367 return DAG.getNode(ISD::OR, DL, VT, X, Y);
40368}
40369
40370// Try to match OR(AND(~MASK,X),AND(MASK,Y)) logic pattern.
40371static bool matchLogicBlend(SDNode *N, SDValue &X, SDValue &Y, SDValue &Mask) {
40372 if (N->getOpcode() != ISD::OR)
40373 return false;
40374
40375 SDValue N0 = N->getOperand(0);
40376 SDValue N1 = N->getOperand(1);
40377
40378 // Canonicalize AND to LHS.
40379 if (N1.getOpcode() == ISD::AND)
40380 std::swap(N0, N1);
40381
40382 // Attempt to match OR(AND(M,Y),ANDNP(M,X)).
40383 if (N0.getOpcode() != ISD::AND || N1.getOpcode() != X86ISD::ANDNP)
40384 return false;
40385
40386 Mask = N1.getOperand(0);
40387 X = N1.getOperand(1);
40388
40389 // Check to see if the mask appeared in both the AND and ANDNP.
40390 if (N0.getOperand(0) == Mask)
40391 Y = N0.getOperand(1);
40392 else if (N0.getOperand(1) == Mask)
40393 Y = N0.getOperand(0);
40394 else
40395 return false;
40396
40397 // TODO: Attempt to match against AND(XOR(-1,M),Y) as well, waiting for
40398 // ANDNP combine allows other combines to happen that prevent matching.
40399 return true;
40400}
40401
40402// Try to fold:
40403// (or (and (m, y), (pandn m, x)))
40404// into:
40405// (vselect m, x, y)
40406// As a special case, try to fold:
40407// (or (and (m, (sub 0, x)), (pandn m, x)))
40408// into:
40409// (sub (xor X, M), M)
40410static SDValue combineLogicBlendIntoPBLENDV(SDNode *N, SelectionDAG &DAG,
40411 const X86Subtarget &Subtarget) {
40412 assert(N->getOpcode() == ISD::OR && "Unexpected Opcode")((N->getOpcode() == ISD::OR && "Unexpected Opcode"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Unexpected Opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40412, __PRETTY_FUNCTION__));
40413
40414 EVT VT = N->getValueType(0);
40415 if (!((VT.is128BitVector() && Subtarget.hasSSE2()) ||
40416 (VT.is256BitVector() && Subtarget.hasInt256())))
40417 return SDValue();
40418
40419 SDValue X, Y, Mask;
40420 if (!matchLogicBlend(N, X, Y, Mask))
40421 return SDValue();
40422
40423 // Validate that X, Y, and Mask are bitcasts, and see through them.
40424 Mask = peekThroughBitcasts(Mask);
40425 X = peekThroughBitcasts(X);
40426 Y = peekThroughBitcasts(Y);
40427
40428 EVT MaskVT = Mask.getValueType();
40429 unsigned EltBits = MaskVT.getScalarSizeInBits();
40430
40431 // TODO: Attempt to handle floating point cases as well?
40432 if (!MaskVT.isInteger() || DAG.ComputeNumSignBits(Mask) != EltBits)
40433 return SDValue();
40434
40435 SDLoc DL(N);
40436
40437 // Attempt to combine to conditional negate: (sub (xor X, M), M)
40438 if (SDValue Res = combineLogicBlendIntoConditionalNegate(VT, Mask, X, Y, DL,
40439 DAG, Subtarget))
40440 return Res;
40441
40442 // PBLENDVB is only available on SSE 4.1.
40443 if (!Subtarget.hasSSE41())
40444 return SDValue();
40445
40446 MVT BlendVT = VT.is256BitVector() ? MVT::v32i8 : MVT::v16i8;
40447
40448 X = DAG.getBitcast(BlendVT, X);
40449 Y = DAG.getBitcast(BlendVT, Y);
40450 Mask = DAG.getBitcast(BlendVT, Mask);
40451 Mask = DAG.getSelect(DL, BlendVT, Mask, Y, X);
40452 return DAG.getBitcast(VT, Mask);
40453}
40454
40455// Helper function for combineOrCmpEqZeroToCtlzSrl
40456// Transforms:
40457// seteq(cmp x, 0)
40458// into:
40459// srl(ctlz x), log2(bitsize(x))
40460// Input pattern is checked by caller.
40461static SDValue lowerX86CmpEqZeroToCtlzSrl(SDValue Op, EVT ExtTy,
40462 SelectionDAG &DAG) {
40463 SDValue Cmp = Op.getOperand(1);
40464 EVT VT = Cmp.getOperand(0).getValueType();
40465 unsigned Log2b = Log2_32(VT.getSizeInBits());
40466 SDLoc dl(Op);
40467 SDValue Clz = DAG.getNode(ISD::CTLZ, dl, VT, Cmp->getOperand(0));
40468 // The result of the shift is true or false, and on X86, the 32-bit
40469 // encoding of shr and lzcnt is more desirable.
40470 SDValue Trunc = DAG.getZExtOrTrunc(Clz, dl, MVT::i32);
40471 SDValue Scc = DAG.getNode(ISD::SRL, dl, MVT::i32, Trunc,
40472 DAG.getConstant(Log2b, dl, MVT::i8));
40473 return DAG.getZExtOrTrunc(Scc, dl, ExtTy);
40474}
40475
40476// Try to transform:
40477// zext(or(setcc(eq, (cmp x, 0)), setcc(eq, (cmp y, 0))))
40478// into:
40479// srl(or(ctlz(x), ctlz(y)), log2(bitsize(x))
40480// Will also attempt to match more generic cases, eg:
40481// zext(or(or(setcc(eq, cmp 0), setcc(eq, cmp 0)), setcc(eq, cmp 0)))
40482// Only applies if the target supports the FastLZCNT feature.
40483static SDValue combineOrCmpEqZeroToCtlzSrl(SDNode *N, SelectionDAG &DAG,
40484 TargetLowering::DAGCombinerInfo &DCI,
40485 const X86Subtarget &Subtarget) {
40486 if (DCI.isBeforeLegalize() || !Subtarget.getTargetLowering()->isCtlzFast())
40487 return SDValue();
40488
40489 auto isORCandidate = [](SDValue N) {
40490 return (N->getOpcode() == ISD::OR && N->hasOneUse());
40491 };
40492
40493 // Check the zero extend is extending to 32-bit or more. The code generated by
40494 // srl(ctlz) for 16-bit or less variants of the pattern would require extra
40495 // instructions to clear the upper bits.
40496 if (!N->hasOneUse() || !N->getSimpleValueType(0).bitsGE(MVT::i32) ||
40497 !isORCandidate(N->getOperand(0)))
40498 return SDValue();
40499
40500 // Check the node matches: setcc(eq, cmp 0)
40501 auto isSetCCCandidate = [](SDValue N) {
40502 return N->getOpcode() == X86ISD::SETCC && N->hasOneUse() &&
40503 X86::CondCode(N->getConstantOperandVal(0)) == X86::COND_E &&
40504 N->getOperand(1).getOpcode() == X86ISD::CMP &&
40505 isNullConstant(N->getOperand(1).getOperand(1)) &&
40506 N->getOperand(1).getValueType().bitsGE(MVT::i32);
40507 };
40508
40509 SDNode *OR = N->getOperand(0).getNode();
40510 SDValue LHS = OR->getOperand(0);
40511 SDValue RHS = OR->getOperand(1);
40512
40513 // Save nodes matching or(or, setcc(eq, cmp 0)).
40514 SmallVector<SDNode *, 2> ORNodes;
40515 while (((isORCandidate(LHS) && isSetCCCandidate(RHS)) ||
40516 (isORCandidate(RHS) && isSetCCCandidate(LHS)))) {
40517 ORNodes.push_back(OR);
40518 OR = (LHS->getOpcode() == ISD::OR) ? LHS.getNode() : RHS.getNode();
40519 LHS = OR->getOperand(0);
40520 RHS = OR->getOperand(1);
40521 }
40522
40523 // The last OR node should match or(setcc(eq, cmp 0), setcc(eq, cmp 0)).
40524 if (!(isSetCCCandidate(LHS) && isSetCCCandidate(RHS)) ||
40525 !isORCandidate(SDValue(OR, 0)))
40526 return SDValue();
40527
40528 // We have a or(setcc(eq, cmp 0), setcc(eq, cmp 0)) pattern, try to lower it
40529 // to
40530 // or(srl(ctlz),srl(ctlz)).
40531 // The dag combiner can then fold it into:
40532 // srl(or(ctlz, ctlz)).
40533 EVT VT = OR->getValueType(0);
40534 SDValue NewLHS = lowerX86CmpEqZeroToCtlzSrl(LHS, VT, DAG);
40535 SDValue Ret, NewRHS;
40536 if (NewLHS && (NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, VT, DAG)))
40537 Ret = DAG.getNode(ISD::OR, SDLoc(OR), VT, NewLHS, NewRHS);
40538
40539 if (!Ret)
40540 return SDValue();
40541
40542 // Try to lower nodes matching the or(or, setcc(eq, cmp 0)) pattern.
40543 while (ORNodes.size() > 0) {
40544 OR = ORNodes.pop_back_val();
40545 LHS = OR->getOperand(0);
40546 RHS = OR->getOperand(1);
40547 // Swap rhs with lhs to match or(setcc(eq, cmp, 0), or).
40548 if (RHS->getOpcode() == ISD::OR)
40549 std::swap(LHS, RHS);
40550 NewRHS = lowerX86CmpEqZeroToCtlzSrl(RHS, VT, DAG);
40551 if (!NewRHS)
40552 return SDValue();
40553 Ret = DAG.getNode(ISD::OR, SDLoc(OR), VT, Ret, NewRHS);
40554 }
40555
40556 if (Ret)
40557 Ret = DAG.getNode(ISD::ZERO_EXTEND, SDLoc(N), N->getValueType(0), Ret);
40558
40559 return Ret;
40560}
40561
40562static SDValue combineOrShiftToFunnelShift(SDNode *N, SelectionDAG &DAG,
40563 const X86Subtarget &Subtarget) {
40564 assert(N->getOpcode() == ISD::OR && "Expected ISD::OR node")((N->getOpcode() == ISD::OR && "Expected ISD::OR node"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::OR && \"Expected ISD::OR node\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40564, __PRETTY_FUNCTION__));
40565 SDValue N0 = N->getOperand(0);
40566 SDValue N1 = N->getOperand(1);
40567 EVT VT = N->getValueType(0);
40568 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40569
40570 if (!TLI.isOperationLegalOrCustom(ISD::FSHL, VT) ||
40571 !TLI.isOperationLegalOrCustom(ISD::FSHR, VT))
40572 return SDValue();
40573
40574 // fold (or (x << c) | (y >> (64 - c))) ==> (shld64 x, y, c)
40575 bool OptForSize = DAG.shouldOptForSize();
40576 unsigned Bits = VT.getScalarSizeInBits();
40577
40578 // SHLD/SHRD instructions have lower register pressure, but on some
40579 // platforms they have higher latency than the equivalent
40580 // series of shifts/or that would otherwise be generated.
40581 // Don't fold (or (x << c) | (y >> (64 - c))) if SHLD/SHRD instructions
40582 // have higher latencies and we are not optimizing for size.
40583 if (!OptForSize && Subtarget.isSHLDSlow())
40584 return SDValue();
40585
40586 if (N0.getOpcode() == ISD::SRL && N1.getOpcode() == ISD::SHL)
40587 std::swap(N0, N1);
40588 if (N0.getOpcode() != ISD::SHL || N1.getOpcode() != ISD::SRL)
40589 return SDValue();
40590 if (!N0.hasOneUse() || !N1.hasOneUse())
40591 return SDValue();
40592
40593 EVT ShiftVT = TLI.getShiftAmountTy(VT, DAG.getDataLayout());
40594
40595 SDValue ShAmt0 = N0.getOperand(1);
40596 if (ShAmt0.getValueType() != ShiftVT)
40597 return SDValue();
40598 SDValue ShAmt1 = N1.getOperand(1);
40599 if (ShAmt1.getValueType() != ShiftVT)
40600 return SDValue();
40601
40602 // Peek through any modulo shift masks.
40603 SDValue ShMsk0;
40604 if (ShAmt0.getOpcode() == ISD::AND &&
40605 isa<ConstantSDNode>(ShAmt0.getOperand(1)) &&
40606 ShAmt0.getConstantOperandAPInt(1) == (Bits - 1)) {
40607 ShMsk0 = ShAmt0;
40608 ShAmt0 = ShAmt0.getOperand(0);
40609 }
40610 SDValue ShMsk1;
40611 if (ShAmt1.getOpcode() == ISD::AND &&
40612 isa<ConstantSDNode>(ShAmt1.getOperand(1)) &&
40613 ShAmt1.getConstantOperandAPInt(1) == (Bits - 1)) {
40614 ShMsk1 = ShAmt1;
40615 ShAmt1 = ShAmt1.getOperand(0);
40616 }
40617
40618 if (ShAmt0.getOpcode() == ISD::TRUNCATE)
40619 ShAmt0 = ShAmt0.getOperand(0);
40620 if (ShAmt1.getOpcode() == ISD::TRUNCATE)
40621 ShAmt1 = ShAmt1.getOperand(0);
40622
40623 SDLoc DL(N);
40624 unsigned Opc = ISD::FSHL;
40625 SDValue Op0 = N0.getOperand(0);
40626 SDValue Op1 = N1.getOperand(0);
40627 if (ShAmt0.getOpcode() == ISD::SUB || ShAmt0.getOpcode() == ISD::XOR) {
40628 Opc = ISD::FSHR;
40629 std::swap(Op0, Op1);
40630 std::swap(ShAmt0, ShAmt1);
40631 std::swap(ShMsk0, ShMsk1);
40632 }
40633
40634 auto GetFunnelShift = [&DAG, &DL, VT, Opc, &ShiftVT](SDValue Op0, SDValue Op1,
40635 SDValue Amt) {
40636 if (Opc == ISD::FSHR)
40637 std::swap(Op0, Op1);
40638 return DAG.getNode(Opc, DL, VT, Op0, Op1,
40639 DAG.getNode(ISD::TRUNCATE, DL, ShiftVT, Amt));
40640 };
40641
40642 // OR( SHL( X, C ), SRL( Y, 32 - C ) ) -> FSHL( X, Y, C )
40643 // OR( SRL( X, C ), SHL( Y, 32 - C ) ) -> FSHR( Y, X, C )
40644 // OR( SHL( X, C ), SRL( SRL( Y, 1 ), XOR( C, 31 ) ) ) -> FSHL( X, Y, C )
40645 // OR( SRL( X, C ), SHL( SHL( Y, 1 ), XOR( C, 31 ) ) ) -> FSHR( Y, X, C )
40646 // OR( SHL( X, AND( C, 31 ) ), SRL( Y, AND( 0 - C, 31 ) ) ) -> FSHL( X, Y, C )
40647 // OR( SRL( X, AND( C, 31 ) ), SHL( Y, AND( 0 - C, 31 ) ) ) -> FSHR( Y, X, C )
40648 if (ShAmt1.getOpcode() == ISD::SUB) {
40649 SDValue Sum = ShAmt1.getOperand(0);
40650 if (auto *SumC = dyn_cast<ConstantSDNode>(Sum)) {
40651 SDValue ShAmt1Op1 = ShAmt1.getOperand(1);
40652 if (ShAmt1Op1.getOpcode() == ISD::AND &&
40653 isa<ConstantSDNode>(ShAmt1Op1.getOperand(1)) &&
40654 ShAmt1Op1.getConstantOperandAPInt(1) == (Bits - 1)) {
40655 ShMsk1 = ShAmt1Op1;
40656 ShAmt1Op1 = ShAmt1Op1.getOperand(0);
40657 }
40658 if (ShAmt1Op1.getOpcode() == ISD::TRUNCATE)
40659 ShAmt1Op1 = ShAmt1Op1.getOperand(0);
40660 if ((SumC->getAPIntValue() == Bits ||
40661 (SumC->getAPIntValue() == 0 && ShMsk1)) &&
40662 ShAmt1Op1 == ShAmt0)
40663 return GetFunnelShift(Op0, Op1, ShAmt0);
40664 }
40665 } else if (auto *ShAmt1C = dyn_cast<ConstantSDNode>(ShAmt1)) {
40666 auto *ShAmt0C = dyn_cast<ConstantSDNode>(ShAmt0);
40667 if (ShAmt0C && (ShAmt0C->getSExtValue() + ShAmt1C->getSExtValue()) == Bits)
40668 return GetFunnelShift(Op0, Op1, ShAmt0);
40669 } else if (ShAmt1.getOpcode() == ISD::XOR) {
40670 SDValue Mask = ShAmt1.getOperand(1);
40671 if (auto *MaskC = dyn_cast<ConstantSDNode>(Mask)) {
40672 unsigned InnerShift = (ISD::FSHL == Opc ? ISD::SRL : ISD::SHL);
40673 SDValue ShAmt1Op0 = ShAmt1.getOperand(0);
40674 if (ShAmt1Op0.getOpcode() == ISD::TRUNCATE)
40675 ShAmt1Op0 = ShAmt1Op0.getOperand(0);
40676 if (MaskC->getSExtValue() == (Bits - 1) &&
40677 (ShAmt1Op0 == ShAmt0 || ShAmt1Op0 == ShMsk0)) {
40678 if (Op1.getOpcode() == InnerShift &&
40679 isa<ConstantSDNode>(Op1.getOperand(1)) &&
40680 Op1.getConstantOperandAPInt(1).isOneValue()) {
40681 return GetFunnelShift(Op0, Op1.getOperand(0), ShAmt0);
40682 }
40683 // Test for ADD( Y, Y ) as an equivalent to SHL( Y, 1 ).
40684 if (InnerShift == ISD::SHL && Op1.getOpcode() == ISD::ADD &&
40685 Op1.getOperand(0) == Op1.getOperand(1)) {
40686 return GetFunnelShift(Op0, Op1.getOperand(0), ShAmt0);
40687 }
40688 }
40689 }
40690 }
40691
40692 return SDValue();
40693}
40694
40695static SDValue combineOr(SDNode *N, SelectionDAG &DAG,
40696 TargetLowering::DAGCombinerInfo &DCI,
40697 const X86Subtarget &Subtarget) {
40698 SDValue N0 = N->getOperand(0);
40699 SDValue N1 = N->getOperand(1);
40700 EVT VT = N->getValueType(0);
40701
40702 // If this is SSE1 only convert to FOR to avoid scalarization.
40703 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32) {
40704 return DAG.getBitcast(MVT::v4i32,
40705 DAG.getNode(X86ISD::FOR, SDLoc(N), MVT::v4f32,
40706 DAG.getBitcast(MVT::v4f32, N0),
40707 DAG.getBitcast(MVT::v4f32, N1)));
40708 }
40709
40710 // Match any-of bool scalar reductions into a bitcast/movmsk + cmp.
40711 // TODO: Support multiple SrcOps.
40712 if (VT == MVT::i1) {
40713 SmallVector<SDValue, 2> SrcOps;
40714 if (matchScalarReduction(SDValue(N, 0), ISD::OR, SrcOps) &&
40715 SrcOps.size() == 1) {
40716 SDLoc dl(N);
40717 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40718 unsigned NumElts = SrcOps[0].getValueType().getVectorNumElements();
40719 EVT MaskVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
40720 SDValue Mask = combineBitcastvxi1(DAG, MaskVT, SrcOps[0], dl, Subtarget);
40721 if (!Mask && TLI.isTypeLegal(SrcOps[0].getValueType()))
40722 Mask = DAG.getBitcast(MaskVT, SrcOps[0]);
40723 if (Mask) {
40724 APInt AllBits = APInt::getNullValue(NumElts);
40725 return DAG.getSetCC(dl, MVT::i1, Mask,
40726 DAG.getConstant(AllBits, dl, MaskVT), ISD::SETNE);
40727 }
40728 }
40729 }
40730
40731 if (DCI.isBeforeLegalizeOps())
40732 return SDValue();
40733
40734 if (SDValue R = combineCompareEqual(N, DAG, DCI, Subtarget))
40735 return R;
40736
40737 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
40738 return FPLogic;
40739
40740 if (SDValue R = canonicalizeBitSelect(N, DAG, Subtarget))
40741 return R;
40742
40743 if (SDValue R = combineLogicBlendIntoPBLENDV(N, DAG, Subtarget))
40744 return R;
40745
40746 if (SDValue R = combineOrShiftToFunnelShift(N, DAG, Subtarget))
40747 return R;
40748
40749 // Attempt to recursively combine an OR of shuffles.
40750 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
40751 SDValue Op(N, 0);
40752 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
40753 return Res;
40754 }
40755
40756 return SDValue();
40757}
40758
40759/// Try to turn tests against the signbit in the form of:
40760/// XOR(TRUNCATE(SRL(X, size(X)-1)), 1)
40761/// into:
40762/// SETGT(X, -1)
40763static SDValue foldXorTruncShiftIntoCmp(SDNode *N, SelectionDAG &DAG) {
40764 // This is only worth doing if the output type is i8 or i1.
40765 EVT ResultType = N->getValueType(0);
40766 if (ResultType != MVT::i8 && ResultType != MVT::i1)
40767 return SDValue();
40768
40769 SDValue N0 = N->getOperand(0);
40770 SDValue N1 = N->getOperand(1);
40771
40772 // We should be performing an xor against a truncated shift.
40773 if (N0.getOpcode() != ISD::TRUNCATE || !N0.hasOneUse())
40774 return SDValue();
40775
40776 // Make sure we are performing an xor against one.
40777 if (!isOneConstant(N1))
40778 return SDValue();
40779
40780 // SetCC on x86 zero extends so only act on this if it's a logical shift.
40781 SDValue Shift = N0.getOperand(0);
40782 if (Shift.getOpcode() != ISD::SRL || !Shift.hasOneUse())
40783 return SDValue();
40784
40785 // Make sure we are truncating from one of i16, i32 or i64.
40786 EVT ShiftTy = Shift.getValueType();
40787 if (ShiftTy != MVT::i16 && ShiftTy != MVT::i32 && ShiftTy != MVT::i64)
40788 return SDValue();
40789
40790 // Make sure the shift amount extracts the sign bit.
40791 if (!isa<ConstantSDNode>(Shift.getOperand(1)) ||
40792 Shift.getConstantOperandAPInt(1) != (ShiftTy.getSizeInBits() - 1))
40793 return SDValue();
40794
40795 // Create a greater-than comparison against -1.
40796 // N.B. Using SETGE against 0 works but we want a canonical looking
40797 // comparison, using SETGT matches up with what TranslateX86CC.
40798 SDLoc DL(N);
40799 SDValue ShiftOp = Shift.getOperand(0);
40800 EVT ShiftOpTy = ShiftOp.getValueType();
40801 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
40802 EVT SetCCResultType = TLI.getSetCCResultType(DAG.getDataLayout(),
40803 *DAG.getContext(), ResultType);
40804 SDValue Cond = DAG.getSetCC(DL, SetCCResultType, ShiftOp,
40805 DAG.getConstant(-1, DL, ShiftOpTy), ISD::SETGT);
40806 if (SetCCResultType != ResultType)
40807 Cond = DAG.getNode(ISD::ZERO_EXTEND, DL, ResultType, Cond);
40808 return Cond;
40809}
40810
40811/// Turn vector tests of the signbit in the form of:
40812/// xor (sra X, elt_size(X)-1), -1
40813/// into:
40814/// pcmpgt X, -1
40815///
40816/// This should be called before type legalization because the pattern may not
40817/// persist after that.
40818static SDValue foldVectorXorShiftIntoCmp(SDNode *N, SelectionDAG &DAG,
40819 const X86Subtarget &Subtarget) {
40820 EVT VT = N->getValueType(0);
40821 if (!VT.isSimple())
40822 return SDValue();
40823
40824 switch (VT.getSimpleVT().SimpleTy) {
40825 default: return SDValue();
40826 case MVT::v16i8:
40827 case MVT::v8i16:
40828 case MVT::v4i32:
40829 case MVT::v2i64: if (!Subtarget.hasSSE2()) return SDValue(); break;
40830 case MVT::v32i8:
40831 case MVT::v16i16:
40832 case MVT::v8i32:
40833 case MVT::v4i64: if (!Subtarget.hasAVX2()) return SDValue(); break;
40834 }
40835
40836 // There must be a shift right algebraic before the xor, and the xor must be a
40837 // 'not' operation.
40838 SDValue Shift = N->getOperand(0);
40839 SDValue Ones = N->getOperand(1);
40840 if (Shift.getOpcode() != ISD::SRA || !Shift.hasOneUse() ||
40841 !ISD::isBuildVectorAllOnes(Ones.getNode()))
40842 return SDValue();
40843
40844 // The shift should be smearing the sign bit across each vector element.
40845 auto *ShiftAmt =
40846 isConstOrConstSplat(Shift.getOperand(1), /*AllowUndefs*/ true);
40847 if (!ShiftAmt ||
40848 ShiftAmt->getAPIntValue() != (Shift.getScalarValueSizeInBits() - 1))
40849 return SDValue();
40850
40851 // Create a greater-than comparison against -1. We don't use the more obvious
40852 // greater-than-or-equal-to-zero because SSE/AVX don't have that instruction.
40853 return DAG.getSetCC(SDLoc(N), VT, Shift.getOperand(0), Ones, ISD::SETGT);
40854}
40855
40856/// Detect patterns of truncation with unsigned saturation:
40857///
40858/// 1. (truncate (umin (x, unsigned_max_of_dest_type)) to dest_type).
40859/// Return the source value x to be truncated or SDValue() if the pattern was
40860/// not matched.
40861///
40862/// 2. (truncate (smin (smax (x, C1), C2)) to dest_type),
40863/// where C1 >= 0 and C2 is unsigned max of destination type.
40864///
40865/// (truncate (smax (smin (x, C2), C1)) to dest_type)
40866/// where C1 >= 0, C2 is unsigned max of destination type and C1 <= C2.
40867///
40868/// These two patterns are equivalent to:
40869/// (truncate (umin (smax(x, C1), unsigned_max_of_dest_type)) to dest_type)
40870/// So return the smax(x, C1) value to be truncated or SDValue() if the
40871/// pattern was not matched.
40872static SDValue detectUSatPattern(SDValue In, EVT VT, SelectionDAG &DAG,
40873 const SDLoc &DL) {
40874 EVT InVT = In.getValueType();
40875
40876 // Saturation with truncation. We truncate from InVT to VT.
40877 assert(InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() &&((InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() &&
"Unexpected types for truncate operation") ? static_cast<
void> (0) : __assert_fail ("InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() && \"Unexpected types for truncate operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40878, __PRETTY_FUNCTION__))
40878 "Unexpected types for truncate operation")((InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() &&
"Unexpected types for truncate operation") ? static_cast<
void> (0) : __assert_fail ("InVT.getScalarSizeInBits() > VT.getScalarSizeInBits() && \"Unexpected types for truncate operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40878, __PRETTY_FUNCTION__));
40879
40880 // Match min/max and return limit value as a parameter.
40881 auto MatchMinMax = [](SDValue V, unsigned Opcode, APInt &Limit) -> SDValue {
40882 if (V.getOpcode() == Opcode &&
40883 ISD::isConstantSplatVector(V.getOperand(1).getNode(), Limit))
40884 return V.getOperand(0);
40885 return SDValue();
40886 };
40887
40888 APInt C1, C2;
40889 if (SDValue UMin = MatchMinMax(In, ISD::UMIN, C2))
40890 // C2 should be equal to UINT32_MAX / UINT16_MAX / UINT8_MAX according
40891 // the element size of the destination type.
40892 if (C2.isMask(VT.getScalarSizeInBits()))
40893 return UMin;
40894
40895 if (SDValue SMin = MatchMinMax(In, ISD::SMIN, C2))
40896 if (MatchMinMax(SMin, ISD::SMAX, C1))
40897 if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()))
40898 return SMin;
40899
40900 if (SDValue SMax = MatchMinMax(In, ISD::SMAX, C1))
40901 if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, C2))
40902 if (C1.isNonNegative() && C2.isMask(VT.getScalarSizeInBits()) &&
40903 C2.uge(C1)) {
40904 return DAG.getNode(ISD::SMAX, DL, InVT, SMin, In.getOperand(1));
40905 }
40906
40907 return SDValue();
40908}
40909
40910/// Detect patterns of truncation with signed saturation:
40911/// (truncate (smin ((smax (x, signed_min_of_dest_type)),
40912/// signed_max_of_dest_type)) to dest_type)
40913/// or:
40914/// (truncate (smax ((smin (x, signed_max_of_dest_type)),
40915/// signed_min_of_dest_type)) to dest_type).
40916/// With MatchPackUS, the smax/smin range is [0, unsigned_max_of_dest_type].
40917/// Return the source value to be truncated or SDValue() if the pattern was not
40918/// matched.
40919static SDValue detectSSatPattern(SDValue In, EVT VT, bool MatchPackUS = false) {
40920 unsigned NumDstBits = VT.getScalarSizeInBits();
40921 unsigned NumSrcBits = In.getScalarValueSizeInBits();
40922 assert(NumSrcBits > NumDstBits && "Unexpected types for truncate operation")((NumSrcBits > NumDstBits && "Unexpected types for truncate operation"
) ? static_cast<void> (0) : __assert_fail ("NumSrcBits > NumDstBits && \"Unexpected types for truncate operation\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40922, __PRETTY_FUNCTION__));
40923
40924 auto MatchMinMax = [](SDValue V, unsigned Opcode,
40925 const APInt &Limit) -> SDValue {
40926 APInt C;
40927 if (V.getOpcode() == Opcode &&
40928 ISD::isConstantSplatVector(V.getOperand(1).getNode(), C) && C == Limit)
40929 return V.getOperand(0);
40930 return SDValue();
40931 };
40932
40933 APInt SignedMax, SignedMin;
40934 if (MatchPackUS) {
40935 SignedMax = APInt::getAllOnesValue(NumDstBits).zext(NumSrcBits);
40936 SignedMin = APInt(NumSrcBits, 0);
40937 } else {
40938 SignedMax = APInt::getSignedMaxValue(NumDstBits).sext(NumSrcBits);
40939 SignedMin = APInt::getSignedMinValue(NumDstBits).sext(NumSrcBits);
40940 }
40941
40942 if (SDValue SMin = MatchMinMax(In, ISD::SMIN, SignedMax))
40943 if (SDValue SMax = MatchMinMax(SMin, ISD::SMAX, SignedMin))
40944 return SMax;
40945
40946 if (SDValue SMax = MatchMinMax(In, ISD::SMAX, SignedMin))
40947 if (SDValue SMin = MatchMinMax(SMax, ISD::SMIN, SignedMax))
40948 return SMin;
40949
40950 return SDValue();
40951}
40952
40953static SDValue combineTruncateWithSat(SDValue In, EVT VT, const SDLoc &DL,
40954 SelectionDAG &DAG,
40955 const X86Subtarget &Subtarget) {
40956 if (!Subtarget.hasSSE2() || !VT.isVector())
40957 return SDValue();
40958
40959 EVT SVT = VT.getVectorElementType();
40960 EVT InVT = In.getValueType();
40961 EVT InSVT = InVT.getVectorElementType();
40962
40963 // If we're clamping a signed 32-bit vector to 0-255 and the 32-bit vector is
40964 // split across two registers. We can use a packusdw+perm to clamp to 0-65535
40965 // and concatenate at the same time. Then we can use a final vpmovuswb to
40966 // clip to 0-255.
40967 if (Subtarget.hasBWI() && !Subtarget.useAVX512Regs() &&
40968 InVT == MVT::v16i32 && VT == MVT::v16i8) {
40969 if (auto USatVal = detectSSatPattern(In, VT, true)) {
40970 // Emit a VPACKUSDW+VPERMQ followed by a VPMOVUSWB.
40971 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKUS, MVT::v16i16, USatVal,
40972 DL, DAG, Subtarget);
40973 assert(Mid && "Failed to pack!")((Mid && "Failed to pack!") ? static_cast<void>
(0) : __assert_fail ("Mid && \"Failed to pack!\"", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 40973, __PRETTY_FUNCTION__));
40974 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, Mid);
40975 }
40976 }
40977
40978 // vXi32 truncate instructions are available with AVX512F.
40979 // vXi16 truncate instructions are only available with AVX512BW.
40980 // For 256-bit or smaller vectors, we require VLX.
40981 // FIXME: We could widen truncates to 512 to remove the VLX restriction.
40982 // If the result type is 256-bits or larger and we have disable 512-bit
40983 // registers, we should go ahead and use the pack instructions if possible.
40984 bool PreferAVX512 = ((Subtarget.hasAVX512() && InSVT == MVT::i32) ||
40985 (Subtarget.hasBWI() && InSVT == MVT::i16)) &&
40986 (InVT.getSizeInBits() > 128) &&
40987 (Subtarget.hasVLX() || InVT.getSizeInBits() > 256) &&
40988 !(!Subtarget.useAVX512Regs() && VT.getSizeInBits() >= 256);
40989
40990 if (isPowerOf2_32(VT.getVectorNumElements()) && !PreferAVX512 &&
40991 VT.getSizeInBits() >= 64 &&
40992 (SVT == MVT::i8 || SVT == MVT::i16) &&
40993 (InSVT == MVT::i16 || InSVT == MVT::i32)) {
40994 if (auto USatVal = detectSSatPattern(In, VT, true)) {
40995 // vXi32 -> vXi8 must be performed as PACKUSWB(PACKSSDW,PACKSSDW).
40996 // Only do this when the result is at least 64 bits or we'll leaving
40997 // dangling PACKSSDW nodes.
40998 if (SVT == MVT::i8 && InSVT == MVT::i32) {
40999 EVT MidVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
41000 VT.getVectorNumElements());
41001 SDValue Mid = truncateVectorWithPACK(X86ISD::PACKSS, MidVT, USatVal, DL,
41002 DAG, Subtarget);
41003 assert(Mid && "Failed to pack!")((Mid && "Failed to pack!") ? static_cast<void>
(0) : __assert_fail ("Mid && \"Failed to pack!\"", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41003, __PRETTY_FUNCTION__));
41004 SDValue V = truncateVectorWithPACK(X86ISD::PACKUS, VT, Mid, DL, DAG,
41005 Subtarget);
41006 assert(V && "Failed to pack!")((V && "Failed to pack!") ? static_cast<void> (
0) : __assert_fail ("V && \"Failed to pack!\"", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41006, __PRETTY_FUNCTION__));
41007 return V;
41008 } else if (SVT == MVT::i8 || Subtarget.hasSSE41())
41009 return truncateVectorWithPACK(X86ISD::PACKUS, VT, USatVal, DL, DAG,
41010 Subtarget);
41011 }
41012 if (auto SSatVal = detectSSatPattern(In, VT))
41013 return truncateVectorWithPACK(X86ISD::PACKSS, VT, SSatVal, DL, DAG,
41014 Subtarget);
41015 }
41016
41017 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41018 if (TLI.isTypeLegal(InVT) && InVT.isVector() && SVT != MVT::i1 &&
41019 Subtarget.hasAVX512() && (InSVT != MVT::i16 || Subtarget.hasBWI())) {
41020 unsigned TruncOpc = 0;
41021 SDValue SatVal;
41022 if (auto SSatVal = detectSSatPattern(In, VT)) {
41023 SatVal = SSatVal;
41024 TruncOpc = X86ISD::VTRUNCS;
41025 } else if (auto USatVal = detectUSatPattern(In, VT, DAG, DL)) {
41026 SatVal = USatVal;
41027 TruncOpc = X86ISD::VTRUNCUS;
41028 }
41029 if (SatVal) {
41030 unsigned ResElts = VT.getVectorNumElements();
41031 // If the input type is less than 512 bits and we don't have VLX, we need
41032 // to widen to 512 bits.
41033 if (!Subtarget.hasVLX() && !InVT.is512BitVector()) {
41034 unsigned NumConcats = 512 / InVT.getSizeInBits();
41035 ResElts *= NumConcats;
41036 SmallVector<SDValue, 4> ConcatOps(NumConcats, DAG.getUNDEF(InVT));
41037 ConcatOps[0] = SatVal;
41038 InVT = EVT::getVectorVT(*DAG.getContext(), InSVT,
41039 NumConcats * InVT.getVectorNumElements());
41040 SatVal = DAG.getNode(ISD::CONCAT_VECTORS, DL, InVT, ConcatOps);
41041 }
41042 // Widen the result if its narrower than 128 bits.
41043 if (ResElts * SVT.getSizeInBits() < 128)
41044 ResElts = 128 / SVT.getSizeInBits();
41045 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), SVT, ResElts);
41046 SDValue Res = DAG.getNode(TruncOpc, DL, TruncVT, SatVal);
41047 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Res,
41048 DAG.getIntPtrConstant(0, DL));
41049 }
41050 }
41051
41052 return SDValue();
41053}
41054
41055/// This function detects the AVG pattern between vectors of unsigned i8/i16,
41056/// which is c = (a + b + 1) / 2, and replace this operation with the efficient
41057/// X86ISD::AVG instruction.
41058static SDValue detectAVGPattern(SDValue In, EVT VT, SelectionDAG &DAG,
41059 const X86Subtarget &Subtarget,
41060 const SDLoc &DL) {
41061 if (!VT.isVector())
41062 return SDValue();
41063 EVT InVT = In.getValueType();
41064 unsigned NumElems = VT.getVectorNumElements();
41065
41066 EVT ScalarVT = VT.getVectorElementType();
41067 if (!((ScalarVT == MVT::i8 || ScalarVT == MVT::i16) &&
41068 NumElems >= 2 && isPowerOf2_32(NumElems)))
41069 return SDValue();
41070
41071 // InScalarVT is the intermediate type in AVG pattern and it should be greater
41072 // than the original input type (i8/i16).
41073 EVT InScalarVT = InVT.getVectorElementType();
41074 if (InScalarVT.getSizeInBits() <= ScalarVT.getSizeInBits())
41075 return SDValue();
41076
41077 if (!Subtarget.hasSSE2())
41078 return SDValue();
41079
41080 // Detect the following pattern:
41081 //
41082 // %1 = zext <N x i8> %a to <N x i32>
41083 // %2 = zext <N x i8> %b to <N x i32>
41084 // %3 = add nuw nsw <N x i32> %1, <i32 1 x N>
41085 // %4 = add nuw nsw <N x i32> %3, %2
41086 // %5 = lshr <N x i32> %N, <i32 1 x N>
41087 // %6 = trunc <N x i32> %5 to <N x i8>
41088 //
41089 // In AVX512, the last instruction can also be a trunc store.
41090 if (In.getOpcode() != ISD::SRL)
41091 return SDValue();
41092
41093 // A lambda checking the given SDValue is a constant vector and each element
41094 // is in the range [Min, Max].
41095 auto IsConstVectorInRange = [](SDValue V, unsigned Min, unsigned Max) {
41096 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(V);
41097 if (!BV || !BV->isConstant())
41098 return false;
41099 for (SDValue Op : V->ops()) {
41100 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op);
41101 if (!C)
41102 return false;
41103 const APInt &Val = C->getAPIntValue();
41104 if (Val.ult(Min) || Val.ugt(Max))
41105 return false;
41106 }
41107 return true;
41108 };
41109
41110 // Check if each element of the vector is right-shifted by one.
41111 auto LHS = In.getOperand(0);
41112 auto RHS = In.getOperand(1);
41113 if (!IsConstVectorInRange(RHS, 1, 1))
41114 return SDValue();
41115 if (LHS.getOpcode() != ISD::ADD)
41116 return SDValue();
41117
41118 // Detect a pattern of a + b + 1 where the order doesn't matter.
41119 SDValue Operands[3];
41120 Operands[0] = LHS.getOperand(0);
41121 Operands[1] = LHS.getOperand(1);
41122
41123 auto AVGBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
41124 ArrayRef<SDValue> Ops) {
41125 return DAG.getNode(X86ISD::AVG, DL, Ops[0].getValueType(), Ops);
41126 };
41127
41128 // Take care of the case when one of the operands is a constant vector whose
41129 // element is in the range [1, 256].
41130 if (IsConstVectorInRange(Operands[1], 1, ScalarVT == MVT::i8 ? 256 : 65536) &&
41131 Operands[0].getOpcode() == ISD::ZERO_EXTEND &&
41132 Operands[0].getOperand(0).getValueType() == VT) {
41133 // The pattern is detected. Subtract one from the constant vector, then
41134 // demote it and emit X86ISD::AVG instruction.
41135 SDValue VecOnes = DAG.getConstant(1, DL, InVT);
41136 Operands[1] = DAG.getNode(ISD::SUB, DL, InVT, Operands[1], VecOnes);
41137 Operands[1] = DAG.getNode(ISD::TRUNCATE, DL, VT, Operands[1]);
41138 return SplitOpsAndApply(DAG, Subtarget, DL, VT,
41139 { Operands[0].getOperand(0), Operands[1] },
41140 AVGBuilder);
41141 }
41142
41143 // Matches 'add like' patterns: add(Op0,Op1) + zext(or(Op0,Op1)).
41144 // Match the or case only if its 'add-like' - can be replaced by an add.
41145 auto FindAddLike = [&](SDValue V, SDValue &Op0, SDValue &Op1) {
41146 if (ISD::ADD == V.getOpcode()) {
41147 Op0 = V.getOperand(0);
41148 Op1 = V.getOperand(1);
41149 return true;
41150 }
41151 if (ISD::ZERO_EXTEND != V.getOpcode())
41152 return false;
41153 V = V.getOperand(0);
41154 if (V.getValueType() != VT || ISD::OR != V.getOpcode() ||
41155 !DAG.haveNoCommonBitsSet(V.getOperand(0), V.getOperand(1)))
41156 return false;
41157 Op0 = V.getOperand(0);
41158 Op1 = V.getOperand(1);
41159 return true;
41160 };
41161
41162 SDValue Op0, Op1;
41163 if (FindAddLike(Operands[0], Op0, Op1))
41164 std::swap(Operands[0], Operands[1]);
41165 else if (!FindAddLike(Operands[1], Op0, Op1))
41166 return SDValue();
41167 Operands[2] = Op0;
41168 Operands[1] = Op1;
41169
41170 // Now we have three operands of two additions. Check that one of them is a
41171 // constant vector with ones, and the other two can be promoted from i8/i16.
41172 for (int i = 0; i < 3; ++i) {
41173 if (!IsConstVectorInRange(Operands[i], 1, 1))
41174 continue;
41175 std::swap(Operands[i], Operands[2]);
41176
41177 // Check if Operands[0] and Operands[1] are results of type promotion.
41178 for (int j = 0; j < 2; ++j)
41179 if (Operands[j].getValueType() != VT) {
41180 if (Operands[j].getOpcode() != ISD::ZERO_EXTEND ||
41181 Operands[j].getOperand(0).getValueType() != VT)
41182 return SDValue();
41183 Operands[j] = Operands[j].getOperand(0);
41184 }
41185
41186 // The pattern is detected, emit X86ISD::AVG instruction(s).
41187 return SplitOpsAndApply(DAG, Subtarget, DL, VT, {Operands[0], Operands[1]},
41188 AVGBuilder);
41189 }
41190
41191 return SDValue();
41192}
41193
41194static SDValue combineLoad(SDNode *N, SelectionDAG &DAG,
41195 TargetLowering::DAGCombinerInfo &DCI,
41196 const X86Subtarget &Subtarget) {
41197 LoadSDNode *Ld = cast<LoadSDNode>(N);
41198 EVT RegVT = Ld->getValueType(0);
41199 EVT MemVT = Ld->getMemoryVT();
41200 SDLoc dl(Ld);
41201 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41202
41203 // For chips with slow 32-byte unaligned loads, break the 32-byte operation
41204 // into two 16-byte operations. Also split non-temporal aligned loads on
41205 // pre-AVX2 targets as 32-byte loads will lower to regular temporal loads.
41206 ISD::LoadExtType Ext = Ld->getExtensionType();
41207 bool Fast;
41208 unsigned Alignment = Ld->getAlignment();
41209 if (RegVT.is256BitVector() && !DCI.isBeforeLegalizeOps() &&
41210 Ext == ISD::NON_EXTLOAD &&
41211 ((Ld->isNonTemporal() && !Subtarget.hasInt256() && Alignment >= 16) ||
41212 (TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), RegVT,
41213 *Ld->getMemOperand(), &Fast) &&
41214 !Fast))) {
41215 unsigned NumElems = RegVT.getVectorNumElements();
41216 if (NumElems < 2)
41217 return SDValue();
41218
41219 unsigned HalfAlign = 16;
41220 SDValue Ptr1 = Ld->getBasePtr();
41221 SDValue Ptr2 = DAG.getMemBasePlusOffset(Ptr1, HalfAlign, dl);
41222 EVT HalfVT = EVT::getVectorVT(*DAG.getContext(), MemVT.getScalarType(),
41223 NumElems / 2);
41224 SDValue Load1 =
41225 DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr1, Ld->getPointerInfo(),
41226 Alignment, Ld->getMemOperand()->getFlags());
41227 SDValue Load2 = DAG.getLoad(HalfVT, dl, Ld->getChain(), Ptr2,
41228 Ld->getPointerInfo().getWithOffset(HalfAlign),
41229 MinAlign(Alignment, HalfAlign),
41230 Ld->getMemOperand()->getFlags());
41231 SDValue TF = DAG.getNode(ISD::TokenFactor, dl, MVT::Other,
41232 Load1.getValue(1), Load2.getValue(1));
41233
41234 SDValue NewVec = DAG.getNode(ISD::CONCAT_VECTORS, dl, RegVT, Load1, Load2);
41235 return DCI.CombineTo(N, NewVec, TF, true);
41236 }
41237
41238 // Bool vector load - attempt to cast to an integer, as we have good
41239 // (vXiY *ext(vXi1 bitcast(iX))) handling.
41240 if (Ext == ISD::NON_EXTLOAD && !Subtarget.hasAVX512() && RegVT.isVector() &&
41241 RegVT.getScalarType() == MVT::i1 && DCI.isBeforeLegalize()) {
41242 unsigned NumElts = RegVT.getVectorNumElements();
41243 EVT IntVT = EVT::getIntegerVT(*DAG.getContext(), NumElts);
41244 if (TLI.isTypeLegal(IntVT)) {
41245 SDValue IntLoad = DAG.getLoad(IntVT, dl, Ld->getChain(), Ld->getBasePtr(),
41246 Ld->getPointerInfo(), Alignment,
41247 Ld->getMemOperand()->getFlags());
41248 SDValue BoolVec = DAG.getBitcast(RegVT, IntLoad);
41249 return DCI.CombineTo(N, BoolVec, IntLoad.getValue(1), true);
41250 }
41251 }
41252
41253 return SDValue();
41254}
41255
41256/// If V is a build vector of boolean constants and exactly one of those
41257/// constants is true, return the operand index of that true element.
41258/// Otherwise, return -1.
41259static int getOneTrueElt(SDValue V) {
41260 // This needs to be a build vector of booleans.
41261 // TODO: Checking for the i1 type matches the IR definition for the mask,
41262 // but the mask check could be loosened to i8 or other types. That might
41263 // also require checking more than 'allOnesValue'; eg, the x86 HW
41264 // instructions only require that the MSB is set for each mask element.
41265 // The ISD::MSTORE comments/definition do not specify how the mask operand
41266 // is formatted.
41267 auto *BV = dyn_cast<BuildVectorSDNode>(V);
41268 if (!BV || BV->getValueType(0).getVectorElementType() != MVT::i1)
41269 return -1;
41270
41271 int TrueIndex = -1;
41272 unsigned NumElts = BV->getValueType(0).getVectorNumElements();
41273 for (unsigned i = 0; i < NumElts; ++i) {
41274 const SDValue &Op = BV->getOperand(i);
41275 if (Op.isUndef())
41276 continue;
41277 auto *ConstNode = dyn_cast<ConstantSDNode>(Op);
41278 if (!ConstNode)
41279 return -1;
41280 if (ConstNode->getAPIntValue().isAllOnesValue()) {
41281 // If we already found a one, this is too many.
41282 if (TrueIndex >= 0)
41283 return -1;
41284 TrueIndex = i;
41285 }
41286 }
41287 return TrueIndex;
41288}
41289
41290/// Given a masked memory load/store operation, return true if it has one mask
41291/// bit set. If it has one mask bit set, then also return the memory address of
41292/// the scalar element to load/store, the vector index to insert/extract that
41293/// scalar element, and the alignment for the scalar memory access.
41294static bool getParamsForOneTrueMaskedElt(MaskedLoadStoreSDNode *MaskedOp,
41295 SelectionDAG &DAG, SDValue &Addr,
41296 SDValue &Index, unsigned &Alignment) {
41297 int TrueMaskElt = getOneTrueElt(MaskedOp->getMask());
41298 if (TrueMaskElt < 0)
41299 return false;
41300
41301 // Get the address of the one scalar element that is specified by the mask
41302 // using the appropriate offset from the base pointer.
41303 EVT EltVT = MaskedOp->getMemoryVT().getVectorElementType();
41304 Addr = MaskedOp->getBasePtr();
41305 if (TrueMaskElt != 0) {
41306 unsigned Offset = TrueMaskElt * EltVT.getStoreSize();
41307 Addr = DAG.getMemBasePlusOffset(Addr, Offset, SDLoc(MaskedOp));
41308 }
41309
41310 Index = DAG.getIntPtrConstant(TrueMaskElt, SDLoc(MaskedOp));
41311 Alignment = MinAlign(MaskedOp->getAlignment(), EltVT.getStoreSize());
41312 return true;
41313}
41314
41315/// If exactly one element of the mask is set for a non-extending masked load,
41316/// it is a scalar load and vector insert.
41317/// Note: It is expected that the degenerate cases of an all-zeros or all-ones
41318/// mask have already been optimized in IR, so we don't bother with those here.
41319static SDValue
41320reduceMaskedLoadToScalarLoad(MaskedLoadSDNode *ML, SelectionDAG &DAG,
41321 TargetLowering::DAGCombinerInfo &DCI) {
41322 assert(ML->isUnindexed() && "Unexpected indexed masked load!")((ML->isUnindexed() && "Unexpected indexed masked load!"
) ? static_cast<void> (0) : __assert_fail ("ML->isUnindexed() && \"Unexpected indexed masked load!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41322, __PRETTY_FUNCTION__));
41323 // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
41324 // However, some target hooks may need to be added to know when the transform
41325 // is profitable. Endianness would also have to be considered.
41326
41327 SDValue Addr, VecIndex;
41328 unsigned Alignment;
41329 if (!getParamsForOneTrueMaskedElt(ML, DAG, Addr, VecIndex, Alignment))
41330 return SDValue();
41331
41332 // Load the one scalar element that is specified by the mask using the
41333 // appropriate offset from the base pointer.
41334 SDLoc DL(ML);
41335 EVT VT = ML->getValueType(0);
41336 EVT EltVT = VT.getVectorElementType();
41337 SDValue Load =
41338 DAG.getLoad(EltVT, DL, ML->getChain(), Addr, ML->getPointerInfo(),
41339 Alignment, ML->getMemOperand()->getFlags());
41340
41341 // Insert the loaded element into the appropriate place in the vector.
41342 SDValue Insert = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT,
41343 ML->getPassThru(), Load, VecIndex);
41344 return DCI.CombineTo(ML, Insert, Load.getValue(1), true);
41345}
41346
41347static SDValue
41348combineMaskedLoadConstantMask(MaskedLoadSDNode *ML, SelectionDAG &DAG,
41349 TargetLowering::DAGCombinerInfo &DCI) {
41350 assert(ML->isUnindexed() && "Unexpected indexed masked load!")((ML->isUnindexed() && "Unexpected indexed masked load!"
) ? static_cast<void> (0) : __assert_fail ("ML->isUnindexed() && \"Unexpected indexed masked load!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41350, __PRETTY_FUNCTION__));
41351 if (!ISD::isBuildVectorOfConstantSDNodes(ML->getMask().getNode()))
41352 return SDValue();
41353
41354 SDLoc DL(ML);
41355 EVT VT = ML->getValueType(0);
41356
41357 // If we are loading the first and last elements of a vector, it is safe and
41358 // always faster to load the whole vector. Replace the masked load with a
41359 // vector load and select.
41360 unsigned NumElts = VT.getVectorNumElements();
41361 BuildVectorSDNode *MaskBV = cast<BuildVectorSDNode>(ML->getMask());
41362 bool LoadFirstElt = !isNullConstant(MaskBV->getOperand(0));
41363 bool LoadLastElt = !isNullConstant(MaskBV->getOperand(NumElts - 1));
41364 if (LoadFirstElt && LoadLastElt) {
41365 SDValue VecLd = DAG.getLoad(VT, DL, ML->getChain(), ML->getBasePtr(),
41366 ML->getMemOperand());
41367 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), VecLd,
41368 ML->getPassThru());
41369 return DCI.CombineTo(ML, Blend, VecLd.getValue(1), true);
41370 }
41371
41372 // Convert a masked load with a constant mask into a masked load and a select.
41373 // This allows the select operation to use a faster kind of select instruction
41374 // (for example, vblendvps -> vblendps).
41375
41376 // Don't try this if the pass-through operand is already undefined. That would
41377 // cause an infinite loop because that's what we're about to create.
41378 if (ML->getPassThru().isUndef())
41379 return SDValue();
41380
41381 if (ISD::isBuildVectorAllZeros(ML->getPassThru().getNode()))
41382 return SDValue();
41383
41384 // The new masked load has an undef pass-through operand. The select uses the
41385 // original pass-through operand.
41386 SDValue NewML = DAG.getMaskedLoad(
41387 VT, DL, ML->getChain(), ML->getBasePtr(), ML->getOffset(), ML->getMask(),
41388 DAG.getUNDEF(VT), ML->getMemoryVT(), ML->getMemOperand(),
41389 ML->getAddressingMode(), ML->getExtensionType());
41390 SDValue Blend = DAG.getSelect(DL, VT, ML->getMask(), NewML,
41391 ML->getPassThru());
41392
41393 return DCI.CombineTo(ML, Blend, NewML.getValue(1), true);
41394}
41395
41396static SDValue combineMaskedLoad(SDNode *N, SelectionDAG &DAG,
41397 TargetLowering::DAGCombinerInfo &DCI,
41398 const X86Subtarget &Subtarget) {
41399 MaskedLoadSDNode *Mld = cast<MaskedLoadSDNode>(N);
41400
41401 // TODO: Expanding load with constant mask may be optimized as well.
41402 if (Mld->isExpandingLoad())
41403 return SDValue();
41404
41405 if (Mld->getExtensionType() == ISD::NON_EXTLOAD) {
41406 if (SDValue ScalarLoad = reduceMaskedLoadToScalarLoad(Mld, DAG, DCI))
41407 return ScalarLoad;
41408 // TODO: Do some AVX512 subsets benefit from this transform?
41409 if (!Subtarget.hasAVX512())
41410 if (SDValue Blend = combineMaskedLoadConstantMask(Mld, DAG, DCI))
41411 return Blend;
41412 }
41413
41414 return SDValue();
41415}
41416
41417/// If exactly one element of the mask is set for a non-truncating masked store,
41418/// it is a vector extract and scalar store.
41419/// Note: It is expected that the degenerate cases of an all-zeros or all-ones
41420/// mask have already been optimized in IR, so we don't bother with those here.
41421static SDValue reduceMaskedStoreToScalarStore(MaskedStoreSDNode *MS,
41422 SelectionDAG &DAG) {
41423 // TODO: This is not x86-specific, so it could be lifted to DAGCombiner.
41424 // However, some target hooks may need to be added to know when the transform
41425 // is profitable. Endianness would also have to be considered.
41426
41427 SDValue Addr, VecIndex;
41428 unsigned Alignment;
41429 if (!getParamsForOneTrueMaskedElt(MS, DAG, Addr, VecIndex, Alignment))
41430 return SDValue();
41431
41432 // Extract the one scalar element that is actually being stored.
41433 SDLoc DL(MS);
41434 EVT VT = MS->getValue().getValueType();
41435 EVT EltVT = VT.getVectorElementType();
41436 SDValue Extract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT,
41437 MS->getValue(), VecIndex);
41438
41439 // Store that element at the appropriate offset from the base pointer.
41440 return DAG.getStore(MS->getChain(), DL, Extract, Addr, MS->getPointerInfo(),
41441 Alignment, MS->getMemOperand()->getFlags());
41442}
41443
41444static SDValue combineMaskedStore(SDNode *N, SelectionDAG &DAG,
41445 TargetLowering::DAGCombinerInfo &DCI,
41446 const X86Subtarget &Subtarget) {
41447 MaskedStoreSDNode *Mst = cast<MaskedStoreSDNode>(N);
41448 if (Mst->isCompressingStore())
41449 return SDValue();
41450
41451 EVT VT = Mst->getValue().getValueType();
41452 SDLoc dl(Mst);
41453 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41454
41455 if (Mst->isTruncatingStore())
41456 return SDValue();
41457
41458 if (SDValue ScalarStore = reduceMaskedStoreToScalarStore(Mst, DAG))
41459 return ScalarStore;
41460
41461 // If the mask value has been legalized to a non-boolean vector, try to
41462 // simplify ops leading up to it. We only demand the MSB of each lane.
41463 SDValue Mask = Mst->getMask();
41464 if (Mask.getScalarValueSizeInBits() != 1) {
41465 APInt DemandedMask(APInt::getSignMask(VT.getScalarSizeInBits()));
41466 if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
41467 return SDValue(N, 0);
41468 }
41469
41470 SDValue Value = Mst->getValue();
41471 if (Value.getOpcode() == ISD::TRUNCATE && Value.getNode()->hasOneUse() &&
41472 TLI.isTruncStoreLegal(Value.getOperand(0).getValueType(),
41473 Mst->getMemoryVT())) {
41474 return DAG.getMaskedStore(Mst->getChain(), SDLoc(N), Value.getOperand(0),
41475 Mst->getBasePtr(), Mst->getOffset(), Mask,
41476 Mst->getMemoryVT(), Mst->getMemOperand(),
41477 Mst->getAddressingMode(), true);
41478 }
41479
41480 return SDValue();
41481}
41482
41483static SDValue combineStore(SDNode *N, SelectionDAG &DAG,
41484 TargetLowering::DAGCombinerInfo &DCI,
41485 const X86Subtarget &Subtarget) {
41486 StoreSDNode *St = cast<StoreSDNode>(N);
41487 EVT StVT = St->getMemoryVT();
41488 SDLoc dl(St);
41489 unsigned Alignment = St->getAlignment();
41490 SDValue StoredVal = St->getValue();
41491 EVT VT = StoredVal.getValueType();
41492 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41493
41494 // Convert a store of vXi1 into a store of iX and a bitcast.
41495 if (!Subtarget.hasAVX512() && VT == StVT && VT.isVector() &&
41496 VT.getVectorElementType() == MVT::i1) {
41497
41498 EVT NewVT = EVT::getIntegerVT(*DAG.getContext(), VT.getVectorNumElements());
41499 StoredVal = DAG.getBitcast(NewVT, StoredVal);
41500
41501 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
41502 St->getPointerInfo(), St->getAlignment(),
41503 St->getMemOperand()->getFlags());
41504 }
41505
41506 // If this is a store of a scalar_to_vector to v1i1, just use a scalar store.
41507 // This will avoid a copy to k-register.
41508 if (VT == MVT::v1i1 && VT == StVT && Subtarget.hasAVX512() &&
41509 StoredVal.getOpcode() == ISD::SCALAR_TO_VECTOR &&
41510 StoredVal.getOperand(0).getValueType() == MVT::i8) {
41511 return DAG.getStore(St->getChain(), dl, StoredVal.getOperand(0),
41512 St->getBasePtr(), St->getPointerInfo(),
41513 St->getAlignment(), St->getMemOperand()->getFlags());
41514 }
41515
41516 // Widen v2i1/v4i1 stores to v8i1.
41517 if ((VT == MVT::v2i1 || VT == MVT::v4i1) && VT == StVT &&
41518 Subtarget.hasAVX512()) {
41519 unsigned NumConcats = 8 / VT.getVectorNumElements();
41520 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT));
41521 Ops[0] = StoredVal;
41522 StoredVal = DAG.getNode(ISD::CONCAT_VECTORS, dl, MVT::v8i1, Ops);
41523 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
41524 St->getPointerInfo(), St->getAlignment(),
41525 St->getMemOperand()->getFlags());
41526 }
41527
41528 // Turn vXi1 stores of constants into a scalar store.
41529 if ((VT == MVT::v8i1 || VT == MVT::v16i1 || VT == MVT::v32i1 ||
41530 VT == MVT::v64i1) && VT == StVT && TLI.isTypeLegal(VT) &&
41531 ISD::isBuildVectorOfConstantSDNodes(StoredVal.getNode())) {
41532 // If its a v64i1 store without 64-bit support, we need two stores.
41533 if (VT == MVT::v64i1 && !Subtarget.is64Bit()) {
41534 SDValue Lo = DAG.getBuildVector(MVT::v32i1, dl,
41535 StoredVal->ops().slice(0, 32));
41536 Lo = combinevXi1ConstantToInteger(Lo, DAG);
41537 SDValue Hi = DAG.getBuildVector(MVT::v32i1, dl,
41538 StoredVal->ops().slice(32, 32));
41539 Hi = combinevXi1ConstantToInteger(Hi, DAG);
41540
41541 SDValue Ptr0 = St->getBasePtr();
41542 SDValue Ptr1 = DAG.getMemBasePlusOffset(Ptr0, 4, dl);
41543
41544 SDValue Ch0 =
41545 DAG.getStore(St->getChain(), dl, Lo, Ptr0, St->getPointerInfo(),
41546 Alignment, St->getMemOperand()->getFlags());
41547 SDValue Ch1 =
41548 DAG.getStore(St->getChain(), dl, Hi, Ptr1,
41549 St->getPointerInfo().getWithOffset(4),
41550 MinAlign(Alignment, 4U),
41551 St->getMemOperand()->getFlags());
41552 return DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Ch0, Ch1);
41553 }
41554
41555 StoredVal = combinevXi1ConstantToInteger(StoredVal, DAG);
41556 return DAG.getStore(St->getChain(), dl, StoredVal, St->getBasePtr(),
41557 St->getPointerInfo(), St->getAlignment(),
41558 St->getMemOperand()->getFlags());
41559 }
41560
41561 // If we are saving a 32-byte vector and 32-byte stores are slow, such as on
41562 // Sandy Bridge, perform two 16-byte stores.
41563 bool Fast;
41564 if (VT.is256BitVector() && StVT == VT &&
41565 TLI.allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
41566 *St->getMemOperand(), &Fast) &&
41567 !Fast) {
41568 unsigned NumElems = VT.getVectorNumElements();
41569 if (NumElems < 2)
41570 return SDValue();
41571
41572 return splitVectorStore(St, DAG);
41573 }
41574
41575 // Split under-aligned vector non-temporal stores.
41576 if (St->isNonTemporal() && StVT == VT && Alignment < VT.getStoreSize()) {
41577 // ZMM/YMM nt-stores - either it can be stored as a series of shorter
41578 // vectors or the legalizer can scalarize it to use MOVNTI.
41579 if (VT.is256BitVector() || VT.is512BitVector()) {
41580 unsigned NumElems = VT.getVectorNumElements();
41581 if (NumElems < 2)
41582 return SDValue();
41583 return splitVectorStore(St, DAG);
41584 }
41585
41586 // XMM nt-stores - scalarize this to f64 nt-stores on SSE4A, else i32/i64
41587 // to use MOVNTI.
41588 if (VT.is128BitVector() && Subtarget.hasSSE2()) {
41589 MVT NTVT = Subtarget.hasSSE4A()
41590 ? MVT::v2f64
41591 : (TLI.isTypeLegal(MVT::i64) ? MVT::v2i64 : MVT::v4i32);
41592 return scalarizeVectorStore(St, NTVT, DAG);
41593 }
41594 }
41595
41596 // Try to optimize v16i16->v16i8 truncating stores when BWI is not
41597 // supported, but avx512f is by extending to v16i32 and truncating.
41598 if (!St->isTruncatingStore() && VT == MVT::v16i8 && !Subtarget.hasBWI() &&
41599 St->getValue().getOpcode() == ISD::TRUNCATE &&
41600 St->getValue().getOperand(0).getValueType() == MVT::v16i16 &&
41601 TLI.isTruncStoreLegal(MVT::v16i32, MVT::v16i8) &&
41602 St->getValue().hasOneUse() && !DCI.isBeforeLegalizeOps()) {
41603 SDValue Ext = DAG.getNode(ISD::ANY_EXTEND, dl, MVT::v16i32, St->getValue());
41604 return DAG.getTruncStore(St->getChain(), dl, Ext, St->getBasePtr(),
41605 MVT::v16i8, St->getMemOperand());
41606 }
41607
41608 // Try to fold a VTRUNCUS or VTRUNCS into a truncating store.
41609 if (!St->isTruncatingStore() && StoredVal.hasOneUse() &&
41610 (StoredVal.getOpcode() == X86ISD::VTRUNCUS ||
41611 StoredVal.getOpcode() == X86ISD::VTRUNCS) &&
41612 TLI.isTruncStoreLegal(StoredVal.getOperand(0).getValueType(), VT)) {
41613 bool IsSigned = StoredVal.getOpcode() == X86ISD::VTRUNCS;
41614 return EmitTruncSStore(IsSigned, St->getChain(),
41615 dl, StoredVal.getOperand(0), St->getBasePtr(),
41616 VT, St->getMemOperand(), DAG);
41617 }
41618
41619 // Optimize trunc store (of multiple scalars) to shuffle and store.
41620 // First, pack all of the elements in one place. Next, store to memory
41621 // in fewer chunks.
41622 if (St->isTruncatingStore() && VT.isVector()) {
41623 // Check if we can detect an AVG pattern from the truncation. If yes,
41624 // replace the trunc store by a normal store with the result of X86ISD::AVG
41625 // instruction.
41626 if (DCI.isBeforeLegalize() || TLI.isTypeLegal(St->getMemoryVT()))
41627 if (SDValue Avg = detectAVGPattern(St->getValue(), St->getMemoryVT(), DAG,
41628 Subtarget, dl))
41629 return DAG.getStore(St->getChain(), dl, Avg, St->getBasePtr(),
41630 St->getPointerInfo(), St->getAlignment(),
41631 St->getMemOperand()->getFlags());
41632
41633 if (TLI.isTruncStoreLegal(VT, StVT)) {
41634 if (SDValue Val = detectSSatPattern(St->getValue(), St->getMemoryVT()))
41635 return EmitTruncSStore(true /* Signed saturation */, St->getChain(),
41636 dl, Val, St->getBasePtr(),
41637 St->getMemoryVT(), St->getMemOperand(), DAG);
41638 if (SDValue Val = detectUSatPattern(St->getValue(), St->getMemoryVT(),
41639 DAG, dl))
41640 return EmitTruncSStore(false /* Unsigned saturation */, St->getChain(),
41641 dl, Val, St->getBasePtr(),
41642 St->getMemoryVT(), St->getMemOperand(), DAG);
41643 }
41644
41645 return SDValue();
41646 }
41647
41648 // Turn load->store of MMX types into GPR load/stores. This avoids clobbering
41649 // the FP state in cases where an emms may be missing.
41650 // A preferable solution to the general problem is to figure out the right
41651 // places to insert EMMS. This qualifies as a quick hack.
41652
41653 // Similarly, turn load->store of i64 into double load/stores in 32-bit mode.
41654 if (VT.getSizeInBits() != 64)
41655 return SDValue();
41656
41657 const Function &F = DAG.getMachineFunction().getFunction();
41658 bool NoImplicitFloatOps = F.hasFnAttribute(Attribute::NoImplicitFloat);
41659 bool F64IsLegal =
41660 !Subtarget.useSoftFloat() && !NoImplicitFloatOps && Subtarget.hasSSE2();
41661 if ((VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit()) &&
41662 isa<LoadSDNode>(St->getValue()) &&
41663 cast<LoadSDNode>(St->getValue())->isSimple() &&
41664 St->getChain().hasOneUse() && St->isSimple()) {
41665 LoadSDNode *Ld = cast<LoadSDNode>(St->getValue().getNode());
41666
41667 if (!ISD::isNormalLoad(Ld))
41668 return SDValue();
41669
41670 // Avoid the transformation if there are multiple uses of the loaded value.
41671 if (!Ld->hasNUsesOfValue(1, 0))
41672 return SDValue();
41673
41674 SDLoc LdDL(Ld);
41675 SDLoc StDL(N);
41676 // Lower to a single movq load/store pair.
41677 SDValue NewLd = DAG.getLoad(MVT::f64, LdDL, Ld->getChain(),
41678 Ld->getBasePtr(), Ld->getMemOperand());
41679
41680 // Make sure new load is placed in same chain order.
41681 DAG.makeEquivalentMemoryOrdering(Ld, NewLd);
41682 return DAG.getStore(St->getChain(), StDL, NewLd, St->getBasePtr(),
41683 St->getMemOperand());
41684 }
41685
41686 // This is similar to the above case, but here we handle a scalar 64-bit
41687 // integer store that is extracted from a vector on a 32-bit target.
41688 // If we have SSE2, then we can treat it like a floating-point double
41689 // to get past legalization. The execution dependencies fixup pass will
41690 // choose the optimal machine instruction for the store if this really is
41691 // an integer or v2f32 rather than an f64.
41692 if (VT == MVT::i64 && F64IsLegal && !Subtarget.is64Bit() &&
41693 St->getOperand(1).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
41694 SDValue OldExtract = St->getOperand(1);
41695 SDValue ExtOp0 = OldExtract.getOperand(0);
41696 unsigned VecSize = ExtOp0.getValueSizeInBits();
41697 EVT VecVT = EVT::getVectorVT(*DAG.getContext(), MVT::f64, VecSize / 64);
41698 SDValue BitCast = DAG.getBitcast(VecVT, ExtOp0);
41699 SDValue NewExtract = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, MVT::f64,
41700 BitCast, OldExtract.getOperand(1));
41701 return DAG.getStore(St->getChain(), dl, NewExtract, St->getBasePtr(),
41702 St->getPointerInfo(), St->getAlignment(),
41703 St->getMemOperand()->getFlags());
41704 }
41705
41706 return SDValue();
41707}
41708
41709/// Return 'true' if this vector operation is "horizontal"
41710/// and return the operands for the horizontal operation in LHS and RHS. A
41711/// horizontal operation performs the binary operation on successive elements
41712/// of its first operand, then on successive elements of its second operand,
41713/// returning the resulting values in a vector. For example, if
41714/// A = < float a0, float a1, float a2, float a3 >
41715/// and
41716/// B = < float b0, float b1, float b2, float b3 >
41717/// then the result of doing a horizontal operation on A and B is
41718/// A horizontal-op B = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >.
41719/// In short, LHS and RHS are inspected to see if LHS op RHS is of the form
41720/// A horizontal-op B, for some already available A and B, and if so then LHS is
41721/// set to A, RHS to B, and the routine returns 'true'.
41722static bool isHorizontalBinOp(SDValue &LHS, SDValue &RHS, SelectionDAG &DAG,
41723 const X86Subtarget &Subtarget,
41724 bool IsCommutative) {
41725 // If either operand is undef, bail out. The binop should be simplified.
41726 if (LHS.isUndef() || RHS.isUndef())
41727 return false;
41728
41729 // Look for the following pattern:
41730 // A = < float a0, float a1, float a2, float a3 >
41731 // B = < float b0, float b1, float b2, float b3 >
41732 // and
41733 // LHS = VECTOR_SHUFFLE A, B, <0, 2, 4, 6>
41734 // RHS = VECTOR_SHUFFLE A, B, <1, 3, 5, 7>
41735 // then LHS op RHS = < a0 op a1, a2 op a3, b0 op b1, b2 op b3 >
41736 // which is A horizontal-op B.
41737
41738 MVT VT = LHS.getSimpleValueType();
41739 assert((VT.is128BitVector() || VT.is256BitVector()) &&(((VT.is128BitVector() || VT.is256BitVector()) && "Unsupported vector type for horizontal add/sub"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector()) && \"Unsupported vector type for horizontal add/sub\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41740, __PRETTY_FUNCTION__))
41740 "Unsupported vector type for horizontal add/sub")(((VT.is128BitVector() || VT.is256BitVector()) && "Unsupported vector type for horizontal add/sub"
) ? static_cast<void> (0) : __assert_fail ("(VT.is128BitVector() || VT.is256BitVector()) && \"Unsupported vector type for horizontal add/sub\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41740, __PRETTY_FUNCTION__));
41741 unsigned NumElts = VT.getVectorNumElements();
41742
41743 // TODO - can we make a general helper method that does all of this for us?
41744 auto GetShuffle = [&](SDValue Op, SDValue &N0, SDValue &N1,
41745 SmallVectorImpl<int> &ShuffleMask) {
41746 if (Op.getOpcode() == ISD::VECTOR_SHUFFLE) {
41747 if (!Op.getOperand(0).isUndef())
41748 N0 = Op.getOperand(0);
41749 if (!Op.getOperand(1).isUndef())
41750 N1 = Op.getOperand(1);
41751 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op)->getMask();
41752 ShuffleMask.append(Mask.begin(), Mask.end());
41753 return;
41754 }
41755 bool UseSubVector = false;
41756 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
41757 Op.getOperand(0).getValueType().is256BitVector() &&
41758 llvm::isNullConstant(Op.getOperand(1))) {
41759 Op = Op.getOperand(0);
41760 UseSubVector = true;
41761 }
41762 bool IsUnary;
41763 SmallVector<SDValue, 2> SrcOps;
41764 SmallVector<int, 16> SrcShuffleMask;
41765 SDValue BC = peekThroughBitcasts(Op);
41766 if (isTargetShuffle(BC.getOpcode()) &&
41767 getTargetShuffleMask(BC.getNode(), BC.getSimpleValueType(), false,
41768 SrcOps, SrcShuffleMask, IsUnary)) {
41769 if (!UseSubVector && SrcShuffleMask.size() == NumElts &&
41770 SrcOps.size() <= 2) {
41771 N0 = SrcOps.size() > 0 ? SrcOps[0] : SDValue();
41772 N1 = SrcOps.size() > 1 ? SrcOps[1] : SDValue();
41773 ShuffleMask.append(SrcShuffleMask.begin(), SrcShuffleMask.end());
41774 }
41775 if (UseSubVector && (SrcShuffleMask.size() == (NumElts * 2)) &&
41776 SrcOps.size() == 1) {
41777 N0 = extract128BitVector(SrcOps[0], 0, DAG, SDLoc(Op));
41778 N1 = extract128BitVector(SrcOps[0], NumElts, DAG, SDLoc(Op));
41779 ArrayRef<int> Mask = ArrayRef<int>(SrcShuffleMask).slice(0, NumElts);
41780 ShuffleMask.append(Mask.begin(), Mask.end());
41781 }
41782 }
41783 };
41784
41785 // View LHS in the form
41786 // LHS = VECTOR_SHUFFLE A, B, LMask
41787 // If LHS is not a shuffle, then pretend it is the identity shuffle:
41788 // LHS = VECTOR_SHUFFLE LHS, undef, <0, 1, ..., N-1>
41789 // NOTE: A default initialized SDValue represents an UNDEF of type VT.
41790 SDValue A, B;
41791 SmallVector<int, 16> LMask;
41792 GetShuffle(LHS, A, B, LMask);
41793
41794 // Likewise, view RHS in the form
41795 // RHS = VECTOR_SHUFFLE C, D, RMask
41796 SDValue C, D;
41797 SmallVector<int, 16> RMask;
41798 GetShuffle(RHS, C, D, RMask);
41799
41800 // At least one of the operands should be a vector shuffle.
41801 unsigned NumShuffles = (LMask.empty() ? 0 : 1) + (RMask.empty() ? 0 : 1);
41802 if (NumShuffles == 0)
41803 return false;
41804
41805 if (LMask.empty()) {
41806 A = LHS;
41807 for (unsigned i = 0; i != NumElts; ++i)
41808 LMask.push_back(i);
41809 }
41810
41811 if (RMask.empty()) {
41812 C = RHS;
41813 for (unsigned i = 0; i != NumElts; ++i)
41814 RMask.push_back(i);
41815 }
41816
41817 // If A and B occur in reverse order in RHS, then canonicalize by commuting
41818 // RHS operands and shuffle mask.
41819 if (A != C) {
41820 std::swap(C, D);
41821 ShuffleVectorSDNode::commuteMask(RMask);
41822 }
41823 // Check that the shuffles are both shuffling the same vectors.
41824 if (!(A == C && B == D))
41825 return false;
41826
41827 // LHS and RHS are now:
41828 // LHS = shuffle A, B, LMask
41829 // RHS = shuffle A, B, RMask
41830 // Check that the masks correspond to performing a horizontal operation.
41831 // AVX defines horizontal add/sub to operate independently on 128-bit lanes,
41832 // so we just repeat the inner loop if this is a 256-bit op.
41833 unsigned Num128BitChunks = VT.getSizeInBits() / 128;
41834 unsigned NumEltsPer128BitChunk = NumElts / Num128BitChunks;
41835 assert((NumEltsPer128BitChunk % 2 == 0) &&(((NumEltsPer128BitChunk % 2 == 0) && "Vector type should have an even number of elements in each lane"
) ? static_cast<void> (0) : __assert_fail ("(NumEltsPer128BitChunk % 2 == 0) && \"Vector type should have an even number of elements in each lane\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41836, __PRETTY_FUNCTION__))
41836 "Vector type should have an even number of elements in each lane")(((NumEltsPer128BitChunk % 2 == 0) && "Vector type should have an even number of elements in each lane"
) ? static_cast<void> (0) : __assert_fail ("(NumEltsPer128BitChunk % 2 == 0) && \"Vector type should have an even number of elements in each lane\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41836, __PRETTY_FUNCTION__));
41837 for (unsigned j = 0; j != NumElts; j += NumEltsPer128BitChunk) {
41838 for (unsigned i = 0; i != NumEltsPer128BitChunk; ++i) {
41839 // Ignore undefined components.
41840 int LIdx = LMask[i + j], RIdx = RMask[i + j];
41841 if (LIdx < 0 || RIdx < 0 ||
41842 (!A.getNode() && (LIdx < (int)NumElts || RIdx < (int)NumElts)) ||
41843 (!B.getNode() && (LIdx >= (int)NumElts || RIdx >= (int)NumElts)))
41844 continue;
41845
41846 // The low half of the 128-bit result must choose from A.
41847 // The high half of the 128-bit result must choose from B,
41848 // unless B is undef. In that case, we are always choosing from A.
41849 unsigned NumEltsPer64BitChunk = NumEltsPer128BitChunk / 2;
41850 unsigned Src = B.getNode() ? i >= NumEltsPer64BitChunk : 0;
41851
41852 // Check that successive elements are being operated on. If not, this is
41853 // not a horizontal operation.
41854 int Index = 2 * (i % NumEltsPer64BitChunk) + NumElts * Src + j;
41855 if (!(LIdx == Index && RIdx == Index + 1) &&
41856 !(IsCommutative && LIdx == Index + 1 && RIdx == Index))
41857 return false;
41858 }
41859 }
41860
41861 LHS = A.getNode() ? A : B; // If A is 'UNDEF', use B for it.
41862 RHS = B.getNode() ? B : A; // If B is 'UNDEF', use A for it.
41863
41864 if (!shouldUseHorizontalOp(LHS == RHS && NumShuffles < 2, DAG, Subtarget))
41865 return false;
41866
41867 LHS = DAG.getBitcast(VT, LHS);
41868 RHS = DAG.getBitcast(VT, RHS);
41869 return true;
41870}
41871
41872/// Do target-specific dag combines on floating-point adds/subs.
41873static SDValue combineFaddFsub(SDNode *N, SelectionDAG &DAG,
41874 const X86Subtarget &Subtarget) {
41875 EVT VT = N->getValueType(0);
41876 SDValue LHS = N->getOperand(0);
41877 SDValue RHS = N->getOperand(1);
41878 bool IsFadd = N->getOpcode() == ISD::FADD;
41879 auto HorizOpcode = IsFadd ? X86ISD::FHADD : X86ISD::FHSUB;
41880 assert((IsFadd || N->getOpcode() == ISD::FSUB) && "Wrong opcode")(((IsFadd || N->getOpcode() == ISD::FSUB) && "Wrong opcode"
) ? static_cast<void> (0) : __assert_fail ("(IsFadd || N->getOpcode() == ISD::FSUB) && \"Wrong opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41880, __PRETTY_FUNCTION__));
41881
41882 // Try to synthesize horizontal add/sub from adds/subs of shuffles.
41883 if (((Subtarget.hasSSE3() && (VT == MVT::v4f32 || VT == MVT::v2f64)) ||
41884 (Subtarget.hasAVX() && (VT == MVT::v8f32 || VT == MVT::v4f64))) &&
41885 isHorizontalBinOp(LHS, RHS, DAG, Subtarget, IsFadd))
41886 return DAG.getNode(HorizOpcode, SDLoc(N), VT, LHS, RHS);
41887
41888 return SDValue();
41889}
41890
41891/// Attempt to pre-truncate inputs to arithmetic ops if it will simplify
41892/// the codegen.
41893/// e.g. TRUNC( BINOP( X, Y ) ) --> BINOP( TRUNC( X ), TRUNC( Y ) )
41894/// TODO: This overlaps with the generic combiner's visitTRUNCATE. Remove
41895/// anything that is guaranteed to be transformed by DAGCombiner.
41896static SDValue combineTruncatedArithmetic(SDNode *N, SelectionDAG &DAG,
41897 const X86Subtarget &Subtarget,
41898 const SDLoc &DL) {
41899 assert(N->getOpcode() == ISD::TRUNCATE && "Wrong opcode")((N->getOpcode() == ISD::TRUNCATE && "Wrong opcode"
) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::TRUNCATE && \"Wrong opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 41899, __PRETTY_FUNCTION__));
41900 SDValue Src = N->getOperand(0);
41901 unsigned SrcOpcode = Src.getOpcode();
41902 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
41903
41904 EVT VT = N->getValueType(0);
41905 EVT SrcVT = Src.getValueType();
41906
41907 auto IsFreeTruncation = [VT](SDValue Op) {
41908 unsigned TruncSizeInBits = VT.getScalarSizeInBits();
41909
41910 // See if this has been extended from a smaller/equal size to
41911 // the truncation size, allowing a truncation to combine with the extend.
41912 unsigned Opcode = Op.getOpcode();
41913 if ((Opcode == ISD::ANY_EXTEND || Opcode == ISD::SIGN_EXTEND ||
41914 Opcode == ISD::ZERO_EXTEND) &&
41915 Op.getOperand(0).getScalarValueSizeInBits() <= TruncSizeInBits)
41916 return true;
41917
41918 // See if this is a single use constant which can be constant folded.
41919 // NOTE: We don't peek throught bitcasts here because there is currently
41920 // no support for constant folding truncate+bitcast+vector_of_constants. So
41921 // we'll just send up with a truncate on both operands which will
41922 // get turned back into (truncate (binop)) causing an infinite loop.
41923 return ISD::isBuildVectorOfConstantSDNodes(Op.getNode());
41924 };
41925
41926 auto TruncateArithmetic = [&](SDValue N0, SDValue N1) {
41927 SDValue Trunc0 = DAG.getNode(ISD::TRUNCATE, DL, VT, N0);
41928 SDValue Trunc1 = DAG.getNode(ISD::TRUNCATE, DL, VT, N1);
41929 return DAG.getNode(SrcOpcode, DL, VT, Trunc0, Trunc1);
41930 };
41931
41932 // Don't combine if the operation has other uses.
41933 if (!Src.hasOneUse())
41934 return SDValue();
41935
41936 // Only support vector truncation for now.
41937 // TODO: i64 scalar math would benefit as well.
41938 if (!VT.isVector())
41939 return SDValue();
41940
41941 // In most cases its only worth pre-truncating if we're only facing the cost
41942 // of one truncation.
41943 // i.e. if one of the inputs will constant fold or the input is repeated.
41944 switch (SrcOpcode) {
41945 case ISD::AND:
41946 case ISD::XOR:
41947 case ISD::OR: {
41948 SDValue Op0 = Src.getOperand(0);
41949 SDValue Op1 = Src.getOperand(1);
41950 if (TLI.isOperationLegalOrPromote(SrcOpcode, VT) &&
41951 (Op0 == Op1 || IsFreeTruncation(Op0) || IsFreeTruncation(Op1)))
41952 return TruncateArithmetic(Op0, Op1);
41953 break;
41954 }
41955
41956 case ISD::MUL:
41957 // X86 is rubbish at scalar and vector i64 multiplies (until AVX512DQ) - its
41958 // better to truncate if we have the chance.
41959 if (SrcVT.getScalarType() == MVT::i64 &&
41960 TLI.isOperationLegal(SrcOpcode, VT) &&
41961 !TLI.isOperationLegal(SrcOpcode, SrcVT))
41962 return TruncateArithmetic(Src.getOperand(0), Src.getOperand(1));
41963 LLVM_FALLTHROUGH[[gnu::fallthrough]];
41964 case ISD::ADD: {
41965 SDValue Op0 = Src.getOperand(0);
41966 SDValue Op1 = Src.getOperand(1);
41967 if (TLI.isOperationLegal(SrcOpcode, VT) &&
41968 (Op0 == Op1 || IsFreeTruncation(Op0) || IsFreeTruncation(Op1)))
41969 return TruncateArithmetic(Op0, Op1);
41970 break;
41971 }
41972 case ISD::SUB: {
41973 // TODO: ISD::SUB We are conservative and require both sides to be freely
41974 // truncatable to avoid interfering with combineSubToSubus.
41975 SDValue Op0 = Src.getOperand(0);
41976 SDValue Op1 = Src.getOperand(1);
41977 if (TLI.isOperationLegal(SrcOpcode, VT) &&
41978 (Op0 == Op1 || (IsFreeTruncation(Op0) && IsFreeTruncation(Op1))))
41979 return TruncateArithmetic(Op0, Op1);
41980 break;
41981 }
41982 }
41983
41984 return SDValue();
41985}
41986
41987/// Truncate using ISD::AND mask and X86ISD::PACKUS.
41988/// e.g. trunc <8 x i32> X to <8 x i16> -->
41989/// MaskX = X & 0xffff (clear high bits to prevent saturation)
41990/// packus (extract_subv MaskX, 0), (extract_subv MaskX, 1)
41991static SDValue combineVectorTruncationWithPACKUS(SDNode *N, const SDLoc &DL,
41992 const X86Subtarget &Subtarget,
41993 SelectionDAG &DAG) {
41994 SDValue In = N->getOperand(0);
41995 EVT InVT = In.getValueType();
41996 EVT OutVT = N->getValueType(0);
41997
41998 APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(),
41999 OutVT.getScalarSizeInBits());
42000 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
42001 return truncateVectorWithPACK(X86ISD::PACKUS, OutVT, In, DL, DAG, Subtarget);
42002}
42003
42004/// Truncate a group of v4i32 into v8i16 using X86ISD::PACKSS.
42005static SDValue combineVectorTruncationWithPACKSS(SDNode *N, const SDLoc &DL,
42006 const X86Subtarget &Subtarget,
42007 SelectionDAG &DAG) {
42008 SDValue In = N->getOperand(0);
42009 EVT InVT = In.getValueType();
42010 EVT OutVT = N->getValueType(0);
42011 In = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, InVT, In,
42012 DAG.getValueType(OutVT));
42013 return truncateVectorWithPACK(X86ISD::PACKSS, OutVT, In, DL, DAG, Subtarget);
42014}
42015
42016/// This function transforms truncation from vXi32/vXi64 to vXi8/vXi16 into
42017/// X86ISD::PACKUS/X86ISD::PACKSS operations. We do it here because after type
42018/// legalization the truncation will be translated into a BUILD_VECTOR with each
42019/// element that is extracted from a vector and then truncated, and it is
42020/// difficult to do this optimization based on them.
42021static SDValue combineVectorTruncation(SDNode *N, SelectionDAG &DAG,
42022 const X86Subtarget &Subtarget) {
42023 EVT OutVT = N->getValueType(0);
42024 if (!OutVT.isVector())
42025 return SDValue();
42026
42027 SDValue In = N->getOperand(0);
42028 if (!In.getValueType().isSimple())
42029 return SDValue();
42030
42031 EVT InVT = In.getValueType();
42032 unsigned NumElems = OutVT.getVectorNumElements();
42033
42034 // TODO: On AVX2, the behavior of X86ISD::PACKUS is different from that on
42035 // SSE2, and we need to take care of it specially.
42036 // AVX512 provides vpmovdb.
42037 if (!Subtarget.hasSSE2() || Subtarget.hasAVX2())
42038 return SDValue();
42039
42040 EVT OutSVT = OutVT.getVectorElementType();
42041 EVT InSVT = InVT.getVectorElementType();
42042 if (!((InSVT == MVT::i32 || InSVT == MVT::i64) &&
42043 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && isPowerOf2_32(NumElems) &&
42044 NumElems >= 8))
42045 return SDValue();
42046
42047 // SSSE3's pshufb results in less instructions in the cases below.
42048 if (Subtarget.hasSSSE3() && NumElems == 8 &&
42049 ((OutSVT == MVT::i8 && InSVT != MVT::i64) ||
42050 (InSVT == MVT::i32 && OutSVT == MVT::i16)))
42051 return SDValue();
42052
42053 SDLoc DL(N);
42054 // SSE2 provides PACKUS for only 2 x v8i16 -> v16i8 and SSE4.1 provides PACKUS
42055 // for 2 x v4i32 -> v8i16. For SSSE3 and below, we need to use PACKSS to
42056 // truncate 2 x v4i32 to v8i16.
42057 if (Subtarget.hasSSE41() || OutSVT == MVT::i8)
42058 return combineVectorTruncationWithPACKUS(N, DL, Subtarget, DAG);
42059 if (InSVT == MVT::i32)
42060 return combineVectorTruncationWithPACKSS(N, DL, Subtarget, DAG);
42061
42062 return SDValue();
42063}
42064
42065/// This function transforms vector truncation of 'extended sign-bits' or
42066/// 'extended zero-bits' values.
42067/// vXi16/vXi32/vXi64 to vXi8/vXi16/vXi32 into X86ISD::PACKSS/PACKUS operations.
42068static SDValue combineVectorSignBitsTruncation(SDNode *N, const SDLoc &DL,
42069 SelectionDAG &DAG,
42070 const X86Subtarget &Subtarget) {
42071 // Requires SSE2.
42072 if (!Subtarget.hasSSE2())
42073 return SDValue();
42074
42075 if (!N->getValueType(0).isVector() || !N->getValueType(0).isSimple())
42076 return SDValue();
42077
42078 SDValue In = N->getOperand(0);
42079 if (!In.getValueType().isSimple())
42080 return SDValue();
42081
42082 MVT VT = N->getValueType(0).getSimpleVT();
42083 MVT SVT = VT.getScalarType();
42084
42085 MVT InVT = In.getValueType().getSimpleVT();
42086 MVT InSVT = InVT.getScalarType();
42087
42088 // Check we have a truncation suited for PACKSS/PACKUS.
42089 if (!VT.is128BitVector() && !VT.is256BitVector())
42090 return SDValue();
42091 if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32)
42092 return SDValue();
42093 if (InSVT != MVT::i16 && InSVT != MVT::i32 && InSVT != MVT::i64)
42094 return SDValue();
42095
42096 // AVX512 has fast truncate, but if the input is already going to be split,
42097 // there's no harm in trying pack.
42098 if (Subtarget.hasAVX512() &&
42099 !(!Subtarget.useAVX512Regs() && VT.is256BitVector() &&
42100 InVT.is512BitVector()))
42101 return SDValue();
42102
42103 unsigned NumPackedSignBits = std::min<unsigned>(SVT.getSizeInBits(), 16);
42104 unsigned NumPackedZeroBits = Subtarget.hasSSE41() ? NumPackedSignBits : 8;
42105
42106 // Use PACKUS if the input has zero-bits that extend all the way to the
42107 // packed/truncated value. e.g. masks, zext_in_reg, etc.
42108 KnownBits Known = DAG.computeKnownBits(In);
42109 unsigned NumLeadingZeroBits = Known.countMinLeadingZeros();
42110 if (NumLeadingZeroBits >= (InSVT.getSizeInBits() - NumPackedZeroBits))
42111 return truncateVectorWithPACK(X86ISD::PACKUS, VT, In, DL, DAG, Subtarget);
42112
42113 // Use PACKSS if the input has sign-bits that extend all the way to the
42114 // packed/truncated value. e.g. Comparison result, sext_in_reg, etc.
42115 unsigned NumSignBits = DAG.ComputeNumSignBits(In);
42116 if (NumSignBits > (InSVT.getSizeInBits() - NumPackedSignBits))
42117 return truncateVectorWithPACK(X86ISD::PACKSS, VT, In, DL, DAG, Subtarget);
42118
42119 return SDValue();
42120}
42121
42122// Try to form a MULHU or MULHS node by looking for
42123// (trunc (srl (mul ext, ext), 16))
42124// TODO: This is X86 specific because we want to be able to handle wide types
42125// before type legalization. But we can only do it if the vector will be
42126// legalized via widening/splitting. Type legalization can't handle promotion
42127// of a MULHU/MULHS. There isn't a way to convey this to the generic DAG
42128// combiner.
42129static SDValue combinePMULH(SDValue Src, EVT VT, const SDLoc &DL,
42130 SelectionDAG &DAG, const X86Subtarget &Subtarget) {
42131 // First instruction should be a right shift of a multiply.
42132 if (Src.getOpcode() != ISD::SRL ||
42133 Src.getOperand(0).getOpcode() != ISD::MUL)
42134 return SDValue();
42135
42136 if (!Subtarget.hasSSE2())
42137 return SDValue();
42138
42139 // Only handle vXi16 types that are at least 128-bits unless they will be
42140 // widened.
42141 if (!VT.isVector() || VT.getVectorElementType() != MVT::i16)
42142 return SDValue();
42143
42144 // Input type should be vXi32.
42145 EVT InVT = Src.getValueType();
42146 if (InVT.getVectorElementType() != MVT::i32)
42147 return SDValue();
42148
42149 // Need a shift by 16.
42150 APInt ShiftAmt;
42151 if (!ISD::isConstantSplatVector(Src.getOperand(1).getNode(), ShiftAmt) ||
42152 ShiftAmt != 16)
42153 return SDValue();
42154
42155 SDValue LHS = Src.getOperand(0).getOperand(0);
42156 SDValue RHS = Src.getOperand(0).getOperand(1);
42157
42158 unsigned ExtOpc = LHS.getOpcode();
42159 if ((ExtOpc != ISD::SIGN_EXTEND && ExtOpc != ISD::ZERO_EXTEND) ||
42160 RHS.getOpcode() != ExtOpc)
42161 return SDValue();
42162
42163 // Peek through the extends.
42164 LHS = LHS.getOperand(0);
42165 RHS = RHS.getOperand(0);
42166
42167 // Ensure the input types match.
42168 if (LHS.getValueType() != VT || RHS.getValueType() != VT)
42169 return SDValue();
42170
42171 unsigned Opc = ExtOpc == ISD::SIGN_EXTEND ? ISD::MULHS : ISD::MULHU;
42172 return DAG.getNode(Opc, DL, VT, LHS, RHS);
42173}
42174
42175// Attempt to match PMADDUBSW, which multiplies corresponding unsigned bytes
42176// from one vector with signed bytes from another vector, adds together
42177// adjacent pairs of 16-bit products, and saturates the result before
42178// truncating to 16-bits.
42179//
42180// Which looks something like this:
42181// (i16 (ssat (add (mul (zext (even elts (i8 A))), (sext (even elts (i8 B)))),
42182// (mul (zext (odd elts (i8 A)), (sext (odd elts (i8 B))))))))
42183static SDValue detectPMADDUBSW(SDValue In, EVT VT, SelectionDAG &DAG,
42184 const X86Subtarget &Subtarget,
42185 const SDLoc &DL) {
42186 if (!VT.isVector() || !Subtarget.hasSSSE3())
42187 return SDValue();
42188
42189 unsigned NumElems = VT.getVectorNumElements();
42190 EVT ScalarVT = VT.getVectorElementType();
42191 if (ScalarVT != MVT::i16 || NumElems < 8 || !isPowerOf2_32(NumElems))
42192 return SDValue();
42193
42194 SDValue SSatVal = detectSSatPattern(In, VT);
42195 if (!SSatVal || SSatVal.getOpcode() != ISD::ADD)
42196 return SDValue();
42197
42198 // Ok this is a signed saturation of an ADD. See if this ADD is adding pairs
42199 // of multiplies from even/odd elements.
42200 SDValue N0 = SSatVal.getOperand(0);
42201 SDValue N1 = SSatVal.getOperand(1);
42202
42203 if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
42204 return SDValue();
42205
42206 SDValue N00 = N0.getOperand(0);
42207 SDValue N01 = N0.getOperand(1);
42208 SDValue N10 = N1.getOperand(0);
42209 SDValue N11 = N1.getOperand(1);
42210
42211 // TODO: Handle constant vectors and use knownbits/computenumsignbits?
42212 // Canonicalize zero_extend to LHS.
42213 if (N01.getOpcode() == ISD::ZERO_EXTEND)
42214 std::swap(N00, N01);
42215 if (N11.getOpcode() == ISD::ZERO_EXTEND)
42216 std::swap(N10, N11);
42217
42218 // Ensure we have a zero_extend and a sign_extend.
42219 if (N00.getOpcode() != ISD::ZERO_EXTEND ||
42220 N01.getOpcode() != ISD::SIGN_EXTEND ||
42221 N10.getOpcode() != ISD::ZERO_EXTEND ||
42222 N11.getOpcode() != ISD::SIGN_EXTEND)
42223 return SDValue();
42224
42225 // Peek through the extends.
42226 N00 = N00.getOperand(0);
42227 N01 = N01.getOperand(0);
42228 N10 = N10.getOperand(0);
42229 N11 = N11.getOperand(0);
42230
42231 // Ensure the extend is from vXi8.
42232 if (N00.getValueType().getVectorElementType() != MVT::i8 ||
42233 N01.getValueType().getVectorElementType() != MVT::i8 ||
42234 N10.getValueType().getVectorElementType() != MVT::i8 ||
42235 N11.getValueType().getVectorElementType() != MVT::i8)
42236 return SDValue();
42237
42238 // All inputs should be build_vectors.
42239 if (N00.getOpcode() != ISD::BUILD_VECTOR ||
42240 N01.getOpcode() != ISD::BUILD_VECTOR ||
42241 N10.getOpcode() != ISD::BUILD_VECTOR ||
42242 N11.getOpcode() != ISD::BUILD_VECTOR)
42243 return SDValue();
42244
42245 // N00/N10 are zero extended. N01/N11 are sign extended.
42246
42247 // For each element, we need to ensure we have an odd element from one vector
42248 // multiplied by the odd element of another vector and the even element from
42249 // one of the same vectors being multiplied by the even element from the
42250 // other vector. So we need to make sure for each element i, this operator
42251 // is being performed:
42252 // A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
42253 SDValue ZExtIn, SExtIn;
42254 for (unsigned i = 0; i != NumElems; ++i) {
42255 SDValue N00Elt = N00.getOperand(i);
42256 SDValue N01Elt = N01.getOperand(i);
42257 SDValue N10Elt = N10.getOperand(i);
42258 SDValue N11Elt = N11.getOperand(i);
42259 // TODO: Be more tolerant to undefs.
42260 if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
42261 N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
42262 N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
42263 N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
42264 return SDValue();
42265 auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
42266 auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
42267 auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
42268 auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
42269 if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
42270 return SDValue();
42271 unsigned IdxN00 = ConstN00Elt->getZExtValue();
42272 unsigned IdxN01 = ConstN01Elt->getZExtValue();
42273 unsigned IdxN10 = ConstN10Elt->getZExtValue();
42274 unsigned IdxN11 = ConstN11Elt->getZExtValue();
42275 // Add is commutative so indices can be reordered.
42276 if (IdxN00 > IdxN10) {
42277 std::swap(IdxN00, IdxN10);
42278 std::swap(IdxN01, IdxN11);
42279 }
42280 // N0 indices be the even element. N1 indices must be the next odd element.
42281 if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
42282 IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
42283 return SDValue();
42284 SDValue N00In = N00Elt.getOperand(0);
42285 SDValue N01In = N01Elt.getOperand(0);
42286 SDValue N10In = N10Elt.getOperand(0);
42287 SDValue N11In = N11Elt.getOperand(0);
42288 // First time we find an input capture it.
42289 if (!ZExtIn) {
42290 ZExtIn = N00In;
42291 SExtIn = N01In;
42292 }
42293 if (ZExtIn != N00In || SExtIn != N01In ||
42294 ZExtIn != N10In || SExtIn != N11In)
42295 return SDValue();
42296 }
42297
42298 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
42299 ArrayRef<SDValue> Ops) {
42300 // Shrink by adding truncate nodes and let DAGCombine fold with the
42301 // sources.
42302 EVT InVT = Ops[0].getValueType();
42303 assert(InVT.getScalarType() == MVT::i8 &&((InVT.getScalarType() == MVT::i8 && "Unexpected scalar element type"
) ? static_cast<void> (0) : __assert_fail ("InVT.getScalarType() == MVT::i8 && \"Unexpected scalar element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42304, __PRETTY_FUNCTION__))
42304 "Unexpected scalar element type")((InVT.getScalarType() == MVT::i8 && "Unexpected scalar element type"
) ? static_cast<void> (0) : __assert_fail ("InVT.getScalarType() == MVT::i8 && \"Unexpected scalar element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42304, __PRETTY_FUNCTION__));
42305 assert(InVT == Ops[1].getValueType() && "Operands' types mismatch")((InVT == Ops[1].getValueType() && "Operands' types mismatch"
) ? static_cast<void> (0) : __assert_fail ("InVT == Ops[1].getValueType() && \"Operands' types mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42305, __PRETTY_FUNCTION__));
42306 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
42307 InVT.getVectorNumElements() / 2);
42308 return DAG.getNode(X86ISD::VPMADDUBSW, DL, ResVT, Ops[0], Ops[1]);
42309 };
42310 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { ZExtIn, SExtIn },
42311 PMADDBuilder);
42312}
42313
42314static SDValue combineTruncate(SDNode *N, SelectionDAG &DAG,
42315 const X86Subtarget &Subtarget) {
42316 EVT VT = N->getValueType(0);
42317 SDValue Src = N->getOperand(0);
42318 SDLoc DL(N);
42319
42320 // Attempt to pre-truncate inputs to arithmetic ops instead.
42321 if (SDValue V = combineTruncatedArithmetic(N, DAG, Subtarget, DL))
42322 return V;
42323
42324 // Try to detect AVG pattern first.
42325 if (SDValue Avg = detectAVGPattern(Src, VT, DAG, Subtarget, DL))
42326 return Avg;
42327
42328 // Try to detect PMADD
42329 if (SDValue PMAdd = detectPMADDUBSW(Src, VT, DAG, Subtarget, DL))
42330 return PMAdd;
42331
42332 // Try to combine truncation with signed/unsigned saturation.
42333 if (SDValue Val = combineTruncateWithSat(Src, VT, DL, DAG, Subtarget))
42334 return Val;
42335
42336 // Try to combine PMULHUW/PMULHW for vXi16.
42337 if (SDValue V = combinePMULH(Src, VT, DL, DAG, Subtarget))
42338 return V;
42339
42340 // The bitcast source is a direct mmx result.
42341 // Detect bitcasts between i32 to x86mmx
42342 if (Src.getOpcode() == ISD::BITCAST && VT == MVT::i32) {
42343 SDValue BCSrc = Src.getOperand(0);
42344 if (BCSrc.getValueType() == MVT::x86mmx)
42345 return DAG.getNode(X86ISD::MMX_MOVD2W, DL, MVT::i32, BCSrc);
42346 }
42347
42348 // Try to truncate extended sign/zero bits with PACKSS/PACKUS.
42349 if (SDValue V = combineVectorSignBitsTruncation(N, DL, DAG, Subtarget))
42350 return V;
42351
42352 return combineVectorTruncation(N, DAG, Subtarget);
42353}
42354
42355static SDValue combineVTRUNC(SDNode *N, SelectionDAG &DAG) {
42356 EVT VT = N->getValueType(0);
42357 SDValue In = N->getOperand(0);
42358 SDLoc DL(N);
42359
42360 if (auto SSatVal = detectSSatPattern(In, VT))
42361 return DAG.getNode(X86ISD::VTRUNCS, DL, VT, SSatVal);
42362 if (auto USatVal = detectUSatPattern(In, VT, DAG, DL))
42363 return DAG.getNode(X86ISD::VTRUNCUS, DL, VT, USatVal);
42364
42365 return SDValue();
42366}
42367
42368/// Returns the negated value if the node \p N flips sign of FP value.
42369///
42370/// FP-negation node may have different forms: FNEG(x), FXOR (x, 0x80000000)
42371/// or FSUB(0, x)
42372/// AVX512F does not have FXOR, so FNEG is lowered as
42373/// (bitcast (xor (bitcast x), (bitcast ConstantFP(0x80000000)))).
42374/// In this case we go though all bitcasts.
42375/// This also recognizes splat of a negated value and returns the splat of that
42376/// value.
42377static SDValue isFNEG(SelectionDAG &DAG, SDNode *N, unsigned Depth = 0) {
42378 if (N->getOpcode() == ISD::FNEG)
42379 return N->getOperand(0);
42380
42381 // Don't recurse exponentially.
42382 if (Depth > SelectionDAG::MaxRecursionDepth)
42383 return SDValue();
42384
42385 unsigned ScalarSize = N->getValueType(0).getScalarSizeInBits();
42386
42387 SDValue Op = peekThroughBitcasts(SDValue(N, 0));
42388 EVT VT = Op->getValueType(0);
42389
42390 // Make sure the element size doesn't change.
42391 if (VT.getScalarSizeInBits() != ScalarSize)
42392 return SDValue();
42393
42394 unsigned Opc = Op.getOpcode();
42395 switch (Opc) {
42396 case ISD::VECTOR_SHUFFLE: {
42397 // For a VECTOR_SHUFFLE(VEC1, VEC2), if the VEC2 is undef, then the negate
42398 // of this is VECTOR_SHUFFLE(-VEC1, UNDEF). The mask can be anything here.
42399 if (!Op.getOperand(1).isUndef())
42400 return SDValue();
42401 if (SDValue NegOp0 = isFNEG(DAG, Op.getOperand(0).getNode(), Depth + 1))
42402 if (NegOp0.getValueType() == VT) // FIXME: Can we do better?
42403 return DAG.getVectorShuffle(VT, SDLoc(Op), NegOp0, DAG.getUNDEF(VT),
42404 cast<ShuffleVectorSDNode>(Op)->getMask());
42405 break;
42406 }
42407 case ISD::INSERT_VECTOR_ELT: {
42408 // Negate of INSERT_VECTOR_ELT(UNDEF, V, INDEX) is INSERT_VECTOR_ELT(UNDEF,
42409 // -V, INDEX).
42410 SDValue InsVector = Op.getOperand(0);
42411 SDValue InsVal = Op.getOperand(1);
42412 if (!InsVector.isUndef())
42413 return SDValue();
42414 if (SDValue NegInsVal = isFNEG(DAG, InsVal.getNode(), Depth + 1))
42415 if (NegInsVal.getValueType() == VT.getVectorElementType()) // FIXME
42416 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(Op), VT, InsVector,
42417 NegInsVal, Op.getOperand(2));
42418 break;
42419 }
42420 case ISD::FSUB:
42421 case ISD::XOR:
42422 case X86ISD::FXOR: {
42423 SDValue Op1 = Op.getOperand(1);
42424 SDValue Op0 = Op.getOperand(0);
42425
42426 // For XOR and FXOR, we want to check if constant
42427 // bits of Op1 are sign bit masks. For FSUB, we
42428 // have to check if constant bits of Op0 are sign
42429 // bit masks and hence we swap the operands.
42430 if (Opc == ISD::FSUB)
42431 std::swap(Op0, Op1);
42432
42433 APInt UndefElts;
42434 SmallVector<APInt, 16> EltBits;
42435 // Extract constant bits and see if they are all
42436 // sign bit masks. Ignore the undef elements.
42437 if (getTargetConstantBitsFromNode(Op1, ScalarSize, UndefElts, EltBits,
42438 /* AllowWholeUndefs */ true,
42439 /* AllowPartialUndefs */ false)) {
42440 for (unsigned I = 0, E = EltBits.size(); I < E; I++)
42441 if (!UndefElts[I] && !EltBits[I].isSignMask())
42442 return SDValue();
42443
42444 return peekThroughBitcasts(Op0);
42445 }
42446 }
42447 }
42448
42449 return SDValue();
42450}
42451
42452static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc,
42453 bool NegRes) {
42454 if (NegMul) {
42455 switch (Opcode) {
42456 default: llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42456);
42457 case ISD::FMA: Opcode = X86ISD::FNMADD; break;
42458 case X86ISD::FMADD_RND: Opcode = X86ISD::FNMADD_RND; break;
42459 case X86ISD::FMSUB: Opcode = X86ISD::FNMSUB; break;
42460 case X86ISD::FMSUB_RND: Opcode = X86ISD::FNMSUB_RND; break;
42461 case X86ISD::FNMADD: Opcode = ISD::FMA; break;
42462 case X86ISD::FNMADD_RND: Opcode = X86ISD::FMADD_RND; break;
42463 case X86ISD::FNMSUB: Opcode = X86ISD::FMSUB; break;
42464 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMSUB_RND; break;
42465 }
42466 }
42467
42468 if (NegAcc) {
42469 switch (Opcode) {
42470 default: llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42470);
42471 case ISD::FMA: Opcode = X86ISD::FMSUB; break;
42472 case X86ISD::FMADD_RND: Opcode = X86ISD::FMSUB_RND; break;
42473 case X86ISD::FMSUB: Opcode = ISD::FMA; break;
42474 case X86ISD::FMSUB_RND: Opcode = X86ISD::FMADD_RND; break;
42475 case X86ISD::FNMADD: Opcode = X86ISD::FNMSUB; break;
42476 case X86ISD::FNMADD_RND: Opcode = X86ISD::FNMSUB_RND; break;
42477 case X86ISD::FNMSUB: Opcode = X86ISD::FNMADD; break;
42478 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FNMADD_RND; break;
42479 case X86ISD::FMADDSUB: Opcode = X86ISD::FMSUBADD; break;
42480 case X86ISD::FMADDSUB_RND: Opcode = X86ISD::FMSUBADD_RND; break;
42481 case X86ISD::FMSUBADD: Opcode = X86ISD::FMADDSUB; break;
42482 case X86ISD::FMSUBADD_RND: Opcode = X86ISD::FMADDSUB_RND; break;
42483 }
42484 }
42485
42486 if (NegRes) {
42487 switch (Opcode) {
42488 default: llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42488);
42489 case ISD::FMA: Opcode = X86ISD::FNMSUB; break;
42490 case X86ISD::FMADD_RND: Opcode = X86ISD::FNMSUB_RND; break;
42491 case X86ISD::FMSUB: Opcode = X86ISD::FNMADD; break;
42492 case X86ISD::FMSUB_RND: Opcode = X86ISD::FNMADD_RND; break;
42493 case X86ISD::FNMADD: Opcode = X86ISD::FMSUB; break;
42494 case X86ISD::FNMADD_RND: Opcode = X86ISD::FMSUB_RND; break;
42495 case X86ISD::FNMSUB: Opcode = ISD::FMA; break;
42496 case X86ISD::FNMSUB_RND: Opcode = X86ISD::FMADD_RND; break;
42497 }
42498 }
42499
42500 return Opcode;
42501}
42502
42503/// Do target-specific dag combines on floating point negations.
42504static SDValue combineFneg(SDNode *N, SelectionDAG &DAG,
42505 const X86Subtarget &Subtarget) {
42506 EVT OrigVT = N->getValueType(0);
42507 SDValue Arg = isFNEG(DAG, N);
42508 if (!Arg)
42509 return SDValue();
42510
42511 EVT VT = Arg.getValueType();
42512 EVT SVT = VT.getScalarType();
42513 SDLoc DL(N);
42514
42515 // Let legalize expand this if it isn't a legal type yet.
42516 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
42517 return SDValue();
42518
42519 // If we're negating a FMUL node on a target with FMA, then we can avoid the
42520 // use of a constant by performing (-0 - A*B) instead.
42521 // FIXME: Check rounding control flags as well once it becomes available.
42522 if (Arg.getOpcode() == ISD::FMUL && (SVT == MVT::f32 || SVT == MVT::f64) &&
42523 Arg->getFlags().hasNoSignedZeros() && Subtarget.hasAnyFMA()) {
42524 SDValue Zero = DAG.getConstantFP(0.0, DL, VT);
42525 SDValue NewNode = DAG.getNode(X86ISD::FNMSUB, DL, VT, Arg.getOperand(0),
42526 Arg.getOperand(1), Zero);
42527 return DAG.getBitcast(OrigVT, NewNode);
42528 }
42529
42530 // If we're negating an FMA node, then we can adjust the
42531 // instruction to include the extra negation.
42532 if (Arg.hasOneUse() && Subtarget.hasAnyFMA()) {
42533 switch (Arg.getOpcode()) {
42534 case ISD::FMA:
42535 case X86ISD::FMSUB:
42536 case X86ISD::FNMADD:
42537 case X86ISD::FNMSUB:
42538 case X86ISD::FMADD_RND:
42539 case X86ISD::FMSUB_RND:
42540 case X86ISD::FNMADD_RND:
42541 case X86ISD::FNMSUB_RND: {
42542 // We can't handle scalar intrinsic node here because it would only
42543 // invert one element and not the whole vector. But we could try to handle
42544 // a negation of the lower element only.
42545 unsigned NewOpcode = negateFMAOpcode(Arg.getOpcode(), false, false, true);
42546 return DAG.getBitcast(OrigVT, DAG.getNode(NewOpcode, DL, VT, Arg->ops()));
42547 }
42548 }
42549 }
42550
42551 return SDValue();
42552}
42553
42554char X86TargetLowering::isNegatibleForFree(SDValue Op, SelectionDAG &DAG,
42555 bool LegalOperations,
42556 bool ForCodeSize,
42557 unsigned Depth) const {
42558 // fneg patterns are removable even if they have multiple uses.
42559 if (isFNEG(DAG, Op.getNode(), Depth))
42560 return 2;
42561
42562 // Don't recurse exponentially.
42563 if (Depth > SelectionDAG::MaxRecursionDepth)
42564 return 0;
42565
42566 EVT VT = Op.getValueType();
42567 EVT SVT = VT.getScalarType();
42568 switch (Op.getOpcode()) {
42569 case ISD::FMA:
42570 case X86ISD::FMSUB:
42571 case X86ISD::FNMADD:
42572 case X86ISD::FNMSUB:
42573 case X86ISD::FMADD_RND:
42574 case X86ISD::FMSUB_RND:
42575 case X86ISD::FNMADD_RND:
42576 case X86ISD::FNMSUB_RND: {
42577 if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) ||
42578 !(SVT == MVT::f32 || SVT == MVT::f64) || !LegalOperations)
42579 break;
42580
42581 // This is always negatible for free but we might be able to remove some
42582 // extra operand negations as well.
42583 for (int i = 0; i != 3; ++i) {
42584 char V = isNegatibleForFree(Op.getOperand(i), DAG, LegalOperations,
42585 ForCodeSize, Depth + 1);
42586 if (V == 2)
42587 return V;
42588 }
42589 return 1;
42590 }
42591 }
42592
42593 return TargetLowering::isNegatibleForFree(Op, DAG, LegalOperations,
42594 ForCodeSize, Depth);
42595}
42596
42597SDValue X86TargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
42598 bool LegalOperations,
42599 bool ForCodeSize,
42600 unsigned Depth) const {
42601 // fneg patterns are removable even if they have multiple uses.
42602 if (SDValue Arg = isFNEG(DAG, Op.getNode(), Depth))
42603 return DAG.getBitcast(Op.getValueType(), Arg);
42604
42605 EVT VT = Op.getValueType();
42606 EVT SVT = VT.getScalarType();
42607 unsigned Opc = Op.getOpcode();
42608 switch (Opc) {
42609 case ISD::FMA:
42610 case X86ISD::FMSUB:
42611 case X86ISD::FNMADD:
42612 case X86ISD::FNMSUB:
42613 case X86ISD::FMADD_RND:
42614 case X86ISD::FMSUB_RND:
42615 case X86ISD::FNMADD_RND:
42616 case X86ISD::FNMSUB_RND: {
42617 if (!Op.hasOneUse() || !Subtarget.hasAnyFMA() || !isTypeLegal(VT) ||
42618 !(SVT == MVT::f32 || SVT == MVT::f64) || !LegalOperations)
42619 break;
42620
42621 // This is always negatible for free but we might be able to remove some
42622 // extra operand negations as well.
42623 SmallVector<SDValue, 4> NewOps(Op.getNumOperands(), SDValue());
42624 for (int i = 0; i != 3; ++i) {
42625 char V = isNegatibleForFree(Op.getOperand(i), DAG, LegalOperations,
42626 ForCodeSize, Depth + 1);
42627 if (V == 2)
42628 NewOps[i] = getNegatedExpression(Op.getOperand(i), DAG, LegalOperations,
42629 ForCodeSize, Depth + 1);
42630 }
42631
42632 bool NegA = !!NewOps[0];
42633 bool NegB = !!NewOps[1];
42634 bool NegC = !!NewOps[2];
42635 unsigned NewOpc = negateFMAOpcode(Opc, NegA != NegB, NegC, true);
42636
42637 // Fill in the non-negated ops with the original values.
42638 for (int i = 0, e = Op.getNumOperands(); i != e; ++i)
42639 if (!NewOps[i])
42640 NewOps[i] = Op.getOperand(i);
42641 return DAG.getNode(NewOpc, SDLoc(Op), VT, NewOps);
42642 }
42643 }
42644
42645 return TargetLowering::getNegatedExpression(Op, DAG, LegalOperations,
42646 ForCodeSize, Depth);
42647}
42648
42649static SDValue lowerX86FPLogicOp(SDNode *N, SelectionDAG &DAG,
42650 const X86Subtarget &Subtarget) {
42651 MVT VT = N->getSimpleValueType(0);
42652 // If we have integer vector types available, use the integer opcodes.
42653 if (!VT.isVector() || !Subtarget.hasSSE2())
42654 return SDValue();
42655
42656 SDLoc dl(N);
42657
42658 unsigned IntBits = VT.getScalarSizeInBits();
42659 MVT IntSVT = MVT::getIntegerVT(IntBits);
42660 MVT IntVT = MVT::getVectorVT(IntSVT, VT.getSizeInBits() / IntBits);
42661
42662 SDValue Op0 = DAG.getBitcast(IntVT, N->getOperand(0));
42663 SDValue Op1 = DAG.getBitcast(IntVT, N->getOperand(1));
42664 unsigned IntOpcode;
42665 switch (N->getOpcode()) {
42666 default: llvm_unreachable("Unexpected FP logic op")::llvm::llvm_unreachable_internal("Unexpected FP logic op", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42666);
42667 case X86ISD::FOR: IntOpcode = ISD::OR; break;
42668 case X86ISD::FXOR: IntOpcode = ISD::XOR; break;
42669 case X86ISD::FAND: IntOpcode = ISD::AND; break;
42670 case X86ISD::FANDN: IntOpcode = X86ISD::ANDNP; break;
42671 }
42672 SDValue IntOp = DAG.getNode(IntOpcode, dl, IntVT, Op0, Op1);
42673 return DAG.getBitcast(VT, IntOp);
42674}
42675
42676
42677/// Fold a xor(setcc cond, val), 1 --> setcc (inverted(cond), val)
42678static SDValue foldXor1SetCC(SDNode *N, SelectionDAG &DAG) {
42679 if (N->getOpcode() != ISD::XOR)
42680 return SDValue();
42681
42682 SDValue LHS = N->getOperand(0);
42683 if (!isOneConstant(N->getOperand(1)) || LHS->getOpcode() != X86ISD::SETCC)
42684 return SDValue();
42685
42686 X86::CondCode NewCC = X86::GetOppositeBranchCondition(
42687 X86::CondCode(LHS->getConstantOperandVal(0)));
42688 SDLoc DL(N);
42689 return getSETCC(NewCC, LHS->getOperand(1), DL, DAG);
42690}
42691
42692static SDValue combineXor(SDNode *N, SelectionDAG &DAG,
42693 TargetLowering::DAGCombinerInfo &DCI,
42694 const X86Subtarget &Subtarget) {
42695 // If this is SSE1 only convert to FXOR to avoid scalarization.
42696 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() &&
42697 N->getValueType(0) == MVT::v4i32) {
42698 return DAG.getBitcast(
42699 MVT::v4i32, DAG.getNode(X86ISD::FXOR, SDLoc(N), MVT::v4f32,
42700 DAG.getBitcast(MVT::v4f32, N->getOperand(0)),
42701 DAG.getBitcast(MVT::v4f32, N->getOperand(1))));
42702 }
42703
42704 if (SDValue Cmp = foldVectorXorShiftIntoCmp(N, DAG, Subtarget))
42705 return Cmp;
42706
42707 if (DCI.isBeforeLegalizeOps())
42708 return SDValue();
42709
42710 if (SDValue SetCC = foldXor1SetCC(N, DAG))
42711 return SetCC;
42712
42713 if (SDValue RV = foldXorTruncShiftIntoCmp(N, DAG))
42714 return RV;
42715
42716 if (SDValue FPLogic = convertIntLogicToFPLogic(N, DAG, Subtarget))
42717 return FPLogic;
42718
42719 return combineFneg(N, DAG, Subtarget);
42720}
42721
42722static SDValue combineBEXTR(SDNode *N, SelectionDAG &DAG,
42723 TargetLowering::DAGCombinerInfo &DCI,
42724 const X86Subtarget &Subtarget) {
42725 SDValue Op0 = N->getOperand(0);
42726 SDValue Op1 = N->getOperand(1);
42727 EVT VT = N->getValueType(0);
42728 unsigned NumBits = VT.getSizeInBits();
42729
42730 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42731
42732 // TODO - Constant Folding.
42733 if (auto *Cst1 = dyn_cast<ConstantSDNode>(Op1)) {
42734 // Reduce Cst1 to the bottom 16-bits.
42735 // NOTE: SimplifyDemandedBits won't do this for constants.
42736 const APInt &Val1 = Cst1->getAPIntValue();
42737 APInt MaskedVal1 = Val1 & 0xFFFF;
42738 if (MaskedVal1 != Val1)
42739 return DAG.getNode(X86ISD::BEXTR, SDLoc(N), VT, Op0,
42740 DAG.getConstant(MaskedVal1, SDLoc(N), VT));
42741 }
42742
42743 // Only bottom 16-bits of the control bits are required.
42744 APInt DemandedMask(APInt::getLowBitsSet(NumBits, 16));
42745 if (TLI.SimplifyDemandedBits(Op1, DemandedMask, DCI))
42746 return SDValue(N, 0);
42747
42748 return SDValue();
42749}
42750
42751static bool isNullFPScalarOrVectorConst(SDValue V) {
42752 return isNullFPConstant(V) || ISD::isBuildVectorAllZeros(V.getNode());
42753}
42754
42755/// If a value is a scalar FP zero or a vector FP zero (potentially including
42756/// undefined elements), return a zero constant that may be used to fold away
42757/// that value. In the case of a vector, the returned constant will not contain
42758/// undefined elements even if the input parameter does. This makes it suitable
42759/// to be used as a replacement operand with operations (eg, bitwise-and) where
42760/// an undef should not propagate.
42761static SDValue getNullFPConstForNullVal(SDValue V, SelectionDAG &DAG,
42762 const X86Subtarget &Subtarget) {
42763 if (!isNullFPScalarOrVectorConst(V))
42764 return SDValue();
42765
42766 if (V.getValueType().isVector())
42767 return getZeroVector(V.getSimpleValueType(), Subtarget, DAG, SDLoc(V));
42768
42769 return V;
42770}
42771
42772static SDValue combineFAndFNotToFAndn(SDNode *N, SelectionDAG &DAG,
42773 const X86Subtarget &Subtarget) {
42774 SDValue N0 = N->getOperand(0);
42775 SDValue N1 = N->getOperand(1);
42776 EVT VT = N->getValueType(0);
42777 SDLoc DL(N);
42778
42779 // Vector types are handled in combineANDXORWithAllOnesIntoANDNP().
42780 if (!((VT == MVT::f32 && Subtarget.hasSSE1()) ||
42781 (VT == MVT::f64 && Subtarget.hasSSE2()) ||
42782 (VT == MVT::v4f32 && Subtarget.hasSSE1() && !Subtarget.hasSSE2())))
42783 return SDValue();
42784
42785 auto isAllOnesConstantFP = [](SDValue V) {
42786 if (V.getSimpleValueType().isVector())
42787 return ISD::isBuildVectorAllOnes(V.getNode());
42788 auto *C = dyn_cast<ConstantFPSDNode>(V);
42789 return C && C->getConstantFPValue()->isAllOnesValue();
42790 };
42791
42792 // fand (fxor X, -1), Y --> fandn X, Y
42793 if (N0.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N0.getOperand(1)))
42794 return DAG.getNode(X86ISD::FANDN, DL, VT, N0.getOperand(0), N1);
42795
42796 // fand X, (fxor Y, -1) --> fandn Y, X
42797 if (N1.getOpcode() == X86ISD::FXOR && isAllOnesConstantFP(N1.getOperand(1)))
42798 return DAG.getNode(X86ISD::FANDN, DL, VT, N1.getOperand(0), N0);
42799
42800 return SDValue();
42801}
42802
42803/// Do target-specific dag combines on X86ISD::FAND nodes.
42804static SDValue combineFAnd(SDNode *N, SelectionDAG &DAG,
42805 const X86Subtarget &Subtarget) {
42806 // FAND(0.0, x) -> 0.0
42807 if (SDValue V = getNullFPConstForNullVal(N->getOperand(0), DAG, Subtarget))
42808 return V;
42809
42810 // FAND(x, 0.0) -> 0.0
42811 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
42812 return V;
42813
42814 if (SDValue V = combineFAndFNotToFAndn(N, DAG, Subtarget))
42815 return V;
42816
42817 return lowerX86FPLogicOp(N, DAG, Subtarget);
42818}
42819
42820/// Do target-specific dag combines on X86ISD::FANDN nodes.
42821static SDValue combineFAndn(SDNode *N, SelectionDAG &DAG,
42822 const X86Subtarget &Subtarget) {
42823 // FANDN(0.0, x) -> x
42824 if (isNullFPScalarOrVectorConst(N->getOperand(0)))
42825 return N->getOperand(1);
42826
42827 // FANDN(x, 0.0) -> 0.0
42828 if (SDValue V = getNullFPConstForNullVal(N->getOperand(1), DAG, Subtarget))
42829 return V;
42830
42831 return lowerX86FPLogicOp(N, DAG, Subtarget);
42832}
42833
42834/// Do target-specific dag combines on X86ISD::FOR and X86ISD::FXOR nodes.
42835static SDValue combineFOr(SDNode *N, SelectionDAG &DAG,
42836 const X86Subtarget &Subtarget) {
42837 assert(N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR)((N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD
::FXOR) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == X86ISD::FOR || N->getOpcode() == X86ISD::FXOR"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42837, __PRETTY_FUNCTION__));
42838
42839 // F[X]OR(0.0, x) -> x
42840 if (isNullFPScalarOrVectorConst(N->getOperand(0)))
42841 return N->getOperand(1);
42842
42843 // F[X]OR(x, 0.0) -> x
42844 if (isNullFPScalarOrVectorConst(N->getOperand(1)))
42845 return N->getOperand(0);
42846
42847 if (SDValue NewVal = combineFneg(N, DAG, Subtarget))
42848 return NewVal;
42849
42850 return lowerX86FPLogicOp(N, DAG, Subtarget);
42851}
42852
42853/// Do target-specific dag combines on X86ISD::FMIN and X86ISD::FMAX nodes.
42854static SDValue combineFMinFMax(SDNode *N, SelectionDAG &DAG) {
42855 assert(N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX)((N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD
::FMAX) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == X86ISD::FMIN || N->getOpcode() == X86ISD::FMAX"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42855, __PRETTY_FUNCTION__));
42856
42857 // FMIN/FMAX are commutative if no NaNs and no negative zeros are allowed.
42858 if (!DAG.getTarget().Options.NoNaNsFPMath ||
42859 !DAG.getTarget().Options.NoSignedZerosFPMath)
42860 return SDValue();
42861
42862 // If we run in unsafe-math mode, then convert the FMAX and FMIN nodes
42863 // into FMINC and FMAXC, which are Commutative operations.
42864 unsigned NewOp = 0;
42865 switch (N->getOpcode()) {
42866 default: llvm_unreachable("unknown opcode")::llvm::llvm_unreachable_internal("unknown opcode", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42866);
42867 case X86ISD::FMIN: NewOp = X86ISD::FMINC; break;
42868 case X86ISD::FMAX: NewOp = X86ISD::FMAXC; break;
42869 }
42870
42871 return DAG.getNode(NewOp, SDLoc(N), N->getValueType(0),
42872 N->getOperand(0), N->getOperand(1));
42873}
42874
42875static SDValue combineFMinNumFMaxNum(SDNode *N, SelectionDAG &DAG,
42876 const X86Subtarget &Subtarget) {
42877 if (Subtarget.useSoftFloat())
42878 return SDValue();
42879
42880 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42881
42882 EVT VT = N->getValueType(0);
42883 if (!((Subtarget.hasSSE1() && VT == MVT::f32) ||
42884 (Subtarget.hasSSE2() && VT == MVT::f64) ||
42885 (VT.isVector() && TLI.isTypeLegal(VT))))
42886 return SDValue();
42887
42888 SDValue Op0 = N->getOperand(0);
42889 SDValue Op1 = N->getOperand(1);
42890 SDLoc DL(N);
42891 auto MinMaxOp = N->getOpcode() == ISD::FMAXNUM ? X86ISD::FMAX : X86ISD::FMIN;
42892
42893 // If we don't have to respect NaN inputs, this is a direct translation to x86
42894 // min/max instructions.
42895 if (DAG.getTarget().Options.NoNaNsFPMath || N->getFlags().hasNoNaNs())
42896 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
42897
42898 // If one of the operands is known non-NaN use the native min/max instructions
42899 // with the non-NaN input as second operand.
42900 if (DAG.isKnownNeverNaN(Op1))
42901 return DAG.getNode(MinMaxOp, DL, VT, Op0, Op1, N->getFlags());
42902 if (DAG.isKnownNeverNaN(Op0))
42903 return DAG.getNode(MinMaxOp, DL, VT, Op1, Op0, N->getFlags());
42904
42905 // If we have to respect NaN inputs, this takes at least 3 instructions.
42906 // Favor a library call when operating on a scalar and minimizing code size.
42907 if (!VT.isVector() && DAG.getMachineFunction().getFunction().hasMinSize())
42908 return SDValue();
42909
42910 EVT SetCCType = TLI.getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(),
42911 VT);
42912
42913 // There are 4 possibilities involving NaN inputs, and these are the required
42914 // outputs:
42915 // Op1
42916 // Num NaN
42917 // ----------------
42918 // Num | Max | Op0 |
42919 // Op0 ----------------
42920 // NaN | Op1 | NaN |
42921 // ----------------
42922 //
42923 // The SSE FP max/min instructions were not designed for this case, but rather
42924 // to implement:
42925 // Min = Op1 < Op0 ? Op1 : Op0
42926 // Max = Op1 > Op0 ? Op1 : Op0
42927 //
42928 // So they always return Op0 if either input is a NaN. However, we can still
42929 // use those instructions for fmaxnum by selecting away a NaN input.
42930
42931 // If either operand is NaN, the 2nd source operand (Op0) is passed through.
42932 SDValue MinOrMax = DAG.getNode(MinMaxOp, DL, VT, Op1, Op0);
42933 SDValue IsOp0Nan = DAG.getSetCC(DL, SetCCType, Op0, Op0, ISD::SETUO);
42934
42935 // If Op0 is a NaN, select Op1. Otherwise, select the max. If both operands
42936 // are NaN, the NaN value of Op1 is the result.
42937 return DAG.getSelect(DL, VT, IsOp0Nan, Op1, MinOrMax);
42938}
42939
42940static SDValue combineX86INT_TO_FP(SDNode *N, SelectionDAG &DAG,
42941 TargetLowering::DAGCombinerInfo &DCI) {
42942 EVT VT = N->getValueType(0);
42943 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
42944
42945 APInt KnownUndef, KnownZero;
42946 APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
42947 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, KnownUndef,
42948 KnownZero, DCI))
42949 return SDValue(N, 0);
42950
42951 // Convert a full vector load into vzload when not all bits are needed.
42952 SDValue In = N->getOperand(0);
42953 MVT InVT = In.getSimpleValueType();
42954 if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
42955 ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
42956 assert(InVT.is128BitVector() && "Expected 128-bit input vector")((InVT.is128BitVector() && "Expected 128-bit input vector"
) ? static_cast<void> (0) : __assert_fail ("InVT.is128BitVector() && \"Expected 128-bit input vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42956, __PRETTY_FUNCTION__));
42957 LoadSDNode *LN = cast<LoadSDNode>(N->getOperand(0));
42958 // Unless the load is volatile or atomic.
42959 if (LN->isSimple()) {
42960 SDLoc dl(N);
42961 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
42962 MVT MemVT = MVT::getIntegerVT(NumBits);
42963 MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
42964 SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
42965 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
42966 SDValue VZLoad =
42967 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, MemVT,
42968 LN->getPointerInfo(),
42969 LN->getAlignment(),
42970 LN->getMemOperand()->getFlags());
42971 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
42972 DAG.getBitcast(InVT, VZLoad));
42973 DCI.CombineTo(N, Convert);
42974 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
42975 return SDValue(N, 0);
42976 }
42977 }
42978
42979 return SDValue();
42980}
42981
42982static SDValue combineCVTP2I_CVTTP2I(SDNode *N, SelectionDAG &DAG,
42983 TargetLowering::DAGCombinerInfo &DCI) {
42984 // FIXME: Handle strict fp nodes.
42985 EVT VT = N->getValueType(0);
42986
42987 // Convert a full vector load into vzload when not all bits are needed.
42988 SDValue In = N->getOperand(0);
42989 MVT InVT = In.getSimpleValueType();
42990 if (VT.getVectorNumElements() < InVT.getVectorNumElements() &&
42991 ISD::isNormalLoad(In.getNode()) && In.hasOneUse()) {
42992 assert(InVT.is128BitVector() && "Expected 128-bit input vector")((InVT.is128BitVector() && "Expected 128-bit input vector"
) ? static_cast<void> (0) : __assert_fail ("InVT.is128BitVector() && \"Expected 128-bit input vector\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 42992, __PRETTY_FUNCTION__));
42993 LoadSDNode *LN = cast<LoadSDNode>(In);
42994 // Unless the load is volatile or atomic.
42995 if (LN->isSimple()) {
42996 SDLoc dl(N);
42997 unsigned NumBits = InVT.getScalarSizeInBits() * VT.getVectorNumElements();
42998 MVT MemVT = MVT::getFloatingPointVT(NumBits);
42999 MVT LoadVT = MVT::getVectorVT(MemVT, 128 / NumBits);
43000 SDVTList Tys = DAG.getVTList(LoadVT, MVT::Other);
43001 SDValue Ops[] = { LN->getChain(), LN->getBasePtr() };
43002 SDValue VZLoad =
43003 DAG.getMemIntrinsicNode(X86ISD::VZEXT_LOAD, dl, Tys, Ops, MemVT,
43004 LN->getPointerInfo(),
43005 LN->getAlignment(),
43006 LN->getMemOperand()->getFlags());
43007 SDValue Convert = DAG.getNode(N->getOpcode(), dl, VT,
43008 DAG.getBitcast(InVT, VZLoad));
43009 DCI.CombineTo(N, Convert);
43010 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), VZLoad.getValue(1));
43011 return SDValue(N, 0);
43012 }
43013 }
43014
43015 return SDValue();
43016}
43017
43018/// Do target-specific dag combines on X86ISD::ANDNP nodes.
43019static SDValue combineAndnp(SDNode *N, SelectionDAG &DAG,
43020 TargetLowering::DAGCombinerInfo &DCI,
43021 const X86Subtarget &Subtarget) {
43022 MVT VT = N->getSimpleValueType(0);
43023
43024 // ANDNP(0, x) -> x
43025 if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode()))
43026 return N->getOperand(1);
43027
43028 // ANDNP(x, 0) -> 0
43029 if (ISD::isBuildVectorAllZeros(N->getOperand(1).getNode()))
43030 return DAG.getConstant(0, SDLoc(N), VT);
43031
43032 // Turn ANDNP back to AND if input is inverted.
43033 if (SDValue Not = IsNOT(N->getOperand(0), DAG))
43034 return DAG.getNode(ISD::AND, SDLoc(N), VT, DAG.getBitcast(VT, Not),
43035 N->getOperand(1));
43036
43037 // Attempt to recursively combine a bitmask ANDNP with shuffles.
43038 if (VT.isVector() && (VT.getScalarSizeInBits() % 8) == 0) {
43039 SDValue Op(N, 0);
43040 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
43041 return Res;
43042 }
43043
43044 return SDValue();
43045}
43046
43047static SDValue combineBT(SDNode *N, SelectionDAG &DAG,
43048 TargetLowering::DAGCombinerInfo &DCI) {
43049 SDValue N0 = N->getOperand(0);
43050 SDValue N1 = N->getOperand(1);
43051
43052 // BT ignores high bits in the bit index operand.
43053 unsigned BitWidth = N1.getValueSizeInBits();
43054 APInt DemandedMask = APInt::getLowBitsSet(BitWidth, Log2_32(BitWidth));
43055 if (SDValue DemandedN1 = DAG.GetDemandedBits(N1, DemandedMask))
43056 return DAG.getNode(X86ISD::BT, SDLoc(N), MVT::i32, N0, DemandedN1);
43057
43058 return SDValue();
43059}
43060
43061// Try to combine sext_in_reg of a cmov of constants by extending the constants.
43062static SDValue combineSextInRegCmov(SDNode *N, SelectionDAG &DAG) {
43063 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG)((N->getOpcode() == ISD::SIGN_EXTEND_INREG) ? static_cast<
void> (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND_INREG"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43063, __PRETTY_FUNCTION__));
43064
43065 EVT DstVT = N->getValueType(0);
43066
43067 SDValue N0 = N->getOperand(0);
43068 SDValue N1 = N->getOperand(1);
43069 EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
43070
43071 if (ExtraVT != MVT::i8 && ExtraVT != MVT::i16)
43072 return SDValue();
43073
43074 // Look through single use any_extends / truncs.
43075 SDValue IntermediateBitwidthOp;
43076 if ((N0.getOpcode() == ISD::ANY_EXTEND || N0.getOpcode() == ISD::TRUNCATE) &&
43077 N0.hasOneUse()) {
43078 IntermediateBitwidthOp = N0;
43079 N0 = N0.getOperand(0);
43080 }
43081
43082 // See if we have a single use cmov.
43083 if (N0.getOpcode() != X86ISD::CMOV || !N0.hasOneUse())
43084 return SDValue();
43085
43086 SDValue CMovOp0 = N0.getOperand(0);
43087 SDValue CMovOp1 = N0.getOperand(1);
43088
43089 // Make sure both operands are constants.
43090 if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
43091 !isa<ConstantSDNode>(CMovOp1.getNode()))
43092 return SDValue();
43093
43094 SDLoc DL(N);
43095
43096 // If we looked through an any_extend/trunc above, add one to the constants.
43097 if (IntermediateBitwidthOp) {
43098 unsigned IntermediateOpc = IntermediateBitwidthOp.getOpcode();
43099 CMovOp0 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp0);
43100 CMovOp1 = DAG.getNode(IntermediateOpc, DL, DstVT, CMovOp1);
43101 }
43102
43103 CMovOp0 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp0, N1);
43104 CMovOp1 = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, DstVT, CMovOp1, N1);
43105
43106 EVT CMovVT = DstVT;
43107 // We do not want i16 CMOV's. Promote to i32 and truncate afterwards.
43108 if (DstVT == MVT::i16) {
43109 CMovVT = MVT::i32;
43110 CMovOp0 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp0);
43111 CMovOp1 = DAG.getNode(ISD::ZERO_EXTEND, DL, CMovVT, CMovOp1);
43112 }
43113
43114 SDValue CMov = DAG.getNode(X86ISD::CMOV, DL, CMovVT, CMovOp0, CMovOp1,
43115 N0.getOperand(2), N0.getOperand(3));
43116
43117 if (CMovVT != DstVT)
43118 CMov = DAG.getNode(ISD::TRUNCATE, DL, DstVT, CMov);
43119
43120 return CMov;
43121}
43122
43123static SDValue combineSignExtendInReg(SDNode *N, SelectionDAG &DAG,
43124 const X86Subtarget &Subtarget) {
43125 assert(N->getOpcode() == ISD::SIGN_EXTEND_INREG)((N->getOpcode() == ISD::SIGN_EXTEND_INREG) ? static_cast<
void> (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND_INREG"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43125, __PRETTY_FUNCTION__));
43126
43127 if (SDValue V = combineSextInRegCmov(N, DAG))
43128 return V;
43129
43130 EVT VT = N->getValueType(0);
43131 SDValue N0 = N->getOperand(0);
43132 SDValue N1 = N->getOperand(1);
43133 EVT ExtraVT = cast<VTSDNode>(N1)->getVT();
43134 SDLoc dl(N);
43135
43136 // The SIGN_EXTEND_INREG to v4i64 is expensive operation on the
43137 // both SSE and AVX2 since there is no sign-extended shift right
43138 // operation on a vector with 64-bit elements.
43139 //(sext_in_reg (v4i64 anyext (v4i32 x )), ExtraVT) ->
43140 // (v4i64 sext (v4i32 sext_in_reg (v4i32 x , ExtraVT)))
43141 if (VT == MVT::v4i64 && (N0.getOpcode() == ISD::ANY_EXTEND ||
43142 N0.getOpcode() == ISD::SIGN_EXTEND)) {
43143 SDValue N00 = N0.getOperand(0);
43144
43145 // EXTLOAD has a better solution on AVX2,
43146 // it may be replaced with X86ISD::VSEXT node.
43147 if (N00.getOpcode() == ISD::LOAD && Subtarget.hasInt256())
43148 if (!ISD::isNormalLoad(N00.getNode()))
43149 return SDValue();
43150
43151 if (N00.getValueType() == MVT::v4i32 && ExtraVT.getSizeInBits() < 128) {
43152 SDValue Tmp = DAG.getNode(ISD::SIGN_EXTEND_INREG, dl, MVT::v4i32,
43153 N00, N1);
43154 return DAG.getNode(ISD::SIGN_EXTEND, dl, MVT::v4i64, Tmp);
43155 }
43156 }
43157 return SDValue();
43158}
43159
43160/// sext(add_nsw(x, C)) --> add(sext(x), C_sext)
43161/// zext(add_nuw(x, C)) --> add(zext(x), C_zext)
43162/// Promoting a sign/zero extension ahead of a no overflow 'add' exposes
43163/// opportunities to combine math ops, use an LEA, or use a complex addressing
43164/// mode. This can eliminate extend, add, and shift instructions.
43165static SDValue promoteExtBeforeAdd(SDNode *Ext, SelectionDAG &DAG,
43166 const X86Subtarget &Subtarget) {
43167 if (Ext->getOpcode() != ISD::SIGN_EXTEND &&
43168 Ext->getOpcode() != ISD::ZERO_EXTEND)
43169 return SDValue();
43170
43171 // TODO: This should be valid for other integer types.
43172 EVT VT = Ext->getValueType(0);
43173 if (VT != MVT::i64)
43174 return SDValue();
43175
43176 SDValue Add = Ext->getOperand(0);
43177 if (Add.getOpcode() != ISD::ADD)
43178 return SDValue();
43179
43180 bool Sext = Ext->getOpcode() == ISD::SIGN_EXTEND;
43181 bool NSW = Add->getFlags().hasNoSignedWrap();
43182 bool NUW = Add->getFlags().hasNoUnsignedWrap();
43183
43184 // We need an 'add nsw' feeding into the 'sext' or 'add nuw' feeding
43185 // into the 'zext'
43186 if ((Sext && !NSW) || (!Sext && !NUW))
43187 return SDValue();
43188
43189 // Having a constant operand to the 'add' ensures that we are not increasing
43190 // the instruction count because the constant is extended for free below.
43191 // A constant operand can also become the displacement field of an LEA.
43192 auto *AddOp1 = dyn_cast<ConstantSDNode>(Add.getOperand(1));
43193 if (!AddOp1)
43194 return SDValue();
43195
43196 // Don't make the 'add' bigger if there's no hope of combining it with some
43197 // other 'add' or 'shl' instruction.
43198 // TODO: It may be profitable to generate simpler LEA instructions in place
43199 // of single 'add' instructions, but the cost model for selecting an LEA
43200 // currently has a high threshold.
43201 bool HasLEAPotential = false;
43202 for (auto *User : Ext->uses()) {
43203 if (User->getOpcode() == ISD::ADD || User->getOpcode() == ISD::SHL) {
43204 HasLEAPotential = true;
43205 break;
43206 }
43207 }
43208 if (!HasLEAPotential)
43209 return SDValue();
43210
43211 // Everything looks good, so pull the '{s|z}ext' ahead of the 'add'.
43212 int64_t AddConstant = Sext ? AddOp1->getSExtValue() : AddOp1->getZExtValue();
43213 SDValue AddOp0 = Add.getOperand(0);
43214 SDValue NewExt = DAG.getNode(Ext->getOpcode(), SDLoc(Ext), VT, AddOp0);
43215 SDValue NewConstant = DAG.getConstant(AddConstant, SDLoc(Add), VT);
43216
43217 // The wider add is guaranteed to not wrap because both operands are
43218 // sign-extended.
43219 SDNodeFlags Flags;
43220 Flags.setNoSignedWrap(NSW);
43221 Flags.setNoUnsignedWrap(NUW);
43222 return DAG.getNode(ISD::ADD, SDLoc(Add), VT, NewExt, NewConstant, Flags);
43223}
43224
43225// If we face {ANY,SIGN,ZERO}_EXTEND that is applied to a CMOV with constant
43226// operands and the result of CMOV is not used anywhere else - promote CMOV
43227// itself instead of promoting its result. This could be beneficial, because:
43228// 1) X86TargetLowering::EmitLoweredSelect later can do merging of two
43229// (or more) pseudo-CMOVs only when they go one-after-another and
43230// getting rid of result extension code after CMOV will help that.
43231// 2) Promotion of constant CMOV arguments is free, hence the
43232// {ANY,SIGN,ZERO}_EXTEND will just be deleted.
43233// 3) 16-bit CMOV encoding is 4 bytes, 32-bit CMOV is 3-byte, so this
43234// promotion is also good in terms of code-size.
43235// (64-bit CMOV is 4-bytes, that's why we don't do 32-bit => 64-bit
43236// promotion).
43237static SDValue combineToExtendCMOV(SDNode *Extend, SelectionDAG &DAG) {
43238 SDValue CMovN = Extend->getOperand(0);
43239 if (CMovN.getOpcode() != X86ISD::CMOV || !CMovN.hasOneUse())
43240 return SDValue();
43241
43242 EVT TargetVT = Extend->getValueType(0);
43243 unsigned ExtendOpcode = Extend->getOpcode();
43244 SDLoc DL(Extend);
43245
43246 EVT VT = CMovN.getValueType();
43247 SDValue CMovOp0 = CMovN.getOperand(0);
43248 SDValue CMovOp1 = CMovN.getOperand(1);
43249
43250 if (!isa<ConstantSDNode>(CMovOp0.getNode()) ||
43251 !isa<ConstantSDNode>(CMovOp1.getNode()))
43252 return SDValue();
43253
43254 // Only extend to i32 or i64.
43255 if (TargetVT != MVT::i32 && TargetVT != MVT::i64)
43256 return SDValue();
43257
43258 // Only extend from i16 unless its a sign_extend from i32. Zext/aext from i32
43259 // are free.
43260 if (VT != MVT::i16 && !(ExtendOpcode == ISD::SIGN_EXTEND && VT == MVT::i32))
43261 return SDValue();
43262
43263 // If this a zero extend to i64, we should only extend to i32 and use a free
43264 // zero extend to finish.
43265 EVT ExtendVT = TargetVT;
43266 if (TargetVT == MVT::i64 && ExtendOpcode != ISD::SIGN_EXTEND)
43267 ExtendVT = MVT::i32;
43268
43269 CMovOp0 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp0);
43270 CMovOp1 = DAG.getNode(ExtendOpcode, DL, ExtendVT, CMovOp1);
43271
43272 SDValue Res = DAG.getNode(X86ISD::CMOV, DL, ExtendVT, CMovOp0, CMovOp1,
43273 CMovN.getOperand(2), CMovN.getOperand(3));
43274
43275 // Finish extending if needed.
43276 if (ExtendVT != TargetVT)
43277 Res = DAG.getNode(ExtendOpcode, DL, TargetVT, Res);
43278
43279 return Res;
43280}
43281
43282// Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
43283// This is more or less the reverse of combineBitcastvxi1.
43284static SDValue
43285combineToExtendBoolVectorInReg(SDNode *N, SelectionDAG &DAG,
43286 TargetLowering::DAGCombinerInfo &DCI,
43287 const X86Subtarget &Subtarget) {
43288 unsigned Opcode = N->getOpcode();
43289 if (Opcode != ISD::SIGN_EXTEND && Opcode != ISD::ZERO_EXTEND &&
43290 Opcode != ISD::ANY_EXTEND)
43291 return SDValue();
43292 if (!DCI.isBeforeLegalizeOps())
43293 return SDValue();
43294 if (!Subtarget.hasSSE2() || Subtarget.hasAVX512())
43295 return SDValue();
43296
43297 SDValue N0 = N->getOperand(0);
43298 EVT VT = N->getValueType(0);
43299 EVT SVT = VT.getScalarType();
43300 EVT InSVT = N0.getValueType().getScalarType();
43301 unsigned EltSizeInBits = SVT.getSizeInBits();
43302
43303 // Input type must be extending a bool vector (bit-casted from a scalar
43304 // integer) to legal integer types.
43305 if (!VT.isVector())
43306 return SDValue();
43307 if (SVT != MVT::i64 && SVT != MVT::i32 && SVT != MVT::i16 && SVT != MVT::i8)
43308 return SDValue();
43309 if (InSVT != MVT::i1 || N0.getOpcode() != ISD::BITCAST)
43310 return SDValue();
43311
43312 SDValue N00 = N0.getOperand(0);
43313 EVT SclVT = N0.getOperand(0).getValueType();
43314 if (!SclVT.isScalarInteger())
43315 return SDValue();
43316
43317 SDLoc DL(N);
43318 SDValue Vec;
43319 SmallVector<int, 32> ShuffleMask;
43320 unsigned NumElts = VT.getVectorNumElements();
43321 assert(NumElts == SclVT.getSizeInBits() && "Unexpected bool vector size")((NumElts == SclVT.getSizeInBits() && "Unexpected bool vector size"
) ? static_cast<void> (0) : __assert_fail ("NumElts == SclVT.getSizeInBits() && \"Unexpected bool vector size\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43321, __PRETTY_FUNCTION__));
43322
43323 // Broadcast the scalar integer to the vector elements.
43324 if (NumElts > EltSizeInBits) {
43325 // If the scalar integer is greater than the vector element size, then we
43326 // must split it down into sub-sections for broadcasting. For example:
43327 // i16 -> v16i8 (i16 -> v8i16 -> v16i8) with 2 sub-sections.
43328 // i32 -> v32i8 (i32 -> v8i32 -> v32i8) with 4 sub-sections.
43329 assert((NumElts % EltSizeInBits) == 0 && "Unexpected integer scale")(((NumElts % EltSizeInBits) == 0 && "Unexpected integer scale"
) ? static_cast<void> (0) : __assert_fail ("(NumElts % EltSizeInBits) == 0 && \"Unexpected integer scale\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43329, __PRETTY_FUNCTION__));
43330 unsigned Scale = NumElts / EltSizeInBits;
43331 EVT BroadcastVT =
43332 EVT::getVectorVT(*DAG.getContext(), SclVT, EltSizeInBits);
43333 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, BroadcastVT, N00);
43334 Vec = DAG.getBitcast(VT, Vec);
43335
43336 for (unsigned i = 0; i != Scale; ++i)
43337 ShuffleMask.append(EltSizeInBits, i);
43338 } else {
43339 // For smaller scalar integers, we can simply any-extend it to the vector
43340 // element size (we don't care about the upper bits) and broadcast it to all
43341 // elements.
43342 SDValue Scl = DAG.getAnyExtOrTrunc(N00, DL, SVT);
43343 Vec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VT, Scl);
43344 ShuffleMask.append(NumElts, 0);
43345 }
43346 Vec = DAG.getVectorShuffle(VT, DL, Vec, Vec, ShuffleMask);
43347
43348 // Now, mask the relevant bit in each element.
43349 SmallVector<SDValue, 32> Bits;
43350 for (unsigned i = 0; i != NumElts; ++i) {
43351 int BitIdx = (i % EltSizeInBits);
43352 APInt Bit = APInt::getBitsSet(EltSizeInBits, BitIdx, BitIdx + 1);
43353 Bits.push_back(DAG.getConstant(Bit, DL, SVT));
43354 }
43355 SDValue BitMask = DAG.getBuildVector(VT, DL, Bits);
43356 Vec = DAG.getNode(ISD::AND, DL, VT, Vec, BitMask);
43357
43358 // Compare against the bitmask and extend the result.
43359 EVT CCVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElts);
43360 Vec = DAG.getSetCC(DL, CCVT, Vec, BitMask, ISD::SETEQ);
43361 Vec = DAG.getSExtOrTrunc(Vec, DL, VT);
43362
43363 // For SEXT, this is now done, otherwise shift the result down for
43364 // zero-extension.
43365 if (Opcode == ISD::SIGN_EXTEND)
43366 return Vec;
43367 return DAG.getNode(ISD::SRL, DL, VT, Vec,
43368 DAG.getConstant(EltSizeInBits - 1, DL, VT));
43369}
43370
43371// Attempt to combine a (sext/zext (setcc)) to a setcc with a xmm/ymm/zmm
43372// result type.
43373static SDValue combineExtSetcc(SDNode *N, SelectionDAG &DAG,
43374 const X86Subtarget &Subtarget) {
43375 SDValue N0 = N->getOperand(0);
43376 EVT VT = N->getValueType(0);
43377 SDLoc dl(N);
43378
43379 // Only do this combine with AVX512 for vector extends.
43380 if (!Subtarget.hasAVX512() || !VT.isVector() || N0.getOpcode() != ISD::SETCC)
43381 return SDValue();
43382
43383 // Only combine legal element types.
43384 EVT SVT = VT.getVectorElementType();
43385 if (SVT != MVT::i8 && SVT != MVT::i16 && SVT != MVT::i32 &&
43386 SVT != MVT::i64 && SVT != MVT::f32 && SVT != MVT::f64)
43387 return SDValue();
43388
43389 // We can only do this if the vector size in 256 bits or less.
43390 unsigned Size = VT.getSizeInBits();
43391 if (Size > 256)
43392 return SDValue();
43393
43394 // Don't fold if the condition code can't be handled by PCMPEQ/PCMPGT since
43395 // that's the only integer compares with we have.
43396 ISD::CondCode CC = cast<CondCodeSDNode>(N0.getOperand(2))->get();
43397 if (ISD::isUnsignedIntSetCC(CC))
43398 return SDValue();
43399
43400 // Only do this combine if the extension will be fully consumed by the setcc.
43401 EVT N00VT = N0.getOperand(0).getValueType();
43402 EVT MatchingVecType = N00VT.changeVectorElementTypeToInteger();
43403 if (Size != MatchingVecType.getSizeInBits())
43404 return SDValue();
43405
43406 SDValue Res = DAG.getSetCC(dl, VT, N0.getOperand(0), N0.getOperand(1), CC);
43407
43408 if (N->getOpcode() == ISD::ZERO_EXTEND)
43409 Res = DAG.getZeroExtendInReg(Res, dl, N0.getValueType().getScalarType());
43410
43411 return Res;
43412}
43413
43414static SDValue combineSext(SDNode *N, SelectionDAG &DAG,
43415 TargetLowering::DAGCombinerInfo &DCI,
43416 const X86Subtarget &Subtarget) {
43417 SDValue N0 = N->getOperand(0);
43418 EVT VT = N->getValueType(0);
43419 EVT InVT = N0.getValueType();
43420 SDLoc DL(N);
43421
43422 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
43423 return NewCMov;
43424
43425 if (!DCI.isBeforeLegalizeOps())
43426 return SDValue();
43427
43428 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
43429 return V;
43430
43431 if (InVT == MVT::i1 && N0.getOpcode() == ISD::XOR &&
43432 isAllOnesConstant(N0.getOperand(1)) && N0.hasOneUse()) {
43433 // Invert and sign-extend a boolean is the same as zero-extend and subtract
43434 // 1 because 0 becomes -1 and 1 becomes 0. The subtract is efficiently
43435 // lowered with an LEA or a DEC. This is the same as: select Bool, 0, -1.
43436 // sext (xor Bool, -1) --> sub (zext Bool), 1
43437 SDValue Zext = DAG.getNode(ISD::ZERO_EXTEND, DL, VT, N0.getOperand(0));
43438 return DAG.getNode(ISD::SUB, DL, VT, Zext, DAG.getConstant(1, DL, VT));
43439 }
43440
43441 if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
43442 return V;
43443
43444 if (VT.isVector())
43445 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
43446 return R;
43447
43448 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
43449 return NewAdd;
43450
43451 return SDValue();
43452}
43453
43454static SDValue combineFMA(SDNode *N, SelectionDAG &DAG,
43455 TargetLowering::DAGCombinerInfo &DCI,
43456 const X86Subtarget &Subtarget) {
43457 SDLoc dl(N);
43458 EVT VT = N->getValueType(0);
43459
43460 // Let legalize expand this if it isn't a legal type yet.
43461 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43462 if (!TLI.isTypeLegal(VT))
43463 return SDValue();
43464
43465 EVT ScalarVT = VT.getScalarType();
43466 if ((ScalarVT != MVT::f32 && ScalarVT != MVT::f64) || !Subtarget.hasAnyFMA())
43467 return SDValue();
43468
43469 SDValue A = N->getOperand(0);
43470 SDValue B = N->getOperand(1);
43471 SDValue C = N->getOperand(2);
43472
43473 auto invertIfNegative = [&DAG, &TLI, &DCI](SDValue &V) {
43474 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
43475 bool LegalOperations = !DCI.isBeforeLegalizeOps();
43476 if (TLI.isNegatibleForFree(V, DAG, LegalOperations, CodeSize) == 2) {
43477 V = TLI.getNegatedExpression(V, DAG, LegalOperations, CodeSize);
43478 return true;
43479 }
43480 // Look through extract_vector_elts. If it comes from an FNEG, create a
43481 // new extract from the FNEG input.
43482 if (V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
43483 isNullConstant(V.getOperand(1))) {
43484 SDValue Vec = V.getOperand(0);
43485 if (TLI.isNegatibleForFree(Vec, DAG, LegalOperations, CodeSize) == 2) {
43486 SDValue NegVal =
43487 TLI.getNegatedExpression(Vec, DAG, LegalOperations, CodeSize);
43488 V = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, SDLoc(V), V.getValueType(),
43489 NegVal, V.getOperand(1));
43490 return true;
43491 }
43492 }
43493
43494 return false;
43495 };
43496
43497 // Do not convert the passthru input of scalar intrinsics.
43498 // FIXME: We could allow negations of the lower element only.
43499 bool NegA = invertIfNegative(A);
43500 bool NegB = invertIfNegative(B);
43501 bool NegC = invertIfNegative(C);
43502
43503 if (!NegA && !NegB && !NegC)
43504 return SDValue();
43505
43506 unsigned NewOpcode =
43507 negateFMAOpcode(N->getOpcode(), NegA != NegB, NegC, false);
43508
43509 if (N->getNumOperands() == 4)
43510 return DAG.getNode(NewOpcode, dl, VT, A, B, C, N->getOperand(3));
43511 return DAG.getNode(NewOpcode, dl, VT, A, B, C);
43512}
43513
43514// Combine FMADDSUB(A, B, FNEG(C)) -> FMSUBADD(A, B, C)
43515// Combine FMSUBADD(A, B, FNEG(C)) -> FMADDSUB(A, B, C)
43516static SDValue combineFMADDSUB(SDNode *N, SelectionDAG &DAG,
43517 TargetLowering::DAGCombinerInfo &DCI) {
43518 SDLoc dl(N);
43519 EVT VT = N->getValueType(0);
43520 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43521 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
43522 bool LegalOperations = !DCI.isBeforeLegalizeOps();
43523
43524 SDValue N2 = N->getOperand(2);
43525 if (TLI.isNegatibleForFree(N2, DAG, LegalOperations, CodeSize) != 2)
43526 return SDValue();
43527
43528 SDValue NegN2 = TLI.getNegatedExpression(N2, DAG, LegalOperations, CodeSize);
43529 unsigned NewOpcode = negateFMAOpcode(N->getOpcode(), false, true, false);
43530
43531 if (N->getNumOperands() == 4)
43532 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
43533 NegN2, N->getOperand(3));
43534 return DAG.getNode(NewOpcode, dl, VT, N->getOperand(0), N->getOperand(1),
43535 NegN2);
43536}
43537
43538static SDValue combineZext(SDNode *N, SelectionDAG &DAG,
43539 TargetLowering::DAGCombinerInfo &DCI,
43540 const X86Subtarget &Subtarget) {
43541 // (i32 zext (and (i8 x86isd::setcc_carry), 1)) ->
43542 // (and (i32 x86isd::setcc_carry), 1)
43543 // This eliminates the zext. This transformation is necessary because
43544 // ISD::SETCC is always legalized to i8.
43545 SDLoc dl(N);
43546 SDValue N0 = N->getOperand(0);
43547 EVT VT = N->getValueType(0);
43548
43549 if (N0.getOpcode() == ISD::AND &&
43550 N0.hasOneUse() &&
43551 N0.getOperand(0).hasOneUse()) {
43552 SDValue N00 = N0.getOperand(0);
43553 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
43554 if (!isOneConstant(N0.getOperand(1)))
43555 return SDValue();
43556 return DAG.getNode(ISD::AND, dl, VT,
43557 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
43558 N00.getOperand(0), N00.getOperand(1)),
43559 DAG.getConstant(1, dl, VT));
43560 }
43561 }
43562
43563 if (N0.getOpcode() == ISD::TRUNCATE &&
43564 N0.hasOneUse() &&
43565 N0.getOperand(0).hasOneUse()) {
43566 SDValue N00 = N0.getOperand(0);
43567 if (N00.getOpcode() == X86ISD::SETCC_CARRY) {
43568 return DAG.getNode(ISD::AND, dl, VT,
43569 DAG.getNode(X86ISD::SETCC_CARRY, dl, VT,
43570 N00.getOperand(0), N00.getOperand(1)),
43571 DAG.getConstant(1, dl, VT));
43572 }
43573 }
43574
43575 if (SDValue NewCMov = combineToExtendCMOV(N, DAG))
43576 return NewCMov;
43577
43578 if (DCI.isBeforeLegalizeOps())
43579 if (SDValue V = combineExtSetcc(N, DAG, Subtarget))
43580 return V;
43581
43582 if (SDValue V = combineToExtendBoolVectorInReg(N, DAG, DCI, Subtarget))
43583 return V;
43584
43585 if (VT.isVector())
43586 if (SDValue R = PromoteMaskArithmetic(N, DAG, Subtarget))
43587 return R;
43588
43589 if (SDValue NewAdd = promoteExtBeforeAdd(N, DAG, Subtarget))
43590 return NewAdd;
43591
43592 if (SDValue R = combineOrCmpEqZeroToCtlzSrl(N, DAG, DCI, Subtarget))
43593 return R;
43594
43595 // TODO: Combine with any target/faux shuffle.
43596 if (N0.getOpcode() == X86ISD::PACKUS && N0.getValueSizeInBits() == 128 &&
43597 VT.getScalarSizeInBits() == N0.getOperand(0).getScalarValueSizeInBits()) {
43598 SDValue N00 = N0.getOperand(0);
43599 SDValue N01 = N0.getOperand(1);
43600 unsigned NumSrcEltBits = N00.getScalarValueSizeInBits();
43601 APInt ZeroMask = APInt::getHighBitsSet(NumSrcEltBits, NumSrcEltBits / 2);
43602 if ((N00.isUndef() || DAG.MaskedValueIsZero(N00, ZeroMask)) &&
43603 (N01.isUndef() || DAG.MaskedValueIsZero(N01, ZeroMask))) {
43604 return concatSubVectors(N00, N01, DAG, dl);
43605 }
43606 }
43607
43608 return SDValue();
43609}
43610
43611/// Recursive helper for combineVectorSizedSetCCEquality() to see if we have a
43612/// recognizable memcmp expansion.
43613static bool isOrXorXorTree(SDValue X, bool Root = true) {
43614 if (X.getOpcode() == ISD::OR)
43615 return isOrXorXorTree(X.getOperand(0), false) &&
43616 isOrXorXorTree(X.getOperand(1), false);
43617 if (Root)
43618 return false;
43619 return X.getOpcode() == ISD::XOR;
43620}
43621
43622/// Recursive helper for combineVectorSizedSetCCEquality() to emit the memcmp
43623/// expansion.
43624template<typename F>
43625static SDValue emitOrXorXorTree(SDValue X, SDLoc &DL, SelectionDAG &DAG,
43626 EVT VecVT, EVT CmpVT, bool HasPT, F SToV) {
43627 SDValue Op0 = X.getOperand(0);
43628 SDValue Op1 = X.getOperand(1);
43629 if (X.getOpcode() == ISD::OR) {
43630 SDValue A = emitOrXorXorTree(Op0, DL, DAG, VecVT, CmpVT, HasPT, SToV);
43631 SDValue B = emitOrXorXorTree(Op1, DL, DAG, VecVT, CmpVT, HasPT, SToV);
43632 if (VecVT != CmpVT)
43633 return DAG.getNode(ISD::OR, DL, CmpVT, A, B);
43634 if (HasPT)
43635 return DAG.getNode(ISD::OR, DL, VecVT, A, B);
43636 return DAG.getNode(ISD::AND, DL, CmpVT, A, B);
43637 } else if (X.getOpcode() == ISD::XOR) {
43638 SDValue A = SToV(Op0);
43639 SDValue B = SToV(Op1);
43640 if (VecVT != CmpVT)
43641 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETNE);
43642 if (HasPT)
43643 return DAG.getNode(ISD::XOR, DL, VecVT, A, B);
43644 return DAG.getSetCC(DL, CmpVT, A, B, ISD::SETEQ);
43645 }
43646 llvm_unreachable("Impossible")::llvm::llvm_unreachable_internal("Impossible", "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43646);
43647}
43648
43649/// Try to map a 128-bit or larger integer comparison to vector instructions
43650/// before type legalization splits it up into chunks.
43651static SDValue combineVectorSizedSetCCEquality(SDNode *SetCC, SelectionDAG &DAG,
43652 const X86Subtarget &Subtarget) {
43653 ISD::CondCode CC = cast<CondCodeSDNode>(SetCC->getOperand(2))->get();
43654 assert((CC == ISD::SETNE || CC == ISD::SETEQ) && "Bad comparison predicate")(((CC == ISD::SETNE || CC == ISD::SETEQ) && "Bad comparison predicate"
) ? static_cast<void> (0) : __assert_fail ("(CC == ISD::SETNE || CC == ISD::SETEQ) && \"Bad comparison predicate\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43654, __PRETTY_FUNCTION__));
43655
43656 // We're looking for an oversized integer equality comparison.
43657 SDValue X = SetCC->getOperand(0);
43658 SDValue Y = SetCC->getOperand(1);
43659 EVT OpVT = X.getValueType();
43660 unsigned OpSize = OpVT.getSizeInBits();
43661 if (!OpVT.isScalarInteger() || OpSize < 128)
43662 return SDValue();
43663
43664 // Ignore a comparison with zero because that gets special treatment in
43665 // EmitTest(). But make an exception for the special case of a pair of
43666 // logically-combined vector-sized operands compared to zero. This pattern may
43667 // be generated by the memcmp expansion pass with oversized integer compares
43668 // (see PR33325).
43669 bool IsOrXorXorTreeCCZero = isNullConstant(Y) && isOrXorXorTree(X);
43670 if (isNullConstant(Y) && !IsOrXorXorTreeCCZero)
43671 return SDValue();
43672
43673 // Don't perform this combine if constructing the vector will be expensive.
43674 auto IsVectorBitCastCheap = [](SDValue X) {
43675 X = peekThroughBitcasts(X);
43676 return isa<ConstantSDNode>(X) || X.getValueType().isVector() ||
43677 X.getOpcode() == ISD::LOAD;
43678 };
43679 if ((!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y)) &&
43680 !IsOrXorXorTreeCCZero)
43681 return SDValue();
43682
43683 EVT VT = SetCC->getValueType(0);
43684 SDLoc DL(SetCC);
43685 bool HasAVX = Subtarget.hasAVX();
43686
43687 // Use XOR (plus OR) and PTEST after SSE4.1 for 128/256-bit operands.
43688 // Use PCMPNEQ (plus OR) and KORTEST for 512-bit operands.
43689 // Otherwise use PCMPEQ (plus AND) and mask testing.
43690 if ((OpSize == 128 && Subtarget.hasSSE2()) ||
43691 (OpSize == 256 && HasAVX) ||
43692 (OpSize == 512 && Subtarget.useAVX512Regs())) {
43693 bool HasPT = Subtarget.hasSSE41();
43694
43695 // PTEST and MOVMSK are slow on Knights Landing and Knights Mill and widened
43696 // vector registers are essentially free. (Technically, widening registers
43697 // prevents load folding, but the tradeoff is worth it.)
43698 bool PreferKOT = Subtarget.preferMaskRegisters();
43699 bool NeedZExt = PreferKOT && !Subtarget.hasVLX() && OpSize != 512;
43700
43701 EVT VecVT = MVT::v16i8;
43702 EVT CmpVT = PreferKOT ? MVT::v16i1 : VecVT;
43703 if (OpSize == 256) {
43704 VecVT = MVT::v32i8;
43705 CmpVT = PreferKOT ? MVT::v32i1 : VecVT;
43706 }
43707 EVT CastVT = VecVT;
43708 bool NeedsAVX512FCast = false;
43709 if (OpSize == 512 || NeedZExt) {
43710 if (Subtarget.hasBWI()) {
43711 VecVT = MVT::v64i8;
43712 CmpVT = MVT::v64i1;
43713 if (OpSize == 512)
43714 CastVT = VecVT;
43715 } else {
43716 VecVT = MVT::v16i32;
43717 CmpVT = MVT::v16i1;
43718 CastVT = OpSize == 512 ? VecVT :
43719 OpSize == 256 ? MVT::v8i32 : MVT::v4i32;
43720 NeedsAVX512FCast = true;
43721 }
43722 }
43723
43724 auto ScalarToVector = [&](SDValue X) -> SDValue {
43725 bool TmpZext = false;
43726 EVT TmpCastVT = CastVT;
43727 if (X.getOpcode() == ISD::ZERO_EXTEND) {
43728 SDValue OrigX = X.getOperand(0);
43729 unsigned OrigSize = OrigX.getScalarValueSizeInBits();
43730 if (OrigSize < OpSize) {
43731 if (OrigSize == 128) {
43732 TmpCastVT = NeedsAVX512FCast ? MVT::v4i32 : MVT::v16i8;
43733 X = OrigX;
43734 TmpZext = true;
43735 } else if (OrigSize == 256) {
43736 TmpCastVT = NeedsAVX512FCast ? MVT::v8i32 : MVT::v32i8;
43737 X = OrigX;
43738 TmpZext = true;
43739 }
43740 }
43741 }
43742 X = DAG.getBitcast(TmpCastVT, X);
43743 if (!NeedZExt && !TmpZext)
43744 return X;
43745 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43746 MVT VecIdxVT = TLI.getVectorIdxTy(DAG.getDataLayout());
43747 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecVT,
43748 DAG.getConstant(0, DL, VecVT), X,
43749 DAG.getConstant(0, DL, VecIdxVT));
43750 };
43751
43752 SDValue Cmp;
43753 if (IsOrXorXorTreeCCZero) {
43754 // This is a bitwise-combined equality comparison of 2 pairs of vectors:
43755 // setcc i128 (or (xor A, B), (xor C, D)), 0, eq|ne
43756 // Use 2 vector equality compares and 'and' the results before doing a
43757 // MOVMSK.
43758 Cmp = emitOrXorXorTree(X, DL, DAG, VecVT, CmpVT, HasPT, ScalarToVector);
43759 } else {
43760 SDValue VecX = ScalarToVector(X);
43761 SDValue VecY = ScalarToVector(Y);
43762 if (VecVT != CmpVT) {
43763 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETNE);
43764 } else if (HasPT) {
43765 Cmp = DAG.getNode(ISD::XOR, DL, VecVT, VecX, VecY);
43766 } else {
43767 Cmp = DAG.getSetCC(DL, CmpVT, VecX, VecY, ISD::SETEQ);
43768 }
43769 }
43770 // AVX512 should emit a setcc that will lower to kortest.
43771 if (VecVT != CmpVT) {
43772 EVT KRegVT = CmpVT == MVT::v64i1 ? MVT::i64 :
43773 CmpVT == MVT::v32i1 ? MVT::i32 : MVT::i16;
43774 return DAG.getSetCC(DL, VT, DAG.getBitcast(KRegVT, Cmp),
43775 DAG.getConstant(0, DL, KRegVT), CC);
43776 }
43777 if (HasPT) {
43778 SDValue BCCmp = DAG.getBitcast(OpSize == 256 ? MVT::v4i64 : MVT::v2i64,
43779 Cmp);
43780 SDValue PT = DAG.getNode(X86ISD::PTEST, DL, MVT::i32, BCCmp, BCCmp);
43781 X86::CondCode X86CC = CC == ISD::SETEQ ? X86::COND_E : X86::COND_NE;
43782 SDValue SetCC = getSETCC(X86CC, PT, DL, DAG);
43783 return DAG.getNode(ISD::TRUNCATE, DL, VT, SetCC.getValue(0));
43784 }
43785 // If all bytes match (bitmask is 0x(FFFF)FFFF), that's equality.
43786 // setcc i128 X, Y, eq --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, eq
43787 // setcc i128 X, Y, ne --> setcc (pmovmskb (pcmpeqb X, Y)), 0xFFFF, ne
43788 // setcc i256 X, Y, eq --> setcc (vpmovmskb (vpcmpeqb X, Y)), 0xFFFFFFFF, eq
43789 // setcc i256 X, Y, ne --> setcc (vpmovmskb (vpcmpeqb X, Y)), 0xFFFFFFFF, ne
43790 SDValue MovMsk = DAG.getNode(X86ISD::MOVMSK, DL, MVT::i32, Cmp);
43791 SDValue FFFFs = DAG.getConstant(OpSize == 128 ? 0xFFFF : 0xFFFFFFFF, DL,
43792 MVT::i32);
43793 return DAG.getSetCC(DL, VT, MovMsk, FFFFs, CC);
43794 }
43795
43796 return SDValue();
43797}
43798
43799static SDValue combineSetCC(SDNode *N, SelectionDAG &DAG,
43800 const X86Subtarget &Subtarget) {
43801 const ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
43802 const SDValue LHS = N->getOperand(0);
43803 const SDValue RHS = N->getOperand(1);
43804 EVT VT = N->getValueType(0);
43805 EVT OpVT = LHS.getValueType();
43806 SDLoc DL(N);
43807
43808 if (CC == ISD::SETNE || CC == ISD::SETEQ) {
43809 // 0-x == y --> x+y == 0
43810 // 0-x != y --> x+y != 0
43811 if (LHS.getOpcode() == ISD::SUB && isNullConstant(LHS.getOperand(0)) &&
43812 LHS.hasOneUse()) {
43813 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, RHS, LHS.getOperand(1));
43814 return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC);
43815 }
43816 // x == 0-y --> x+y == 0
43817 // x != 0-y --> x+y != 0
43818 if (RHS.getOpcode() == ISD::SUB && isNullConstant(RHS.getOperand(0)) &&
43819 RHS.hasOneUse()) {
43820 SDValue Add = DAG.getNode(ISD::ADD, DL, OpVT, LHS, RHS.getOperand(1));
43821 return DAG.getSetCC(DL, VT, Add, DAG.getConstant(0, DL, OpVT), CC);
43822 }
43823
43824 if (SDValue V = combineVectorSizedSetCCEquality(N, DAG, Subtarget))
43825 return V;
43826 }
43827
43828 if (VT.isVector() && VT.getVectorElementType() == MVT::i1 &&
43829 (CC == ISD::SETNE || CC == ISD::SETEQ || ISD::isSignedIntSetCC(CC))) {
43830 // Using temporaries to avoid messing up operand ordering for later
43831 // transformations if this doesn't work.
43832 SDValue Op0 = LHS;
43833 SDValue Op1 = RHS;
43834 ISD::CondCode TmpCC = CC;
43835 // Put build_vector on the right.
43836 if (Op0.getOpcode() == ISD::BUILD_VECTOR) {
43837 std::swap(Op0, Op1);
43838 TmpCC = ISD::getSetCCSwappedOperands(TmpCC);
43839 }
43840
43841 bool IsSEXT0 =
43842 (Op0.getOpcode() == ISD::SIGN_EXTEND) &&
43843 (Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1);
43844 bool IsVZero1 = ISD::isBuildVectorAllZeros(Op1.getNode());
43845
43846 if (IsSEXT0 && IsVZero1) {
43847 assert(VT == Op0.getOperand(0).getValueType() &&((VT == Op0.getOperand(0).getValueType() && "Uexpected operand type"
) ? static_cast<void> (0) : __assert_fail ("VT == Op0.getOperand(0).getValueType() && \"Uexpected operand type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43848, __PRETTY_FUNCTION__))
43848 "Uexpected operand type")((VT == Op0.getOperand(0).getValueType() && "Uexpected operand type"
) ? static_cast<void> (0) : __assert_fail ("VT == Op0.getOperand(0).getValueType() && \"Uexpected operand type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43848, __PRETTY_FUNCTION__));
43849 if (TmpCC == ISD::SETGT)
43850 return DAG.getConstant(0, DL, VT);
43851 if (TmpCC == ISD::SETLE)
43852 return DAG.getConstant(1, DL, VT);
43853 if (TmpCC == ISD::SETEQ || TmpCC == ISD::SETGE)
43854 return DAG.getNOT(DL, Op0.getOperand(0), VT);
43855
43856 assert((TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) &&(((TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) && "Unexpected condition code!"
) ? static_cast<void> (0) : __assert_fail ("(TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) && \"Unexpected condition code!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43857, __PRETTY_FUNCTION__))
43857 "Unexpected condition code!")(((TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) && "Unexpected condition code!"
) ? static_cast<void> (0) : __assert_fail ("(TmpCC == ISD::SETNE || TmpCC == ISD::SETLT) && \"Unexpected condition code!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43857, __PRETTY_FUNCTION__));
43858 return Op0.getOperand(0);
43859 }
43860 }
43861
43862 // If we have AVX512, but not BWI and this is a vXi16/vXi8 setcc, just
43863 // pre-promote its result type since vXi1 vectors don't get promoted
43864 // during type legalization.
43865 // NOTE: The element count check is to ignore operand types that need to
43866 // go through type promotion to a 128-bit vector.
43867 if (Subtarget.hasAVX512() && !Subtarget.hasBWI() && VT.isVector() &&
43868 VT.getVectorElementType() == MVT::i1 &&
43869 (OpVT.getVectorElementType() == MVT::i8 ||
43870 OpVT.getVectorElementType() == MVT::i16)) {
43871 SDValue Setcc = DAG.getSetCC(DL, OpVT, LHS, RHS, CC);
43872 return DAG.getNode(ISD::TRUNCATE, DL, VT, Setcc);
43873 }
43874
43875 // For an SSE1-only target, lower a comparison of v4f32 to X86ISD::CMPP early
43876 // to avoid scalarization via legalization because v4i32 is not a legal type.
43877 if (Subtarget.hasSSE1() && !Subtarget.hasSSE2() && VT == MVT::v4i32 &&
43878 LHS.getValueType() == MVT::v4f32)
43879 return LowerVSETCC(SDValue(N, 0), Subtarget, DAG);
43880
43881 return SDValue();
43882}
43883
43884static SDValue combineMOVMSK(SDNode *N, SelectionDAG &DAG,
43885 TargetLowering::DAGCombinerInfo &DCI,
43886 const X86Subtarget &Subtarget) {
43887 SDValue Src = N->getOperand(0);
43888 MVT SrcVT = Src.getSimpleValueType();
43889 MVT VT = N->getSimpleValueType(0);
43890 unsigned NumBits = VT.getScalarSizeInBits();
43891 unsigned NumElts = SrcVT.getVectorNumElements();
43892
43893 // Perform constant folding.
43894 if (ISD::isBuildVectorOfConstantSDNodes(Src.getNode())) {
43895 assert(VT == MVT::i32 && "Unexpected result type")((VT == MVT::i32 && "Unexpected result type") ? static_cast
<void> (0) : __assert_fail ("VT == MVT::i32 && \"Unexpected result type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 43895, __PRETTY_FUNCTION__));
43896 APInt Imm(32, 0);
43897 for (unsigned Idx = 0, e = Src.getNumOperands(); Idx < e; ++Idx) {
43898 if (!Src.getOperand(Idx).isUndef() &&
43899 Src.getConstantOperandAPInt(Idx).isNegative())
43900 Imm.setBit(Idx);
43901 }
43902 return DAG.getConstant(Imm, SDLoc(N), VT);
43903 }
43904
43905 // Look through int->fp bitcasts that don't change the element width.
43906 unsigned EltWidth = SrcVT.getScalarSizeInBits();
43907 if (Subtarget.hasSSE2() && Src.getOpcode() == ISD::BITCAST &&
43908 Src.getOperand(0).getScalarValueSizeInBits() == EltWidth)
43909 return DAG.getNode(X86ISD::MOVMSK, SDLoc(N), VT, Src.getOperand(0));
43910
43911 // Fold movmsk(not(x)) -> not(movmsk) to improve folding of movmsk results
43912 // with scalar comparisons.
43913 if (SDValue NotSrc = IsNOT(Src, DAG)) {
43914 SDLoc DL(N);
43915 APInt NotMask = APInt::getLowBitsSet(NumBits, NumElts);
43916 NotSrc = DAG.getBitcast(SrcVT, NotSrc);
43917 return DAG.getNode(ISD::XOR, DL, VT,
43918 DAG.getNode(X86ISD::MOVMSK, DL, VT, NotSrc),
43919 DAG.getConstant(NotMask, DL, VT));
43920 }
43921
43922 // Simplify the inputs.
43923 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43924 APInt DemandedMask(APInt::getAllOnesValue(NumBits));
43925 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
43926 return SDValue(N, 0);
43927
43928 return SDValue();
43929}
43930
43931static SDValue combineX86GatherScatter(SDNode *N, SelectionDAG &DAG,
43932 TargetLowering::DAGCombinerInfo &DCI) {
43933 // With vector masks we only demand the upper bit of the mask.
43934 SDValue Mask = cast<X86MaskedGatherScatterSDNode>(N)->getMask();
43935 if (Mask.getScalarValueSizeInBits() != 1) {
43936 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
43937 APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
43938 if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
43939 return SDValue(N, 0);
43940 }
43941
43942 return SDValue();
43943}
43944
43945static SDValue combineGatherScatter(SDNode *N, SelectionDAG &DAG,
43946 TargetLowering::DAGCombinerInfo &DCI) {
43947 SDLoc DL(N);
43948 auto *GorS = cast<MaskedGatherScatterSDNode>(N);
43949 SDValue Chain = GorS->getChain();
43950 SDValue Index = GorS->getIndex();
43951 SDValue Mask = GorS->getMask();
43952 SDValue Base = GorS->getBasePtr();
43953 SDValue Scale = GorS->getScale();
43954
43955 if (DCI.isBeforeLegalize()) {
43956 unsigned IndexWidth = Index.getScalarValueSizeInBits();
43957
43958 // Shrink constant indices if they are larger than 32-bits.
43959 // Only do this before legalize types since v2i64 could become v2i32.
43960 // FIXME: We could check that the type is legal if we're after legalize
43961 // types, but then we would need to construct test cases where that happens.
43962 // FIXME: We could support more than just constant vectors, but we need to
43963 // careful with costing. A truncate that can be optimized out would be fine.
43964 // Otherwise we might only want to create a truncate if it avoids a split.
43965 if (auto *BV = dyn_cast<BuildVectorSDNode>(Index)) {
43966 if (BV->isConstant() && IndexWidth > 32 &&
43967 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
43968 unsigned NumElts = Index.getValueType().getVectorNumElements();
43969 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
43970 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
43971 if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
43972 SDValue Ops[] = { Chain, Gather->getPassThru(),
43973 Mask, Base, Index, Scale } ;
43974 return DAG.getMaskedGather(Gather->getVTList(),
43975 Gather->getMemoryVT(), DL, Ops,
43976 Gather->getMemOperand(),
43977 Gather->getIndexType());
43978 }
43979 auto *Scatter = cast<MaskedScatterSDNode>(GorS);
43980 SDValue Ops[] = { Chain, Scatter->getValue(),
43981 Mask, Base, Index, Scale };
43982 return DAG.getMaskedScatter(Scatter->getVTList(),
43983 Scatter->getMemoryVT(), DL,
43984 Ops, Scatter->getMemOperand(),
43985 Scatter->getIndexType());
43986 }
43987 }
43988
43989 // Shrink any sign/zero extends from 32 or smaller to larger than 32 if
43990 // there are sufficient sign bits. Only do this before legalize types to
43991 // avoid creating illegal types in truncate.
43992 if ((Index.getOpcode() == ISD::SIGN_EXTEND ||
43993 Index.getOpcode() == ISD::ZERO_EXTEND) &&
43994 IndexWidth > 32 &&
43995 Index.getOperand(0).getScalarValueSizeInBits() <= 32 &&
43996 DAG.ComputeNumSignBits(Index) > (IndexWidth - 32)) {
43997 unsigned NumElts = Index.getValueType().getVectorNumElements();
43998 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32, NumElts);
43999 Index = DAG.getNode(ISD::TRUNCATE, DL, NewVT, Index);
44000 if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
44001 SDValue Ops[] = { Chain, Gather->getPassThru(),
44002 Mask, Base, Index, Scale } ;
44003 return DAG.getMaskedGather(Gather->getVTList(),
44004 Gather->getMemoryVT(), DL, Ops,
44005 Gather->getMemOperand(),
44006 Gather->getIndexType());
44007 }
44008 auto *Scatter = cast<MaskedScatterSDNode>(GorS);
44009 SDValue Ops[] = { Chain, Scatter->getValue(),
44010 Mask, Base, Index, Scale };
44011 return DAG.getMaskedScatter(Scatter->getVTList(),
44012 Scatter->getMemoryVT(), DL,
44013 Ops, Scatter->getMemOperand(),
44014 Scatter->getIndexType());
44015 }
44016 }
44017
44018 if (DCI.isBeforeLegalizeOps()) {
44019 unsigned IndexWidth = Index.getScalarValueSizeInBits();
44020
44021 // Make sure the index is either i32 or i64
44022 if (IndexWidth != 32 && IndexWidth != 64) {
44023 MVT EltVT = IndexWidth > 32 ? MVT::i64 : MVT::i32;
44024 EVT IndexVT = EVT::getVectorVT(*DAG.getContext(), EltVT,
44025 Index.getValueType().getVectorNumElements());
44026 Index = DAG.getSExtOrTrunc(Index, DL, IndexVT);
44027 if (auto *Gather = dyn_cast<MaskedGatherSDNode>(GorS)) {
44028 SDValue Ops[] = { Chain, Gather->getPassThru(),
44029 Mask, Base, Index, Scale } ;
44030 return DAG.getMaskedGather(Gather->getVTList(),
44031 Gather->getMemoryVT(), DL, Ops,
44032 Gather->getMemOperand(),
44033 Gather->getIndexType());
44034 }
44035 auto *Scatter = cast<MaskedScatterSDNode>(GorS);
44036 SDValue Ops[] = { Chain, Scatter->getValue(),
44037 Mask, Base, Index, Scale };
44038 return DAG.getMaskedScatter(Scatter->getVTList(),
44039 Scatter->getMemoryVT(), DL,
44040 Ops, Scatter->getMemOperand(),
44041 Scatter->getIndexType());
44042 }
44043 }
44044
44045 // With vector masks we only demand the upper bit of the mask.
44046 if (Mask.getScalarValueSizeInBits() != 1) {
44047 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
44048 APInt DemandedMask(APInt::getSignMask(Mask.getScalarValueSizeInBits()));
44049 if (TLI.SimplifyDemandedBits(Mask, DemandedMask, DCI))
44050 return SDValue(N, 0);
44051 }
44052
44053 return SDValue();
44054}
44055
44056// Optimize RES = X86ISD::SETCC CONDCODE, EFLAG_INPUT
44057static SDValue combineX86SetCC(SDNode *N, SelectionDAG &DAG,
44058 const X86Subtarget &Subtarget) {
44059 SDLoc DL(N);
44060 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(0));
44061 SDValue EFLAGS = N->getOperand(1);
44062
44063 // Try to simplify the EFLAGS and condition code operands.
44064 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget))
44065 return getSETCC(CC, Flags, DL, DAG);
44066
44067 return SDValue();
44068}
44069
44070/// Optimize branch condition evaluation.
44071static SDValue combineBrCond(SDNode *N, SelectionDAG &DAG,
44072 const X86Subtarget &Subtarget) {
44073 SDLoc DL(N);
44074 SDValue EFLAGS = N->getOperand(3);
44075 X86::CondCode CC = X86::CondCode(N->getConstantOperandVal(2));
44076
44077 // Try to simplify the EFLAGS and condition code operands.
44078 // Make sure to not keep references to operands, as combineSetCCEFLAGS can
44079 // RAUW them under us.
44080 if (SDValue Flags = combineSetCCEFLAGS(EFLAGS, CC, DAG, Subtarget)) {
44081 SDValue Cond = DAG.getTargetConstant(CC, DL, MVT::i8);
44082 return DAG.getNode(X86ISD::BRCOND, DL, N->getVTList(), N->getOperand(0),
44083 N->getOperand(1), Cond, Flags);
44084 }
44085
44086 return SDValue();
44087}
44088
44089static SDValue combineVectorCompareAndMaskUnaryOp(SDNode *N,
44090 SelectionDAG &DAG) {
44091 // Take advantage of vector comparisons producing 0 or -1 in each lane to
44092 // optimize away operation when it's from a constant.
44093 //
44094 // The general transformation is:
44095 // UNARYOP(AND(VECTOR_CMP(x,y), constant)) -->
44096 // AND(VECTOR_CMP(x,y), constant2)
44097 // constant2 = UNARYOP(constant)
44098
44099 // Early exit if this isn't a vector operation, the operand of the
44100 // unary operation isn't a bitwise AND, or if the sizes of the operations
44101 // aren't the same.
44102 EVT VT = N->getValueType(0);
44103 bool IsStrict = N->isStrictFPOpcode();
44104 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
44105 if (!VT.isVector() || Op0->getOpcode() != ISD::AND ||
44106 Op0->getOperand(0)->getOpcode() != ISD::SETCC ||
44107 VT.getSizeInBits() != Op0.getValueSizeInBits())
44108 return SDValue();
44109
44110 // Now check that the other operand of the AND is a constant. We could
44111 // make the transformation for non-constant splats as well, but it's unclear
44112 // that would be a benefit as it would not eliminate any operations, just
44113 // perform one more step in scalar code before moving to the vector unit.
44114 if (auto *BV = dyn_cast<BuildVectorSDNode>(Op0.getOperand(1))) {
44115 // Bail out if the vector isn't a constant.
44116 if (!BV->isConstant())
44117 return SDValue();
44118
44119 // Everything checks out. Build up the new and improved node.
44120 SDLoc DL(N);
44121 EVT IntVT = BV->getValueType(0);
44122 // Create a new constant of the appropriate type for the transformed
44123 // DAG.
44124 SDValue SourceConst;
44125 if (IsStrict)
44126 SourceConst = DAG.getNode(N->getOpcode(), DL, {VT, MVT::Other},
44127 {N->getOperand(0), SDValue(BV, 0)});
44128 else
44129 SourceConst = DAG.getNode(N->getOpcode(), DL, VT, SDValue(BV, 0));
44130 // The AND node needs bitcasts to/from an integer vector type around it.
44131 SDValue MaskConst = DAG.getBitcast(IntVT, SourceConst);
44132 SDValue NewAnd = DAG.getNode(ISD::AND, DL, IntVT, Op0->getOperand(0),
44133 MaskConst);
44134 SDValue Res = DAG.getBitcast(VT, NewAnd);
44135 if (IsStrict)
44136 return DAG.getMergeValues({Res, SourceConst.getValue(1)}, DL);
44137 return Res;
44138 }
44139
44140 return SDValue();
44141}
44142
44143/// If we are converting a value to floating-point, try to replace scalar
44144/// truncate of an extracted vector element with a bitcast. This tries to keep
44145/// the sequence on XMM registers rather than moving between vector and GPRs.
44146static SDValue combineToFPTruncExtElt(SDNode *N, SelectionDAG &DAG) {
44147 // TODO: This is currently only used by combineSIntToFP, but it is generalized
44148 // to allow being called by any similar cast opcode.
44149 // TODO: Consider merging this into lowering: vectorizeExtractedCast().
44150 SDValue Trunc = N->getOperand(0);
44151 if (!Trunc.hasOneUse() || Trunc.getOpcode() != ISD::TRUNCATE)
44152 return SDValue();
44153
44154 SDValue ExtElt = Trunc.getOperand(0);
44155 if (!ExtElt.hasOneUse() || ExtElt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
44156 !isNullConstant(ExtElt.getOperand(1)))
44157 return SDValue();
44158
44159 EVT TruncVT = Trunc.getValueType();
44160 EVT SrcVT = ExtElt.getValueType();
44161 unsigned DestWidth = TruncVT.getSizeInBits();
44162 unsigned SrcWidth = SrcVT.getSizeInBits();
44163 if (SrcWidth % DestWidth != 0)
44164 return SDValue();
44165
44166 // inttofp (trunc (extelt X, 0)) --> inttofp (extelt (bitcast X), 0)
44167 EVT SrcVecVT = ExtElt.getOperand(0).getValueType();
44168 unsigned VecWidth = SrcVecVT.getSizeInBits();
44169 unsigned NumElts = VecWidth / DestWidth;
44170 EVT BitcastVT = EVT::getVectorVT(*DAG.getContext(), TruncVT, NumElts);
44171 SDValue BitcastVec = DAG.getBitcast(BitcastVT, ExtElt.getOperand(0));
44172 SDLoc DL(N);
44173 SDValue NewExtElt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, TruncVT,
44174 BitcastVec, ExtElt.getOperand(1));
44175 return DAG.getNode(N->getOpcode(), DL, N->getValueType(0), NewExtElt);
44176}
44177
44178static SDValue combineUIntToFP(SDNode *N, SelectionDAG &DAG,
44179 const X86Subtarget &Subtarget) {
44180 bool IsStrict = N->isStrictFPOpcode();
44181 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
44182 EVT VT = N->getValueType(0);
44183 EVT InVT = Op0.getValueType();
44184
44185 // UINT_TO_FP(vXi1) -> SINT_TO_FP(ZEXT(vXi1 to vXi32))
44186 // UINT_TO_FP(vXi8) -> SINT_TO_FP(ZEXT(vXi8 to vXi32))
44187 // UINT_TO_FP(vXi16) -> SINT_TO_FP(ZEXT(vXi16 to vXi32))
44188 if (InVT.isVector() && InVT.getScalarSizeInBits() < 32) {
44189 SDLoc dl(N);
44190 EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
44191 InVT.getVectorNumElements());
44192 SDValue P = DAG.getNode(ISD::ZERO_EXTEND, dl, DstVT, Op0);
44193
44194 // UINT_TO_FP isn't legal without AVX512 so use SINT_TO_FP.
44195 if (IsStrict)
44196 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
44197 {N->getOperand(0), P});
44198 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
44199 }
44200
44201 // Since UINT_TO_FP is legal (it's marked custom), dag combiner won't
44202 // optimize it to a SINT_TO_FP when the sign bit is known zero. Perform
44203 // the optimization here.
44204 if (DAG.SignBitIsZero(Op0)) {
44205 if (IsStrict)
44206 return DAG.getNode(ISD::STRICT_SINT_TO_FP, SDLoc(N), {VT, MVT::Other},
44207 {N->getOperand(0), Op0});
44208 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);
44209 }
44210
44211 return SDValue();
44212}
44213
44214static SDValue combineSIntToFP(SDNode *N, SelectionDAG &DAG,
44215 TargetLowering::DAGCombinerInfo &DCI,
44216 const X86Subtarget &Subtarget) {
44217 // First try to optimize away the conversion entirely when it's
44218 // conditionally from a constant. Vectors only.
44219 bool IsStrict = N->isStrictFPOpcode();
44220 if (SDValue Res = combineVectorCompareAndMaskUnaryOp(N, DAG))
44221 return Res;
44222
44223 // Now move on to more general possibilities.
44224 SDValue Op0 = N->getOperand(IsStrict ? 1 : 0);
44225 EVT VT = N->getValueType(0);
44226 EVT InVT = Op0.getValueType();
44227
44228 // SINT_TO_FP(vXi1) -> SINT_TO_FP(SEXT(vXi1 to vXi32))
44229 // SINT_TO_FP(vXi8) -> SINT_TO_FP(SEXT(vXi8 to vXi32))
44230 // SINT_TO_FP(vXi16) -> SINT_TO_FP(SEXT(vXi16 to vXi32))
44231 if (InVT.isVector() && InVT.getScalarSizeInBits() < 32) {
44232 SDLoc dl(N);
44233 EVT DstVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
44234 InVT.getVectorNumElements());
44235 SDValue P = DAG.getNode(ISD::SIGN_EXTEND, dl, DstVT, Op0);
44236 if (IsStrict)
44237 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
44238 {N->getOperand(0), P});
44239 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, P);
44240 }
44241
44242 // Without AVX512DQ we only support i64 to float scalar conversion. For both
44243 // vectors and scalars, see if we know that the upper bits are all the sign
44244 // bit, in which case we can truncate the input to i32 and convert from that.
44245 if (InVT.getScalarSizeInBits() > 32 && !Subtarget.hasDQI()) {
44246 unsigned BitWidth = InVT.getScalarSizeInBits();
44247 unsigned NumSignBits = DAG.ComputeNumSignBits(Op0);
44248 if (NumSignBits >= (BitWidth - 31)) {
44249 EVT TruncVT = MVT::i32;
44250 if (InVT.isVector())
44251 TruncVT = EVT::getVectorVT(*DAG.getContext(), TruncVT,
44252 InVT.getVectorNumElements());
44253 SDLoc dl(N);
44254 if (DCI.isBeforeLegalize() || TruncVT != MVT::v2i32) {
44255 SDValue Trunc = DAG.getNode(ISD::TRUNCATE, dl, TruncVT, Op0);
44256 if (IsStrict)
44257 return DAG.getNode(ISD::STRICT_SINT_TO_FP, dl, {VT, MVT::Other},
44258 {N->getOperand(0), Trunc});
44259 return DAG.getNode(ISD::SINT_TO_FP, dl, VT, Trunc);
44260 }
44261 // If we're after legalize and the type is v2i32 we need to shuffle and
44262 // use CVTSI2P.
44263 assert(InVT == MVT::v2i64 && "Unexpected VT!")((InVT == MVT::v2i64 && "Unexpected VT!") ? static_cast
<void> (0) : __assert_fail ("InVT == MVT::v2i64 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44263, __PRETTY_FUNCTION__));
44264 SDValue Cast = DAG.getBitcast(MVT::v4i32, Op0);
44265 SDValue Shuf = DAG.getVectorShuffle(MVT::v4i32, dl, Cast, Cast,
44266 { 0, 2, -1, -1 });
44267 if (IsStrict)
44268 return DAG.getNode(X86ISD::STRICT_CVTSI2P, dl, {VT, MVT::Other},
44269 {N->getOperand(0), Shuf});
44270 return DAG.getNode(X86ISD::CVTSI2P, dl, VT, Shuf);
44271 }
44272 }
44273
44274 // Transform (SINT_TO_FP (i64 ...)) into an x87 operation if we have
44275 // a 32-bit target where SSE doesn't support i64->FP operations.
44276 if (!Subtarget.useSoftFloat() && Subtarget.hasX87() &&
44277 Op0.getOpcode() == ISD::LOAD) {
44278 LoadSDNode *Ld = cast<LoadSDNode>(Op0.getNode());
44279 EVT LdVT = Ld->getValueType(0);
44280
44281 // This transformation is not supported if the result type is f16 or f128.
44282 if (VT == MVT::f16 || VT == MVT::f128)
44283 return SDValue();
44284
44285 // If we have AVX512DQ we can use packed conversion instructions unless
44286 // the VT is f80.
44287 if (Subtarget.hasDQI() && VT != MVT::f80)
44288 return SDValue();
44289
44290 if (Ld->isSimple() && !VT.isVector() &&
44291 ISD::isNON_EXTLoad(Op0.getNode()) && Op0.hasOneUse() &&
44292 !Subtarget.is64Bit() && LdVT == MVT::i64) {
44293 std::pair<SDValue, SDValue> Tmp = Subtarget.getTargetLowering()->BuildFILD(
44294 SDValue(N, 0), LdVT, Ld->getChain(), Op0, DAG);
44295 DAG.ReplaceAllUsesOfValueWith(Op0.getValue(1), Tmp.second);
44296 return Tmp.first;
44297 }
44298 }
44299
44300 if (IsStrict)
44301 return SDValue();
44302
44303 if (SDValue V = combineToFPTruncExtElt(N, DAG))
44304 return V;
44305
44306 return SDValue();
44307}
44308
44309static bool needCarryOrOverflowFlag(SDValue Flags) {
44310 assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!")((Flags.getValueType() == MVT::i32 && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("Flags.getValueType() == MVT::i32 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44310, __PRETTY_FUNCTION__));
44311
44312 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end();
44313 UI != UE; ++UI) {
44314 SDNode *User = *UI;
44315
44316 X86::CondCode CC;
44317 switch (User->getOpcode()) {
44318 default:
44319 // Be conservative.
44320 return true;
44321 case X86ISD::SETCC:
44322 case X86ISD::SETCC_CARRY:
44323 CC = (X86::CondCode)User->getConstantOperandVal(0);
44324 break;
44325 case X86ISD::BRCOND:
44326 CC = (X86::CondCode)User->getConstantOperandVal(2);
44327 break;
44328 case X86ISD::CMOV:
44329 CC = (X86::CondCode)User->getConstantOperandVal(2);
44330 break;
44331 }
44332
44333 switch (CC) {
44334 default: break;
44335 case X86::COND_A: case X86::COND_AE:
44336 case X86::COND_B: case X86::COND_BE:
44337 case X86::COND_O: case X86::COND_NO:
44338 case X86::COND_G: case X86::COND_GE:
44339 case X86::COND_L: case X86::COND_LE:
44340 return true;
44341 }
44342 }
44343
44344 return false;
44345}
44346
44347static bool onlyZeroFlagUsed(SDValue Flags) {
44348 assert(Flags.getValueType() == MVT::i32 && "Unexpected VT!")((Flags.getValueType() == MVT::i32 && "Unexpected VT!"
) ? static_cast<void> (0) : __assert_fail ("Flags.getValueType() == MVT::i32 && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44348, __PRETTY_FUNCTION__));
44349
44350 for (SDNode::use_iterator UI = Flags->use_begin(), UE = Flags->use_end();
44351 UI != UE; ++UI) {
44352 SDNode *User = *UI;
44353
44354 unsigned CCOpNo;
44355 switch (User->getOpcode()) {
44356 default:
44357 // Be conservative.
44358 return false;
44359 case X86ISD::SETCC: CCOpNo = 0; break;
44360 case X86ISD::SETCC_CARRY: CCOpNo = 0; break;
44361 case X86ISD::BRCOND: CCOpNo = 2; break;
44362 case X86ISD::CMOV: CCOpNo = 2; break;
44363 }
44364
44365 X86::CondCode CC = (X86::CondCode)User->getConstantOperandVal(CCOpNo);
44366 if (CC != X86::COND_E && CC != X86::COND_NE)
44367 return false;
44368 }
44369
44370 return true;
44371}
44372
44373static SDValue combineCMP(SDNode *N, SelectionDAG &DAG) {
44374 // Only handle test patterns.
44375 if (!isNullConstant(N->getOperand(1)))
44376 return SDValue();
44377
44378 // If we have a CMP of a truncated binop, see if we can make a smaller binop
44379 // and use its flags directly.
44380 // TODO: Maybe we should try promoting compares that only use the zero flag
44381 // first if we can prove the upper bits with computeKnownBits?
44382 SDLoc dl(N);
44383 SDValue Op = N->getOperand(0);
44384 EVT VT = Op.getValueType();
44385
44386 // If we have a constant logical shift that's only used in a comparison
44387 // against zero turn it into an equivalent AND. This allows turning it into
44388 // a TEST instruction later.
44389 if ((Op.getOpcode() == ISD::SRL || Op.getOpcode() == ISD::SHL) &&
44390 Op.hasOneUse() && isa<ConstantSDNode>(Op.getOperand(1)) &&
44391 onlyZeroFlagUsed(SDValue(N, 0))) {
44392 unsigned BitWidth = VT.getSizeInBits();
44393 const APInt &ShAmt = Op.getConstantOperandAPInt(1);
44394 if (ShAmt.ult(BitWidth)) { // Avoid undefined shifts.
44395 unsigned MaskBits = BitWidth - ShAmt.getZExtValue();
44396 APInt Mask = Op.getOpcode() == ISD::SRL
44397 ? APInt::getHighBitsSet(BitWidth, MaskBits)
44398 : APInt::getLowBitsSet(BitWidth, MaskBits);
44399 if (Mask.isSignedIntN(32)) {
44400 Op = DAG.getNode(ISD::AND, dl, VT, Op.getOperand(0),
44401 DAG.getConstant(Mask, dl, VT));
44402 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
44403 DAG.getConstant(0, dl, VT));
44404 }
44405 }
44406 }
44407
44408 // Look for a truncate with a single use.
44409 if (Op.getOpcode() != ISD::TRUNCATE || !Op.hasOneUse())
44410 return SDValue();
44411
44412 Op = Op.getOperand(0);
44413
44414 // Arithmetic op can only have one use.
44415 if (!Op.hasOneUse())
44416 return SDValue();
44417
44418 unsigned NewOpc;
44419 switch (Op.getOpcode()) {
44420 default: return SDValue();
44421 case ISD::AND:
44422 // Skip and with constant. We have special handling for and with immediate
44423 // during isel to generate test instructions.
44424 if (isa<ConstantSDNode>(Op.getOperand(1)))
44425 return SDValue();
44426 NewOpc = X86ISD::AND;
44427 break;
44428 case ISD::OR: NewOpc = X86ISD::OR; break;
44429 case ISD::XOR: NewOpc = X86ISD::XOR; break;
44430 case ISD::ADD:
44431 // If the carry or overflow flag is used, we can't truncate.
44432 if (needCarryOrOverflowFlag(SDValue(N, 0)))
44433 return SDValue();
44434 NewOpc = X86ISD::ADD;
44435 break;
44436 case ISD::SUB:
44437 // If the carry or overflow flag is used, we can't truncate.
44438 if (needCarryOrOverflowFlag(SDValue(N, 0)))
44439 return SDValue();
44440 NewOpc = X86ISD::SUB;
44441 break;
44442 }
44443
44444 // We found an op we can narrow. Truncate its inputs.
44445 SDValue Op0 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(0));
44446 SDValue Op1 = DAG.getNode(ISD::TRUNCATE, dl, VT, Op.getOperand(1));
44447
44448 // Use a X86 specific opcode to avoid DAG combine messing with it.
44449 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
44450 Op = DAG.getNode(NewOpc, dl, VTs, Op0, Op1);
44451
44452 // For AND, keep a CMP so that we can match the test pattern.
44453 if (NewOpc == X86ISD::AND)
44454 return DAG.getNode(X86ISD::CMP, dl, MVT::i32, Op,
44455 DAG.getConstant(0, dl, VT));
44456
44457 // Return the flags.
44458 return Op.getValue(1);
44459}
44460
44461static SDValue combineX86AddSub(SDNode *N, SelectionDAG &DAG,
44462 TargetLowering::DAGCombinerInfo &DCI) {
44463 assert((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) &&(((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode
()) && "Expected X86ISD::ADD or X86ISD::SUB") ? static_cast
<void> (0) : __assert_fail ("(X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) && \"Expected X86ISD::ADD or X86ISD::SUB\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44464, __PRETTY_FUNCTION__))
44464 "Expected X86ISD::ADD or X86ISD::SUB")(((X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode
()) && "Expected X86ISD::ADD or X86ISD::SUB") ? static_cast
<void> (0) : __assert_fail ("(X86ISD::ADD == N->getOpcode() || X86ISD::SUB == N->getOpcode()) && \"Expected X86ISD::ADD or X86ISD::SUB\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44464, __PRETTY_FUNCTION__));
44465
44466 SDLoc DL(N);
44467 SDValue LHS = N->getOperand(0);
44468 SDValue RHS = N->getOperand(1);
44469 MVT VT = LHS.getSimpleValueType();
44470 unsigned GenericOpc = X86ISD::ADD == N->getOpcode() ? ISD::ADD : ISD::SUB;
44471
44472 // If we don't use the flag result, simplify back to a generic ADD/SUB.
44473 if (!N->hasAnyUseOfValue(1)) {
44474 SDValue Res = DAG.getNode(GenericOpc, DL, VT, LHS, RHS);
44475 return DAG.getMergeValues({Res, DAG.getConstant(0, DL, MVT::i32)}, DL);
44476 }
44477
44478 // Fold any similar generic ADD/SUB opcodes to reuse this node.
44479 auto MatchGeneric = [&](SDValue N0, SDValue N1, bool Negate) {
44480 SDValue Ops[] = {N0, N1};
44481 SDVTList VTs = DAG.getVTList(N->getValueType(0));
44482 if (SDNode *GenericAddSub = DAG.getNodeIfExists(GenericOpc, VTs, Ops)) {
44483 SDValue Op(N, 0);
44484 if (Negate)
44485 Op = DAG.getNode(ISD::SUB, DL, VT, DAG.getConstant(0, DL, VT), Op);
44486 DCI.CombineTo(GenericAddSub, Op);
44487 }
44488 };
44489 MatchGeneric(LHS, RHS, false);
44490 MatchGeneric(RHS, LHS, X86ISD::SUB == N->getOpcode());
44491
44492 return SDValue();
44493}
44494
44495static SDValue combineSBB(SDNode *N, SelectionDAG &DAG) {
44496 if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
44497 MVT VT = N->getSimpleValueType(0);
44498 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
44499 return DAG.getNode(X86ISD::SBB, SDLoc(N), VTs,
44500 N->getOperand(0), N->getOperand(1),
44501 Flags);
44502 }
44503
44504 // Fold SBB(SUB(X,Y),0,Carry) -> SBB(X,Y,Carry)
44505 // iff the flag result is dead.
44506 SDValue Op0 = N->getOperand(0);
44507 SDValue Op1 = N->getOperand(1);
44508 if (Op0.getOpcode() == ISD::SUB && isNullConstant(Op1) &&
44509 !N->hasAnyUseOfValue(1))
44510 return DAG.getNode(X86ISD::SBB, SDLoc(N), N->getVTList(), Op0.getOperand(0),
44511 Op0.getOperand(1), N->getOperand(2));
44512
44513 return SDValue();
44514}
44515
44516// Optimize RES, EFLAGS = X86ISD::ADC LHS, RHS, EFLAGS
44517static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
44518 TargetLowering::DAGCombinerInfo &DCI) {
44519 // If the LHS and RHS of the ADC node are zero, then it can't overflow and
44520 // the result is either zero or one (depending on the input carry bit).
44521 // Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
44522 if (X86::isZeroNode(N->getOperand(0)) &&
44523 X86::isZeroNode(N->getOperand(1)) &&
44524 // We don't have a good way to replace an EFLAGS use, so only do this when
44525 // dead right now.
44526 SDValue(N, 1).use_empty()) {
44527 SDLoc DL(N);
44528 EVT VT = N->getValueType(0);
44529 SDValue CarryOut = DAG.getConstant(0, DL, N->getValueType(1));
44530 SDValue Res1 =
44531 DAG.getNode(ISD::AND, DL, VT,
44532 DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
44533 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
44534 N->getOperand(2)),
44535 DAG.getConstant(1, DL, VT));
44536 return DCI.CombineTo(N, Res1, CarryOut);
44537 }
44538
44539 if (SDValue Flags = combineCarryThroughADD(N->getOperand(2), DAG)) {
44540 MVT VT = N->getSimpleValueType(0);
44541 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
44542 return DAG.getNode(X86ISD::ADC, SDLoc(N), VTs,
44543 N->getOperand(0), N->getOperand(1),
44544 Flags);
44545 }
44546
44547 return SDValue();
44548}
44549
44550/// If this is an add or subtract where one operand is produced by a cmp+setcc,
44551/// then try to convert it to an ADC or SBB. This replaces TEST+SET+{ADD/SUB}
44552/// with CMP+{ADC, SBB}.
44553static SDValue combineAddOrSubToADCOrSBB(SDNode *N, SelectionDAG &DAG) {
44554 bool IsSub = N->getOpcode() == ISD::SUB;
44555 SDValue X = N->getOperand(0);
44556 SDValue Y = N->getOperand(1);
44557
44558 // If this is an add, canonicalize a zext operand to the RHS.
44559 // TODO: Incomplete? What if both sides are zexts?
44560 if (!IsSub && X.getOpcode() == ISD::ZERO_EXTEND &&
44561 Y.getOpcode() != ISD::ZERO_EXTEND)
44562 std::swap(X, Y);
44563
44564 // Look through a one-use zext.
44565 bool PeekedThroughZext = false;
44566 if (Y.getOpcode() == ISD::ZERO_EXTEND && Y.hasOneUse()) {
44567 Y = Y.getOperand(0);
44568 PeekedThroughZext = true;
44569 }
44570
44571 // If this is an add, canonicalize a setcc operand to the RHS.
44572 // TODO: Incomplete? What if both sides are setcc?
44573 // TODO: Should we allow peeking through a zext of the other operand?
44574 if (!IsSub && !PeekedThroughZext && X.getOpcode() == X86ISD::SETCC &&
44575 Y.getOpcode() != X86ISD::SETCC)
44576 std::swap(X, Y);
44577
44578 if (Y.getOpcode() != X86ISD::SETCC || !Y.hasOneUse())
44579 return SDValue();
44580
44581 SDLoc DL(N);
44582 EVT VT = N->getValueType(0);
44583 X86::CondCode CC = (X86::CondCode)Y.getConstantOperandVal(0);
44584
44585 // If X is -1 or 0, then we have an opportunity to avoid constants required in
44586 // the general case below.
44587 auto *ConstantX = dyn_cast<ConstantSDNode>(X);
44588 if (ConstantX) {
44589 if ((!IsSub && CC == X86::COND_AE && ConstantX->isAllOnesValue()) ||
44590 (IsSub && CC == X86::COND_B && ConstantX->isNullValue())) {
44591 // This is a complicated way to get -1 or 0 from the carry flag:
44592 // -1 + SETAE --> -1 + (!CF) --> CF ? -1 : 0 --> SBB %eax, %eax
44593 // 0 - SETB --> 0 - (CF) --> CF ? -1 : 0 --> SBB %eax, %eax
44594 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
44595 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
44596 Y.getOperand(1));
44597 }
44598
44599 if ((!IsSub && CC == X86::COND_BE && ConstantX->isAllOnesValue()) ||
44600 (IsSub && CC == X86::COND_A && ConstantX->isNullValue())) {
44601 SDValue EFLAGS = Y->getOperand(1);
44602 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.hasOneUse() &&
44603 EFLAGS.getValueType().isInteger() &&
44604 !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
44605 // Swap the operands of a SUB, and we have the same pattern as above.
44606 // -1 + SETBE (SUB A, B) --> -1 + SETAE (SUB B, A) --> SUB + SBB
44607 // 0 - SETA (SUB A, B) --> 0 - SETB (SUB B, A) --> SUB + SBB
44608 SDValue NewSub = DAG.getNode(
44609 X86ISD::SUB, SDLoc(EFLAGS), EFLAGS.getNode()->getVTList(),
44610 EFLAGS.getOperand(1), EFLAGS.getOperand(0));
44611 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
44612 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
44613 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
44614 NewEFLAGS);
44615 }
44616 }
44617 }
44618
44619 if (CC == X86::COND_B) {
44620 // X + SETB Z --> adc X, 0
44621 // X - SETB Z --> sbb X, 0
44622 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
44623 DAG.getVTList(VT, MVT::i32), X,
44624 DAG.getConstant(0, DL, VT), Y.getOperand(1));
44625 }
44626
44627 if (CC == X86::COND_A) {
44628 SDValue EFLAGS = Y->getOperand(1);
44629 // Try to convert COND_A into COND_B in an attempt to facilitate
44630 // materializing "setb reg".
44631 //
44632 // Do not flip "e > c", where "c" is a constant, because Cmp instruction
44633 // cannot take an immediate as its first operand.
44634 //
44635 if (EFLAGS.getOpcode() == X86ISD::SUB && EFLAGS.getNode()->hasOneUse() &&
44636 EFLAGS.getValueType().isInteger() &&
44637 !isa<ConstantSDNode>(EFLAGS.getOperand(1))) {
44638 SDValue NewSub = DAG.getNode(X86ISD::SUB, SDLoc(EFLAGS),
44639 EFLAGS.getNode()->getVTList(),
44640 EFLAGS.getOperand(1), EFLAGS.getOperand(0));
44641 SDValue NewEFLAGS = SDValue(NewSub.getNode(), EFLAGS.getResNo());
44642 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL,
44643 DAG.getVTList(VT, MVT::i32), X,
44644 DAG.getConstant(0, DL, VT), NewEFLAGS);
44645 }
44646 }
44647
44648 if (CC != X86::COND_E && CC != X86::COND_NE)
44649 return SDValue();
44650
44651 SDValue Cmp = Y.getOperand(1);
44652 if (Cmp.getOpcode() != X86ISD::CMP || !Cmp.hasOneUse() ||
44653 !X86::isZeroNode(Cmp.getOperand(1)) ||
44654 !Cmp.getOperand(0).getValueType().isInteger())
44655 return SDValue();
44656
44657 SDValue Z = Cmp.getOperand(0);
44658 EVT ZVT = Z.getValueType();
44659
44660 // If X is -1 or 0, then we have an opportunity to avoid constants required in
44661 // the general case below.
44662 if (ConstantX) {
44663 // 'neg' sets the carry flag when Z != 0, so create 0 or -1 using 'sbb' with
44664 // fake operands:
44665 // 0 - (Z != 0) --> sbb %eax, %eax, (neg Z)
44666 // -1 + (Z == 0) --> sbb %eax, %eax, (neg Z)
44667 if ((IsSub && CC == X86::COND_NE && ConstantX->isNullValue()) ||
44668 (!IsSub && CC == X86::COND_E && ConstantX->isAllOnesValue())) {
44669 SDValue Zero = DAG.getConstant(0, DL, ZVT);
44670 SDVTList X86SubVTs = DAG.getVTList(ZVT, MVT::i32);
44671 SDValue Neg = DAG.getNode(X86ISD::SUB, DL, X86SubVTs, Zero, Z);
44672 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
44673 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8),
44674 SDValue(Neg.getNode(), 1));
44675 }
44676
44677 // cmp with 1 sets the carry flag when Z == 0, so create 0 or -1 using 'sbb'
44678 // with fake operands:
44679 // 0 - (Z == 0) --> sbb %eax, %eax, (cmp Z, 1)
44680 // -1 + (Z != 0) --> sbb %eax, %eax, (cmp Z, 1)
44681 if ((IsSub && CC == X86::COND_E && ConstantX->isNullValue()) ||
44682 (!IsSub && CC == X86::COND_NE && ConstantX->isAllOnesValue())) {
44683 SDValue One = DAG.getConstant(1, DL, ZVT);
44684 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One);
44685 return DAG.getNode(X86ISD::SETCC_CARRY, DL, VT,
44686 DAG.getTargetConstant(X86::COND_B, DL, MVT::i8), Cmp1);
44687 }
44688 }
44689
44690 // (cmp Z, 1) sets the carry flag if Z is 0.
44691 SDValue One = DAG.getConstant(1, DL, ZVT);
44692 SDValue Cmp1 = DAG.getNode(X86ISD::CMP, DL, MVT::i32, Z, One);
44693
44694 // Add the flags type for ADC/SBB nodes.
44695 SDVTList VTs = DAG.getVTList(VT, MVT::i32);
44696
44697 // X - (Z != 0) --> sub X, (zext(setne Z, 0)) --> adc X, -1, (cmp Z, 1)
44698 // X + (Z != 0) --> add X, (zext(setne Z, 0)) --> sbb X, -1, (cmp Z, 1)
44699 if (CC == X86::COND_NE)
44700 return DAG.getNode(IsSub ? X86ISD::ADC : X86ISD::SBB, DL, VTs, X,
44701 DAG.getConstant(-1ULL, DL, VT), Cmp1);
44702
44703 // X - (Z == 0) --> sub X, (zext(sete Z, 0)) --> sbb X, 0, (cmp Z, 1)
44704 // X + (Z == 0) --> add X, (zext(sete Z, 0)) --> adc X, 0, (cmp Z, 1)
44705 return DAG.getNode(IsSub ? X86ISD::SBB : X86ISD::ADC, DL, VTs, X,
44706 DAG.getConstant(0, DL, VT), Cmp1);
44707}
44708
44709static SDValue combineLoopMAddPattern(SDNode *N, SelectionDAG &DAG,
44710 const X86Subtarget &Subtarget) {
44711 if (!Subtarget.hasSSE2())
44712 return SDValue();
44713
44714 EVT VT = N->getValueType(0);
44715
44716 // If the vector size is less than 128, or greater than the supported RegSize,
44717 // do not use PMADD.
44718 if (!VT.isVector() || VT.getVectorNumElements() < 8)
44719 return SDValue();
44720
44721 SDValue Op0 = N->getOperand(0);
44722 SDValue Op1 = N->getOperand(1);
44723
44724 auto UsePMADDWD = [&](SDValue Op) {
44725 ShrinkMode Mode;
44726 return Op.getOpcode() == ISD::MUL &&
44727 canReduceVMulWidth(Op.getNode(), DAG, Mode) &&
44728 Mode != ShrinkMode::MULU16 &&
44729 (!Subtarget.hasSSE41() ||
44730 (Op->isOnlyUserOf(Op.getOperand(0).getNode()) &&
44731 Op->isOnlyUserOf(Op.getOperand(1).getNode())));
44732 };
44733
44734 SDValue MulOp, OtherOp;
44735 if (UsePMADDWD(Op0)) {
44736 MulOp = Op0;
44737 OtherOp = Op1;
44738 } else if (UsePMADDWD(Op1)) {
44739 MulOp = Op1;
44740 OtherOp = Op0;
44741 } else
44742 return SDValue();
44743
44744 SDLoc DL(N);
44745 EVT ReducedVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
44746 VT.getVectorNumElements());
44747 EVT MAddVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
44748 VT.getVectorNumElements() / 2);
44749
44750 // Shrink the operands of mul.
44751 SDValue N0 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, MulOp->getOperand(0));
44752 SDValue N1 = DAG.getNode(ISD::TRUNCATE, DL, ReducedVT, MulOp->getOperand(1));
44753
44754 // Madd vector size is half of the original vector size
44755 auto PMADDWDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
44756 ArrayRef<SDValue> Ops) {
44757 MVT OpVT = MVT::getVectorVT(MVT::i32, Ops[0].getValueSizeInBits() / 32);
44758 return DAG.getNode(X86ISD::VPMADDWD, DL, OpVT, Ops);
44759 };
44760 SDValue Madd = SplitOpsAndApply(DAG, Subtarget, DL, MAddVT, { N0, N1 },
44761 PMADDWDBuilder);
44762 // Fill the rest of the output with 0
44763 SDValue Zero = DAG.getConstant(0, DL, Madd.getSimpleValueType());
44764 SDValue Concat = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Madd, Zero);
44765
44766 // Preserve the reduction flag on the ADD. We may need to revisit for the
44767 // other operand.
44768 SDNodeFlags Flags;
44769 Flags.setVectorReduction(true);
44770 return DAG.getNode(ISD::ADD, DL, VT, Concat, OtherOp, Flags);
44771}
44772
44773static SDValue combineLoopSADPattern(SDNode *N, SelectionDAG &DAG,
44774 const X86Subtarget &Subtarget) {
44775 if (!Subtarget.hasSSE2())
44776 return SDValue();
44777
44778 SDLoc DL(N);
44779 EVT VT = N->getValueType(0);
44780
44781 // TODO: There's nothing special about i32, any integer type above i16 should
44782 // work just as well.
44783 if (!VT.isVector() || !VT.isSimple() ||
44784 !(VT.getVectorElementType() == MVT::i32))
44785 return SDValue();
44786
44787 unsigned RegSize = 128;
44788 if (Subtarget.useBWIRegs())
44789 RegSize = 512;
44790 else if (Subtarget.hasAVX())
44791 RegSize = 256;
44792
44793 // We only handle v16i32 for SSE2 / v32i32 for AVX / v64i32 for AVX512.
44794 // TODO: We should be able to handle larger vectors by splitting them before
44795 // feeding them into several SADs, and then reducing over those.
44796 if (VT.getSizeInBits() / 4 > RegSize)
44797 return SDValue();
44798
44799 // We know N is a reduction add. To match SAD, we need one of the operands to
44800 // be an ABS.
44801 SDValue AbsOp = N->getOperand(0);
44802 SDValue OtherOp = N->getOperand(1);
44803 if (AbsOp.getOpcode() != ISD::ABS)
44804 std::swap(AbsOp, OtherOp);
44805 if (AbsOp.getOpcode() != ISD::ABS)
44806 return SDValue();
44807
44808 // Check whether we have an abs-diff pattern feeding into the select.
44809 SDValue SadOp0, SadOp1;
44810 if(!detectZextAbsDiff(AbsOp, SadOp0, SadOp1))
44811 return SDValue();
44812
44813 // SAD pattern detected. Now build a SAD instruction and an addition for
44814 // reduction. Note that the number of elements of the result of SAD is less
44815 // than the number of elements of its input. Therefore, we could only update
44816 // part of elements in the reduction vector.
44817 SDValue Sad = createPSADBW(DAG, SadOp0, SadOp1, DL, Subtarget);
44818
44819 // The output of PSADBW is a vector of i64.
44820 // We need to turn the vector of i64 into a vector of i32.
44821 // If the reduction vector is at least as wide as the psadbw result, just
44822 // bitcast. If it's narrower which can only occur for v2i32, bits 127:16 of
44823 // the PSADBW will be zero. If we promote/ narrow vectors, truncate the v2i64
44824 // result to v2i32 which will be removed by type legalization. If we/ widen
44825 // narrow vectors then we bitcast to v4i32 and extract v2i32.
44826 MVT ResVT = MVT::getVectorVT(MVT::i32, Sad.getValueSizeInBits() / 32);
44827 Sad = DAG.getNode(ISD::BITCAST, DL, ResVT, Sad);
44828
44829 if (VT.getSizeInBits() > ResVT.getSizeInBits()) {
44830 // Fill the upper elements with zero to match the add width.
44831 assert(VT.getSizeInBits() % ResVT.getSizeInBits() == 0 && "Unexpected VTs")((VT.getSizeInBits() % ResVT.getSizeInBits() == 0 && "Unexpected VTs"
) ? static_cast<void> (0) : __assert_fail ("VT.getSizeInBits() % ResVT.getSizeInBits() == 0 && \"Unexpected VTs\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44831, __PRETTY_FUNCTION__));
44832 unsigned NumConcats = VT.getSizeInBits() / ResVT.getSizeInBits();
44833 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getConstant(0, DL, ResVT));
44834 Ops[0] = Sad;
44835 Sad = DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Ops);
44836 } else if (VT.getSizeInBits() < ResVT.getSizeInBits()) {
44837 Sad = DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Sad,
44838 DAG.getIntPtrConstant(0, DL));
44839 }
44840
44841 // Preserve the reduction flag on the ADD. We may need to revisit for the
44842 // other operand.
44843 SDNodeFlags Flags;
44844 Flags.setVectorReduction(true);
44845 return DAG.getNode(ISD::ADD, DL, VT, Sad, OtherOp, Flags);
44846}
44847
44848static SDValue matchPMADDWD(SelectionDAG &DAG, SDValue Op0, SDValue Op1,
44849 const SDLoc &DL, EVT VT,
44850 const X86Subtarget &Subtarget) {
44851 // Example of pattern we try to detect:
44852 // t := (v8i32 mul (sext (v8i16 x0), (sext (v8i16 x1))))
44853 //(add (build_vector (extract_elt t, 0),
44854 // (extract_elt t, 2),
44855 // (extract_elt t, 4),
44856 // (extract_elt t, 6)),
44857 // (build_vector (extract_elt t, 1),
44858 // (extract_elt t, 3),
44859 // (extract_elt t, 5),
44860 // (extract_elt t, 7)))
44861
44862 if (!Subtarget.hasSSE2())
44863 return SDValue();
44864
44865 if (Op0.getOpcode() != ISD::BUILD_VECTOR ||
44866 Op1.getOpcode() != ISD::BUILD_VECTOR)
44867 return SDValue();
44868
44869 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
44870 VT.getVectorNumElements() < 4 ||
44871 !isPowerOf2_32(VT.getVectorNumElements()))
44872 return SDValue();
44873
44874 // Check if one of Op0,Op1 is of the form:
44875 // (build_vector (extract_elt Mul, 0),
44876 // (extract_elt Mul, 2),
44877 // (extract_elt Mul, 4),
44878 // ...
44879 // the other is of the form:
44880 // (build_vector (extract_elt Mul, 1),
44881 // (extract_elt Mul, 3),
44882 // (extract_elt Mul, 5),
44883 // ...
44884 // and identify Mul.
44885 SDValue Mul;
44886 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; i += 2) {
44887 SDValue Op0L = Op0->getOperand(i), Op1L = Op1->getOperand(i),
44888 Op0H = Op0->getOperand(i + 1), Op1H = Op1->getOperand(i + 1);
44889 // TODO: Be more tolerant to undefs.
44890 if (Op0L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
44891 Op1L.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
44892 Op0H.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
44893 Op1H.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
44894 return SDValue();
44895 auto *Const0L = dyn_cast<ConstantSDNode>(Op0L->getOperand(1));
44896 auto *Const1L = dyn_cast<ConstantSDNode>(Op1L->getOperand(1));
44897 auto *Const0H = dyn_cast<ConstantSDNode>(Op0H->getOperand(1));
44898 auto *Const1H = dyn_cast<ConstantSDNode>(Op1H->getOperand(1));
44899 if (!Const0L || !Const1L || !Const0H || !Const1H)
44900 return SDValue();
44901 unsigned Idx0L = Const0L->getZExtValue(), Idx1L = Const1L->getZExtValue(),
44902 Idx0H = Const0H->getZExtValue(), Idx1H = Const1H->getZExtValue();
44903 // Commutativity of mul allows factors of a product to reorder.
44904 if (Idx0L > Idx1L)
44905 std::swap(Idx0L, Idx1L);
44906 if (Idx0H > Idx1H)
44907 std::swap(Idx0H, Idx1H);
44908 // Commutativity of add allows pairs of factors to reorder.
44909 if (Idx0L > Idx0H) {
44910 std::swap(Idx0L, Idx0H);
44911 std::swap(Idx1L, Idx1H);
44912 }
44913 if (Idx0L != 2 * i || Idx1L != 2 * i + 1 || Idx0H != 2 * i + 2 ||
44914 Idx1H != 2 * i + 3)
44915 return SDValue();
44916 if (!Mul) {
44917 // First time an extract_elt's source vector is visited. Must be a MUL
44918 // with 2X number of vector elements than the BUILD_VECTOR.
44919 // Both extracts must be from same MUL.
44920 Mul = Op0L->getOperand(0);
44921 if (Mul->getOpcode() != ISD::MUL ||
44922 Mul.getValueType().getVectorNumElements() != 2 * e)
44923 return SDValue();
44924 }
44925 // Check that the extract is from the same MUL previously seen.
44926 if (Mul != Op0L->getOperand(0) || Mul != Op1L->getOperand(0) ||
44927 Mul != Op0H->getOperand(0) || Mul != Op1H->getOperand(0))
44928 return SDValue();
44929 }
44930
44931 // Check if the Mul source can be safely shrunk.
44932 ShrinkMode Mode;
44933 if (!canReduceVMulWidth(Mul.getNode(), DAG, Mode) ||
44934 Mode == ShrinkMode::MULU16)
44935 return SDValue();
44936
44937 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
44938 ArrayRef<SDValue> Ops) {
44939 // Shrink by adding truncate nodes and let DAGCombine fold with the
44940 // sources.
44941 EVT InVT = Ops[0].getValueType();
44942 assert(InVT.getScalarType() == MVT::i32 &&((InVT.getScalarType() == MVT::i32 && "Unexpected scalar element type"
) ? static_cast<void> (0) : __assert_fail ("InVT.getScalarType() == MVT::i32 && \"Unexpected scalar element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44943, __PRETTY_FUNCTION__))
44943 "Unexpected scalar element type")((InVT.getScalarType() == MVT::i32 && "Unexpected scalar element type"
) ? static_cast<void> (0) : __assert_fail ("InVT.getScalarType() == MVT::i32 && \"Unexpected scalar element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44943, __PRETTY_FUNCTION__));
44944 assert(InVT == Ops[1].getValueType() && "Operands' types mismatch")((InVT == Ops[1].getValueType() && "Operands' types mismatch"
) ? static_cast<void> (0) : __assert_fail ("InVT == Ops[1].getValueType() && \"Operands' types mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 44944, __PRETTY_FUNCTION__));
44945 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
44946 InVT.getVectorNumElements() / 2);
44947 EVT TruncVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16,
44948 InVT.getVectorNumElements());
44949 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT,
44950 DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Ops[0]),
44951 DAG.getNode(ISD::TRUNCATE, DL, TruncVT, Ops[1]));
44952 };
44953 return SplitOpsAndApply(DAG, Subtarget, DL, VT,
44954 { Mul.getOperand(0), Mul.getOperand(1) },
44955 PMADDBuilder);
44956}
44957
44958// Attempt to turn this pattern into PMADDWD.
44959// (mul (add (sext (build_vector)), (sext (build_vector))),
44960// (add (sext (build_vector)), (sext (build_vector)))
44961static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDValue N0, SDValue N1,
44962 const SDLoc &DL, EVT VT,
44963 const X86Subtarget &Subtarget) {
44964 if (!Subtarget.hasSSE2())
44965 return SDValue();
44966
44967 if (N0.getOpcode() != ISD::MUL || N1.getOpcode() != ISD::MUL)
44968 return SDValue();
44969
44970 if (!VT.isVector() || VT.getVectorElementType() != MVT::i32 ||
44971 VT.getVectorNumElements() < 4 ||
44972 !isPowerOf2_32(VT.getVectorNumElements()))
44973 return SDValue();
44974
44975 SDValue N00 = N0.getOperand(0);
44976 SDValue N01 = N0.getOperand(1);
44977 SDValue N10 = N1.getOperand(0);
44978 SDValue N11 = N1.getOperand(1);
44979
44980 // All inputs need to be sign extends.
44981 // TODO: Support ZERO_EXTEND from known positive?
44982 if (N00.getOpcode() != ISD::SIGN_EXTEND ||
44983 N01.getOpcode() != ISD::SIGN_EXTEND ||
44984 N10.getOpcode() != ISD::SIGN_EXTEND ||
44985 N11.getOpcode() != ISD::SIGN_EXTEND)
44986 return SDValue();
44987
44988 // Peek through the extends.
44989 N00 = N00.getOperand(0);
44990 N01 = N01.getOperand(0);
44991 N10 = N10.getOperand(0);
44992 N11 = N11.getOperand(0);
44993
44994 // Must be extending from vXi16.
44995 EVT InVT = N00.getValueType();
44996 if (InVT.getVectorElementType() != MVT::i16 || N01.getValueType() != InVT ||
44997 N10.getValueType() != InVT || N11.getValueType() != InVT)
44998 return SDValue();
44999
45000 // All inputs should be build_vectors.
45001 if (N00.getOpcode() != ISD::BUILD_VECTOR ||
45002 N01.getOpcode() != ISD::BUILD_VECTOR ||
45003 N10.getOpcode() != ISD::BUILD_VECTOR ||
45004 N11.getOpcode() != ISD::BUILD_VECTOR)
45005 return SDValue();
45006
45007 // For each element, we need to ensure we have an odd element from one vector
45008 // multiplied by the odd element of another vector and the even element from
45009 // one of the same vectors being multiplied by the even element from the
45010 // other vector. So we need to make sure for each element i, this operator
45011 // is being performed:
45012 // A[2 * i] * B[2 * i] + A[2 * i + 1] * B[2 * i + 1]
45013 SDValue In0, In1;
45014 for (unsigned i = 0; i != N00.getNumOperands(); ++i) {
45015 SDValue N00Elt = N00.getOperand(i);
45016 SDValue N01Elt = N01.getOperand(i);
45017 SDValue N10Elt = N10.getOperand(i);
45018 SDValue N11Elt = N11.getOperand(i);
45019 // TODO: Be more tolerant to undefs.
45020 if (N00Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
45021 N01Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
45022 N10Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
45023 N11Elt.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
45024 return SDValue();
45025 auto *ConstN00Elt = dyn_cast<ConstantSDNode>(N00Elt.getOperand(1));
45026 auto *ConstN01Elt = dyn_cast<ConstantSDNode>(N01Elt.getOperand(1));
45027 auto *ConstN10Elt = dyn_cast<ConstantSDNode>(N10Elt.getOperand(1));
45028 auto *ConstN11Elt = dyn_cast<ConstantSDNode>(N11Elt.getOperand(1));
45029 if (!ConstN00Elt || !ConstN01Elt || !ConstN10Elt || !ConstN11Elt)
45030 return SDValue();
45031 unsigned IdxN00 = ConstN00Elt->getZExtValue();
45032 unsigned IdxN01 = ConstN01Elt->getZExtValue();
45033 unsigned IdxN10 = ConstN10Elt->getZExtValue();
45034 unsigned IdxN11 = ConstN11Elt->getZExtValue();
45035 // Add is commutative so indices can be reordered.
45036 if (IdxN00 > IdxN10) {
45037 std::swap(IdxN00, IdxN10);
45038 std::swap(IdxN01, IdxN11);
45039 }
45040 // N0 indices be the even element. N1 indices must be the next odd element.
45041 if (IdxN00 != 2 * i || IdxN10 != 2 * i + 1 ||
45042 IdxN01 != 2 * i || IdxN11 != 2 * i + 1)
45043 return SDValue();
45044 SDValue N00In = N00Elt.getOperand(0);
45045 SDValue N01In = N01Elt.getOperand(0);
45046 SDValue N10In = N10Elt.getOperand(0);
45047 SDValue N11In = N11Elt.getOperand(0);
45048 // First time we find an input capture it.
45049 if (!In0) {
45050 In0 = N00In;
45051 In1 = N01In;
45052 }
45053 // Mul is commutative so the input vectors can be in any order.
45054 // Canonicalize to make the compares easier.
45055 if (In0 != N00In)
45056 std::swap(N00In, N01In);
45057 if (In0 != N10In)
45058 std::swap(N10In, N11In);
45059 if (In0 != N00In || In1 != N01In || In0 != N10In || In1 != N11In)
45060 return SDValue();
45061 }
45062
45063 auto PMADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
45064 ArrayRef<SDValue> Ops) {
45065 // Shrink by adding truncate nodes and let DAGCombine fold with the
45066 // sources.
45067 EVT OpVT = Ops[0].getValueType();
45068 assert(OpVT.getScalarType() == MVT::i16 &&((OpVT.getScalarType() == MVT::i16 && "Unexpected scalar element type"
) ? static_cast<void> (0) : __assert_fail ("OpVT.getScalarType() == MVT::i16 && \"Unexpected scalar element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45069, __PRETTY_FUNCTION__))
45069 "Unexpected scalar element type")((OpVT.getScalarType() == MVT::i16 && "Unexpected scalar element type"
) ? static_cast<void> (0) : __assert_fail ("OpVT.getScalarType() == MVT::i16 && \"Unexpected scalar element type\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45069, __PRETTY_FUNCTION__));
45070 assert(OpVT == Ops[1].getValueType() && "Operands' types mismatch")((OpVT == Ops[1].getValueType() && "Operands' types mismatch"
) ? static_cast<void> (0) : __assert_fail ("OpVT == Ops[1].getValueType() && \"Operands' types mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45070, __PRETTY_FUNCTION__));
45071 EVT ResVT = EVT::getVectorVT(*DAG.getContext(), MVT::i32,
45072 OpVT.getVectorNumElements() / 2);
45073 return DAG.getNode(X86ISD::VPMADDWD, DL, ResVT, Ops[0], Ops[1]);
45074 };
45075 return SplitOpsAndApply(DAG, Subtarget, DL, VT, { In0, In1 },
45076 PMADDBuilder);
45077}
45078
45079static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
45080 TargetLowering::DAGCombinerInfo &DCI,
45081 const X86Subtarget &Subtarget) {
45082 const SDNodeFlags Flags = N->getFlags();
45083 if (Flags.hasVectorReduction()) {
45084 if (SDValue Sad = combineLoopSADPattern(N, DAG, Subtarget))
45085 return Sad;
45086 if (SDValue MAdd = combineLoopMAddPattern(N, DAG, Subtarget))
45087 return MAdd;
45088 }
45089 EVT VT = N->getValueType(0);
45090 SDValue Op0 = N->getOperand(0);
45091 SDValue Op1 = N->getOperand(1);
45092
45093 if (SDValue MAdd = matchPMADDWD(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))
45094 return MAdd;
45095 if (SDValue MAdd = matchPMADDWD_2(DAG, Op0, Op1, SDLoc(N), VT, Subtarget))
45096 return MAdd;
45097
45098 // Try to synthesize horizontal adds from adds of shuffles.
45099 if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 ||
45100 VT == MVT::v8i32) &&
45101 Subtarget.hasSSSE3() &&
45102 isHorizontalBinOp(Op0, Op1, DAG, Subtarget, true)) {
45103 auto HADDBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
45104 ArrayRef<SDValue> Ops) {
45105 return DAG.getNode(X86ISD::HADD, DL, Ops[0].getValueType(), Ops);
45106 };
45107 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1},
45108 HADDBuilder);
45109 }
45110
45111 // If vectors of i1 are legal, turn (add (zext (vXi1 X)), Y) into
45112 // (sub Y, (sext (vXi1 X))).
45113 // FIXME: We have the (sub Y, (zext (vXi1 X))) -> (add (sext (vXi1 X)), Y) in
45114 // generic DAG combine without a legal type check, but adding this there
45115 // caused regressions.
45116 if (VT.isVector()) {
45117 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45118 if (Op0.getOpcode() == ISD::ZERO_EXTEND &&
45119 Op0.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
45120 TLI.isTypeLegal(Op0.getOperand(0).getValueType())) {
45121 SDLoc DL(N);
45122 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op0.getOperand(0));
45123 return DAG.getNode(ISD::SUB, DL, VT, Op1, SExt);
45124 }
45125
45126 if (Op1.getOpcode() == ISD::ZERO_EXTEND &&
45127 Op1.getOperand(0).getValueType().getVectorElementType() == MVT::i1 &&
45128 TLI.isTypeLegal(Op1.getOperand(0).getValueType())) {
45129 SDLoc DL(N);
45130 SDValue SExt = DAG.getNode(ISD::SIGN_EXTEND, DL, VT, Op1.getOperand(0));
45131 return DAG.getNode(ISD::SUB, DL, VT, Op0, SExt);
45132 }
45133 }
45134
45135 return combineAddOrSubToADCOrSBB(N, DAG);
45136}
45137
45138static SDValue combineSubToSubus(SDNode *N, SelectionDAG &DAG,
45139 const X86Subtarget &Subtarget) {
45140 SDValue Op0 = N->getOperand(0);
45141 SDValue Op1 = N->getOperand(1);
45142 EVT VT = N->getValueType(0);
45143
45144 if (!VT.isVector())
45145 return SDValue();
45146
45147 // PSUBUS is supported, starting from SSE2, but truncation for v8i32
45148 // is only worth it with SSSE3 (PSHUFB).
45149 EVT EltVT = VT.getVectorElementType();
45150 if (!(Subtarget.hasSSE2() && (EltVT == MVT::i8 || EltVT == MVT::i16)) &&
45151 !(Subtarget.hasSSSE3() && (VT == MVT::v8i32 || VT == MVT::v8i64)) &&
45152 !(Subtarget.useBWIRegs() && (VT == MVT::v16i32)))
45153 return SDValue();
45154
45155 SDValue SubusLHS, SubusRHS;
45156 // Try to find umax(a,b) - b or a - umin(a,b) patterns
45157 // they may be converted to subus(a,b).
45158 // TODO: Need to add IR canonicalization for this code.
45159 if (Op0.getOpcode() == ISD::UMAX) {
45160 SubusRHS = Op1;
45161 SDValue MaxLHS = Op0.getOperand(0);
45162 SDValue MaxRHS = Op0.getOperand(1);
45163 if (MaxLHS == Op1)
45164 SubusLHS = MaxRHS;
45165 else if (MaxRHS == Op1)
45166 SubusLHS = MaxLHS;
45167 else
45168 return SDValue();
45169 } else if (Op1.getOpcode() == ISD::UMIN) {
45170 SubusLHS = Op0;
45171 SDValue MinLHS = Op1.getOperand(0);
45172 SDValue MinRHS = Op1.getOperand(1);
45173 if (MinLHS == Op0)
45174 SubusRHS = MinRHS;
45175 else if (MinRHS == Op0)
45176 SubusRHS = MinLHS;
45177 else
45178 return SDValue();
45179 } else
45180 return SDValue();
45181
45182 // PSUBUS doesn't support v8i32/v8i64/v16i32, but it can be enabled with
45183 // special preprocessing in some cases.
45184 if (EltVT == MVT::i8 || EltVT == MVT::i16)
45185 return DAG.getNode(ISD::USUBSAT, SDLoc(N), VT, SubusLHS, SubusRHS);
45186
45187 assert((VT == MVT::v8i32 || VT == MVT::v16i32 || VT == MVT::v8i64) &&(((VT == MVT::v8i32 || VT == MVT::v16i32 || VT == MVT::v8i64)
&& "Unexpected VT!") ? static_cast<void> (0) :
__assert_fail ("(VT == MVT::v8i32 || VT == MVT::v16i32 || VT == MVT::v8i64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45188, __PRETTY_FUNCTION__))
45188 "Unexpected VT!")(((VT == MVT::v8i32 || VT == MVT::v16i32 || VT == MVT::v8i64)
&& "Unexpected VT!") ? static_cast<void> (0) :
__assert_fail ("(VT == MVT::v8i32 || VT == MVT::v16i32 || VT == MVT::v8i64) && \"Unexpected VT!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45188, __PRETTY_FUNCTION__));
45189
45190 // Special preprocessing case can be only applied
45191 // if the value was zero extended from 16 bit,
45192 // so we require first 16 bits to be zeros for 32 bit
45193 // values, or first 48 bits for 64 bit values.
45194 KnownBits Known = DAG.computeKnownBits(SubusLHS);
45195 unsigned NumZeros = Known.countMinLeadingZeros();
45196 if ((VT == MVT::v8i64 && NumZeros < 48) || NumZeros < 16)
45197 return SDValue();
45198
45199 EVT ExtType = SubusLHS.getValueType();
45200 EVT ShrinkedType;
45201 if (VT == MVT::v8i32 || VT == MVT::v8i64)
45202 ShrinkedType = MVT::v8i16;
45203 else
45204 ShrinkedType = NumZeros >= 24 ? MVT::v16i8 : MVT::v16i16;
45205
45206 // If SubusLHS is zeroextended - truncate SubusRHS to it's
45207 // size SubusRHS = umin(0xFFF.., SubusRHS).
45208 SDValue SaturationConst =
45209 DAG.getConstant(APInt::getLowBitsSet(ExtType.getScalarSizeInBits(),
45210 ShrinkedType.getScalarSizeInBits()),
45211 SDLoc(SubusLHS), ExtType);
45212 SDValue UMin = DAG.getNode(ISD::UMIN, SDLoc(SubusLHS), ExtType, SubusRHS,
45213 SaturationConst);
45214 SDValue NewSubusLHS =
45215 DAG.getZExtOrTrunc(SubusLHS, SDLoc(SubusLHS), ShrinkedType);
45216 SDValue NewSubusRHS = DAG.getZExtOrTrunc(UMin, SDLoc(SubusRHS), ShrinkedType);
45217 SDValue Psubus = DAG.getNode(ISD::USUBSAT, SDLoc(N), ShrinkedType,
45218 NewSubusLHS, NewSubusRHS);
45219
45220 // Zero extend the result, it may be used somewhere as 32 bit,
45221 // if not zext and following trunc will shrink.
45222 return DAG.getZExtOrTrunc(Psubus, SDLoc(N), ExtType);
45223}
45224
45225static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
45226 TargetLowering::DAGCombinerInfo &DCI,
45227 const X86Subtarget &Subtarget) {
45228 SDValue Op0 = N->getOperand(0);
45229 SDValue Op1 = N->getOperand(1);
45230
45231 // X86 can't encode an immediate LHS of a sub. See if we can push the
45232 // negation into a preceding instruction.
45233 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op0)) {
45234 // If the RHS of the sub is a XOR with one use and a constant, invert the
45235 // immediate. Then add one to the LHS of the sub so we can turn
45236 // X-Y -> X+~Y+1, saving one register.
45237 if (Op1->hasOneUse() && Op1.getOpcode() == ISD::XOR &&
45238 isa<ConstantSDNode>(Op1.getOperand(1))) {
45239 const APInt &XorC = Op1.getConstantOperandAPInt(1);
45240 EVT VT = Op0.getValueType();
45241 SDValue NewXor = DAG.getNode(ISD::XOR, SDLoc(Op1), VT,
45242 Op1.getOperand(0),
45243 DAG.getConstant(~XorC, SDLoc(Op1), VT));
45244 return DAG.getNode(ISD::ADD, SDLoc(N), VT, NewXor,
45245 DAG.getConstant(C->getAPIntValue() + 1, SDLoc(N), VT));
45246 }
45247 }
45248
45249 // Try to synthesize horizontal subs from subs of shuffles.
45250 EVT VT = N->getValueType(0);
45251 if ((VT == MVT::v8i16 || VT == MVT::v4i32 || VT == MVT::v16i16 ||
45252 VT == MVT::v8i32) &&
45253 Subtarget.hasSSSE3() &&
45254 isHorizontalBinOp(Op0, Op1, DAG, Subtarget, false)) {
45255 auto HSUBBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
45256 ArrayRef<SDValue> Ops) {
45257 return DAG.getNode(X86ISD::HSUB, DL, Ops[0].getValueType(), Ops);
45258 };
45259 return SplitOpsAndApply(DAG, Subtarget, SDLoc(N), VT, {Op0, Op1},
45260 HSUBBuilder);
45261 }
45262
45263 // Try to create PSUBUS if SUB's argument is max/min
45264 if (SDValue V = combineSubToSubus(N, DAG, Subtarget))
45265 return V;
45266
45267 return combineAddOrSubToADCOrSBB(N, DAG);
45268}
45269
45270static SDValue combineVectorCompare(SDNode *N, SelectionDAG &DAG,
45271 const X86Subtarget &Subtarget) {
45272 MVT VT = N->getSimpleValueType(0);
45273 SDLoc DL(N);
45274
45275 if (N->getOperand(0) == N->getOperand(1)) {
45276 if (N->getOpcode() == X86ISD::PCMPEQ)
45277 return DAG.getConstant(-1, DL, VT);
45278 if (N->getOpcode() == X86ISD::PCMPGT)
45279 return DAG.getConstant(0, DL, VT);
45280 }
45281
45282 return SDValue();
45283}
45284
45285/// Helper that combines an array of subvector ops as if they were the operands
45286/// of a ISD::CONCAT_VECTORS node, but may have come from another source (e.g.
45287/// ISD::INSERT_SUBVECTOR). The ops are assumed to be of the same type.
45288static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT,
45289 ArrayRef<SDValue> Ops, SelectionDAG &DAG,
45290 TargetLowering::DAGCombinerInfo &DCI,
45291 const X86Subtarget &Subtarget) {
45292 assert(Subtarget.hasAVX() && "AVX assumed for concat_vectors")((Subtarget.hasAVX() && "AVX assumed for concat_vectors"
) ? static_cast<void> (0) : __assert_fail ("Subtarget.hasAVX() && \"AVX assumed for concat_vectors\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45292, __PRETTY_FUNCTION__));
45293
45294 if (llvm::all_of(Ops, [](SDValue Op) { return Op.isUndef(); }))
45295 return DAG.getUNDEF(VT);
45296
45297 if (llvm::all_of(Ops, [](SDValue Op) {
45298 return ISD::isBuildVectorAllZeros(Op.getNode());
45299 }))
45300 return getZeroVector(VT, Subtarget, DAG, DL);
45301
45302 SDValue Op0 = Ops[0];
45303
45304 // Fold subvector loads into one.
45305 // If needed, look through bitcasts to get to the load.
45306 if (auto *FirstLd = dyn_cast<LoadSDNode>(peekThroughBitcasts(Op0))) {
45307 bool Fast;
45308 const X86TargetLowering *TLI = Subtarget.getTargetLowering();
45309 if (TLI->allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT,
45310 *FirstLd->getMemOperand(), &Fast) &&
45311 Fast) {
45312 if (SDValue Ld =
45313 EltsFromConsecutiveLoads(VT, Ops, DL, DAG, Subtarget, false))
45314 return Ld;
45315 }
45316 }
45317
45318 // Repeated subvectors.
45319 if (llvm::all_of(Ops, [Op0](SDValue Op) { return Op == Op0; })) {
45320 // If this broadcast/subv_broadcast is inserted into both halves, use a
45321 // larger broadcast/subv_broadcast.
45322 if (Op0.getOpcode() == X86ISD::VBROADCAST ||
45323 Op0.getOpcode() == X86ISD::SUBV_BROADCAST)
45324 return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0));
45325
45326 // concat_vectors(movddup(x),movddup(x)) -> broadcast(x)
45327 if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 &&
45328 (Subtarget.hasAVX2() || MayFoldLoad(Op0.getOperand(0))))
45329 return DAG.getNode(X86ISD::VBROADCAST, DL, VT,
45330 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64,
45331 Op0.getOperand(0),
45332 DAG.getIntPtrConstant(0, DL)));
45333
45334 // concat_vectors(scalar_to_vector(x),scalar_to_vector(x)) -> broadcast(x)
45335 if (Op0.getOpcode() == ISD::SCALAR_TO_VECTOR &&
45336 (Subtarget.hasAVX2() ||
45337 (VT.getScalarSizeInBits() >= 32 && MayFoldLoad(Op0.getOperand(0)))) &&
45338 Op0.getOperand(0).getValueType() == VT.getScalarType())
45339 return DAG.getNode(X86ISD::VBROADCAST, DL, VT, Op0.getOperand(0));
45340 }
45341
45342 bool IsSplat = llvm::all_of(Ops, [&Op0](SDValue Op) { return Op == Op0; });
45343
45344 // Repeated opcode.
45345 // TODO - combineX86ShufflesRecursively should handle shuffle concatenation
45346 // but it currently struggles with different vector widths.
45347 if (llvm::all_of(Ops, [Op0](SDValue Op) {
45348 return Op.getOpcode() == Op0.getOpcode();
45349 })) {
45350 unsigned NumOps = Ops.size();
45351 switch (Op0.getOpcode()) {
45352 case X86ISD::PSHUFHW:
45353 case X86ISD::PSHUFLW:
45354 case X86ISD::PSHUFD:
45355 if (!IsSplat && NumOps == 2 && VT.is256BitVector() &&
45356 Subtarget.hasInt256() && Op0.getOperand(1) == Ops[1].getOperand(1)) {
45357 SmallVector<SDValue, 2> Src;
45358 for (unsigned i = 0; i != NumOps; ++i)
45359 Src.push_back(Ops[i].getOperand(0));
45360 return DAG.getNode(Op0.getOpcode(), DL, VT,
45361 DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Src),
45362 Op0.getOperand(1));
45363 }
45364 LLVM_FALLTHROUGH[[gnu::fallthrough]];
45365 case X86ISD::VPERMILPI:
45366 // TODO - add support for vXf64/vXi64 shuffles.
45367 if (!IsSplat && NumOps == 2 && (VT == MVT::v8f32 || VT == MVT::v8i32) &&
45368 Subtarget.hasAVX() && Op0.getOperand(1) == Ops[1].getOperand(1)) {
45369 SmallVector<SDValue, 2> Src;
45370 for (unsigned i = 0; i != NumOps; ++i)
45371 Src.push_back(DAG.getBitcast(MVT::v4f32, Ops[i].getOperand(0)));
45372 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8f32, Src);
45373 Res = DAG.getNode(X86ISD::VPERMILPI, DL, MVT::v8f32, Res,
45374 Op0.getOperand(1));
45375 return DAG.getBitcast(VT, Res);
45376 }
45377 break;
45378 case X86ISD::PACKUS:
45379 if (NumOps == 2 && VT.is256BitVector() && Subtarget.hasInt256()) {
45380 SmallVector<SDValue, 2> LHS, RHS;
45381 for (unsigned i = 0; i != NumOps; ++i) {
45382 LHS.push_back(Ops[i].getOperand(0));
45383 RHS.push_back(Ops[i].getOperand(1));
45384 }
45385 MVT SrcVT = Op0.getOperand(0).getSimpleValueType();
45386 SrcVT = MVT::getVectorVT(SrcVT.getScalarType(),
45387 NumOps * SrcVT.getVectorNumElements());
45388 return DAG.getNode(Op0.getOpcode(), DL, VT,
45389 DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, LHS),
45390 DAG.getNode(ISD::CONCAT_VECTORS, DL, SrcVT, RHS));
45391 }
45392 break;
45393 }
45394 }
45395
45396 return SDValue();
45397}
45398
45399static SDValue combineConcatVectors(SDNode *N, SelectionDAG &DAG,
45400 TargetLowering::DAGCombinerInfo &DCI,
45401 const X86Subtarget &Subtarget) {
45402 EVT VT = N->getValueType(0);
45403 EVT SrcVT = N->getOperand(0).getValueType();
45404 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45405
45406 // Don't do anything for i1 vectors.
45407 if (VT.getVectorElementType() == MVT::i1)
45408 return SDValue();
45409
45410 if (Subtarget.hasAVX() && TLI.isTypeLegal(VT) && TLI.isTypeLegal(SrcVT)) {
45411 SmallVector<SDValue, 4> Ops(N->op_begin(), N->op_end());
45412 if (SDValue R = combineConcatVectorOps(SDLoc(N), VT.getSimpleVT(), Ops, DAG,
45413 DCI, Subtarget))
45414 return R;
45415 }
45416
45417 return SDValue();
45418}
45419
45420static SDValue combineInsertSubvector(SDNode *N, SelectionDAG &DAG,
45421 TargetLowering::DAGCombinerInfo &DCI,
45422 const X86Subtarget &Subtarget) {
45423 if (DCI.isBeforeLegalizeOps())
45424 return SDValue();
45425
45426 MVT OpVT = N->getSimpleValueType(0);
45427
45428 bool IsI1Vector = OpVT.getVectorElementType() == MVT::i1;
45429
45430 SDLoc dl(N);
45431 SDValue Vec = N->getOperand(0);
45432 SDValue SubVec = N->getOperand(1);
45433
45434 uint64_t IdxVal = N->getConstantOperandVal(2);
45435 MVT SubVecVT = SubVec.getSimpleValueType();
45436
45437 if (Vec.isUndef() && SubVec.isUndef())
45438 return DAG.getUNDEF(OpVT);
45439
45440 // Inserting undefs/zeros into zeros/undefs is a zero vector.
45441 if ((Vec.isUndef() || ISD::isBuildVectorAllZeros(Vec.getNode())) &&
45442 (SubVec.isUndef() || ISD::isBuildVectorAllZeros(SubVec.getNode())))
45443 return getZeroVector(OpVT, Subtarget, DAG, dl);
45444
45445 if (ISD::isBuildVectorAllZeros(Vec.getNode())) {
45446 // If we're inserting into a zero vector and then into a larger zero vector,
45447 // just insert into the larger zero vector directly.
45448 if (SubVec.getOpcode() == ISD::INSERT_SUBVECTOR &&
45449 ISD::isBuildVectorAllZeros(SubVec.getOperand(0).getNode())) {
45450 uint64_t Idx2Val = SubVec.getConstantOperandVal(2);
45451 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
45452 getZeroVector(OpVT, Subtarget, DAG, dl),
45453 SubVec.getOperand(1),
45454 DAG.getIntPtrConstant(IdxVal + Idx2Val, dl));
45455 }
45456
45457 // If we're inserting into a zero vector and our input was extracted from an
45458 // insert into a zero vector of the same type and the extraction was at
45459 // least as large as the original insertion. Just insert the original
45460 // subvector into a zero vector.
45461 if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR && IdxVal == 0 &&
45462 isNullConstant(SubVec.getOperand(1)) &&
45463 SubVec.getOperand(0).getOpcode() == ISD::INSERT_SUBVECTOR) {
45464 SDValue Ins = SubVec.getOperand(0);
45465 if (isNullConstant(Ins.getOperand(2)) &&
45466 ISD::isBuildVectorAllZeros(Ins.getOperand(0).getNode()) &&
45467 Ins.getOperand(1).getValueSizeInBits() <= SubVecVT.getSizeInBits())
45468 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
45469 getZeroVector(OpVT, Subtarget, DAG, dl),
45470 Ins.getOperand(1), N->getOperand(2));
45471 }
45472 }
45473
45474 // Stop here if this is an i1 vector.
45475 if (IsI1Vector)
45476 return SDValue();
45477
45478 // If this is an insert of an extract, combine to a shuffle. Don't do this
45479 // if the insert or extract can be represented with a subregister operation.
45480 if (SubVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
45481 SubVec.getOperand(0).getSimpleValueType() == OpVT &&
45482 (IdxVal != 0 || !Vec.isUndef())) {
45483 int ExtIdxVal = SubVec.getConstantOperandVal(1);
45484 if (ExtIdxVal != 0) {
45485 int VecNumElts = OpVT.getVectorNumElements();
45486 int SubVecNumElts = SubVecVT.getVectorNumElements();
45487 SmallVector<int, 64> Mask(VecNumElts);
45488 // First create an identity shuffle mask.
45489 for (int i = 0; i != VecNumElts; ++i)
45490 Mask[i] = i;
45491 // Now insert the extracted portion.
45492 for (int i = 0; i != SubVecNumElts; ++i)
45493 Mask[i + IdxVal] = i + ExtIdxVal + VecNumElts;
45494
45495 return DAG.getVectorShuffle(OpVT, dl, Vec, SubVec.getOperand(0), Mask);
45496 }
45497 }
45498
45499 // Match concat_vector style patterns.
45500 SmallVector<SDValue, 2> SubVectorOps;
45501 if (collectConcatOps(N, SubVectorOps)) {
45502 if (SDValue Fold =
45503 combineConcatVectorOps(dl, OpVT, SubVectorOps, DAG, DCI, Subtarget))
45504 return Fold;
45505
45506 // If we're inserting all zeros into the upper half, change this to
45507 // a concat with zero. We will match this to a move
45508 // with implicit upper bit zeroing during isel.
45509 // We do this here because we don't want combineConcatVectorOps to
45510 // create INSERT_SUBVECTOR from CONCAT_VECTORS.
45511 if (SubVectorOps.size() == 2 &&
45512 ISD::isBuildVectorAllZeros(SubVectorOps[1].getNode()))
45513 return DAG.getNode(ISD::INSERT_SUBVECTOR, dl, OpVT,
45514 getZeroVector(OpVT, Subtarget, DAG, dl),
45515 SubVectorOps[0], DAG.getIntPtrConstant(0, dl));
45516 }
45517
45518 // If this is a broadcast insert into an upper undef, use a larger broadcast.
45519 if (Vec.isUndef() && IdxVal != 0 && SubVec.getOpcode() == X86ISD::VBROADCAST)
45520 return DAG.getNode(X86ISD::VBROADCAST, dl, OpVT, SubVec.getOperand(0));
45521
45522 // If this is a broadcast load inserted into an upper undef, use a larger
45523 // broadcast load.
45524 if (Vec.isUndef() && IdxVal != 0 && SubVec.hasOneUse() &&
45525 SubVec.getOpcode() == X86ISD::VBROADCAST_LOAD) {
45526 auto *MemIntr = cast<MemIntrinsicSDNode>(SubVec);
45527 SDVTList Tys = DAG.getVTList(OpVT, MVT::Other);
45528 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
45529 SDValue BcastLd =
45530 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, dl, Tys, Ops,
45531 MemIntr->getMemoryVT(),
45532 MemIntr->getMemOperand());
45533 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
45534 return BcastLd;
45535 }
45536
45537 return SDValue();
45538}
45539
45540/// If we are extracting a subvector of a vector select and the select condition
45541/// is composed of concatenated vectors, try to narrow the select width. This
45542/// is a common pattern for AVX1 integer code because 256-bit selects may be
45543/// legal, but there is almost no integer math/logic available for 256-bit.
45544/// This function should only be called with legal types (otherwise, the calls
45545/// to get simple value types will assert).
45546static SDValue narrowExtractedVectorSelect(SDNode *Ext, SelectionDAG &DAG) {
45547 SDValue Sel = peekThroughBitcasts(Ext->getOperand(0));
45548 SmallVector<SDValue, 4> CatOps;
45549 if (Sel.getOpcode() != ISD::VSELECT ||
45550 !collectConcatOps(Sel.getOperand(0).getNode(), CatOps))
45551 return SDValue();
45552
45553 // Note: We assume simple value types because this should only be called with
45554 // legal operations/types.
45555 // TODO: This can be extended to handle extraction to 256-bits.
45556 MVT VT = Ext->getSimpleValueType(0);
45557 if (!VT.is128BitVector())
45558 return SDValue();
45559
45560 MVT SelCondVT = Sel.getOperand(0).getSimpleValueType();
45561 if (!SelCondVT.is256BitVector() && !SelCondVT.is512BitVector())
45562 return SDValue();
45563
45564 MVT WideVT = Ext->getOperand(0).getSimpleValueType();
45565 MVT SelVT = Sel.getSimpleValueType();
45566 assert((SelVT.is256BitVector() || SelVT.is512BitVector()) &&(((SelVT.is256BitVector() || SelVT.is512BitVector()) &&
"Unexpected vector type with legal operations") ? static_cast
<void> (0) : __assert_fail ("(SelVT.is256BitVector() || SelVT.is512BitVector()) && \"Unexpected vector type with legal operations\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45567, __PRETTY_FUNCTION__))
45567 "Unexpected vector type with legal operations")(((SelVT.is256BitVector() || SelVT.is512BitVector()) &&
"Unexpected vector type with legal operations") ? static_cast
<void> (0) : __assert_fail ("(SelVT.is256BitVector() || SelVT.is512BitVector()) && \"Unexpected vector type with legal operations\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45567, __PRETTY_FUNCTION__));
45568
45569 unsigned SelElts = SelVT.getVectorNumElements();
45570 unsigned CastedElts = WideVT.getVectorNumElements();
45571 unsigned ExtIdx = cast<ConstantSDNode>(Ext->getOperand(1))->getZExtValue();
45572 if (SelElts % CastedElts == 0) {
45573 // The select has the same or more (narrower) elements than the extract
45574 // operand. The extraction index gets scaled by that factor.
45575 ExtIdx *= (SelElts / CastedElts);
45576 } else if (CastedElts % SelElts == 0) {
45577 // The select has less (wider) elements than the extract operand. Make sure
45578 // that the extraction index can be divided evenly.
45579 unsigned IndexDivisor = CastedElts / SelElts;
45580 if (ExtIdx % IndexDivisor != 0)
45581 return SDValue();
45582 ExtIdx /= IndexDivisor;
45583 } else {
45584 llvm_unreachable("Element count of simple vector types are not divisible?")::llvm::llvm_unreachable_internal("Element count of simple vector types are not divisible?"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 45584);
45585 }
45586
45587 unsigned NarrowingFactor = WideVT.getSizeInBits() / VT.getSizeInBits();
45588 unsigned NarrowElts = SelElts / NarrowingFactor;
45589 MVT NarrowSelVT = MVT::getVectorVT(SelVT.getVectorElementType(), NarrowElts);
45590 SDLoc DL(Ext);
45591 SDValue ExtCond = extract128BitVector(Sel.getOperand(0), ExtIdx, DAG, DL);
45592 SDValue ExtT = extract128BitVector(Sel.getOperand(1), ExtIdx, DAG, DL);
45593 SDValue ExtF = extract128BitVector(Sel.getOperand(2), ExtIdx, DAG, DL);
45594 SDValue NarrowSel = DAG.getSelect(DL, NarrowSelVT, ExtCond, ExtT, ExtF);
45595 return DAG.getBitcast(VT, NarrowSel);
45596}
45597
45598static SDValue combineExtractSubvector(SDNode *N, SelectionDAG &DAG,
45599 TargetLowering::DAGCombinerInfo &DCI,
45600 const X86Subtarget &Subtarget) {
45601 // For AVX1 only, if we are extracting from a 256-bit and+not (which will
45602 // eventually get combined/lowered into ANDNP) with a concatenated operand,
45603 // split the 'and' into 128-bit ops to avoid the concatenate and extract.
45604 // We let generic combining take over from there to simplify the
45605 // insert/extract and 'not'.
45606 // This pattern emerges during AVX1 legalization. We handle it before lowering
45607 // to avoid complications like splitting constant vector loads.
45608
45609 // Capture the original wide type in the likely case that we need to bitcast
45610 // back to this type.
45611 if (!N->getValueType(0).isSimple())
45612 return SDValue();
45613
45614 MVT VT = N->getSimpleValueType(0);
45615 SDValue InVec = N->getOperand(0);
45616 SDValue InVecBC = peekThroughBitcasts(InVec);
45617 EVT InVecVT = InVec.getValueType();
45618 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45619
45620 if (Subtarget.hasAVX() && !Subtarget.hasAVX2() &&
45621 TLI.isTypeLegal(InVecVT) &&
45622 InVecVT.getSizeInBits() == 256 && InVecBC.getOpcode() == ISD::AND) {
45623 auto isConcatenatedNot = [] (SDValue V) {
45624 V = peekThroughBitcasts(V);
45625 if (!isBitwiseNot(V))
45626 return false;
45627 SDValue NotOp = V->getOperand(0);
45628 return peekThroughBitcasts(NotOp).getOpcode() == ISD::CONCAT_VECTORS;
45629 };
45630 if (isConcatenatedNot(InVecBC.getOperand(0)) ||
45631 isConcatenatedNot(InVecBC.getOperand(1))) {
45632 // extract (and v4i64 X, (not (concat Y1, Y2))), n -> andnp v2i64 X(n), Y1
45633 SDValue Concat = split256IntArith(InVecBC, DAG);
45634 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(N), VT,
45635 DAG.getBitcast(InVecVT, Concat), N->getOperand(1));
45636 }
45637 }
45638
45639 if (DCI.isBeforeLegalizeOps())
45640 return SDValue();
45641
45642 if (SDValue V = narrowExtractedVectorSelect(N, DAG))
45643 return V;
45644
45645 unsigned IdxVal = cast<ConstantSDNode>(N->getOperand(1))->getZExtValue();
45646
45647 if (ISD::isBuildVectorAllZeros(InVec.getNode()))
45648 return getZeroVector(VT, Subtarget, DAG, SDLoc(N));
45649
45650 if (ISD::isBuildVectorAllOnes(InVec.getNode())) {
45651 if (VT.getScalarType() == MVT::i1)
45652 return DAG.getConstant(1, SDLoc(N), VT);
45653 return getOnesVector(VT, DAG, SDLoc(N));
45654 }
45655
45656 if (InVec.getOpcode() == ISD::BUILD_VECTOR)
45657 return DAG.getBuildVector(
45658 VT, SDLoc(N),
45659 InVec.getNode()->ops().slice(IdxVal, VT.getVectorNumElements()));
45660
45661 // If we are extracting from an insert into a zero vector, replace with a
45662 // smaller insert into zero if we don't access less than the original
45663 // subvector. Don't do this for i1 vectors.
45664 if (VT.getVectorElementType() != MVT::i1 &&
45665 InVec.getOpcode() == ISD::INSERT_SUBVECTOR && IdxVal == 0 &&
45666 InVec.hasOneUse() && isNullConstant(InVec.getOperand(2)) &&
45667 ISD::isBuildVectorAllZeros(InVec.getOperand(0).getNode()) &&
45668 InVec.getOperand(1).getValueSizeInBits() <= VT.getSizeInBits()) {
45669 SDLoc DL(N);
45670 return DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT,
45671 getZeroVector(VT, Subtarget, DAG, DL),
45672 InVec.getOperand(1), InVec.getOperand(2));
45673 }
45674
45675 // If we're extracting from a broadcast then we're better off just
45676 // broadcasting to the smaller type directly, assuming this is the only use.
45677 // As its a broadcast we don't care about the extraction index.
45678 if (InVec.getOpcode() == X86ISD::VBROADCAST && InVec.hasOneUse() &&
45679 InVec.getOperand(0).getValueSizeInBits() <= VT.getSizeInBits())
45680 return DAG.getNode(X86ISD::VBROADCAST, SDLoc(N), VT, InVec.getOperand(0));
45681
45682 if (InVec.getOpcode() == X86ISD::VBROADCAST_LOAD && InVec.hasOneUse()) {
45683 auto *MemIntr = cast<MemIntrinsicSDNode>(InVec);
45684 if (MemIntr->getMemoryVT().getSizeInBits() <= VT.getSizeInBits()) {
45685 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
45686 SDValue Ops[] = { MemIntr->getChain(), MemIntr->getBasePtr() };
45687 SDValue BcastLd =
45688 DAG.getMemIntrinsicNode(X86ISD::VBROADCAST_LOAD, SDLoc(N), Tys, Ops,
45689 MemIntr->getMemoryVT(),
45690 MemIntr->getMemOperand());
45691 DAG.ReplaceAllUsesOfValueWith(SDValue(MemIntr, 1), BcastLd.getValue(1));
45692 return BcastLd;
45693 }
45694 }
45695
45696 // If we're extracting the lowest subvector and we're the only user,
45697 // we may be able to perform this with a smaller vector width.
45698 if (IdxVal == 0 && InVec.hasOneUse()) {
45699 unsigned InOpcode = InVec.getOpcode();
45700 if (VT == MVT::v2f64 && InVecVT == MVT::v4f64) {
45701 // v2f64 CVTDQ2PD(v4i32).
45702 if (InOpcode == ISD::SINT_TO_FP &&
45703 InVec.getOperand(0).getValueType() == MVT::v4i32) {
45704 return DAG.getNode(X86ISD::CVTSI2P, SDLoc(N), VT, InVec.getOperand(0));
45705 }
45706 // v2f64 CVTUDQ2PD(v4i32).
45707 if (InOpcode == ISD::UINT_TO_FP && Subtarget.hasVLX() &&
45708 InVec.getOperand(0).getValueType() == MVT::v4i32) {
45709 return DAG.getNode(X86ISD::CVTUI2P, SDLoc(N), VT, InVec.getOperand(0));
45710 }
45711 // v2f64 CVTPS2PD(v4f32).
45712 if (InOpcode == ISD::FP_EXTEND &&
45713 InVec.getOperand(0).getValueType() == MVT::v4f32) {
45714 return DAG.getNode(X86ISD::VFPEXT, SDLoc(N), VT, InVec.getOperand(0));
45715 }
45716 }
45717 if ((InOpcode == ISD::ANY_EXTEND ||
45718 InOpcode == ISD::ANY_EXTEND_VECTOR_INREG ||
45719 InOpcode == ISD::ZERO_EXTEND ||
45720 InOpcode == ISD::ZERO_EXTEND_VECTOR_INREG ||
45721 InOpcode == ISD::SIGN_EXTEND ||
45722 InOpcode == ISD::SIGN_EXTEND_VECTOR_INREG) &&
45723 VT.is128BitVector() &&
45724 InVec.getOperand(0).getSimpleValueType().is128BitVector()) {
45725 unsigned ExtOp = getOpcode_EXTEND_VECTOR_INREG(InOpcode);
45726 return DAG.getNode(ExtOp, SDLoc(N), VT, InVec.getOperand(0));
45727 }
45728 if (InOpcode == ISD::VSELECT &&
45729 InVec.getOperand(0).getValueType().is256BitVector() &&
45730 InVec.getOperand(1).getValueType().is256BitVector() &&
45731 InVec.getOperand(2).getValueType().is256BitVector()) {
45732 SDLoc DL(N);
45733 SDValue Ext0 = extractSubVector(InVec.getOperand(0), 0, DAG, DL, 128);
45734 SDValue Ext1 = extractSubVector(InVec.getOperand(1), 0, DAG, DL, 128);
45735 SDValue Ext2 = extractSubVector(InVec.getOperand(2), 0, DAG, DL, 128);
45736 return DAG.getNode(InOpcode, DL, VT, Ext0, Ext1, Ext2);
45737 }
45738 }
45739
45740 return SDValue();
45741}
45742
45743static SDValue combineScalarToVector(SDNode *N, SelectionDAG &DAG) {
45744 EVT VT = N->getValueType(0);
45745 SDValue Src = N->getOperand(0);
45746 SDLoc DL(N);
45747
45748 // If this is a scalar to vector to v1i1 from an AND with 1, bypass the and.
45749 // This occurs frequently in our masked scalar intrinsic code and our
45750 // floating point select lowering with AVX512.
45751 // TODO: SimplifyDemandedBits instead?
45752 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::AND && Src.hasOneUse())
45753 if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1)))
45754 if (C->getAPIntValue().isOneValue())
45755 return DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v1i1,
45756 Src.getOperand(0));
45757
45758 // Combine scalar_to_vector of an extract_vector_elt into an extract_subvec.
45759 if (VT == MVT::v1i1 && Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
45760 Src.hasOneUse() && Src.getOperand(0).getValueType().isVector() &&
45761 Src.getOperand(0).getValueType().getVectorElementType() == MVT::i1)
45762 if (auto *C = dyn_cast<ConstantSDNode>(Src.getOperand(1)))
45763 if (C->isNullValue())
45764 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, VT, Src.getOperand(0),
45765 Src.getOperand(1));
45766
45767 // Reduce v2i64 to v4i32 if we don't need the upper bits.
45768 // TODO: Move to DAGCombine?
45769 if (VT == MVT::v2i64 && Src.getOpcode() == ISD::ANY_EXTEND &&
45770 Src.getValueType() == MVT::i64 && Src.hasOneUse() &&
45771 Src.getOperand(0).getScalarValueSizeInBits() <= 32)
45772 return DAG.getBitcast(
45773 VT, DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v4i32,
45774 DAG.getAnyExtOrTrunc(Src.getOperand(0), DL, MVT::i32)));
45775
45776 return SDValue();
45777}
45778
45779// Simplify PMULDQ and PMULUDQ operations.
45780static SDValue combinePMULDQ(SDNode *N, SelectionDAG &DAG,
45781 TargetLowering::DAGCombinerInfo &DCI,
45782 const X86Subtarget &Subtarget) {
45783 SDValue LHS = N->getOperand(0);
45784 SDValue RHS = N->getOperand(1);
45785
45786 // Canonicalize constant to RHS.
45787 if (DAG.isConstantIntBuildVectorOrConstantInt(LHS) &&
45788 !DAG.isConstantIntBuildVectorOrConstantInt(RHS))
45789 return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0), RHS, LHS);
45790
45791 // Multiply by zero.
45792 // Don't return RHS as it may contain UNDEFs.
45793 if (ISD::isBuildVectorAllZeros(RHS.getNode()))
45794 return DAG.getConstant(0, SDLoc(N), N->getValueType(0));
45795
45796 // PMULDQ/PMULUDQ only uses lower 32 bits from each vector element.
45797 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45798 if (TLI.SimplifyDemandedBits(SDValue(N, 0), APInt::getAllOnesValue(64), DCI))
45799 return SDValue(N, 0);
45800
45801 // If the input is an extend_invec and the SimplifyDemandedBits call didn't
45802 // convert it to any_extend_invec, due to the LegalOperations check, do the
45803 // conversion directly to a vector shuffle manually. This exposes combine
45804 // opportunities missed by combineExtInVec not calling
45805 // combineX86ShufflesRecursively on SSE4.1 targets.
45806 // FIXME: This is basically a hack around several other issues related to
45807 // ANY_EXTEND_VECTOR_INREG.
45808 if (N->getValueType(0) == MVT::v2i64 && LHS.hasOneUse() &&
45809 (LHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
45810 LHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
45811 LHS.getOperand(0).getValueType() == MVT::v4i32) {
45812 SDLoc dl(N);
45813 LHS = DAG.getVectorShuffle(MVT::v4i32, dl, LHS.getOperand(0),
45814 LHS.getOperand(0), { 0, -1, 1, -1 });
45815 LHS = DAG.getBitcast(MVT::v2i64, LHS);
45816 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
45817 }
45818 if (N->getValueType(0) == MVT::v2i64 && RHS.hasOneUse() &&
45819 (RHS.getOpcode() == ISD::ZERO_EXTEND_VECTOR_INREG ||
45820 RHS.getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG) &&
45821 RHS.getOperand(0).getValueType() == MVT::v4i32) {
45822 SDLoc dl(N);
45823 RHS = DAG.getVectorShuffle(MVT::v4i32, dl, RHS.getOperand(0),
45824 RHS.getOperand(0), { 0, -1, 1, -1 });
45825 RHS = DAG.getBitcast(MVT::v2i64, RHS);
45826 return DAG.getNode(N->getOpcode(), dl, MVT::v2i64, LHS, RHS);
45827 }
45828
45829 return SDValue();
45830}
45831
45832static SDValue combineExtInVec(SDNode *N, SelectionDAG &DAG,
45833 TargetLowering::DAGCombinerInfo &DCI,
45834 const X86Subtarget &Subtarget) {
45835 EVT VT = N->getValueType(0);
45836 SDValue In = N->getOperand(0);
45837 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45838
45839 // Try to merge vector loads and extend_inreg to an extload.
45840 if (!DCI.isBeforeLegalizeOps() && ISD::isNormalLoad(In.getNode()) &&
45841 In.hasOneUse()) {
45842 auto *Ld = cast<LoadSDNode>(In);
45843 if (Ld->isSimple()) {
45844 MVT SVT = In.getSimpleValueType().getVectorElementType();
45845 ISD::LoadExtType Ext = N->getOpcode() == ISD::SIGN_EXTEND_VECTOR_INREG ? ISD::SEXTLOAD : ISD::ZEXTLOAD;
45846 EVT MemVT = EVT::getVectorVT(*DAG.getContext(), SVT,
45847 VT.getVectorNumElements());
45848 if (TLI.isLoadExtLegal(Ext, VT, MemVT)) {
45849 SDValue Load =
45850 DAG.getExtLoad(Ext, SDLoc(N), VT, Ld->getChain(), Ld->getBasePtr(),
45851 Ld->getPointerInfo(), MemVT, Ld->getAlignment(),
45852 Ld->getMemOperand()->getFlags());
45853 DAG.ReplaceAllUsesOfValueWith(SDValue(Ld, 1), Load.getValue(1));
45854 return Load;
45855 }
45856 }
45857 }
45858
45859 // Attempt to combine as a shuffle.
45860 // TODO: SSE41 support
45861 if (Subtarget.hasAVX() && N->getOpcode() != ISD::SIGN_EXTEND_VECTOR_INREG) {
45862 SDValue Op(N, 0);
45863 if (TLI.isTypeLegal(VT) && TLI.isTypeLegal(In.getValueType()))
45864 if (SDValue Res = combineX86ShufflesRecursively(Op, DAG, Subtarget))
45865 return Res;
45866 }
45867
45868 return SDValue();
45869}
45870
45871static SDValue combineKSHIFT(SDNode *N, SelectionDAG &DAG,
45872 TargetLowering::DAGCombinerInfo &DCI) {
45873 EVT VT = N->getValueType(0);
45874
45875 if (ISD::isBuildVectorAllZeros(N->getOperand(0).getNode()))
45876 return DAG.getConstant(0, SDLoc(N), VT);
45877
45878 APInt KnownUndef, KnownZero;
45879 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
45880 APInt DemandedElts = APInt::getAllOnesValue(VT.getVectorNumElements());
45881 if (TLI.SimplifyDemandedVectorElts(SDValue(N, 0), DemandedElts, KnownUndef,
45882 KnownZero, DCI))
45883 return SDValue(N, 0);
45884
45885 return SDValue();
45886}
45887
45888SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
45889 DAGCombinerInfo &DCI) const {
45890 SelectionDAG &DAG = DCI.DAG;
45891 switch (N->getOpcode()) {
45892 default: break;
45893 case ISD::SCALAR_TO_VECTOR:
45894 return combineScalarToVector(N, DAG);
45895 case ISD::EXTRACT_VECTOR_ELT:
45896 case X86ISD::PEXTRW:
45897 case X86ISD::PEXTRB:
45898 return combineExtractVectorElt(N, DAG, DCI, Subtarget);
45899 case ISD::CONCAT_VECTORS:
45900 return combineConcatVectors(N, DAG, DCI, Subtarget);
45901 case ISD::INSERT_SUBVECTOR:
45902 return combineInsertSubvector(N, DAG, DCI, Subtarget);
45903 case ISD::EXTRACT_SUBVECTOR:
45904 return combineExtractSubvector(N, DAG, DCI, Subtarget);
45905 case ISD::VSELECT:
45906 case ISD::SELECT:
45907 case X86ISD::BLENDV: return combineSelect(N, DAG, DCI, Subtarget);
45908 case ISD::BITCAST: return combineBitcast(N, DAG, DCI, Subtarget);
45909 case X86ISD::CMOV: return combineCMov(N, DAG, DCI, Subtarget);
45910 case X86ISD::CMP: return combineCMP(N, DAG);
45911 case ISD::ADD: return combineAdd(N, DAG, DCI, Subtarget);
45912 case ISD::SUB: return combineSub(N, DAG, DCI, Subtarget);
45913 case X86ISD::ADD:
45914 case X86ISD::SUB: return combineX86AddSub(N, DAG, DCI);
45915 case X86ISD::SBB: return combineSBB(N, DAG);
45916 case X86ISD::ADC: return combineADC(N, DAG, DCI);
45917 case ISD::MUL: return combineMul(N, DAG, DCI, Subtarget);
45918 case ISD::SHL: return combineShiftLeft(N, DAG);
45919 case ISD::SRA: return combineShiftRightArithmetic(N, DAG);
45920 case ISD::SRL: return combineShiftRightLogical(N, DAG, DCI);
45921 case ISD::AND: return combineAnd(N, DAG, DCI, Subtarget);
45922 case ISD::OR: return combineOr(N, DAG, DCI, Subtarget);
45923 case ISD::XOR: return combineXor(N, DAG, DCI, Subtarget);
45924 case X86ISD::BEXTR: return combineBEXTR(N, DAG, DCI, Subtarget);
45925 case ISD::LOAD: return combineLoad(N, DAG, DCI, Subtarget);
45926 case ISD::MLOAD: return combineMaskedLoad(N, DAG, DCI, Subtarget);
45927 case ISD::STORE: return combineStore(N, DAG, DCI, Subtarget);
45928 case ISD::MSTORE: return combineMaskedStore(N, DAG, DCI, Subtarget);
45929 case ISD::SINT_TO_FP:
45930 case ISD::STRICT_SINT_TO_FP:
45931 return combineSIntToFP(N, DAG, DCI, Subtarget);
45932 case ISD::UINT_TO_FP:
45933 case ISD::STRICT_UINT_TO_FP:
45934 return combineUIntToFP(N, DAG, Subtarget);
45935 case ISD::FADD:
45936 case ISD::FSUB: return combineFaddFsub(N, DAG, Subtarget);
45937 case ISD::FNEG: return combineFneg(N, DAG, Subtarget);
45938 case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
45939 case X86ISD::VTRUNC: return combineVTRUNC(N, DAG);
45940 case X86ISD::ANDNP: return combineAndnp(N, DAG, DCI, Subtarget);
45941 case X86ISD::FAND: return combineFAnd(N, DAG, Subtarget);
45942 case X86ISD::FANDN: return combineFAndn(N, DAG, Subtarget);
45943 case X86ISD::FXOR:
45944 case X86ISD::FOR: return combineFOr(N, DAG, Subtarget);
45945 case X86ISD::FMIN:
45946 case X86ISD::FMAX: return combineFMinFMax(N, DAG);
45947 case ISD::FMINNUM:
45948 case ISD::FMAXNUM: return combineFMinNumFMaxNum(N, DAG, Subtarget);
45949 case X86ISD::CVTSI2P:
45950 case X86ISD::CVTUI2P: return combineX86INT_TO_FP(N, DAG, DCI);
45951 case X86ISD::CVTP2SI:
45952 case X86ISD::CVTP2UI:
45953 case X86ISD::CVTTP2SI:
45954 case X86ISD::CVTTP2UI: return combineCVTP2I_CVTTP2I(N, DAG, DCI);
45955 case X86ISD::BT: return combineBT(N, DAG, DCI);
45956 case ISD::ANY_EXTEND:
45957 case ISD::ZERO_EXTEND: return combineZext(N, DAG, DCI, Subtarget);
45958 case ISD::SIGN_EXTEND: return combineSext(N, DAG, DCI, Subtarget);
45959 case ISD::SIGN_EXTEND_INREG: return combineSignExtendInReg(N, DAG, Subtarget);
45960 case ISD::ANY_EXTEND_VECTOR_INREG:
45961 case ISD::SIGN_EXTEND_VECTOR_INREG:
45962 case ISD::ZERO_EXTEND_VECTOR_INREG: return combineExtInVec(N, DAG, DCI,
45963 Subtarget);
45964 case ISD::SETCC: return combineSetCC(N, DAG, Subtarget);
45965 case X86ISD::SETCC: return combineX86SetCC(N, DAG, Subtarget);
45966 case X86ISD::BRCOND: return combineBrCond(N, DAG, Subtarget);
45967 case X86ISD::PACKSS:
45968 case X86ISD::PACKUS: return combineVectorPack(N, DAG, DCI, Subtarget);
45969 case X86ISD::VSHL:
45970 case X86ISD::VSRA:
45971 case X86ISD::VSRL:
45972 return combineVectorShiftVar(N, DAG, DCI, Subtarget);
45973 case X86ISD::VSHLI:
45974 case X86ISD::VSRAI:
45975 case X86ISD::VSRLI:
45976 return combineVectorShiftImm(N, DAG, DCI, Subtarget);
45977 case X86ISD::PINSRB:
45978 case X86ISD::PINSRW: return combineVectorInsert(N, DAG, DCI, Subtarget);
45979 case X86ISD::SHUFP: // Handle all target specific shuffles
45980 case X86ISD::INSERTPS:
45981 case X86ISD::EXTRQI:
45982 case X86ISD::INSERTQI:
45983 case X86ISD::PALIGNR:
45984 case X86ISD::VSHLDQ:
45985 case X86ISD::VSRLDQ:
45986 case X86ISD::BLENDI:
45987 case X86ISD::UNPCKH:
45988 case X86ISD::UNPCKL:
45989 case X86ISD::MOVHLPS:
45990 case X86ISD::MOVLHPS:
45991 case X86ISD::PSHUFB:
45992 case X86ISD::PSHUFD:
45993 case X86ISD::PSHUFHW:
45994 case X86ISD::PSHUFLW:
45995 case X86ISD::MOVSHDUP:
45996 case X86ISD::MOVSLDUP:
45997 case X86ISD::MOVDDUP:
45998 case X86ISD::MOVSS:
45999 case X86ISD::MOVSD:
46000 case X86ISD::VBROADCAST:
46001 case X86ISD::VPPERM:
46002 case X86ISD::VPERMI:
46003 case X86ISD::VPERMV:
46004 case X86ISD::VPERMV3:
46005 case X86ISD::VPERMIL2:
46006 case X86ISD::VPERMILPI:
46007 case X86ISD::VPERMILPV:
46008 case X86ISD::VPERM2X128:
46009 case X86ISD::SHUF128:
46010 case X86ISD::VZEXT_MOVL:
46011 case ISD::VECTOR_SHUFFLE: return combineShuffle(N, DAG, DCI,Subtarget);
46012 case X86ISD::FMADD_RND:
46013 case X86ISD::FMSUB:
46014 case X86ISD::FMSUB_RND:
46015 case X86ISD::FNMADD:
46016 case X86ISD::FNMADD_RND:
46017 case X86ISD::FNMSUB:
46018 case X86ISD::FNMSUB_RND:
46019 case ISD::FMA: return combineFMA(N, DAG, DCI, Subtarget);
46020 case X86ISD::FMADDSUB_RND:
46021 case X86ISD::FMSUBADD_RND:
46022 case X86ISD::FMADDSUB:
46023 case X86ISD::FMSUBADD: return combineFMADDSUB(N, DAG, DCI);
46024 case X86ISD::MOVMSK: return combineMOVMSK(N, DAG, DCI, Subtarget);
46025 case X86ISD::MGATHER:
46026 case X86ISD::MSCATTER: return combineX86GatherScatter(N, DAG, DCI);
46027 case ISD::MGATHER:
46028 case ISD::MSCATTER: return combineGatherScatter(N, DAG, DCI);
46029 case X86ISD::PCMPEQ:
46030 case X86ISD::PCMPGT: return combineVectorCompare(N, DAG, Subtarget);
46031 case X86ISD::PMULDQ:
46032 case X86ISD::PMULUDQ: return combinePMULDQ(N, DAG, DCI, Subtarget);
46033 case X86ISD::KSHIFTL:
46034 case X86ISD::KSHIFTR: return combineKSHIFT(N, DAG, DCI);
46035 }
46036
46037 return SDValue();
46038}
46039
46040bool X86TargetLowering::isTypeDesirableForOp(unsigned Opc, EVT VT) const {
46041 if (!isTypeLegal(VT))
46042 return false;
46043
46044 // There are no vXi8 shifts.
46045 if (Opc == ISD::SHL && VT.isVector() && VT.getVectorElementType() == MVT::i8)
46046 return false;
46047
46048 // TODO: Almost no 8-bit ops are desirable because they have no actual
46049 // size/speed advantages vs. 32-bit ops, but they do have a major
46050 // potential disadvantage by causing partial register stalls.
46051 //
46052 // 8-bit multiply/shl is probably not cheaper than 32-bit multiply/shl, and
46053 // we have specializations to turn 32-bit multiply/shl into LEA or other ops.
46054 // Also, see the comment in "IsDesirableToPromoteOp" - where we additionally
46055 // check for a constant operand to the multiply.
46056 if ((Opc == ISD::MUL || Opc == ISD::SHL) && VT == MVT::i8)
46057 return false;
46058
46059 // i16 instruction encodings are longer and some i16 instructions are slow,
46060 // so those are not desirable.
46061 if (VT == MVT::i16) {
46062 switch (Opc) {
46063 default:
46064 break;
46065 case ISD::LOAD:
46066 case ISD::SIGN_EXTEND:
46067 case ISD::ZERO_EXTEND:
46068 case ISD::ANY_EXTEND:
46069 case ISD::SHL:
46070 case ISD::SRA:
46071 case ISD::SRL:
46072 case ISD::SUB:
46073 case ISD::ADD:
46074 case ISD::MUL:
46075 case ISD::AND:
46076 case ISD::OR:
46077 case ISD::XOR:
46078 return false;
46079 }
46080 }
46081
46082 // Any legal type not explicitly accounted for above here is desirable.
46083 return true;
46084}
46085
46086SDValue X86TargetLowering::expandIndirectJTBranch(const SDLoc& dl,
46087 SDValue Value, SDValue Addr,
46088 SelectionDAG &DAG) const {
46089 const Module *M = DAG.getMachineFunction().getMMI().getModule();
46090 Metadata *IsCFProtectionSupported = M->getModuleFlag("cf-protection-branch");
46091 if (IsCFProtectionSupported) {
46092 // In case control-flow branch protection is enabled, we need to add
46093 // notrack prefix to the indirect branch.
46094 // In order to do that we create NT_BRIND SDNode.
46095 // Upon ISEL, the pattern will convert it to jmp with NoTrack prefix.
46096 return DAG.getNode(X86ISD::NT_BRIND, dl, MVT::Other, Value, Addr);
46097 }
46098
46099 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, DAG);
46100}
46101
46102bool X86TargetLowering::IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const {
46103 EVT VT = Op.getValueType();
46104 bool Is8BitMulByConstant = VT == MVT::i8 && Op.getOpcode() == ISD::MUL &&
46105 isa<ConstantSDNode>(Op.getOperand(1));
46106
46107 // i16 is legal, but undesirable since i16 instruction encodings are longer
46108 // and some i16 instructions are slow.
46109 // 8-bit multiply-by-constant can usually be expanded to something cheaper
46110 // using LEA and/or other ALU ops.
46111 if (VT != MVT::i16 && !Is8BitMulByConstant)
46112 return false;
46113
46114 auto IsFoldableRMW = [](SDValue Load, SDValue Op) {
46115 if (!Op.hasOneUse())
46116 return false;
46117 SDNode *User = *Op->use_begin();
46118 if (!ISD::isNormalStore(User))
46119 return false;
46120 auto *Ld = cast<LoadSDNode>(Load);
46121 auto *St = cast<StoreSDNode>(User);
46122 return Ld->getBasePtr() == St->getBasePtr();
46123 };
46124
46125 auto IsFoldableAtomicRMW = [](SDValue Load, SDValue Op) {
46126 if (!Load.hasOneUse() || Load.getOpcode() != ISD::ATOMIC_LOAD)
46127 return false;
46128 if (!Op.hasOneUse())
46129 return false;
46130 SDNode *User = *Op->use_begin();
46131 if (User->getOpcode() != ISD::ATOMIC_STORE)
46132 return false;
46133 auto *Ld = cast<AtomicSDNode>(Load);
46134 auto *St = cast<AtomicSDNode>(User);
46135 return Ld->getBasePtr() == St->getBasePtr();
46136 };
46137
46138 bool Commute = false;
46139 switch (Op.getOpcode()) {
46140 default: return false;
46141 case ISD::SIGN_EXTEND:
46142 case ISD::ZERO_EXTEND:
46143 case ISD::ANY_EXTEND:
46144 break;
46145 case ISD::SHL:
46146 case ISD::SRA:
46147 case ISD::SRL: {
46148 SDValue N0 = Op.getOperand(0);
46149 // Look out for (store (shl (load), x)).
46150 if (MayFoldLoad(N0) && IsFoldableRMW(N0, Op))
46151 return false;
46152 break;
46153 }
46154 case ISD::ADD:
46155 case ISD::MUL:
46156 case ISD::AND:
46157 case ISD::OR:
46158 case ISD::XOR:
46159 Commute = true;
46160 LLVM_FALLTHROUGH[[gnu::fallthrough]];
46161 case ISD::SUB: {
46162 SDValue N0 = Op.getOperand(0);
46163 SDValue N1 = Op.getOperand(1);
46164 // Avoid disabling potential load folding opportunities.
46165 if (MayFoldLoad(N1) &&
46166 (!Commute || !isa<ConstantSDNode>(N0) ||
46167 (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N1, Op))))
46168 return false;
46169 if (MayFoldLoad(N0) &&
46170 ((Commute && !isa<ConstantSDNode>(N1)) ||
46171 (Op.getOpcode() != ISD::MUL && IsFoldableRMW(N0, Op))))
46172 return false;
46173 if (IsFoldableAtomicRMW(N0, Op) ||
46174 (Commute && IsFoldableAtomicRMW(N1, Op)))
46175 return false;
46176 }
46177 }
46178
46179 PVT = MVT::i32;
46180 return true;
46181}
46182
46183bool X86TargetLowering::
46184 isDesirableToCombineBuildVectorToShuffleTruncate(
46185 ArrayRef<int> ShuffleMask, EVT SrcVT, EVT TruncVT) const {
46186
46187 assert(SrcVT.getVectorNumElements() == ShuffleMask.size() &&((SrcVT.getVectorNumElements() == ShuffleMask.size() &&
"Element count mismatch") ? static_cast<void> (0) : __assert_fail
("SrcVT.getVectorNumElements() == ShuffleMask.size() && \"Element count mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 46188, __PRETTY_FUNCTION__))
46188 "Element count mismatch")((SrcVT.getVectorNumElements() == ShuffleMask.size() &&
"Element count mismatch") ? static_cast<void> (0) : __assert_fail
("SrcVT.getVectorNumElements() == ShuffleMask.size() && \"Element count mismatch\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 46188, __PRETTY_FUNCTION__));
46189 assert(((Subtarget.getTargetLowering()->isShuffleMaskLegal(ShuffleMask
, SrcVT) && "Shuffle Mask expected to be legal") ? static_cast
<void> (0) : __assert_fail ("Subtarget.getTargetLowering()->isShuffleMaskLegal(ShuffleMask, SrcVT) && \"Shuffle Mask expected to be legal\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 46191, __PRETTY_FUNCTION__))
46190 Subtarget.getTargetLowering()->isShuffleMaskLegal(ShuffleMask, SrcVT) &&((Subtarget.getTargetLowering()->isShuffleMaskLegal(ShuffleMask
, SrcVT) && "Shuffle Mask expected to be legal") ? static_cast
<void> (0) : __assert_fail ("Subtarget.getTargetLowering()->isShuffleMaskLegal(ShuffleMask, SrcVT) && \"Shuffle Mask expected to be legal\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 46191, __PRETTY_FUNCTION__))
46191 "Shuffle Mask expected to be legal")((Subtarget.getTargetLowering()->isShuffleMaskLegal(ShuffleMask
, SrcVT) && "Shuffle Mask expected to be legal") ? static_cast
<void> (0) : __assert_fail ("Subtarget.getTargetLowering()->isShuffleMaskLegal(ShuffleMask, SrcVT) && \"Shuffle Mask expected to be legal\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 46191, __PRETTY_FUNCTION__));
46192
46193 // For 32-bit elements VPERMD is better than shuffle+truncate.
46194 // TODO: After we improve lowerBuildVector, add execption for VPERMW.
46195 if (SrcVT.getScalarSizeInBits() == 32 || !Subtarget.hasAVX2())
46196 return false;
46197
46198 if (is128BitLaneCrossingShuffleMask(SrcVT.getSimpleVT(), ShuffleMask))
46199 return false;
46200
46201 return true;
46202}
46203
46204//===----------------------------------------------------------------------===//
46205// X86 Inline Assembly Support
46206//===----------------------------------------------------------------------===//
46207
46208// Helper to match a string separated by whitespace.
46209static bool matchAsm(StringRef S, ArrayRef<const char *> Pieces) {
46210 S = S.substr(S.find_first_not_of(" \t")); // Skip leading whitespace.
46211
46212 for (StringRef Piece : Pieces) {
46213 if (!S.startswith(Piece)) // Check if the piece matches.
46214 return false;
46215
46216 S = S.substr(Piece.size());
46217 StringRef::size_type Pos = S.find_first_not_of(" \t");
46218 if (Pos == 0) // We matched a prefix.
46219 return false;
46220
46221 S = S.substr(Pos);
46222 }
46223
46224 return S.empty();
46225}
46226
46227static bool clobbersFlagRegisters(const SmallVector<StringRef, 4> &AsmPieces) {
46228
46229 if (AsmPieces.size() == 3 || AsmPieces.size() == 4) {
46230 if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{cc}") &&
46231 std::count(AsmPieces.begin(), AsmPieces.end(), "~{flags}") &&
46232 std::count(AsmPieces.begin(), AsmPieces.end(), "~{fpsr}")) {
46233
46234 if (AsmPieces.size() == 3)
46235 return true;
46236 else if (std::count(AsmPieces.begin(), AsmPieces.end(), "~{dirflag}"))
46237 return true;
46238 }
46239 }
46240 return false;
46241}
46242
46243bool X86TargetLowering::ExpandInlineAsm(CallInst *CI) const {
46244 InlineAsm *IA = cast<InlineAsm>(CI->getCalledValue());
46245
46246 const std::string &AsmStr = IA->getAsmString();
46247
46248 IntegerType *Ty = dyn_cast<IntegerType>(CI->getType());
46249 if (!Ty || Ty->getBitWidth() % 16 != 0)
46250 return false;
46251
46252 // TODO: should remove alternatives from the asmstring: "foo {a|b}" -> "foo a"
46253 SmallVector<StringRef, 4> AsmPieces;
46254 SplitString(AsmStr, AsmPieces, ";\n");
46255
46256 switch (AsmPieces.size()) {
46257 default: return false;
46258 case 1:
46259 // FIXME: this should verify that we are targeting a 486 or better. If not,
46260 // we will turn this bswap into something that will be lowered to logical
46261 // ops instead of emitting the bswap asm. For now, we don't support 486 or
46262 // lower so don't worry about this.
46263 // bswap $0
46264 if (matchAsm(AsmPieces[0], {"bswap", "$0"}) ||
46265 matchAsm(AsmPieces[0], {"bswapl", "$0"}) ||
46266 matchAsm(AsmPieces[0], {"bswapq", "$0"}) ||
46267 matchAsm(AsmPieces[0], {"bswap", "${0:q}"}) ||
46268 matchAsm(AsmPieces[0], {"bswapl", "${0:q}"}) ||
46269 matchAsm(AsmPieces[0], {"bswapq", "${0:q}"})) {
46270 // No need to check constraints, nothing other than the equivalent of
46271 // "=r,0" would be valid here.
46272 return IntrinsicLowering::LowerToByteSwap(CI);
46273 }
46274
46275 // rorw $$8, ${0:w} --> llvm.bswap.i16
46276 if (CI->getType()->isIntegerTy(16) &&
46277 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
46278 (matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) ||
46279 matchAsm(AsmPieces[0], {"rolw", "$$8,", "${0:w}"}))) {
46280 AsmPieces.clear();
46281 StringRef ConstraintsStr = IA->getConstraintString();
46282 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
46283 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
46284 if (clobbersFlagRegisters(AsmPieces))
46285 return IntrinsicLowering::LowerToByteSwap(CI);
46286 }
46287 break;
46288 case 3:
46289 if (CI->getType()->isIntegerTy(32) &&
46290 IA->getConstraintString().compare(0, 5, "=r,0,") == 0 &&
46291 matchAsm(AsmPieces[0], {"rorw", "$$8,", "${0:w}"}) &&
46292 matchAsm(AsmPieces[1], {"rorl", "$$16,", "$0"}) &&
46293 matchAsm(AsmPieces[2], {"rorw", "$$8,", "${0:w}"})) {
46294 AsmPieces.clear();
46295 StringRef ConstraintsStr = IA->getConstraintString();
46296 SplitString(StringRef(ConstraintsStr).substr(5), AsmPieces, ",");
46297 array_pod_sort(AsmPieces.begin(), AsmPieces.end());
46298 if (clobbersFlagRegisters(AsmPieces))
46299 return IntrinsicLowering::LowerToByteSwap(CI);
46300 }
46301
46302 if (CI->getType()->isIntegerTy(64)) {
46303 InlineAsm::ConstraintInfoVector Constraints = IA->ParseConstraints();
46304 if (Constraints.size() >= 2 &&
46305 Constraints[0].Codes.size() == 1 && Constraints[0].Codes[0] == "A" &&
46306 Constraints[1].Codes.size() == 1 && Constraints[1].Codes[0] == "0") {
46307 // bswap %eax / bswap %edx / xchgl %eax, %edx -> llvm.bswap.i64
46308 if (matchAsm(AsmPieces[0], {"bswap", "%eax"}) &&
46309 matchAsm(AsmPieces[1], {"bswap", "%edx"}) &&
46310 matchAsm(AsmPieces[2], {"xchgl", "%eax,", "%edx"}))
46311 return IntrinsicLowering::LowerToByteSwap(CI);
46312 }
46313 }
46314 break;
46315 }
46316 return false;
46317}
46318
46319static X86::CondCode parseConstraintCode(llvm::StringRef Constraint) {
46320 X86::CondCode Cond = StringSwitch<X86::CondCode>(Constraint)
46321 .Case("{@cca}", X86::COND_A)
46322 .Case("{@ccae}", X86::COND_AE)
46323 .Case("{@ccb}", X86::COND_B)
46324 .Case("{@ccbe}", X86::COND_BE)
46325 .Case("{@ccc}", X86::COND_B)
46326 .Case("{@cce}", X86::COND_E)
46327 .Case("{@ccz}", X86::COND_E)
46328 .Case("{@ccg}", X86::COND_G)
46329 .Case("{@ccge}", X86::COND_GE)
46330 .Case("{@ccl}", X86::COND_L)
46331 .Case("{@ccle}", X86::COND_LE)
46332 .Case("{@ccna}", X86::COND_BE)
46333 .Case("{@ccnae}", X86::COND_B)
46334 .Case("{@ccnb}", X86::COND_AE)
46335 .Case("{@ccnbe}", X86::COND_A)
46336 .Case("{@ccnc}", X86::COND_AE)
46337 .Case("{@ccne}", X86::COND_NE)
46338 .Case("{@ccnz}", X86::COND_NE)
46339 .Case("{@ccng}", X86::COND_LE)
46340 .Case("{@ccnge}", X86::COND_L)
46341 .Case("{@ccnl}", X86::COND_GE)
46342 .Case("{@ccnle}", X86::COND_G)
46343 .Case("{@ccno}", X86::COND_NO)
46344 .Case("{@ccnp}", X86::COND_P)
46345 .Case("{@ccns}", X86::COND_NS)
46346 .Case("{@cco}", X86::COND_O)
46347 .Case("{@ccp}", X86::COND_P)
46348 .Case("{@ccs}", X86::COND_S)
46349 .Default(X86::COND_INVALID);
46350 return Cond;
46351}
46352
46353/// Given a constraint letter, return the type of constraint for this target.
46354X86TargetLowering::ConstraintType
46355X86TargetLowering::getConstraintType(StringRef Constraint) const {
46356 if (Constraint.size() == 1) {
46357 switch (Constraint[0]) {
46358 case 'R':
46359 case 'q':
46360 case 'Q':
46361 case 'f':
46362 case 't':
46363 case 'u':
46364 case 'y':
46365 case 'x':
46366 case 'v':
46367 case 'Y':
46368 case 'l':
46369 case 'k': // AVX512 masking registers.
46370 return C_RegisterClass;
46371 case 'a':
46372 case 'b':
46373 case 'c':
46374 case 'd':
46375 case 'S':
46376 case 'D':
46377 case 'A':
46378 return C_Register;
46379 case 'I':
46380 case 'J':
46381 case 'K':
46382 case 'N':
46383 case 'G':
46384 case 'L':
46385 case 'M':
46386 return C_Immediate;
46387 case 'C':
46388 case 'e':
46389 case 'Z':
46390 return C_Other;
46391 default:
46392 break;
46393 }
46394 }
46395 else if (Constraint.size() == 2) {
46396 switch (Constraint[0]) {
46397 default:
46398 break;
46399 case 'Y':
46400 switch (Constraint[1]) {
46401 default:
46402 break;
46403 case 'z':
46404 case '0':
46405 return C_Register;
46406 case 'i':
46407 case 'm':
46408 case 'k':
46409 case 't':
46410 case '2':
46411 return C_RegisterClass;
46412 }
46413 }
46414 } else if (parseConstraintCode(Constraint) != X86::COND_INVALID)
46415 return C_Other;
46416 return TargetLowering::getConstraintType(Constraint);
46417}
46418
46419/// Examine constraint type and operand type and determine a weight value.
46420/// This object must already have been set up with the operand type
46421/// and the current alternative constraint selected.
46422TargetLowering::ConstraintWeight
46423 X86TargetLowering::getSingleConstraintMatchWeight(
46424 AsmOperandInfo &info, const char *constraint) const {
46425 ConstraintWeight weight = CW_Invalid;
46426 Value *CallOperandVal = info.CallOperandVal;
46427 // If we don't have a value, we can't do a match,
46428 // but allow it at the lowest weight.
46429 if (!CallOperandVal)
46430 return CW_Default;
46431 Type *type = CallOperandVal->getType();
46432 // Look at the constraint type.
46433 switch (*constraint) {
46434 default:
46435 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
46436 LLVM_FALLTHROUGH[[gnu::fallthrough]];
46437 case 'R':
46438 case 'q':
46439 case 'Q':
46440 case 'a':
46441 case 'b':
46442 case 'c':
46443 case 'd':
46444 case 'S':
46445 case 'D':
46446 case 'A':
46447 if (CallOperandVal->getType()->isIntegerTy())
46448 weight = CW_SpecificReg;
46449 break;
46450 case 'f':
46451 case 't':
46452 case 'u':
46453 if (type->isFloatingPointTy())
46454 weight = CW_SpecificReg;
46455 break;
46456 case 'y':
46457 if (type->isX86_MMXTy() && Subtarget.hasMMX())
46458 weight = CW_SpecificReg;
46459 break;
46460 case 'Y': {
46461 unsigned Size = StringRef(constraint).size();
46462 // Pick 'i' as the next char as 'Yi' and 'Y' are synonymous, when matching 'Y'
46463 char NextChar = Size == 2 ? constraint[1] : 'i';
46464 if (Size > 2)
46465 break;
46466 switch (NextChar) {
46467 default:
46468 return CW_Invalid;
46469 // XMM0
46470 case 'z':
46471 case '0':
46472 if ((type->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1())
46473 return CW_SpecificReg;
46474 return CW_Invalid;
46475 // Conditional OpMask regs (AVX512)
46476 case 'k':
46477 if ((type->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
46478 return CW_Register;
46479 return CW_Invalid;
46480 // Any MMX reg
46481 case 'm':
46482 if (type->isX86_MMXTy() && Subtarget.hasMMX())
46483 return weight;
46484 return CW_Invalid;
46485 // Any SSE reg when ISA >= SSE2, same as 'Y'
46486 case 'i':
46487 case 't':
46488 case '2':
46489 if (!Subtarget.hasSSE2())
46490 return CW_Invalid;
46491 break;
46492 }
46493 // Fall through (handle "Y" constraint).
46494 LLVM_FALLTHROUGH[[gnu::fallthrough]];
46495 }
46496 case 'v':
46497 if ((type->getPrimitiveSizeInBits() == 512) && Subtarget.hasAVX512())
46498 weight = CW_Register;
46499 LLVM_FALLTHROUGH[[gnu::fallthrough]];
46500 case 'x':
46501 if (((type->getPrimitiveSizeInBits() == 128) && Subtarget.hasSSE1()) ||
46502 ((type->getPrimitiveSizeInBits() == 256) && Subtarget.hasAVX()))
46503 weight = CW_Register;
46504 break;
46505 case 'k':
46506 // Enable conditional vector operations using %k<#> registers.
46507 if ((type->getPrimitiveSizeInBits() == 64) && Subtarget.hasAVX512())
46508 weight = CW_Register;
46509 break;
46510 case 'I':
46511 if (ConstantInt *C = dyn_cast<ConstantInt>(info.CallOperandVal)) {
46512 if (C->getZExtValue() <= 31)
46513 weight = CW_Constant;
46514 }
46515 break;
46516 case 'J':
46517 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
46518 if (C->getZExtValue() <= 63)
46519 weight = CW_Constant;
46520 }
46521 break;
46522 case 'K':
46523 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
46524 if ((C->getSExtValue() >= -0x80) && (C->getSExtValue() <= 0x7f))
46525 weight = CW_Constant;
46526 }
46527 break;
46528 case 'L':
46529 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
46530 if ((C->getZExtValue() == 0xff) || (C->getZExtValue() == 0xffff))
46531 weight = CW_Constant;
46532 }
46533 break;
46534 case 'M':
46535 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
46536 if (C->getZExtValue() <= 3)
46537 weight = CW_Constant;
46538 }
46539 break;
46540 case 'N':
46541 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
46542 if (C->getZExtValue() <= 0xff)
46543 weight = CW_Constant;
46544 }
46545 break;
46546 case 'G':
46547 case 'C':
46548 if (isa<ConstantFP>(CallOperandVal)) {
46549 weight = CW_Constant;
46550 }
46551 break;
46552 case 'e':
46553 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
46554 if ((C->getSExtValue() >= -0x80000000LL) &&
46555 (C->getSExtValue() <= 0x7fffffffLL))
46556 weight = CW_Constant;
46557 }
46558 break;
46559 case 'Z':
46560 if (ConstantInt *C = dyn_cast<ConstantInt>(CallOperandVal)) {
46561 if (C->getZExtValue() <= 0xffffffff)
46562 weight = CW_Constant;
46563 }
46564 break;
46565 }
46566 return weight;
46567}
46568
46569/// Try to replace an X constraint, which matches anything, with another that
46570/// has more specific requirements based on the type of the corresponding
46571/// operand.
46572const char *X86TargetLowering::
46573LowerXConstraint(EVT ConstraintVT) const {
46574 // FP X constraints get lowered to SSE1/2 registers if available, otherwise
46575 // 'f' like normal targets.
46576 if (ConstraintVT.isFloatingPoint()) {
46577 if (Subtarget.hasSSE2())
46578 return "Y";
46579 if (Subtarget.hasSSE1())
46580 return "x";
46581 }
46582
46583 return TargetLowering::LowerXConstraint(ConstraintVT);
46584}
46585
46586// Lower @cc targets via setcc.
46587SDValue X86TargetLowering::LowerAsmOutputForConstraint(
46588 SDValue &Chain, SDValue &Flag, SDLoc DL, const AsmOperandInfo &OpInfo,
46589 SelectionDAG &DAG) const {
46590 X86::CondCode Cond = parseConstraintCode(OpInfo.ConstraintCode);
46591 if (Cond == X86::COND_INVALID)
46592 return SDValue();
46593 // Check that return type is valid.
46594 if (OpInfo.ConstraintVT.isVector() || !OpInfo.ConstraintVT.isInteger() ||
46595 OpInfo.ConstraintVT.getSizeInBits() < 8)
46596 report_fatal_error("Flag output operand is of invalid type");
46597
46598 // Get EFLAGS register. Only update chain when copyfrom is glued.
46599 if (Flag.getNode()) {
46600 Flag = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32, Flag);
46601 Chain = Flag.getValue(1);
46602 } else
46603 Flag = DAG.getCopyFromReg(Chain, DL, X86::EFLAGS, MVT::i32);
46604 // Extract CC code.
46605 SDValue CC = getSETCC(Cond, Flag, DL, DAG);
46606 // Extend to 32-bits
46607 SDValue Result = DAG.getNode(ISD::ZERO_EXTEND, DL, OpInfo.ConstraintVT, CC);
46608
46609 return Result;
46610}
46611
46612/// Lower the specified operand into the Ops vector.
46613/// If it is invalid, don't add anything to Ops.
46614void X86TargetLowering::LowerAsmOperandForConstraint(SDValue Op,
46615 std::string &Constraint,
46616 std::vector<SDValue>&Ops,
46617 SelectionDAG &DAG) const {
46618 SDValue Result;
46619
46620 // Only support length 1 constraints for now.
46621 if (Constraint.length() > 1) return;
46622
46623 char ConstraintLetter = Constraint[0];
46624 switch (ConstraintLetter) {
46625 default: break;
46626 case 'I':
46627 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46628 if (C->getZExtValue() <= 31) {
46629 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
46630 Op.getValueType());
46631 break;
46632 }
46633 }
46634 return;
46635 case 'J':
46636 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46637 if (C->getZExtValue() <= 63) {
46638 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
46639 Op.getValueType());
46640 break;
46641 }
46642 }
46643 return;
46644 case 'K':
46645 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46646 if (isInt<8>(C->getSExtValue())) {
46647 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
46648 Op.getValueType());
46649 break;
46650 }
46651 }
46652 return;
46653 case 'L':
46654 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46655 if (C->getZExtValue() == 0xff || C->getZExtValue() == 0xffff ||
46656 (Subtarget.is64Bit() && C->getZExtValue() == 0xffffffff)) {
46657 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op),
46658 Op.getValueType());
46659 break;
46660 }
46661 }
46662 return;
46663 case 'M':
46664 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46665 if (C->getZExtValue() <= 3) {
46666 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
46667 Op.getValueType());
46668 break;
46669 }
46670 }
46671 return;
46672 case 'N':
46673 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46674 if (C->getZExtValue() <= 255) {
46675 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
46676 Op.getValueType());
46677 break;
46678 }
46679 }
46680 return;
46681 case 'O':
46682 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46683 if (C->getZExtValue() <= 127) {
46684 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
46685 Op.getValueType());
46686 break;
46687 }
46688 }
46689 return;
46690 case 'e': {
46691 // 32-bit signed value
46692 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46693 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
46694 C->getSExtValue())) {
46695 // Widen to 64 bits here to get it sign extended.
46696 Result = DAG.getTargetConstant(C->getSExtValue(), SDLoc(Op), MVT::i64);
46697 break;
46698 }
46699 // FIXME gcc accepts some relocatable values here too, but only in certain
46700 // memory models; it's complicated.
46701 }
46702 return;
46703 }
46704 case 'Z': {
46705 // 32-bit unsigned value
46706 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Op)) {
46707 if (ConstantInt::isValueValidForType(Type::getInt32Ty(*DAG.getContext()),
46708 C->getZExtValue())) {
46709 Result = DAG.getTargetConstant(C->getZExtValue(), SDLoc(Op),
46710 Op.getValueType());
46711 break;
46712 }
46713 }
46714 // FIXME gcc accepts some relocatable values here too, but only in certain
46715 // memory models; it's complicated.
46716 return;
46717 }
46718 case 'i': {
46719 // Literal immediates are always ok.
46720 if (ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Op)) {
46721 bool IsBool = CST->getConstantIntValue()->getBitWidth() == 1;
46722 BooleanContent BCont = getBooleanContents(MVT::i64);
46723 ISD::NodeType ExtOpc = IsBool ? getExtendForContent(BCont)
46724 : ISD::SIGN_EXTEND;
46725 int64_t ExtVal = ExtOpc == ISD::ZERO_EXTEND ? CST->getZExtValue()
46726 : CST->getSExtValue();
46727 Result = DAG.getTargetConstant(ExtVal, SDLoc(Op), MVT::i64);
46728 break;
46729 }
46730
46731 // In any sort of PIC mode addresses need to be computed at runtime by
46732 // adding in a register or some sort of table lookup. These can't
46733 // be used as immediates.
46734 if (Subtarget.isPICStyleGOT() || Subtarget.isPICStyleStubPIC())
46735 return;
46736
46737 // If we are in non-pic codegen mode, we allow the address of a global (with
46738 // an optional displacement) to be used with 'i'.
46739 if (auto *GA = dyn_cast<GlobalAddressSDNode>(Op))
46740 // If we require an extra load to get this address, as in PIC mode, we
46741 // can't accept it.
46742 if (isGlobalStubReference(
46743 Subtarget.classifyGlobalReference(GA->getGlobal())))
46744 return;
46745 break;
46746 }
46747 }
46748
46749 if (Result.getNode()) {
46750 Ops.push_back(Result);
46751 return;
46752 }
46753 return TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
46754}
46755
46756/// Check if \p RC is a general purpose register class.
46757/// I.e., GR* or one of their variant.
46758static bool isGRClass(const TargetRegisterClass &RC) {
46759 return RC.hasSuperClassEq(&X86::GR8RegClass) ||
46760 RC.hasSuperClassEq(&X86::GR16RegClass) ||
46761 RC.hasSuperClassEq(&X86::GR32RegClass) ||
46762 RC.hasSuperClassEq(&X86::GR64RegClass) ||
46763 RC.hasSuperClassEq(&X86::LOW32_ADDR_ACCESS_RBPRegClass);
46764}
46765
46766/// Check if \p RC is a vector register class.
46767/// I.e., FR* / VR* or one of their variant.
46768static bool isFRClass(const TargetRegisterClass &RC) {
46769 return RC.hasSuperClassEq(&X86::FR32XRegClass) ||
46770 RC.hasSuperClassEq(&X86::FR64XRegClass) ||
46771 RC.hasSuperClassEq(&X86::VR128XRegClass) ||
46772 RC.hasSuperClassEq(&X86::VR256XRegClass) ||
46773 RC.hasSuperClassEq(&X86::VR512RegClass);
46774}
46775
46776/// Check if \p RC is a mask register class.
46777/// I.e., VK* or one of their variant.
46778static bool isVKClass(const TargetRegisterClass &RC) {
46779 return RC.hasSuperClassEq(&X86::VK1RegClass) ||
46780 RC.hasSuperClassEq(&X86::VK2RegClass) ||
46781 RC.hasSuperClassEq(&X86::VK4RegClass) ||
46782 RC.hasSuperClassEq(&X86::VK8RegClass) ||
46783 RC.hasSuperClassEq(&X86::VK16RegClass) ||
46784 RC.hasSuperClassEq(&X86::VK32RegClass) ||
46785 RC.hasSuperClassEq(&X86::VK64RegClass);
46786}
46787
46788std::pair<unsigned, const TargetRegisterClass *>
46789X86TargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
46790 StringRef Constraint,
46791 MVT VT) const {
46792 // First, see if this is a constraint that directly corresponds to an LLVM
46793 // register class.
46794 if (Constraint.size() == 1) {
46795 // GCC Constraint Letters
46796 switch (Constraint[0]) {
46797 default: break;
46798 // 'A' means [ER]AX + [ER]DX.
46799 case 'A':
46800 if (Subtarget.is64Bit())
46801 return std::make_pair(X86::RAX, &X86::GR64_ADRegClass);
46802 assert((Subtarget.is32Bit() || Subtarget.is16Bit()) &&(((Subtarget.is32Bit() || Subtarget.is16Bit()) && "Expecting 64, 32 or 16 bit subtarget"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.is32Bit() || Subtarget.is16Bit()) && \"Expecting 64, 32 or 16 bit subtarget\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 46803, __PRETTY_FUNCTION__))
46803 "Expecting 64, 32 or 16 bit subtarget")(((Subtarget.is32Bit() || Subtarget.is16Bit()) && "Expecting 64, 32 or 16 bit subtarget"
) ? static_cast<void> (0) : __assert_fail ("(Subtarget.is32Bit() || Subtarget.is16Bit()) && \"Expecting 64, 32 or 16 bit subtarget\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 46803, __PRETTY_FUNCTION__));
46804 return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
46805
46806 // TODO: Slight differences here in allocation order and leaving
46807 // RIP in the class. Do they matter any more here than they do
46808 // in the normal allocation?
46809 case 'k':
46810 if (Subtarget.hasAVX512()) {
46811 if (VT == MVT::i1)
46812 return std::make_pair(0U, &X86::VK1RegClass);
46813 if (VT == MVT::i8)
46814 return std::make_pair(0U, &X86::VK8RegClass);
46815 if (VT == MVT::i16)
46816 return std::make_pair(0U, &X86::VK16RegClass);
46817 }
46818 if (Subtarget.hasBWI()) {
46819 if (VT == MVT::i32)
46820 return std::make_pair(0U, &X86::VK32RegClass);
46821 if (VT == MVT::i64)
46822 return std::make_pair(0U, &X86::VK64RegClass);
46823 }
46824 break;
46825 case 'q': // GENERAL_REGS in 64-bit mode, Q_REGS in 32-bit mode.
46826 if (Subtarget.is64Bit()) {
46827 if (VT == MVT::i32 || VT == MVT::f32)
46828 return std::make_pair(0U, &X86::GR32RegClass);
46829 if (VT == MVT::i16)
46830 return std::make_pair(0U, &X86::GR16RegClass);
46831 if (VT == MVT::i8 || VT == MVT::i1)
46832 return std::make_pair(0U, &X86::GR8RegClass);
46833 if (VT == MVT::i64 || VT == MVT::f64)
46834 return std::make_pair(0U, &X86::GR64RegClass);
46835 break;
46836 }
46837 LLVM_FALLTHROUGH[[gnu::fallthrough]];
46838 // 32-bit fallthrough
46839 case 'Q': // Q_REGS
46840 if (VT == MVT::i32 || VT == MVT::f32)
46841 return std::make_pair(0U, &X86::GR32_ABCDRegClass);
46842 if (VT == MVT::i16)
46843 return std::make_pair(0U, &X86::GR16_ABCDRegClass);
46844 if (VT == MVT::i8 || VT == MVT::i1)
46845 return std::make_pair(0U, &X86::GR8_ABCD_LRegClass);
46846 if (VT == MVT::i64)
46847 return std::make_pair(0U, &X86::GR64_ABCDRegClass);
46848 break;
46849 case 'r': // GENERAL_REGS
46850 case 'l': // INDEX_REGS
46851 if (VT == MVT::i8 || VT == MVT::i1)
46852 return std::make_pair(0U, &X86::GR8RegClass);
46853 if (VT == MVT::i16)
46854 return std::make_pair(0U, &X86::GR16RegClass);
46855 if (VT == MVT::i32 || VT == MVT::f32 || !Subtarget.is64Bit())
46856 return std::make_pair(0U, &X86::GR32RegClass);
46857 return std::make_pair(0U, &X86::GR64RegClass);
46858 case 'R': // LEGACY_REGS
46859 if (VT == MVT::i8 || VT == MVT::i1)
46860 return std::make_pair(0U, &X86::GR8_NOREXRegClass);
46861 if (VT == MVT::i16)
46862 return std::make_pair(0U, &X86::GR16_NOREXRegClass);
46863 if (VT == MVT::i32 || !Subtarget.is64Bit())
46864 return std::make_pair(0U, &X86::GR32_NOREXRegClass);
46865 return std::make_pair(0U, &X86::GR64_NOREXRegClass);
46866 case 'f': // FP Stack registers.
46867 // If SSE is enabled for this VT, use f80 to ensure the isel moves the
46868 // value to the correct fpstack register class.
46869 if (VT == MVT::f32 && !isScalarFPTypeInSSEReg(VT))
46870 return std::make_pair(0U, &X86::RFP32RegClass);
46871 if (VT == MVT::f64 && !isScalarFPTypeInSSEReg(VT))
46872 return std::make_pair(0U, &X86::RFP64RegClass);
46873 return std::make_pair(0U, &X86::RFP80RegClass);
46874 case 'y': // MMX_REGS if MMX allowed.
46875 if (!Subtarget.hasMMX()) break;
46876 return std::make_pair(0U, &X86::VR64RegClass);
46877 case 'Y': // SSE_REGS if SSE2 allowed
46878 if (!Subtarget.hasSSE2()) break;
46879 LLVM_FALLTHROUGH[[gnu::fallthrough]];
46880 case 'v':
46881 case 'x': // SSE_REGS if SSE1 allowed or AVX_REGS if AVX allowed
46882 if (!Subtarget.hasSSE1()) break;
46883 bool VConstraint = (Constraint[0] == 'v');
46884
46885 switch (VT.SimpleTy) {
46886 default: break;
46887 // Scalar SSE types.
46888 case MVT::f32:
46889 case MVT::i32:
46890 if (VConstraint && Subtarget.hasVLX())
46891 return std::make_pair(0U, &X86::FR32XRegClass);
46892 return std::make_pair(0U, &X86::FR32RegClass);
46893 case MVT::f64:
46894 case MVT::i64:
46895 if (VConstraint && Subtarget.hasVLX())
46896 return std::make_pair(0U, &X86::FR64XRegClass);
46897 return std::make_pair(0U, &X86::FR64RegClass);
46898 // TODO: Handle i128 in FR128RegClass after it is tested well.
46899 // Vector types and fp128.
46900 case MVT::f128:
46901 case MVT::v16i8:
46902 case MVT::v8i16:
46903 case MVT::v4i32:
46904 case MVT::v2i64:
46905 case MVT::v4f32:
46906 case MVT::v2f64:
46907 if (VConstraint && Subtarget.hasVLX())
46908 return std::make_pair(0U, &X86::VR128XRegClass);
46909 return std::make_pair(0U, &X86::VR128RegClass);
46910 // AVX types.
46911 case MVT::v32i8:
46912 case MVT::v16i16:
46913 case MVT::v8i32:
46914 case MVT::v4i64:
46915 case MVT::v8f32:
46916 case MVT::v4f64:
46917 if (VConstraint && Subtarget.hasVLX())
46918 return std::make_pair(0U, &X86::VR256XRegClass);
46919 if (Subtarget.hasAVX())
46920 return std::make_pair(0U, &X86::VR256RegClass);
46921 break;
46922 case MVT::v8f64:
46923 case MVT::v16f32:
46924 case MVT::v16i32:
46925 case MVT::v8i64:
46926 if (!Subtarget.hasAVX512()) break;
46927 if (VConstraint)
46928 return std::make_pair(0U, &X86::VR512RegClass);
46929 return std::make_pair(0U, &X86::VR512_0_15RegClass);
46930 }
46931 break;
46932 }
46933 } else if (Constraint.size() == 2 && Constraint[0] == 'Y') {
46934 switch (Constraint[1]) {
46935 default:
46936 break;
46937 case 'i':
46938 case 't':
46939 case '2':
46940 return getRegForInlineAsmConstraint(TRI, "Y", VT);
46941 case 'm':
46942 if (!Subtarget.hasMMX()) break;
46943 return std::make_pair(0U, &X86::VR64RegClass);
46944 case 'z':
46945 case '0':
46946 if (!Subtarget.hasSSE1()) break;
46947 return std::make_pair(X86::XMM0, &X86::VR128RegClass);
46948 case 'k':
46949 // This register class doesn't allocate k0 for masked vector operation.
46950 if (Subtarget.hasAVX512()) {
46951 if (VT == MVT::i1)
46952 return std::make_pair(0U, &X86::VK1WMRegClass);
46953 if (VT == MVT::i8)
46954 return std::make_pair(0U, &X86::VK8WMRegClass);
46955 if (VT == MVT::i16)
46956 return std::make_pair(0U, &X86::VK16WMRegClass);
46957 }
46958 if (Subtarget.hasBWI()) {
46959 if (VT == MVT::i32)
46960 return std::make_pair(0U, &X86::VK32WMRegClass);
46961 if (VT == MVT::i64)
46962 return std::make_pair(0U, &X86::VK64WMRegClass);
46963 }
46964 break;
46965 }
46966 }
46967
46968 if (parseConstraintCode(Constraint) != X86::COND_INVALID)
46969 return std::make_pair(0U, &X86::GR32RegClass);
46970
46971 // Use the default implementation in TargetLowering to convert the register
46972 // constraint into a member of a register class.
46973 std::pair<unsigned, const TargetRegisterClass*> Res;
46974 Res = TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
46975
46976 // Not found as a standard register?
46977 if (!Res.second) {
46978 // Map st(0) -> st(7) -> ST0
46979 if (Constraint.size() == 7 && Constraint[0] == '{' &&
46980 tolower(Constraint[1]) == 's' && tolower(Constraint[2]) == 't' &&
46981 Constraint[3] == '(' &&
46982 (Constraint[4] >= '0' && Constraint[4] <= '7') &&
46983 Constraint[5] == ')' && Constraint[6] == '}') {
46984 // st(7) is not allocatable and thus not a member of RFP80. Return
46985 // singleton class in cases where we have a reference to it.
46986 if (Constraint[4] == '7')
46987 return std::make_pair(X86::FP7, &X86::RFP80_7RegClass);
46988 return std::make_pair(X86::FP0 + Constraint[4] - '0',
46989 &X86::RFP80RegClass);
46990 }
46991
46992 // GCC allows "st(0)" to be called just plain "st".
46993 if (StringRef("{st}").equals_lower(Constraint))
46994 return std::make_pair(X86::FP0, &X86::RFP80RegClass);
46995
46996 // flags -> EFLAGS
46997 if (StringRef("{flags}").equals_lower(Constraint))
46998 return std::make_pair(X86::EFLAGS, &X86::CCRRegClass);
46999
47000 // dirflag -> DF
47001 if (StringRef("{dirflag}").equals_lower(Constraint))
47002 return std::make_pair(X86::DF, &X86::DFCCRRegClass);
47003
47004 // fpsr -> FPSW
47005 if (StringRef("{fpsr}").equals_lower(Constraint))
47006 return std::make_pair(X86::FPSW, &X86::FPCCRRegClass);
47007
47008 return Res;
47009 }
47010
47011 // Make sure it isn't a register that requires 64-bit mode.
47012 if (!Subtarget.is64Bit() &&
47013 (isFRClass(*Res.second) || isGRClass(*Res.second)) &&
47014 TRI->getEncodingValue(Res.first) >= 8) {
47015 // Register requires REX prefix, but we're in 32-bit mode.
47016 return std::make_pair(0, nullptr);
47017 }
47018
47019 // Make sure it isn't a register that requires AVX512.
47020 if (!Subtarget.hasAVX512() && isFRClass(*Res.second) &&
47021 TRI->getEncodingValue(Res.first) & 0x10) {
47022 // Register requires EVEX prefix.
47023 return std::make_pair(0, nullptr);
47024 }
47025
47026 // Otherwise, check to see if this is a register class of the wrong value
47027 // type. For example, we want to map "{ax},i32" -> {eax}, we don't want it to
47028 // turn into {ax},{dx}.
47029 // MVT::Other is used to specify clobber names.
47030 if (TRI->isTypeLegalForClass(*Res.second, VT) || VT == MVT::Other)
47031 return Res; // Correct type already, nothing to do.
47032
47033 // Get a matching integer of the correct size. i.e. "ax" with MVT::32 should
47034 // return "eax". This should even work for things like getting 64bit integer
47035 // registers when given an f64 type.
47036 const TargetRegisterClass *Class = Res.second;
47037 // The generic code will match the first register class that contains the
47038 // given register. Thus, based on the ordering of the tablegened file,
47039 // the "plain" GR classes might not come first.
47040 // Therefore, use a helper method.
47041 if (isGRClass(*Class)) {
47042 unsigned Size = VT.getSizeInBits();
47043 if (Size == 1) Size = 8;
47044 unsigned DestReg = getX86SubSuperRegisterOrZero(Res.first, Size);
47045 if (DestReg > 0) {
47046 bool is64Bit = Subtarget.is64Bit();
47047 const TargetRegisterClass *RC =
47048 Size == 8 ? (is64Bit ? &X86::GR8RegClass : &X86::GR8_NOREXRegClass)
47049 : Size == 16 ? (is64Bit ? &X86::GR16RegClass : &X86::GR16_NOREXRegClass)
47050 : Size == 32 ? (is64Bit ? &X86::GR32RegClass : &X86::GR32_NOREXRegClass)
47051 : Size == 64 ? (is64Bit ? &X86::GR64RegClass : nullptr)
47052 : nullptr;
47053 if (Size == 64 && !is64Bit) {
47054 // Model GCC's behavior here and select a fixed pair of 32-bit
47055 // registers.
47056 switch (DestReg) {
47057 case X86::RAX:
47058 return std::make_pair(X86::EAX, &X86::GR32_ADRegClass);
47059 case X86::RDX:
47060 return std::make_pair(X86::EDX, &X86::GR32_DCRegClass);
47061 case X86::RCX:
47062 return std::make_pair(X86::ECX, &X86::GR32_CBRegClass);
47063 case X86::RBX:
47064 return std::make_pair(X86::EBX, &X86::GR32_BSIRegClass);
47065 case X86::RSI:
47066 return std::make_pair(X86::ESI, &X86::GR32_SIDIRegClass);
47067 case X86::RDI:
47068 return std::make_pair(X86::EDI, &X86::GR32_DIBPRegClass);
47069 case X86::RBP:
47070 return std::make_pair(X86::EBP, &X86::GR32_BPSPRegClass);
47071 default:
47072 return std::make_pair(0, nullptr);
47073 }
47074 }
47075 if (RC && RC->contains(DestReg))
47076 return std::make_pair(DestReg, RC);
47077 return Res;
47078 }
47079 // No register found/type mismatch.
47080 return std::make_pair(0, nullptr);
47081 } else if (isFRClass(*Class)) {
47082 // Handle references to XMM physical registers that got mapped into the
47083 // wrong class. This can happen with constraints like {xmm0} where the
47084 // target independent register mapper will just pick the first match it can
47085 // find, ignoring the required type.
47086
47087 // TODO: Handle f128 and i128 in FR128RegClass after it is tested well.
47088 if (VT == MVT::f32 || VT == MVT::i32)
47089 Res.second = &X86::FR32XRegClass;
47090 else if (VT == MVT::f64 || VT == MVT::i64)
47091 Res.second = &X86::FR64XRegClass;
47092 else if (TRI->isTypeLegalForClass(X86::VR128XRegClass, VT))
47093 Res.second = &X86::VR128XRegClass;
47094 else if (TRI->isTypeLegalForClass(X86::VR256XRegClass, VT))
47095 Res.second = &X86::VR256XRegClass;
47096 else if (TRI->isTypeLegalForClass(X86::VR512RegClass, VT))
47097 Res.second = &X86::VR512RegClass;
47098 else {
47099 // Type mismatch and not a clobber: Return an error;
47100 Res.first = 0;
47101 Res.second = nullptr;
47102 }
47103 } else if (isVKClass(*Class)) {
47104 if (VT == MVT::i1)
47105 Res.second = &X86::VK1RegClass;
47106 else if (VT == MVT::i8)
47107 Res.second = &X86::VK8RegClass;
47108 else if (VT == MVT::i16)
47109 Res.second = &X86::VK16RegClass;
47110 else if (VT == MVT::i32)
47111 Res.second = &X86::VK32RegClass;
47112 else if (VT == MVT::i64)
47113 Res.second = &X86::VK64RegClass;
47114 else {
47115 // Type mismatch and not a clobber: Return an error;
47116 Res.first = 0;
47117 Res.second = nullptr;
47118 }
47119 }
47120
47121 return Res;
47122}
47123
47124int X86TargetLowering::getScalingFactorCost(const DataLayout &DL,
47125 const AddrMode &AM, Type *Ty,
47126 unsigned AS) const {
47127 // Scaling factors are not free at all.
47128 // An indexed folded instruction, i.e., inst (reg1, reg2, scale),
47129 // will take 2 allocations in the out of order engine instead of 1
47130 // for plain addressing mode, i.e. inst (reg1).
47131 // E.g.,
47132 // vaddps (%rsi,%rdx), %ymm0, %ymm1
47133 // Requires two allocations (one for the load, one for the computation)
47134 // whereas:
47135 // vaddps (%rsi), %ymm0, %ymm1
47136 // Requires just 1 allocation, i.e., freeing allocations for other operations
47137 // and having less micro operations to execute.
47138 //
47139 // For some X86 architectures, this is even worse because for instance for
47140 // stores, the complex addressing mode forces the instruction to use the
47141 // "load" ports instead of the dedicated "store" port.
47142 // E.g., on Haswell:
47143 // vmovaps %ymm1, (%r8, %rdi) can use port 2 or 3.
47144 // vmovaps %ymm1, (%r8) can use port 2, 3, or 7.
47145 if (isLegalAddressingMode(DL, AM, Ty, AS))
47146 // Scale represents reg2 * scale, thus account for 1
47147 // as soon as we use a second register.
47148 return AM.Scale != 0;
47149 return -1;
47150}
47151
47152bool X86TargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
47153 // Integer division on x86 is expensive. However, when aggressively optimizing
47154 // for code size, we prefer to use a div instruction, as it is usually smaller
47155 // than the alternative sequence.
47156 // The exception to this is vector division. Since x86 doesn't have vector
47157 // integer division, leaving the division as-is is a loss even in terms of
47158 // size, because it will have to be scalarized, while the alternative code
47159 // sequence can be performed in vector form.
47160 bool OptSize =
47161 Attr.hasAttribute(AttributeList::FunctionIndex, Attribute::MinSize);
47162 return OptSize && !VT.isVector();
47163}
47164
47165void X86TargetLowering::initializeSplitCSR(MachineBasicBlock *Entry) const {
47166 if (!Subtarget.is64Bit())
47167 return;
47168
47169 // Update IsSplitCSR in X86MachineFunctionInfo.
47170 X86MachineFunctionInfo *AFI =
47171 Entry->getParent()->getInfo<X86MachineFunctionInfo>();
47172 AFI->setIsSplitCSR(true);
47173}
47174
47175void X86TargetLowering::insertCopiesSplitCSR(
47176 MachineBasicBlock *Entry,
47177 const SmallVectorImpl<MachineBasicBlock *> &Exits) const {
47178 const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
47179 const MCPhysReg *IStart = TRI->getCalleeSavedRegsViaCopy(Entry->getParent());
47180 if (!IStart)
47181 return;
47182
47183 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
47184 MachineRegisterInfo *MRI = &Entry->getParent()->getRegInfo();
47185 MachineBasicBlock::iterator MBBI = Entry->begin();
47186 for (const MCPhysReg *I = IStart; *I; ++I) {
47187 const TargetRegisterClass *RC = nullptr;
47188 if (X86::GR64RegClass.contains(*I))
47189 RC = &X86::GR64RegClass;
47190 else
47191 llvm_unreachable("Unexpected register class in CSRsViaCopy!")::llvm::llvm_unreachable_internal("Unexpected register class in CSRsViaCopy!"
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 47191);
47192
47193 Register NewVR = MRI->createVirtualRegister(RC);
47194 // Create copy from CSR to a virtual register.
47195 // FIXME: this currently does not emit CFI pseudo-instructions, it works
47196 // fine for CXX_FAST_TLS since the C++-style TLS access functions should be
47197 // nounwind. If we want to generalize this later, we may need to emit
47198 // CFI pseudo-instructions.
47199 assert(((Entry->getParent()->getFunction().hasFnAttribute(Attribute
::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"
) ? static_cast<void> (0) : __assert_fail ("Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) && \"Function should be nounwind in insertCopiesSplitCSR!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 47201, __PRETTY_FUNCTION__))
47200 Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) &&((Entry->getParent()->getFunction().hasFnAttribute(Attribute
::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"
) ? static_cast<void> (0) : __assert_fail ("Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) && \"Function should be nounwind in insertCopiesSplitCSR!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 47201, __PRETTY_FUNCTION__))
47201 "Function should be nounwind in insertCopiesSplitCSR!")((Entry->getParent()->getFunction().hasFnAttribute(Attribute
::NoUnwind) && "Function should be nounwind in insertCopiesSplitCSR!"
) ? static_cast<void> (0) : __assert_fail ("Entry->getParent()->getFunction().hasFnAttribute(Attribute::NoUnwind) && \"Function should be nounwind in insertCopiesSplitCSR!\""
, "/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86ISelLowering.cpp"
, 47201, __PRETTY_FUNCTION__));
47202 Entry->addLiveIn(*I);
47203 BuildMI(*Entry, MBBI, DebugLoc(), TII->get(TargetOpcode::COPY), NewVR)
47204 .addReg(*I);
47205
47206 // Insert the copy-back instructions right before the terminator.
47207 for (auto *Exit : Exits)
47208 BuildMI(*Exit, Exit->getFirstTerminator(), DebugLoc(),
47209 TII->get(TargetOpcode::COPY), *I)
47210 .addReg(NewVR);
47211 }
47212}
47213
47214bool X86TargetLowering::supportSwiftError() const {
47215 return Subtarget.is64Bit();
47216}
47217
47218/// Returns the name of the symbol used to emit stack probes or the empty
47219/// string if not applicable.
47220StringRef
47221X86TargetLowering::getStackProbeSymbolName(MachineFunction &MF) const {
47222 // If the function specifically requests stack probes, emit them.
47223 if (MF.getFunction().hasFnAttribute("probe-stack"))
47224 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString();
47225
47226 // Generally, if we aren't on Windows, the platform ABI does not include
47227 // support for stack probes, so don't emit them.
47228 if (!Subtarget.isOSWindows() || Subtarget.isTargetMachO() ||
47229 MF.getFunction().hasFnAttribute("no-stack-arg-probe"))
47230 return "";
47231
47232 // We need a stack probe to conform to the Windows ABI. Choose the right
47233 // symbol.
47234 if (Subtarget.is64Bit())
47235 return Subtarget.isTargetCygMing() ? "___chkstk_ms" : "__chkstk";
47236 return Subtarget.isTargetCygMing() ? "_alloca" : "_chkstk";
47237}
47238
47239unsigned
47240X86TargetLowering::getStackProbeSize(MachineFunction &MF) const {
47241 // The default stack probe size is 4096 if the function has no stackprobesize
47242 // attribute.
47243 unsigned StackProbeSize = 4096;
47244 const Function &Fn = MF.getFunction();
47245 if (Fn.hasFnAttribute("stack-probe-size"))
47246 Fn.getFnAttribute("stack-probe-size")
47247 .getValueAsString()
47248 .getAsInteger(0, StackProbeSize);
47249 return StackProbeSize;
47250}
←

/build/llvm-toolchain-snapshot-10~++20200110111110+a1cc19b5814/llvm/lib/Target/X86/X86Subtarget.h

1//===-- X86Subtarget.h - Define Subtarget for the X86 ----------*- C++ -*--===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the X86 specific subclass of TargetSubtargetInfo.
10//
11//===----------------------------------------------------------------------===//
12
13#ifndef LLVM_LIB_TARGET_X86_X86SUBTARGET_H
14#define LLVM_LIB_TARGET_X86_X86SUBTARGET_H
15
16#include "X86FrameLowering.h"
17#include "X86ISelLowering.h"
18#include "X86InstrInfo.h"
19#include "X86SelectionDAGInfo.h"
20#include "llvm/ADT/StringRef.h"
21#include "llvm/ADT/Triple.h"
22#include "llvm/CodeGen/GlobalISel/CallLowering.h"
23#include "llvm/CodeGen/GlobalISel/InstructionSelector.h"
24#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h"
25#include "llvm/CodeGen/GlobalISel/RegisterBankInfo.h"
26#include "llvm/CodeGen/TargetSubtargetInfo.h"
27#include "llvm/IR/CallingConv.h"
28#include "llvm/Target/TargetMachine.h"
29#include <climits>
30#include <memory>
31
32#define GET_SUBTARGETINFO_HEADER
33#include "X86GenSubtargetInfo.inc"
34
35namespace llvm {
36
37class GlobalValue;
38
39/// The X86 backend supports a number of different styles of PIC.
40///
41namespace PICStyles {
42
43enum class Style {
44 StubPIC, // Used on i386-darwin in pic mode.
45 GOT, // Used on 32 bit elf on when in pic mode.
46 RIPRel, // Used on X86-64 when in pic mode.
47 None // Set when not in pic mode.
48};
49
50} // end namespace PICStyles
51
52class X86Subtarget final : public X86GenSubtargetInfo {
53public:
54 // NOTE: Do not add anything new to this list. Coarse, CPU name based flags
55 // are not a good idea. We should be migrating away from these.
56 enum X86ProcFamilyEnum {
57 Others,
58 IntelAtom,
59 IntelSLM
60 };
61
62protected:
63 enum X86SSEEnum {
64 NoSSE, SSE1, SSE2, SSE3, SSSE3, SSE41, SSE42, AVX, AVX2, AVX512F
65 };
66
67 enum X863DNowEnum {
68 NoThreeDNow, MMX, ThreeDNow, ThreeDNowA
69 };
70
71 /// X86 processor family: Intel Atom, and others
72 X86ProcFamilyEnum X86ProcFamily = Others;
73
74 /// Which PIC style to use
75 PICStyles::Style PICStyle;
76
77 const TargetMachine &TM;
78
79 /// SSE1, SSE2, SSE3, SSSE3, SSE41, SSE42, or none supported.
80 X86SSEEnum X86SSELevel = NoSSE;
81
82 /// MMX, 3DNow, 3DNow Athlon, or none supported.
83 X863DNowEnum X863DNowLevel = NoThreeDNow;
84
85 /// True if the processor supports X87 instructions.
86 bool HasX87 = false;
87
88 /// True if the processor supports CMPXCHG8B.
89 bool HasCmpxchg8b = false;
90
91 /// True if this processor has NOPL instruction
92 /// (generally pentium pro+).
93 bool HasNOPL = false;
94
95 /// True if this processor has conditional move instructions
96 /// (generally pentium pro+).
97 bool HasCMov = false;
98
99 /// True if the processor supports X86-64 instructions.
100 bool HasX86_64 = false;
101
102 /// True if the processor supports POPCNT.
103 bool HasPOPCNT = false;
104
105 /// True if the processor supports SSE4A instructions.
106 bool HasSSE4A = false;
107
108 /// Target has AES instructions
109 bool HasAES = false;
110 bool HasVAES = false;
111
112 /// Target has FXSAVE/FXRESTOR instructions
113 bool HasFXSR = false;
114
115 /// Target has XSAVE instructions
116 bool HasXSAVE = false;
117
118 /// Target has XSAVEOPT instructions
119 bool HasXSAVEOPT = false;
120
121 /// Target has XSAVEC instructions
122 bool HasXSAVEC = false;
123
124 /// Target has XSAVES instructions
125 bool HasXSAVES = false;
126
127 /// Target has carry-less multiplication
128 bool HasPCLMUL = false;
129 bool HasVPCLMULQDQ = false;
130
131 /// Target has Galois Field Arithmetic instructions
132 bool HasGFNI = false;
133
134 /// Target has 3-operand fused multiply-add
135 bool HasFMA = false;
136
137 /// Target has 4-operand fused multiply-add
138 bool HasFMA4 = false;
139
140 /// Target has XOP instructions
141 bool HasXOP = false;
142
143 /// Target has TBM instructions.
144 bool HasTBM = false;
145
146 /// Target has LWP instructions
147 bool HasLWP = false;
148
149 /// True if the processor has the MOVBE instruction.
150 bool HasMOVBE = false;
151
152 /// True if the processor has the RDRAND instruction.
153 bool HasRDRAND = false;
154
155 /// Processor has 16-bit floating point conversion instructions.
156 bool HasF16C = false;
157
158 /// Processor has FS/GS base insturctions.
159 bool HasFSGSBase = false;
160
161 /// Processor has LZCNT instruction.
162 bool HasLZCNT = false;
163
164 /// Processor has BMI1 instructions.
165 bool HasBMI = false;
166
167 /// Processor has BMI2 instructions.
168 bool HasBMI2 = false;
169
170 /// Processor has VBMI instructions.
171 bool HasVBMI = false;
172
173 /// Processor has VBMI2 instructions.
174 bool HasVBMI2 = false;
175
176 /// Processor has Integer Fused Multiply Add
177 bool HasIFMA = false;
178
179 /// Processor has RTM instructions.
180 bool HasRTM = false;
181
182 /// Processor has ADX instructions.
183 bool HasADX = false;
184
185 /// Processor has SHA instructions.
186 bool HasSHA = false;
187
188 /// Processor has PRFCHW instructions.
189 bool HasPRFCHW = false;
190
191 /// Processor has RDSEED instructions.
192 bool HasRDSEED = false;
193
194 /// Processor has LAHF/SAHF instructions.
195 bool HasLAHFSAHF = false;
196
197 /// Processor has MONITORX/MWAITX instructions.
198 bool HasMWAITX = false;
199
200 /// Processor has Cache Line Zero instruction
201 bool HasCLZERO = false;
202
203 /// Processor has Cache Line Demote instruction
204 bool HasCLDEMOTE = false;
205
206 /// Processor has MOVDIRI instruction (direct store integer).
207 bool HasMOVDIRI = false;
208
209 /// Processor has MOVDIR64B instruction (direct store 64 bytes).
210 bool HasMOVDIR64B = false;
211
212 /// Processor has ptwrite instruction.
213 bool HasPTWRITE = false;
214
215 /// Processor has Prefetch with intent to Write instruction
216 bool HasPREFETCHWT1 = false;
217
218 /// True if SHLD instructions are slow.
219 bool IsSHLDSlow = false;
220
221 /// True if the PMULLD instruction is slow compared to PMULLW/PMULHW and
222 // PMULUDQ.
223 bool IsPMULLDSlow = false;
224
225 /// True if the PMADDWD instruction is slow compared to PMULLD.
226 bool IsPMADDWDSlow = false;
227
228 /// True if unaligned memory accesses of 16-bytes are slow.
229 bool IsUAMem16Slow = false;
230
231 /// True if unaligned memory accesses of 32-bytes are slow.
232 bool IsUAMem32Slow = false;
233
234 /// True if SSE operations can have unaligned memory operands.
235 /// This may require setting a configuration bit in the processor.
236 bool HasSSEUnalignedMem = false;
237
238 /// True if this processor has the CMPXCHG16B instruction;
239 /// this is true for most x86-64 chips, but not the first AMD chips.
240 bool HasCmpxchg16b = false;
241
242 /// True if the LEA instruction should be used for adjusting
243 /// the stack pointer. This is an optimization for Intel Atom processors.
244 bool UseLeaForSP = false;
245
246 /// True if POPCNT instruction has a false dependency on the destination register.
247 bool HasPOPCNTFalseDeps = false;
248
249 /// True if LZCNT/TZCNT instructions have a false dependency on the destination register.
250 bool HasLZCNTFalseDeps = false;
251
252 /// True if its preferable to combine to a single shuffle using a variable
253 /// mask over multiple fixed shuffles.
254 bool HasFastVariableShuffle = false;
255
256 /// True if vzeroupper instructions should be inserted after code that uses
257 /// ymm or zmm registers.
258 bool InsertVZEROUPPER = false;
259
260 /// True if there is no performance penalty for writing NOPs with up to
261 /// 11 bytes.
262 bool HasFast11ByteNOP = false;
263
264 /// True if there is no performance penalty for writing NOPs with up to
265 /// 15 bytes.
266 bool HasFast15ByteNOP = false;
267
268 /// True if gather is reasonably fast. This is true for Skylake client and
269 /// all AVX-512 CPUs.
270 bool HasFastGather = false;
271
272 /// True if hardware SQRTSS instruction is at least as fast (latency) as
273 /// RSQRTSS followed by a Newton-Raphson iteration.
274 bool HasFastScalarFSQRT = false;
275
276 /// True if hardware SQRTPS/VSQRTPS instructions are at least as fast
277 /// (throughput) as RSQRTPS/VRSQRTPS followed by a Newton-Raphson iteration.
278 bool HasFastVectorFSQRT = false;
279
280 /// True if 8-bit divisions are significantly faster than
281 /// 32-bit divisions and should be used when possible.
282 bool HasSlowDivide32 = false;
283
284 /// True if 32-bit divides are significantly faster than
285 /// 64-bit divisions and should be used when possible.
286 bool HasSlowDivide64 = false;
287
288 /// True if LZCNT instruction is fast.
289 bool HasFastLZCNT = false;
290
291 /// True if SHLD based rotate is fast.
292 bool HasFastSHLDRotate = false;
293
294 /// True if the processor supports macrofusion.
295 bool HasMacroFusion = false;
296
297 /// True if the processor supports branch fusion.
298 bool HasBranchFusion = false;
299
300 /// True if the processor has enhanced REP MOVSB/STOSB.
301 bool HasERMSB = false;
302
303 /// True if the short functions should be padded to prevent
304 /// a stall when returning too early.
305 bool PadShortFunctions = false;
306
307 /// True if two memory operand instructions should use a temporary register
308 /// instead.
309 bool SlowTwoMemOps = false;
310
311 /// True if the LEA instruction inputs have to be ready at address generation
312 /// (AG) time.
313 bool LEAUsesAG = false;
314
315 /// True if the LEA instruction with certain arguments is slow
316 bool SlowLEA = false;
317
318 /// True if the LEA instruction has all three source operands: base, index,
319 /// and offset or if the LEA instruction uses base and index registers where
320 /// the base is EBP, RBP,or R13
321 bool Slow3OpsLEA = false;
322
323 /// True if INC and DEC instructions are slow when writing to flags
324 bool SlowIncDec = false;
325
326 /// Processor has AVX-512 PreFetch Instructions
327 bool HasPFI = false;
328
329 /// Processor has AVX-512 Exponential and Reciprocal Instructions
330 bool HasERI = false;
331
332 /// Processor has AVX-512 Conflict Detection Instructions
333 bool HasCDI = false;
334
335 /// Processor has AVX-512 population count Instructions
336 bool HasVPOPCNTDQ = false;
337
338 /// Processor has AVX-512 Doubleword and Quadword instructions
339 bool HasDQI = false;
340
341 /// Processor has AVX-512 Byte and Word instructions
342 bool HasBWI = false;
343
344 /// Processor has AVX-512 Vector Length eXtenstions
345 bool HasVLX = false;
346
347 /// Processor has PKU extenstions
348 bool HasPKU = false;
349
350 /// Processor has AVX-512 Vector Neural Network Instructions
351 bool HasVNNI = false;
352
353 /// Processor has AVX-512 bfloat16 floating-point extensions
354 bool HasBF16 = false;
355
356 /// Processor supports ENQCMD instructions
357 bool HasENQCMD = false;
358
359 /// Processor has AVX-512 Bit Algorithms instructions
360 bool HasBITALG = false;
361
362 /// Processor has AVX-512 vp2intersect instructions
363 bool HasVP2INTERSECT = false;
364
365 /// Deprecated flag for MPX instructions.
366 bool DeprecatedHasMPX = false;
367
368 /// Processor supports CET SHSTK - Control-Flow Enforcement Technology
369 /// using Shadow Stack
370 bool HasSHSTK = false;
371
372 /// Processor supports Invalidate Process-Context Identifier
373 bool HasINVPCID = false;
374
375 /// Processor has Software Guard Extensions
376 bool HasSGX = false;
377
378 /// Processor supports Flush Cache Line instruction
379 bool HasCLFLUSHOPT = false;
380
381 /// Processor supports Cache Line Write Back instruction
382 bool HasCLWB = false;
383
384 /// Processor supports Write Back No Invalidate instruction
385 bool HasWBNOINVD = false;
386
387 /// Processor support RDPID instruction
388 bool HasRDPID = false;
389
390 /// Processor supports WaitPKG instructions
391 bool HasWAITPKG = false;
392
393 /// Processor supports PCONFIG instruction
394 bool HasPCONFIG = false;
395
396 /// Processor has a single uop BEXTR implementation.
397 bool HasFastBEXTR = false;
398
399 /// Try harder to combine to horizontal vector ops if they are fast.
400 bool HasFastHorizontalOps = false;
401
402 /// Prefer a left/right scalar logical shifts pair over a shift+and pair.
403 bool HasFastScalarShiftMasks = false;
404
405 /// Prefer a left/right vector logical shifts pair over a shift+and pair.
406 bool HasFastVectorShiftMasks = false;
407
408 /// Use a retpoline thunk rather than indirect calls to block speculative
409 /// execution.
410 bool UseRetpolineIndirectCalls = false;
411
412 /// Use a retpoline thunk or remove any indirect branch to block speculative
413 /// execution.
414 bool UseRetpolineIndirectBranches = false;
415
416 /// Deprecated flag, query `UseRetpolineIndirectCalls` and
417 /// `UseRetpolineIndirectBranches` instead.
418 bool DeprecatedUseRetpoline = false;
419
420 /// When using a retpoline thunk, call an externally provided thunk rather
421 /// than emitting one inside the compiler.
422 bool UseRetpolineExternalThunk = false;
423
424 /// Use software floating point for code generation.
425 bool UseSoftFloat = false;
426
427 /// Use alias analysis during code generation.
428 bool UseAA = false;
429
430 /// The minimum alignment known to hold of the stack frame on
431 /// entry to the function and which must be maintained by every function.
432 Align stackAlignment = Align(4);
433
434 /// Max. memset / memcpy size that is turned into rep/movs, rep/stos ops.
435 ///
436 // FIXME: this is a known good value for Yonah. How about others?
437 unsigned MaxInlineSizeThreshold = 128;
438
439 /// Indicates target prefers 128 bit instructions.
440 bool Prefer128Bit = false;
441
442 /// Indicates target prefers 256 bit instructions.
443 bool Prefer256Bit = false;
444
445 /// Indicates target prefers AVX512 mask registers.
446 bool PreferMaskRegisters = false;
447
448 /// Threeway branch is profitable in this subtarget.
449 bool ThreewayBranchProfitable = false;
450
451 /// Use Goldmont specific floating point div/sqrt costs.
452 bool UseGLMDivSqrtCosts = false;
453
454 /// What processor and OS we're targeting.
455 Triple TargetTriple;
456
457 /// GlobalISel related APIs.
458 std::unique_ptr<CallLowering> CallLoweringInfo;
459 std::unique_ptr<LegalizerInfo> Legalizer;
460 std::unique_ptr<RegisterBankInfo> RegBankInfo;
461 std::unique_ptr<InstructionSelector> InstSelector;
462
463private:
464 /// Override the stack alignment.
465 MaybeAlign StackAlignOverride;
466
467 /// Preferred vector width from function attribute.
468 unsigned PreferVectorWidthOverride;
469
470 /// Resolved preferred vector width from function attribute and subtarget
471 /// features.
472 unsigned PreferVectorWidth = UINT32_MAX(4294967295U);
473
474 /// Required vector width from function attribute.
475 unsigned RequiredVectorWidth;
476
477 /// True if compiling for 64-bit, false for 16-bit or 32-bit.
478 bool In64BitMode;
479
480 /// True if compiling for 32-bit, false for 16-bit or 64-bit.
481 bool In32BitMode;
482
483 /// True if compiling for 16-bit, false for 32-bit or 64-bit.
484 bool In16BitMode;
485
486 /// Contains the Overhead of gather\scatter instructions
487 int GatherOverhead = 1024;
488 int ScatterOverhead = 1024;
489
490 X86SelectionDAGInfo TSInfo;
491 // Ordering here is important. X86InstrInfo initializes X86RegisterInfo which
492 // X86TargetLowering needs.
493 X86InstrInfo InstrInfo;
494 X86TargetLowering TLInfo;
495 X86FrameLowering FrameLowering;
496
497public:
498 /// This constructor initializes the data members to match that
499 /// of the specified triple.
500 ///
501 X86Subtarget(const Triple &TT, StringRef CPU, StringRef FS,
502 const X86TargetMachine &TM, MaybeAlign StackAlignOverride,
503 unsigned PreferVectorWidthOverride,
504 unsigned RequiredVectorWidth);
505
506 const X86TargetLowering *getTargetLowering() const override {
507 return &TLInfo;
508 }
509
510 const X86InstrInfo *getInstrInfo() const override { return &InstrInfo; }
511
512 const X86FrameLowering *getFrameLowering() const override {
513 return &FrameLowering;
514 }
515
516 const X86SelectionDAGInfo *getSelectionDAGInfo() const override {
517 return &TSInfo;
518 }
519
520 const X86RegisterInfo *getRegisterInfo() const override {
521 return &getInstrInfo()->getRegisterInfo();
522 }
523
524 /// Returns the minimum alignment known to hold of the
525 /// stack frame on entry to the function and which must be maintained by every
526 /// function for this subtarget.
527 Align getStackAlignment() const { return stackAlignment; }
528
529 /// Returns the maximum memset / memcpy size
530 /// that still makes it profitable to inline the call.
531 unsigned getMaxInlineSizeThreshold() const { return MaxInlineSizeThreshold; }
532
533 /// ParseSubtargetFeatures - Parses features string setting specified
534 /// subtarget options. Definition of function is auto generated by tblgen.
535 void ParseSubtargetFeatures(StringRef CPU, StringRef FS);
536
537 /// Methods used by Global ISel
538 const CallLowering *getCallLowering() const override;
539 InstructionSelector *getInstructionSelector() const override;
540 const LegalizerInfo *getLegalizerInfo() const override;
541 const RegisterBankInfo *getRegBankInfo() const override;
542
543private:
544 /// Initialize the full set of dependencies so we can use an initializer
545 /// list for X86Subtarget.
546 X86Subtarget &initializeSubtargetDependencies(StringRef CPU, StringRef FS);
547 void initSubtargetFeatures(StringRef CPU, StringRef FS);
548
549public:
550 /// Is this x86_64? (disregarding specific ABI / programming model)
551 bool is64Bit() const {
552 return In64BitMode;
553 }
554
555 bool is32Bit() const {
556 return In32BitMode;
557 }
558
559 bool is16Bit() const {
560 return In16BitMode;
561 }
562
563 /// Is this x86_64 with the ILP32 programming model (x32 ABI)?
564 bool isTarget64BitILP32() const {
565 return In64BitMode && (TargetTriple.getEnvironment() == Triple::GNUX32 ||
566 TargetTriple.isOSNaCl());
567 }
568
569 /// Is this x86_64 with the LP64 programming model (standard AMD64, no x32)?
570 bool isTarget64BitLP64() const {
571 return In64BitMode && (TargetTriple.getEnvironment() != Triple::GNUX32 &&
572 !TargetTriple.isOSNaCl());
573 }
574
575 PICStyles::Style getPICStyle() const { return PICStyle; }
576 void setPICStyle(PICStyles::Style Style) { PICStyle = Style; }
577
578 bool hasX87() const { return HasX87; }
579 bool hasCmpxchg8b() const { return HasCmpxchg8b; }
580 bool hasNOPL() const { return HasNOPL; }
581 // SSE codegen depends on cmovs, and all SSE1+ processors support them.
582 // All 64-bit processors support cmov.
583 bool hasCMov() const { return HasCMov || X86SSELevel >= SSE1 || is64Bit(); }
584 bool hasSSE1() const { return X86SSELevel >= SSE1; }
585 bool hasSSE2() const { return X86SSELevel >= SSE2; }
586 bool hasSSE3() const { return X86SSELevel >= SSE3; }
587 bool hasSSSE3() const { return X86SSELevel >= SSSE3; }
588 bool hasSSE41() const { return X86SSELevel >= SSE41; }
589 bool hasSSE42() const { return X86SSELevel >= SSE42; }
590 bool hasAVX() const { return X86SSELevel >= AVX; }
591 bool hasAVX2() const { return X86SSELevel >= AVX2; }
20
←
Assuming field 'X86SSELevel' is < AVX2
→
21
←
Returning zero, which participates in a condition later
→
592 bool hasAVX512() const { return X86SSELevel >= AVX512F; }
593 bool hasInt256() const { return hasAVX2(); }
594 bool hasSSE4A() const { return HasSSE4A; }
595 bool hasMMX() const { return X863DNowLevel >= MMX; }
596 bool has3DNow() const { return X863DNowLevel >= ThreeDNow; }
597 bool has3DNowA() const { return X863DNowLevel >= ThreeDNowA; }
598 bool hasPOPCNT() const { return HasPOPCNT; }
599 bool hasAES() const { return HasAES; }
600 bool hasVAES() const { return HasVAES; }
601 bool hasFXSR() const { return HasFXSR; }
602 bool hasXSAVE() const { return HasXSAVE; }
603 bool hasXSAVEOPT() const { return HasXSAVEOPT; }
604 bool hasXSAVEC() const { return HasXSAVEC; }
605 bool hasXSAVES() const { return HasXSAVES; }
606 bool hasPCLMUL() const { return HasPCLMUL; }
607 bool hasVPCLMULQDQ() const { return HasVPCLMULQDQ; }
608 bool hasGFNI() const { return HasGFNI; }
609 // Prefer FMA4 to FMA - its better for commutation/memory folding and
610 // has equal or better performance on all supported targets.
611 bool hasFMA() const { return HasFMA; }
612 bool hasFMA4() const { return HasFMA4; }
613 bool hasAnyFMA() const { return hasFMA() || hasFMA4(); }
614 bool hasXOP() const { return HasXOP; }
615 bool hasTBM() const { return HasTBM; }
616 bool hasLWP() const { return HasLWP; }
617 bool hasMOVBE() const { return HasMOVBE; }
618 bool hasRDRAND() const { return HasRDRAND; }
619 bool hasF16C() const { return HasF16C; }
620 bool hasFSGSBase() const { return HasFSGSBase; }
621 bool hasLZCNT() const { return HasLZCNT; }
622 bool hasBMI() const { return HasBMI; }
623 bool hasBMI2() const { return HasBMI2; }
624 bool hasVBMI() const { return HasVBMI; }
625 bool hasVBMI2() const { return HasVBMI2; }
626 bool hasIFMA() const { return HasIFMA; }
627 bool hasRTM() const { return HasRTM; }
628 bool hasADX() const { return HasADX; }
629 bool hasSHA() const { return HasSHA; }
630 bool hasPRFCHW() const { return HasPRFCHW || HasPREFETCHWT1; }
631 bool hasPREFETCHWT1() const { return HasPREFETCHWT1; }
632 bool hasSSEPrefetch() const {
633 // We implicitly enable these when we have a write prefix supporting cache
634 // level OR if we have prfchw, but don't already have a read prefetch from
635 // 3dnow.
636 return hasSSE1() || (hasPRFCHW() && !has3DNow()) || hasPREFETCHWT1();
637 }
638 bool hasRDSEED() const { return HasRDSEED; }
639 bool hasLAHFSAHF() const { return HasLAHFSAHF; }
640 bool hasMWAITX() const { return HasMWAITX; }
641 bool hasCLZERO() const { return HasCLZERO; }
642 bool hasCLDEMOTE() const { return HasCLDEMOTE; }
643 bool hasMOVDIRI() const { return HasMOVDIRI; }
644 bool hasMOVDIR64B() const { return HasMOVDIR64B; }
645 bool hasPTWRITE() const { return HasPTWRITE; }
646 bool isSHLDSlow() const { return IsSHLDSlow; }
647 bool isPMULLDSlow() const { return IsPMULLDSlow; }
648 bool isPMADDWDSlow() const { return IsPMADDWDSlow; }
649 bool isUnalignedMem16Slow() const { return IsUAMem16Slow; }
650 bool isUnalignedMem32Slow() const { return IsUAMem32Slow; }
651 int getGatherOverhead() const { return GatherOverhead; }
652 int getScatterOverhead() const { return ScatterOverhead; }
653 bool hasSSEUnalignedMem() const { return HasSSEUnalignedMem; }
654 bool hasCmpxchg16b() const { return HasCmpxchg16b && is64Bit(); }
655 bool useLeaForSP() const { return UseLeaForSP; }
656 bool hasPOPCNTFalseDeps() const { return HasPOPCNTFalseDeps; }
657 bool hasLZCNTFalseDeps() const { return HasLZCNTFalseDeps; }
658 bool hasFastVariableShuffle() const {
659 return HasFastVariableShuffle;
660 }
661 bool insertVZEROUPPER() const { return InsertVZEROUPPER; }
662 bool hasFastGather() const { return HasFastGather; }
663 bool hasFastScalarFSQRT() const { return HasFastScalarFSQRT; }
664 bool hasFastVectorFSQRT() const { return HasFastVectorFSQRT; }
665 bool hasFastLZCNT() const { return HasFastLZCNT; }
666 bool hasFastSHLDRotate() const { return HasFastSHLDRotate; }
667 bool hasFastBEXTR() const { return HasFastBEXTR; }
668 bool hasFastHorizontalOps() const { return HasFastHorizontalOps; }
669 bool hasFastScalarShiftMasks() const { return HasFastScalarShiftMasks; }
670 bool hasFastVectorShiftMasks() const { return HasFastVectorShiftMasks; }
671 bool hasMacroFusion() const { return HasMacroFusion; }
672 bool hasBranchFusion() const { return HasBranchFusion; }
673 bool hasERMSB() const { return HasERMSB; }
674 bool hasSlowDivide32() const { return HasSlowDivide32; }
675 bool hasSlowDivide64() const { return HasSlowDivide64; }
676 bool padShortFunctions() const { return PadShortFunctions; }
677 bool slowTwoMemOps() const { return SlowTwoMemOps; }
678 bool LEAusesAG() const { return LEAUsesAG; }
679 bool slowLEA() const { return SlowLEA; }
680 bool slow3OpsLEA() const { return Slow3OpsLEA; }
681 bool slowIncDec() const { return SlowIncDec; }
682 bool hasCDI() const { return HasCDI; }
683 bool hasVPOPCNTDQ() const { return HasVPOPCNTDQ; }
684 bool hasPFI() const { return HasPFI; }
685 bool hasERI() const { return HasERI; }
686 bool hasDQI() const { return HasDQI; }
687 bool hasBWI() const { return HasBWI; }
688 bool hasVLX() const { return HasVLX; }
689 bool hasPKU() const { return HasPKU; }
690 bool hasVNNI() const { return HasVNNI; }
691 bool hasBF16() const { return HasBF16; }
692 bool hasVP2INTERSECT() const { return HasVP2INTERSECT; }
693 bool hasBITALG() const { return HasBITALG; }
694 bool hasSHSTK() const { return HasSHSTK; }
695 bool hasCLFLUSHOPT() const { return HasCLFLUSHOPT; }
696 bool hasCLWB() const { return HasCLWB; }
697 bool hasWBNOINVD() const { return HasWBNOINVD; }
698 bool hasRDPID() const { return HasRDPID; }
699 bool hasWAITPKG() const { return HasWAITPKG; }
700 bool hasPCONFIG() const { return HasPCONFIG; }
701 bool hasSGX() const { return HasSGX; }
702 bool threewayBranchProfitable() const { return ThreewayBranchProfitable; }
703 bool hasINVPCID() const { return HasINVPCID; }
704 bool hasENQCMD() const { return HasENQCMD; }
705 bool useRetpolineIndirectCalls() const { return UseRetpolineIndirectCalls; }
706 bool useRetpolineIndirectBranches() const {
707 return UseRetpolineIndirectBranches;
708 }
709 bool useRetpolineExternalThunk() const { return UseRetpolineExternalThunk; }
710 bool preferMaskRegisters() const { return PreferMaskRegisters; }
711 bool useGLMDivSqrtCosts() const { return UseGLMDivSqrtCosts; }
712
713 unsigned getPreferVectorWidth() const { return PreferVectorWidth; }
714 unsigned getRequiredVectorWidth() const { return RequiredVectorWidth; }
715
716 // Helper functions to determine when we should allow widening to 512-bit
717 // during codegen.
718 // TODO: Currently we're always allowing widening on CPUs without VLX,
719 // because for many cases we don't have a better option.
720 bool canExtendTo512DQ() const {
721 return hasAVX512() && (!hasVLX() || getPreferVectorWidth() >= 512);
722 }
723 bool canExtendTo512BW() const {
724 return hasBWI() && canExtendTo512DQ();
725 }
726
727 // If there are no 512-bit vectors and we prefer not to use 512-bit registers,
728 // disable them in the legalizer.
729 bool useAVX512Regs() const {
730 return hasAVX512() && (canExtendTo512DQ() || RequiredVectorWidth > 256);
731 }
732
733 bool useBWIRegs() const {
734 return hasBWI() && useAVX512Regs();
735 }
736
737 bool isXRaySupported() const override { return is64Bit(); }
738
739 X86ProcFamilyEnum getProcFamily() const { return X86ProcFamily; }
740
741 /// TODO: to be removed later and replaced with suitable properties
742 bool isAtom() const { return X86ProcFamily == IntelAtom; }
743 bool isSLM() const { return X86ProcFamily == IntelSLM; }
744 bool useSoftFloat() const { return UseSoftFloat; }
745 bool useAA() const override { return UseAA; }
746
747 /// Use mfence if we have SSE2 or we're on x86-64 (even if we asked for
748 /// no-sse2). There isn't any reason to disable it if the target processor
749 /// supports it.
750 bool hasMFence() const { return hasSSE2() || is64Bit(); }
751
752 const Triple &getTargetTriple() const { return TargetTriple; }
753
754 bool isTargetDarwin() const { return TargetTriple.isOSDarwin(); }
755 bool isTargetFreeBSD() const { return TargetTriple.isOSFreeBSD(); }
756 bool isTargetDragonFly() const { return TargetTriple.isOSDragonFly(); }
757 bool isTargetSolaris() const { return TargetTriple.isOSSolaris(); }
758 bool isTargetPS4() const { return TargetTriple.isPS4CPU(); }
759
760 bool isTargetELF() const { return TargetTriple.isOSBinFormatELF(); }
761 bool isTargetCOFF() const { return TargetTriple.isOSBinFormatCOFF(); }
762 bool isTargetMachO() const { return TargetTriple.isOSBinFormatMachO(); }
763
764 bool isTargetLinux() const { return TargetTriple.isOSLinux(); }
765 bool isTargetKFreeBSD() const { return TargetTriple.isOSKFreeBSD(); }
766 bool isTargetGlibc() const { return TargetTriple.isOSGlibc(); }
767 bool isTargetAndroid() const { return TargetTriple.isAndroid(); }
768 bool isTargetNaCl() const { return TargetTriple.isOSNaCl(); }
769 bool isTargetNaCl32() const { return isTargetNaCl() && !is64Bit(); }
770 bool isTargetNaCl64() const { return isTargetNaCl() && is64Bit(); }
771 bool isTargetMCU() const { return TargetTriple.isOSIAMCU(); }
772 bool isTargetFuchsia() const { return TargetTriple.isOSFuchsia(); }
773
774 bool isTargetWindowsMSVC() const {
775 return TargetTriple.isWindowsMSVCEnvironment();
776 }
777
778 bool isTargetWindowsCoreCLR() const {
779 return TargetTriple.isWindowsCoreCLREnvironment();
780 }
781
782 bool isTargetWindowsCygwin() const {
783 return TargetTriple.isWindowsCygwinEnvironment();
784 }
785
786 bool isTargetWindowsGNU() const {
787 return TargetTriple.isWindowsGNUEnvironment();
788 }
789
790 bool isTargetWindowsItanium() const {
791 return TargetTriple.isWindowsItaniumEnvironment();
792 }
793
794 bool isTargetCygMing() const { return TargetTriple.isOSCygMing(); }
795
796 bool isOSWindows() const { return TargetTriple.isOSWindows(); }
797
798 bool isTargetWin64() const { return In64BitMode && isOSWindows(); }
799
800 bool isTargetWin32() const { return !In64BitMode && isOSWindows(); }
801
802 bool isPICStyleGOT() const { return PICStyle == PICStyles::Style::GOT; }
803 bool isPICStyleRIPRel() const { return PICStyle == PICStyles::Style::RIPRel; }
804
805 bool isPICStyleStubPIC() const {
806 return PICStyle == PICStyles::Style::StubPIC;
807 }
808
809 bool isPositionIndependent() const { return TM.isPositionIndependent(); }
810
811 bool isCallingConvWin64(CallingConv::ID CC) const {
812 switch (CC) {
813 // On Win64, all these conventions just use the default convention.
814 case CallingConv::C:
815 case CallingConv::Fast:
816 case CallingConv::Tail:
817 case CallingConv::Swift:
818 case CallingConv::X86_FastCall:
819 case CallingConv::X86_StdCall:
820 case CallingConv::X86_ThisCall:
821 case CallingConv::X86_VectorCall:
822 case CallingConv::Intel_OCL_BI:
823 return isTargetWin64();
824 // This convention allows using the Win64 convention on other targets.
825 case CallingConv::Win64:
826 return true;
827 // This convention allows using the SysV convention on Windows targets.
828 case CallingConv::X86_64_SysV:
829 return false;
830 // Otherwise, who knows what this is.
831 default:
832 return false;
833 }
834 }
835
836 /// Classify a global variable reference for the current subtarget according
837 /// to how we should reference it in a non-pcrel context.
838 unsigned char classifyLocalReference(const GlobalValue *GV) const;
839
840 unsigned char classifyGlobalReference(const GlobalValue *GV,
841 const Module &M) const;
842 unsigned char classifyGlobalReference(const GlobalValue *GV) const;
843
844 /// Classify a global function reference for the current subtarget.
845 unsigned char classifyGlobalFunctionReference(const GlobalValue *GV,
846 const Module &M) const;
847 unsigned char classifyGlobalFunctionReference(const GlobalValue *GV) const;
848
849 /// Classify a blockaddress reference for the current subtarget according to
850 /// how we should reference it in a non-pcrel context.
851 unsigned char classifyBlockAddressReference() const;
852
853 /// Return true if the subtarget allows calls to immediate address.
854 bool isLegalToCallImmediateAddr() const;
855
856 /// If we are using retpolines, we need to expand indirectbr to avoid it
857 /// lowering to an actual indirect jump.
858 bool enableIndirectBrExpand() const override {
859 return useRetpolineIndirectBranches();
860 }
861
862 /// Enable the MachineScheduler pass for all X86 subtargets.
863 bool enableMachineScheduler() const override { return true; }
864
865 bool enableEarlyIfConversion() const override;
866
867 void getPostRAMutations(std::vector<std::unique_ptr<ScheduleDAGMutation>>
868 &Mutations) const override;
869
870 AntiDepBreakMode getAntiDepBreakMode() const override {
871 return TargetSubtargetInfo::ANTIDEP_CRITICAL;
872 }
873
874 bool enableAdvancedRASplitCost() const override { return true; }
875};
876
877} // end namespace llvm
878
879#endif // LLVM_LIB_TARGET_X86_X86SUBTARGET_H