LLVM  9.0.0svn
X86ISelLowering.cpp
Go to the documentation of this file.
1 //===-- X86ISelLowering.cpp - X86 DAG Lowering Implementation -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that X86 uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "X86ISelLowering.h"
15 #include "Utils/X86ShuffleDecode.h"
16 #include "X86CallingConv.h"
17 #include "X86FrameLowering.h"
18 #include "X86InstrBuilder.h"
19 #include "X86IntrinsicsInfo.h"
20 #include "X86MachineFunctionInfo.h"
21 #include "X86TargetMachine.h"
22 #include "X86TargetObjectFile.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/ADT/SmallSet.h"
25 #include "llvm/ADT/Statistic.h"
26 #include "llvm/ADT/StringExtras.h"
27 #include "llvm/ADT/StringSwitch.h"
28 #include "llvm/Analysis/EHPersonalities.h"
29 #include "llvm/CodeGen/IntrinsicLowering.h"
30 #include "llvm/CodeGen/MachineFrameInfo.h"
31 #include "llvm/CodeGen/MachineFunction.h"
32 #include "llvm/CodeGen/MachineInstrBuilder.h"
33 #include "llvm/CodeGen/MachineJumpTableInfo.h"
34 #include "llvm/CodeGen/MachineModuleInfo.h"
35 #include "llvm/CodeGen/MachineRegisterInfo.h"
36 #include "llvm/CodeGen/TargetLowering.h"
37 #include "llvm/CodeGen/WinEHFuncInfo.h"
38 #include "llvm/IR/CallSite.h"
39 #include "llvm/IR/CallingConv.h"
40 #include "llvm/IR/Constants.h"
41 #include "llvm/IR/DerivedTypes.h"
42 #include "llvm/IR/DiagnosticInfo.h"
43 #include "llvm/IR/Function.h"
44 #include "llvm/IR/GlobalAlias.h"
45 #include "llvm/IR/GlobalVariable.h"
46 #include "llvm/IR/Instructions.h"
47 #include "llvm/IR/Intrinsics.h"
48 #include "llvm/MC/MCAsmInfo.h"
49 #include "llvm/MC/MCContext.h"
50 #include "llvm/MC/MCExpr.h"
51 #include "llvm/MC/MCSymbol.h"
52 #include "llvm/Support/CommandLine.h"
53 #include "llvm/Support/Debug.h"
54 #include "llvm/Support/ErrorHandling.h"
55 #include "llvm/Support/KnownBits.h"
56 #include "llvm/Support/MathExtras.h"
57 #include "llvm/Target/TargetOptions.h"
58 #include <algorithm>
59 #include <bitset>
60 #include <cctype>
61 #include <numeric>
62 using namespace llvm;
63 
64 #define DEBUG_TYPE "x86-isel"
65 
66 STATISTIC(NumTailCalls, "Number of tail calls");
67 
68 static cl::opt<bool> ExperimentalVectorWideningLegalization(
69  "x86-experimental-vector-widening-legalization", cl::init(false),
70  cl::desc("Enable an experimental vector type legalization through widening "
71  "rather than promotion."),
72  cl::Hidden);
73 
74 static cl::opt<int> ExperimentalPrefLoopAlignment(
75  "x86-experimental-pref-loop-alignment", cl::init(4),
76  cl::desc("Sets the preferable loop alignment for experiments "
77  "(the last x86-experimental-pref-loop-alignment bits"
78  " of the loop header PC will be 0)."),
79  cl::Hidden);
80 
81 static cl::opt<bool> MulConstantOptimization(
82  "mul-constant-optimization", cl::init(true),
83  cl::desc("Replace 'mul x, Const' with more effective instructions like "
84  "SHIFT, LEA, etc."),
85  cl::Hidden);
86 
87 /// Call this when the user attempts to do something unsupported, like
88 /// returning a double without SSE2 enabled on x86_64. This is not fatal, unlike
89 /// report_fatal_error, so calling code should attempt to recover without
90 /// crashing.
91 static void errorUnsupported(SelectionDAG &DAG, const SDLoc &dl,
92  const char *Msg) {
93  MachineFunction &MF = DAG.getMachineFunction();
94  DAG.getContext()->diagnose(
95  DiagnosticInfoUnsupported(MF.getFunction(), Msg, dl.getDebugLoc()));
96 }
97 
98 X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
99  const X86Subtarget &STI)
100  : TargetLowering(TM), Subtarget(STI) {
101  bool UseX87 = !Subtarget.useSoftFloat() && Subtarget.hasX87();
102  X86ScalarSSEf64 = Subtarget.hasSSE2();
103  X86ScalarSSEf32 = Subtarget.hasSSE1();
104  MVT PtrVT = MVT::getIntegerVT(TM.getPointerSizeInBits(0));
105 
106  // Set up the TargetLowering object.
107 
108  // X86 is weird. It always uses i8 for shift amounts and setcc results.
109  setBooleanContents(ZeroOrOneBooleanContent);
110  // X86-SSE is even stranger. It uses -1 or 0 for vector masks.
111  setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
112 
113  // For 64-bit, since we have so many registers, use the ILP scheduler.
114  // For 32-bit, use the register pressure specific scheduling.
115  // For Atom, always use ILP scheduling.
116  if (Subtarget.isAtom())
117  setSchedulingPreference(Sched::ILP);
118  else if (Subtarget.is64Bit())
119  setSchedulingPreference(Sched::ILP);
120  else
121  setSchedulingPreference(Sched::RegPressure);
122  const X86RegisterInfo *RegInfo = Subtarget.getRegisterInfo();
123  setStackPointerRegisterToSaveRestore(RegInfo->getStackRegister());
124 
125  // Bypass expensive divides and use cheaper ones.
126  if (TM.getOptLevel() >= CodeGenOpt::Default) {
127  if (Subtarget.hasSlowDivide32())
128  addBypassSlowDiv(32, 8);
129  if (Subtarget.hasSlowDivide64() && Subtarget.is64Bit())
130  addBypassSlowDiv(64, 32);
131  }
132 
133  if (Subtarget.isTargetKnownWindowsMSVC() ||
134  Subtarget.isTargetWindowsItanium()) {
135  // Setup Windows compiler runtime calls.
136  setLibcallName(RTLIB::SDIV_I64, "_alldiv");
137  setLibcallName(RTLIB::UDIV_I64, "_aulldiv");
138  setLibcallName(RTLIB::SREM_I64, "_allrem");
139  setLibcallName(RTLIB::UREM_I64, "_aullrem");
140  setLibcallName(RTLIB::MUL_I64, "_allmul");
141  setLibcallCallingConv(RTLIB::SDIV_I64, CallingConv::X86_StdCall);
142  setLibcallCallingConv(RTLIB::UDIV_I64, CallingConv::X86_StdCall);
143  setLibcallCallingConv(RTLIB::SREM_I64, CallingConv::X86_StdCall);
144  setLibcallCallingConv(RTLIB::UREM_I64, CallingConv::X86_StdCall);
145  setLibcallCallingConv(RTLIB::MUL_I64, CallingConv::X86_StdCall);
146  }
147 
148  if (Subtarget.isTargetDarwin()) {
149  // Darwin should use _setjmp/_longjmp instead of setjmp/longjmp.
150  setUseUnderscoreSetJmp(false);
151  setUseUnderscoreLongJmp(false);
152  } else if (Subtarget.isTargetWindowsGNU()) {
153  // MS runtime is weird: it exports _setjmp, but longjmp!
154  setUseUnderscoreSetJmp(true);
155  setUseUnderscoreLongJmp(false);
156  } else {
157  setUseUnderscoreSetJmp(true);
158  setUseUnderscoreLongJmp(true);
159  }
160 
161  // If we don't have cmpxchg8b(meaing this is a 386/486), limit atomic size to
162  // 32 bits so the AtomicExpandPass will expand it so we don't need cmpxchg8b.
163  // FIXME: Should we be limitting the atomic size on other configs? Default is
164  // 1024.
165  if (!Subtarget.hasCmpxchg8b())
166  setMaxAtomicSizeInBitsSupported(32);
167 
168  // Set up the register classes.
169  addRegisterClass(MVT::i8, &X86::GR8RegClass);
170  addRegisterClass(MVT::i16, &X86::GR16RegClass);
171  addRegisterClass(MVT::i32, &X86::GR32RegClass);
172  if (Subtarget.is64Bit())
173  addRegisterClass(MVT::i64, &X86::GR64RegClass);
174 
175  for (MVT VT : MVT::integer_valuetypes())
176  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::i1, Promote);
177 
178  // We don't accept any truncstore of integer registers.
179  setTruncStoreAction(MVT::i64, MVT::i32, Expand);
180  setTruncStoreAction(MVT::i64, MVT::i16, Expand);
181  setTruncStoreAction(MVT::i64, MVT::i8 , Expand);
182  setTruncStoreAction(MVT::i32, MVT::i16, Expand);
183  setTruncStoreAction(MVT::i32, MVT::i8 , Expand);
184  setTruncStoreAction(MVT::i16, MVT::i8, Expand);
185 
186  setTruncStoreAction(MVT::f64, MVT::f32, Expand);
187 
188  // SETOEQ and SETUNE require checking two conditions.
189  setCondCodeAction(ISD::SETOEQ, MVT::f32, Expand);
190  setCondCodeAction(ISD::SETOEQ, MVT::f64, Expand);
191  setCondCodeAction(ISD::SETOEQ, MVT::f80, Expand);
192  setCondCodeAction(ISD::SETUNE, MVT::f32, Expand);
193  setCondCodeAction(ISD::SETUNE, MVT::f64, Expand);
194  setCondCodeAction(ISD::SETUNE, MVT::f80, Expand);
195 
196  // Integer absolute.
197  if (Subtarget.hasCMov()) {
198  setOperationAction(ISD::ABS , MVT::i16 , Custom);
199  setOperationAction(ISD::ABS , MVT::i32 , Custom);
200  }
201  setOperationAction(ISD::ABS , MVT::i64 , Custom);
202 
203  // Funnel shifts.
204  for (auto ShiftOp : {ISD::FSHL, ISD::FSHR}) {
205  setOperationAction(ShiftOp , MVT::i16 , Custom);
206  setOperationAction(ShiftOp , MVT::i32 , Custom);
207  if (Subtarget.is64Bit())
208  setOperationAction(ShiftOp , MVT::i64 , Custom);
209  }
210 
211  // Promote all UINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have this
212  // operation.
213  setOperationAction(ISD::UINT_TO_FP , MVT::i1 , Promote);
214  setOperationAction(ISD::UINT_TO_FP , MVT::i8 , Promote);
215  setOperationAction(ISD::UINT_TO_FP , MVT::i16 , Promote);
216 
217  if (Subtarget.is64Bit()) {
218  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512())
219  // f32/f64 are legal, f80 is custom.
220  setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom);
221  else
222  setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Promote);
223  setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom);
224  } else if (!Subtarget.useSoftFloat()) {
225  // We have an algorithm for SSE2->double, and we turn this into a
226  // 64-bit FILD followed by conditional FADD for other targets.
227  setOperationAction(ISD::UINT_TO_FP , MVT::i64 , Custom);
228  // We have an algorithm for SSE2, and we turn this into a 64-bit
229  // FILD or VCVTUSI2SS/SD for other targets.
230  setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Custom);
231  } else {
232  setOperationAction(ISD::UINT_TO_FP , MVT::i32 , Expand);
233  }
234 
235  // Promote i1/i8 SINT_TO_FP to larger SINT_TO_FP's, as X86 doesn't have
236  // this operation.
237  setOperationAction(ISD::SINT_TO_FP , MVT::i1 , Promote);
238  setOperationAction(ISD::SINT_TO_FP , MVT::i8 , Promote);
239 
240  if (!Subtarget.useSoftFloat()) {
241  // SSE has no i16 to fp conversion, only i32.
242  if (X86ScalarSSEf32) {
243  setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
244  // f32 and f64 cases are Legal, f80 case is not
245  setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
246  } else {
247  setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Custom);
248  setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Custom);
249  }
250  } else {
251  setOperationAction(ISD::SINT_TO_FP , MVT::i16 , Promote);
252  setOperationAction(ISD::SINT_TO_FP , MVT::i32 , Expand);
253  }
254 
255  // Promote i1/i8 FP_TO_SINT to larger FP_TO_SINTS's, as X86 doesn't have
256  // this operation.
257  setOperationAction(ISD::FP_TO_SINT , MVT::i1 , Promote);
258  setOperationAction(ISD::FP_TO_SINT , MVT::i8 , Promote);
259 
260  if (!Subtarget.useSoftFloat()) {
261  // In 32-bit mode these are custom lowered. In 64-bit mode F32 and F64
262  // are Legal, f80 is custom lowered.
263  setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Custom);
264  setOperationAction(ISD::SINT_TO_FP , MVT::i64 , Custom);
265 
266  setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Custom);
267  setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Custom);
268  } else {
269  setOperationAction(ISD::FP_TO_SINT , MVT::i16 , Promote);
270  setOperationAction(ISD::FP_TO_SINT , MVT::i32 , Expand);
271  setOperationAction(ISD::FP_TO_SINT , MVT::i64 , Expand);
272  }
273 
274  // Handle FP_TO_UINT by promoting the destination to a larger signed
275  // conversion.
276  setOperationAction(ISD::FP_TO_UINT , MVT::i1 , Promote);
277  setOperationAction(ISD::FP_TO_UINT , MVT::i8 , Promote);
278  setOperationAction(ISD::FP_TO_UINT , MVT::i16 , Promote);
279 
280  if (Subtarget.is64Bit()) {
281  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
282  // FP_TO_UINT-i32/i64 is legal for f32/f64, but custom for f80.
283  setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom);
284  setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom);
285  } else {
286  setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Promote);
287  setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Expand);
288  }
289  } else if (!Subtarget.useSoftFloat()) {
290  // Since AVX is a superset of SSE3, only check for SSE here.
291  if (Subtarget.hasSSE1() && !Subtarget.hasSSE3())
292  // Expand FP_TO_UINT into a select.
293  // FIXME: We would like to use a Custom expander here eventually to do
294  // the optimal thing for SSE vs. the default expansion in the legalizer.
295  setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Expand);
296  else
297  // With AVX512 we can use vcvts[ds]2usi for f32/f64->i32, f80 is custom.
298  // With SSE3 we can use fisttpll to convert to a signed i64; without
299  // SSE, we're stuck with a fistpll.
300  setOperationAction(ISD::FP_TO_UINT , MVT::i32 , Custom);
301 
302  setOperationAction(ISD::FP_TO_UINT , MVT::i64 , Custom);
303  }
304 
305  // TODO: when we have SSE, these could be more efficient, by using movd/movq.
306  if (!X86ScalarSSEf64) {
307  setOperationAction(ISD::BITCAST , MVT::f32 , Expand);
308  setOperationAction(ISD::BITCAST , MVT::i32 , Expand);
309  if (Subtarget.is64Bit()) {
310  setOperationAction(ISD::BITCAST , MVT::f64 , Expand);
311  // Without SSE, i64->f64 goes through memory.
312  setOperationAction(ISD::BITCAST , MVT::i64 , Expand);
313  }
314  } else if (!Subtarget.is64Bit())
315  setOperationAction(ISD::BITCAST , MVT::i64 , Custom);
316 
317  // Scalar integer divide and remainder are lowered to use operations that
318  // produce two results, to match the available instructions. This exposes
319  // the two-result form to trivial CSE, which is able to combine x/y and x%y
320  // into a single instruction.
321  //
322  // Scalar integer multiply-high is also lowered to use two-result
323  // operations, to match the available instructions. However, plain multiply
324  // (low) operations are left as Legal, as there are single-result
325  // instructions for this in x86. Using the two-result multiply instructions
326  // when both high and low results are needed must be arranged by dagcombine.
327  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
328  setOperationAction(ISD::MULHS, VT, Expand);
329  setOperationAction(ISD::MULHU, VT, Expand);
330  setOperationAction(ISD::SDIV, VT, Expand);
331  setOperationAction(ISD::UDIV, VT, Expand);
332  setOperationAction(ISD::SREM, VT, Expand);
333  setOperationAction(ISD::UREM, VT, Expand);
334  }
335 
336  setOperationAction(ISD::BR_JT , MVT::Other, Expand);
337  setOperationAction(ISD::BRCOND , MVT::Other, Custom);
338  for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128,
339  MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
340  setOperationAction(ISD::BR_CC, VT, Expand);
341  setOperationAction(ISD::SELECT_CC, VT, Expand);
342  }
343  if (Subtarget.is64Bit())
344  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i32, Legal);
345  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i16 , Legal);
346  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i8 , Legal);
347  setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1 , Expand);
348  setOperationAction(ISD::FP_ROUND_INREG , MVT::f32 , Expand);
349 
350  setOperationAction(ISD::FREM , MVT::f32 , Expand);
351  setOperationAction(ISD::FREM , MVT::f64 , Expand);
352  setOperationAction(ISD::FREM , MVT::f80 , Expand);
353  setOperationAction(ISD::FLT_ROUNDS_ , MVT::i32 , Custom);
354 
355  // Promote the i8 variants and force them on up to i32 which has a shorter
356  // encoding.
357  setOperationPromotedToType(ISD::CTTZ , MVT::i8 , MVT::i32);
358  setOperationPromotedToType(ISD::CTTZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
359  if (!Subtarget.hasBMI()) {
360  setOperationAction(ISD::CTTZ , MVT::i16 , Custom);
361  setOperationAction(ISD::CTTZ , MVT::i32 , Custom);
362  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i16 , Legal);
363  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i32 , Legal);
364  if (Subtarget.is64Bit()) {
365  setOperationAction(ISD::CTTZ , MVT::i64 , Custom);
366  setOperationAction(ISD::CTTZ_ZERO_UNDEF, MVT::i64, Legal);
367  }
368  }
369 
370  if (Subtarget.hasLZCNT()) {
371  // When promoting the i8 variants, force them to i32 for a shorter
372  // encoding.
373  setOperationPromotedToType(ISD::CTLZ , MVT::i8 , MVT::i32);
374  setOperationPromotedToType(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , MVT::i32);
375  } else {
376  setOperationAction(ISD::CTLZ , MVT::i8 , Custom);
377  setOperationAction(ISD::CTLZ , MVT::i16 , Custom);
378  setOperationAction(ISD::CTLZ , MVT::i32 , Custom);
379  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i8 , Custom);
380  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i16 , Custom);
381  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i32 , Custom);
382  if (Subtarget.is64Bit()) {
383  setOperationAction(ISD::CTLZ , MVT::i64 , Custom);
384  setOperationAction(ISD::CTLZ_ZERO_UNDEF, MVT::i64, Custom);
385  }
386  }
387 
388  // Special handling for half-precision floating point conversions.
389  // If we don't have F16C support, then lower half float conversions
390  // into library calls.
391  if (Subtarget.useSoftFloat() || !Subtarget.hasF16C()) {
392  setOperationAction(ISD::FP16_TO_FP, MVT::f32, Expand);
393  setOperationAction(ISD::FP_TO_FP16, MVT::f32, Expand);
394  }
395 
396  // There's never any support for operations beyond MVT::f32.
397  setOperationAction(ISD::FP16_TO_FP, MVT::f64, Expand);
398  setOperationAction(ISD::FP16_TO_FP, MVT::f80, Expand);
399  setOperationAction(ISD::FP_TO_FP16, MVT::f64, Expand);
400  setOperationAction(ISD::FP_TO_FP16, MVT::f80, Expand);
401 
402  setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
403  setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
404  setLoadExtAction(ISD::EXTLOAD, MVT::f80, MVT::f16, Expand);
405  setTruncStoreAction(MVT::f32, MVT::f16, Expand);
406  setTruncStoreAction(MVT::f64, MVT::f16, Expand);
407  setTruncStoreAction(MVT::f80, MVT::f16, Expand);
408 
409  if (Subtarget.hasPOPCNT()) {
410  setOperationPromotedToType(ISD::CTPOP, MVT::i8, MVT::i32);
411  } else {
412  setOperationAction(ISD::CTPOP , MVT::i8 , Expand);
413  setOperationAction(ISD::CTPOP , MVT::i16 , Expand);
414  setOperationAction(ISD::CTPOP , MVT::i32 , Expand);
415  if (Subtarget.is64Bit())
416  setOperationAction(ISD::CTPOP , MVT::i64 , Expand);
417  else
418  setOperationAction(ISD::CTPOP , MVT::i64 , Custom);
419  }
420 
421  setOperationAction(ISD::READCYCLECOUNTER , MVT::i64 , Custom);
422 
423  if (!Subtarget.hasMOVBE())
424  setOperationAction(ISD::BSWAP , MVT::i16 , Expand);
425 
426  // These should be promoted to a larger select which is supported.
427  setOperationAction(ISD::SELECT , MVT::i1 , Promote);
428  // X86 wants to expand cmov itself.
429  for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
430  setOperationAction(ISD::SELECT, VT, Custom);
431  setOperationAction(ISD::SETCC, VT, Custom);
432  }
433  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
434  if (VT == MVT::i64 && !Subtarget.is64Bit())
435  continue;
436  setOperationAction(ISD::SELECT, VT, Custom);
437  setOperationAction(ISD::SETCC, VT, Custom);
438  }
439 
440  // Custom action for SELECT MMX and expand action for SELECT_CC MMX
441  setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
442  setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
443 
444  setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
445  // NOTE: EH_SJLJ_SETJMP/_LONGJMP are not recommended, since
446  // LLVM/Clang supports zero-cost DWARF and SEH exception handling.
447  setOperationAction(ISD::EH_SJLJ_SETJMP, MVT::i32, Custom);
448  setOperationAction(ISD::EH_SJLJ_LONGJMP, MVT::Other, Custom);
449  setOperationAction(ISD::EH_SJLJ_SETUP_DISPATCH, MVT::Other, Custom);
450  if (TM.Options.ExceptionModel == ExceptionHandling::SjLj)
451  setLibcallName(RTLIB::UNWIND_RESUME, "_Unwind_SjLj_Resume");
452 
453  // Darwin ABI issue.
454  for (auto VT : { MVT::i32, MVT::i64 }) {
455  if (VT == MVT::i64 && !Subtarget.is64Bit())
456  continue;
457  setOperationAction(ISD::ConstantPool , VT, Custom);
458  setOperationAction(ISD::JumpTable , VT, Custom);
459  setOperationAction(ISD::GlobalAddress , VT, Custom);
460  setOperationAction(ISD::GlobalTLSAddress, VT, Custom);
461  setOperationAction(ISD::ExternalSymbol , VT, Custom);
462  setOperationAction(ISD::BlockAddress , VT, Custom);
463  }
464 
465  // 64-bit shl, sra, srl (iff 32-bit x86)
466  for (auto VT : { MVT::i32, MVT::i64 }) {
467  if (VT == MVT::i64 && !Subtarget.is64Bit())
468  continue;
469  setOperationAction(ISD::SHL_PARTS, VT, Custom);
470  setOperationAction(ISD::SRA_PARTS, VT, Custom);
471  setOperationAction(ISD::SRL_PARTS, VT, Custom);
472  }
473 
474  if (Subtarget.hasSSEPrefetch() || Subtarget.has3DNow())
475  setOperationAction(ISD::PREFETCH , MVT::Other, Legal);
476 
477  setOperationAction(ISD::ATOMIC_FENCE , MVT::Other, Custom);
478 
479  // Expand certain atomics
480  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
481  setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, VT, Custom);
482  setOperationAction(ISD::ATOMIC_LOAD_SUB, VT, Custom);
483  setOperationAction(ISD::ATOMIC_LOAD_ADD, VT, Custom);
484  setOperationAction(ISD::ATOMIC_LOAD_OR, VT, Custom);
485  setOperationAction(ISD::ATOMIC_LOAD_XOR, VT, Custom);
486  setOperationAction(ISD::ATOMIC_LOAD_AND, VT, Custom);
487  setOperationAction(ISD::ATOMIC_STORE, VT, Custom);
488  }
489 
490  if (!Subtarget.is64Bit())
491  setOperationAction(ISD::ATOMIC_LOAD, MVT::i64, Custom);
492 
493  if (Subtarget.hasCmpxchg16b()) {
494  setOperationAction(ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS, MVT::i128, Custom);
495  }
496 
497  // FIXME - use subtarget debug flags
498  if (!Subtarget.isTargetDarwin() && !Subtarget.isTargetELF() &&
499  !Subtarget.isTargetCygMing() && !Subtarget.isTargetWin64() &&
500  TM.Options.ExceptionModel != ExceptionHandling::SjLj) {
501  setOperationAction(ISD::EH_LABEL, MVT::Other, Expand);
502  }
503 
504  setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i32, Custom);
505  setOperationAction(ISD::FRAME_TO_ARGS_OFFSET, MVT::i64, Custom);
506 
507  setOperationAction(ISD::INIT_TRAMPOLINE, MVT::Other, Custom);
508  setOperationAction(ISD::ADJUST_TRAMPOLINE, MVT::Other, Custom);
509 
510  setOperationAction(ISD::TRAP, MVT::Other, Legal);
511  setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
512 
513  // VASTART needs to be custom lowered to use the VarArgsFrameIndex
514  setOperationAction(ISD::VASTART , MVT::Other, Custom);
515  setOperationAction(ISD::VAEND , MVT::Other, Expand);
516  bool Is64Bit = Subtarget.is64Bit();
517  setOperationAction(ISD::VAARG, MVT::Other, Is64Bit ? Custom : Expand);
518  setOperationAction(ISD::VACOPY, MVT::Other, Is64Bit ? Custom : Expand);
519 
520  setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
521  setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
522 
523  setOperationAction(ISD::DYNAMIC_STACKALLOC, PtrVT, Custom);
524 
525  // GC_TRANSITION_START and GC_TRANSITION_END need custom lowering.
526  setOperationAction(ISD::GC_TRANSITION_START, MVT::Other, Custom);
527  setOperationAction(ISD::GC_TRANSITION_END, MVT::Other, Custom);
528 
529  if (!Subtarget.useSoftFloat() && X86ScalarSSEf64) {
530  // f32 and f64 use SSE.
531  // Set up the FP register classes.
532  addRegisterClass(MVT::f32, Subtarget.hasAVX512() ? &X86::FR32XRegClass
533  : &X86::FR32RegClass);
534  addRegisterClass(MVT::f64, Subtarget.hasAVX512() ? &X86::FR64XRegClass
535  : &X86::FR64RegClass);
536 
537  for (auto VT : { MVT::f32, MVT::f64 }) {
538  // Use ANDPD to simulate FABS.
539  setOperationAction(ISD::FABS, VT, Custom);
540 
541  // Use XORP to simulate FNEG.
542  setOperationAction(ISD::FNEG, VT, Custom);
543 
544  // Use ANDPD and ORPD to simulate FCOPYSIGN.
545  setOperationAction(ISD::FCOPYSIGN, VT, Custom);
546 
547  // These might be better off as horizontal vector ops.
548  setOperationAction(ISD::FADD, VT, Custom);
549  setOperationAction(ISD::FSUB, VT, Custom);
550 
551  // We don't support sin/cos/fmod
552  setOperationAction(ISD::FSIN , VT, Expand);
553  setOperationAction(ISD::FCOS , VT, Expand);
554  setOperationAction(ISD::FSINCOS, VT, Expand);
555  }
556 
557  // Lower this to MOVMSK plus an AND.
558  setOperationAction(ISD::FGETSIGN, MVT::i64, Custom);
559  setOperationAction(ISD::FGETSIGN, MVT::i32, Custom);
560 
561  } else if (!useSoftFloat() && X86ScalarSSEf32 && (UseX87 || Is64Bit)) {
562  // Use SSE for f32, x87 for f64.
563  // Set up the FP register classes.
564  addRegisterClass(MVT::f32, &X86::FR32RegClass);
565  if (UseX87)
566  addRegisterClass(MVT::f64, &X86::RFP64RegClass);
567 
568  // Use ANDPS to simulate FABS.
569  setOperationAction(ISD::FABS , MVT::f32, Custom);
570 
571  // Use XORP to simulate FNEG.
572  setOperationAction(ISD::FNEG , MVT::f32, Custom);
573 
574  if (UseX87)
575  setOperationAction(ISD::UNDEF, MVT::f64, Expand);
576 
577  // Use ANDPS and ORPS to simulate FCOPYSIGN.
578  if (UseX87)
579  setOperationAction(ISD::FCOPYSIGN, MVT::f64, Expand);
580  setOperationAction(ISD::FCOPYSIGN, MVT::f32, Custom);
581 
582  // We don't support sin/cos/fmod
583  setOperationAction(ISD::FSIN , MVT::f32, Expand);
584  setOperationAction(ISD::FCOS , MVT::f32, Expand);
585  setOperationAction(ISD::FSINCOS, MVT::f32, Expand);
586 
587  if (UseX87) {
588  // Always expand sin/cos functions even though x87 has an instruction.
589  setOperationAction(ISD::FSIN, MVT::f64, Expand);
590  setOperationAction(ISD::FCOS, MVT::f64, Expand);
591  setOperationAction(ISD::FSINCOS, MVT::f64, Expand);
592  }
593  } else if (UseX87) {
594  // f32 and f64 in x87.
595  // Set up the FP register classes.
596  addRegisterClass(MVT::f64, &X86::RFP64RegClass);
597  addRegisterClass(MVT::f32, &X86::RFP32RegClass);
598 
599  for (auto VT : { MVT::f32, MVT::f64 }) {
600  setOperationAction(ISD::UNDEF, VT, Expand);
601  setOperationAction(ISD::FCOPYSIGN, VT, Expand);
602 
603  // Always expand sin/cos functions even though x87 has an instruction.
604  setOperationAction(ISD::FSIN , VT, Expand);
605  setOperationAction(ISD::FCOS , VT, Expand);
606  setOperationAction(ISD::FSINCOS, VT, Expand);
607  }
608  }
609 
610  // Expand FP32 immediates into loads from the stack, save special cases.
611  if (isTypeLegal(MVT::f32)) {
612  if (UseX87 && (getRegClassFor(MVT::f32) == &X86::RFP32RegClass)) {
613  addLegalFPImmediate(APFloat(+0.0f)); // FLD0
614  addLegalFPImmediate(APFloat(+1.0f)); // FLD1
615  addLegalFPImmediate(APFloat(-0.0f)); // FLD0/FCHS
616  addLegalFPImmediate(APFloat(-1.0f)); // FLD1/FCHS
617  } else // SSE immediates.
618  addLegalFPImmediate(APFloat(+0.0f)); // xorps
619  }
620  // Expand FP64 immediates into loads from the stack, save special cases.
621  if (isTypeLegal(MVT::f64)) {
622  if (UseX87 && getRegClassFor(MVT::f64) == &X86::RFP64RegClass) {
623  addLegalFPImmediate(APFloat(+0.0)); // FLD0
624  addLegalFPImmediate(APFloat(+1.0)); // FLD1
625  addLegalFPImmediate(APFloat(-0.0)); // FLD0/FCHS
626  addLegalFPImmediate(APFloat(-1.0)); // FLD1/FCHS
627  } else // SSE immediates.
628  addLegalFPImmediate(APFloat(+0.0)); // xorpd
629  }
630 
631  // We don't support FMA.
632  setOperationAction(ISD::FMA, MVT::f64, Expand);
633  setOperationAction(ISD::FMA, MVT::f32, Expand);
634 
635  // Long double always uses X87, except f128 in MMX.
636  if (UseX87) {
637  if (Subtarget.is64Bit() && Subtarget.hasMMX()) {
638  addRegisterClass(MVT::f128, Subtarget.hasVLX() ? &X86::VR128XRegClass
639  : &X86::VR128RegClass);
640  ValueTypeActions.setTypeAction(MVT::f128, TypeSoftenFloat);
641  setOperationAction(ISD::FABS , MVT::f128, Custom);
642  setOperationAction(ISD::FNEG , MVT::f128, Custom);
643  setOperationAction(ISD::FCOPYSIGN, MVT::f128, Custom);
644  }
645 
646  addRegisterClass(MVT::f80, &X86::RFP80RegClass);
647  setOperationAction(ISD::UNDEF, MVT::f80, Expand);
648  setOperationAction(ISD::FCOPYSIGN, MVT::f80, Expand);
649  {
650  APFloat TmpFlt = APFloat::getZero(APFloat::x87DoubleExtended());
651  addLegalFPImmediate(TmpFlt); // FLD0
652  TmpFlt.changeSign();
653  addLegalFPImmediate(TmpFlt); // FLD0/FCHS
654 
655  bool ignored;
656  APFloat TmpFlt2(+1.0);
657  TmpFlt2.convert(APFloat::x87DoubleExtended(), APFloat::rmNearestTiesToEven,
658  &ignored);
659  addLegalFPImmediate(TmpFlt2); // FLD1
660  TmpFlt2.changeSign();
661  addLegalFPImmediate(TmpFlt2); // FLD1/FCHS
662  }
663 
664  // Always expand sin/cos functions even though x87 has an instruction.
665  setOperationAction(ISD::FSIN , MVT::f80, Expand);
666  setOperationAction(ISD::FCOS , MVT::f80, Expand);
667  setOperationAction(ISD::FSINCOS, MVT::f80, Expand);
668 
669  setOperationAction(ISD::FFLOOR, MVT::f80, Expand);
670  setOperationAction(ISD::FCEIL, MVT::f80, Expand);
671  setOperationAction(ISD::FTRUNC, MVT::f80, Expand);
672  setOperationAction(ISD::FRINT, MVT::f80, Expand);
673  setOperationAction(ISD::FNEARBYINT, MVT::f80, Expand);
674  setOperationAction(ISD::FMA, MVT::f80, Expand);
675  setOperationAction(ISD::LROUND, MVT::f80, Expand);
676  setOperationAction(ISD::LLROUND, MVT::f80, Expand);
677  }
678 
679  // Always use a library call for pow.
680  setOperationAction(ISD::FPOW , MVT::f32 , Expand);
681  setOperationAction(ISD::FPOW , MVT::f64 , Expand);
682  setOperationAction(ISD::FPOW , MVT::f80 , Expand);
683 
684  setOperationAction(ISD::FLOG, MVT::f80, Expand);
685  setOperationAction(ISD::FLOG2, MVT::f80, Expand);
686  setOperationAction(ISD::FLOG10, MVT::f80, Expand);
687  setOperationAction(ISD::FEXP, MVT::f80, Expand);
688  setOperationAction(ISD::FEXP2, MVT::f80, Expand);
689  setOperationAction(ISD::FMINNUM, MVT::f80, Expand);
690  setOperationAction(ISD::FMAXNUM, MVT::f80, Expand);
691 
692  // Some FP actions are always expanded for vector types.
693  for (auto VT : { MVT::v4f32, MVT::v8f32, MVT::v16f32,
694  MVT::v2f64, MVT::v4f64, MVT::v8f64 }) {
695  setOperationAction(ISD::FSIN, VT, Expand);
696  setOperationAction(ISD::FSINCOS, VT, Expand);
697  setOperationAction(ISD::FCOS, VT, Expand);
698  setOperationAction(ISD::FREM, VT, Expand);
699  setOperationAction(ISD::FCOPYSIGN, VT, Expand);
700  setOperationAction(ISD::FPOW, VT, Expand);
701  setOperationAction(ISD::FLOG, VT, Expand);
702  setOperationAction(ISD::FLOG2, VT, Expand);
703  setOperationAction(ISD::FLOG10, VT, Expand);
704  setOperationAction(ISD::FEXP, VT, Expand);
705  setOperationAction(ISD::FEXP2, VT, Expand);
706  }
707 
708  // First set operation action for all vector types to either promote
709  // (for widening) or expand (for scalarization). Then we will selectively
710  // turn on ones that can be effectively codegen'd.
711  for (MVT VT : MVT::vector_valuetypes()) {
712  setOperationAction(ISD::SDIV, VT, Expand);
713  setOperationAction(ISD::UDIV, VT, Expand);
714  setOperationAction(ISD::SREM, VT, Expand);
715  setOperationAction(ISD::UREM, VT, Expand);
716  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT,Expand);
717  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Expand);
718  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT,Expand);
719  setOperationAction(ISD::INSERT_SUBVECTOR, VT,Expand);
720  setOperationAction(ISD::FMA, VT, Expand);
721  setOperationAction(ISD::FFLOOR, VT, Expand);
722  setOperationAction(ISD::FCEIL, VT, Expand);
723  setOperationAction(ISD::FTRUNC, VT, Expand);
724  setOperationAction(ISD::FRINT, VT, Expand);
725  setOperationAction(ISD::FNEARBYINT, VT, Expand);
726  setOperationAction(ISD::SMUL_LOHI, VT, Expand);
727  setOperationAction(ISD::MULHS, VT, Expand);
728  setOperationAction(ISD::UMUL_LOHI, VT, Expand);
729  setOperationAction(ISD::MULHU, VT, Expand);
730  setOperationAction(ISD::SDIVREM, VT, Expand);
731  setOperationAction(ISD::UDIVREM, VT, Expand);
732  setOperationAction(ISD::CTPOP, VT, Expand);
733  setOperationAction(ISD::CTTZ, VT, Expand);
734  setOperationAction(ISD::CTLZ, VT, Expand);
735  setOperationAction(ISD::ROTL, VT, Expand);
736  setOperationAction(ISD::ROTR, VT, Expand);
737  setOperationAction(ISD::BSWAP, VT, Expand);
738  setOperationAction(ISD::SETCC, VT, Expand);
739  setOperationAction(ISD::FP_TO_UINT, VT, Expand);
740  setOperationAction(ISD::FP_TO_SINT, VT, Expand);
741  setOperationAction(ISD::UINT_TO_FP, VT, Expand);
742  setOperationAction(ISD::SINT_TO_FP, VT, Expand);
743  setOperationAction(ISD::SIGN_EXTEND_INREG, VT,Expand);
744  setOperationAction(ISD::TRUNCATE, VT, Expand);
745  setOperationAction(ISD::SIGN_EXTEND, VT, Expand);
746  setOperationAction(ISD::ZERO_EXTEND, VT, Expand);
747  setOperationAction(ISD::ANY_EXTEND, VT, Expand);
748  setOperationAction(ISD::SELECT_CC, VT, Expand);
749  for (MVT InnerVT : MVT::vector_valuetypes()) {
750  setTruncStoreAction(InnerVT, VT, Expand);
751 
752  setLoadExtAction(ISD::SEXTLOAD, InnerVT, VT, Expand);
753  setLoadExtAction(ISD::ZEXTLOAD, InnerVT, VT, Expand);
754 
755  // N.b. ISD::EXTLOAD legality is basically ignored except for i1-like
756  // types, we have to deal with them whether we ask for Expansion or not.
757  // Setting Expand causes its own optimisation problems though, so leave
758  // them legal.
759  if (VT.getVectorElementType() == MVT::i1)
760  setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
761 
762  // EXTLOAD for MVT::f16 vectors is not legal because f16 vectors are
763  // split/scalarized right now.
764  if (VT.getVectorElementType() == MVT::f16)
765  setLoadExtAction(ISD::EXTLOAD, InnerVT, VT, Expand);
766  }
767  }
768 
769  // FIXME: In order to prevent SSE instructions being expanded to MMX ones
770  // with -msoft-float, disable use of MMX as well.
771  if (!Subtarget.useSoftFloat() && Subtarget.hasMMX()) {
772  addRegisterClass(MVT::x86mmx, &X86::VR64RegClass);
773  // No operations on x86mmx supported, everything uses intrinsics.
774  }
775 
776  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE1()) {
777  addRegisterClass(MVT::v4f32, Subtarget.hasVLX() ? &X86::VR128XRegClass
778  : &X86::VR128RegClass);
779 
780  setOperationAction(ISD::FNEG, MVT::v4f32, Custom);
781  setOperationAction(ISD::FABS, MVT::v4f32, Custom);
782  setOperationAction(ISD::FCOPYSIGN, MVT::v4f32, Custom);
783  setOperationAction(ISD::BUILD_VECTOR, MVT::v4f32, Custom);
784  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v4f32, Custom);
785  setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
786  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
787  setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
788  setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Custom);
789  }
790 
791  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
792  addRegisterClass(MVT::v2f64, Subtarget.hasVLX() ? &X86::VR128XRegClass
793  : &X86::VR128RegClass);
794 
795  // FIXME: Unfortunately, -soft-float and -no-implicit-float mean XMM
796  // registers cannot be used even for integer operations.
797  addRegisterClass(MVT::v16i8, Subtarget.hasVLX() ? &X86::VR128XRegClass
798  : &X86::VR128RegClass);
799  addRegisterClass(MVT::v8i16, Subtarget.hasVLX() ? &X86::VR128XRegClass
800  : &X86::VR128RegClass);
801  addRegisterClass(MVT::v4i32, Subtarget.hasVLX() ? &X86::VR128XRegClass
802  : &X86::VR128RegClass);
803  addRegisterClass(MVT::v2i64, Subtarget.hasVLX() ? &X86::VR128XRegClass
804  : &X86::VR128RegClass);
805 
806  for (auto VT : { MVT::v2i8, MVT::v4i8, MVT::v8i8,
807  MVT::v2i16, MVT::v4i16, MVT::v2i32 }) {
808  setOperationAction(ISD::SDIV, VT, Custom);
809  setOperationAction(ISD::SREM, VT, Custom);
810  setOperationAction(ISD::UDIV, VT, Custom);
811  setOperationAction(ISD::UREM, VT, Custom);
812  }
813 
814  setOperationAction(ISD::MUL, MVT::v2i8, Custom);
815  setOperationAction(ISD::MUL, MVT::v2i16, Custom);
816  setOperationAction(ISD::MUL, MVT::v2i32, Custom);
817  setOperationAction(ISD::MUL, MVT::v4i8, Custom);
818  setOperationAction(ISD::MUL, MVT::v4i16, Custom);
819  setOperationAction(ISD::MUL, MVT::v8i8, Custom);
820 
821  setOperationAction(ISD::MUL, MVT::v16i8, Custom);
822  setOperationAction(ISD::MUL, MVT::v4i32, Custom);
823  setOperationAction(ISD::MUL, MVT::v2i64, Custom);
824  setOperationAction(ISD::MULHU, MVT::v4i32, Custom);
825  setOperationAction(ISD::MULHS, MVT::v4i32, Custom);
826  setOperationAction(ISD::MULHU, MVT::v16i8, Custom);
827  setOperationAction(ISD::MULHS, MVT::v16i8, Custom);
828  setOperationAction(ISD::MULHU, MVT::v8i16, Legal);
829  setOperationAction(ISD::MULHS, MVT::v8i16, Legal);
830  setOperationAction(ISD::MUL, MVT::v8i16, Legal);
831  setOperationAction(ISD::FNEG, MVT::v2f64, Custom);
832  setOperationAction(ISD::FABS, MVT::v2f64, Custom);
833  setOperationAction(ISD::FCOPYSIGN, MVT::v2f64, Custom);
834 
835  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
836  setOperationAction(ISD::SMAX, VT, VT == MVT::v8i16 ? Legal : Custom);
837  setOperationAction(ISD::SMIN, VT, VT == MVT::v8i16 ? Legal : Custom);
838  setOperationAction(ISD::UMAX, VT, VT == MVT::v16i8 ? Legal : Custom);
839  setOperationAction(ISD::UMIN, VT, VT == MVT::v16i8 ? Legal : Custom);
840  }
841 
842  setOperationAction(ISD::UADDSAT, MVT::v16i8, Legal);
843  setOperationAction(ISD::SADDSAT, MVT::v16i8, Legal);
844  setOperationAction(ISD::USUBSAT, MVT::v16i8, Legal);
845  setOperationAction(ISD::SSUBSAT, MVT::v16i8, Legal);
846  setOperationAction(ISD::UADDSAT, MVT::v8i16, Legal);
847  setOperationAction(ISD::SADDSAT, MVT::v8i16, Legal);
848  setOperationAction(ISD::USUBSAT, MVT::v8i16, Legal);
849  setOperationAction(ISD::SSUBSAT, MVT::v8i16, Legal);
850  setOperationAction(ISD::UADDSAT, MVT::v4i32, Custom);
851  setOperationAction(ISD::USUBSAT, MVT::v4i32, Custom);
852  setOperationAction(ISD::UADDSAT, MVT::v2i64, Custom);
853  setOperationAction(ISD::USUBSAT, MVT::v2i64, Custom);
854 
855  if (!ExperimentalVectorWideningLegalization) {
856  // Use widening instead of promotion.
857  for (auto VT : { MVT::v8i8, MVT::v4i8, MVT::v2i8,
858  MVT::v4i16, MVT::v2i16 }) {
859  setOperationAction(ISD::UADDSAT, VT, Custom);
860  setOperationAction(ISD::SADDSAT, VT, Custom);
861  setOperationAction(ISD::USUBSAT, VT, Custom);
862  setOperationAction(ISD::SSUBSAT, VT, Custom);
863  }
864  }
865 
866  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v8i16, Custom);
867  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4i32, Custom);
868  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v4f32, Custom);
869 
870  // Provide custom widening for v2f32 setcc. This is really for VLX when
871  // setcc result type returns v2i1/v4i1 vector for v2f32/v4f32 leading to
872  // type legalization changing the result type to v4i1 during widening.
873  // It works fine for SSE2 and is probably faster so no need to qualify with
874  // VLX support.
875  setOperationAction(ISD::SETCC, MVT::v2i32, Custom);
876 
877  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
878  setOperationAction(ISD::SETCC, VT, Custom);
879  setOperationAction(ISD::CTPOP, VT, Custom);
880  setOperationAction(ISD::ABS, VT, Custom);
881 
882  // The condition codes aren't legal in SSE/AVX and under AVX512 we use
883  // setcc all the way to isel and prefer SETGT in some isel patterns.
884  setCondCodeAction(ISD::SETLT, VT, Custom);
885  setCondCodeAction(ISD::SETLE, VT, Custom);
886  }
887 
888  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
889  setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
890  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
891  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
892  setOperationAction(ISD::VSELECT, VT, Custom);
893  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
894  }
895 
896  // We support custom legalizing of sext and anyext loads for specific
897  // memory vector types which we can load as a scalar (or sequence of
898  // scalars) and extend in-register to a legal 128-bit vector type. For sext
899  // loads these must work with a single scalar load.
900  for (MVT VT : MVT::integer_vector_valuetypes()) {
901  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i8, Custom);
902  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i16, Custom);
903  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2i32, Custom);
904  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i8, Custom);
905  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4i16, Custom);
906  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8i8, Custom);
907  }
908 
909  for (auto VT : { MVT::v2f64, MVT::v2i64 }) {
910  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
911  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
912  setOperationAction(ISD::VSELECT, VT, Custom);
913 
914  if (VT == MVT::v2i64 && !Subtarget.is64Bit())
915  continue;
916 
917  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
918  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
919  }
920 
921  // Custom lower v2i64 and v2f64 selects.
922  setOperationAction(ISD::SELECT, MVT::v2f64, Custom);
923  setOperationAction(ISD::SELECT, MVT::v2i64, Custom);
924  setOperationAction(ISD::SELECT, MVT::v4i32, Custom);
925  setOperationAction(ISD::SELECT, MVT::v8i16, Custom);
926  setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
927 
928  setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Legal);
929  setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
930  setOperationAction(ISD::FP_TO_SINT, MVT::v2i16, Custom);
931 
932  // Custom legalize these to avoid over promotion or custom promotion.
933  setOperationAction(ISD::FP_TO_SINT, MVT::v2i8, Custom);
934  setOperationAction(ISD::FP_TO_SINT, MVT::v4i8, Custom);
935  setOperationAction(ISD::FP_TO_SINT, MVT::v8i8, Custom);
936  setOperationAction(ISD::FP_TO_SINT, MVT::v2i16, Custom);
937  setOperationAction(ISD::FP_TO_SINT, MVT::v4i16, Custom);
938  setOperationAction(ISD::FP_TO_UINT, MVT::v2i8, Custom);
939  setOperationAction(ISD::FP_TO_UINT, MVT::v4i8, Custom);
940  setOperationAction(ISD::FP_TO_UINT, MVT::v8i8, Custom);
941  setOperationAction(ISD::FP_TO_UINT, MVT::v2i16, Custom);
942  setOperationAction(ISD::FP_TO_UINT, MVT::v4i16, Custom);
943 
944  // By marking FP_TO_SINT v8i16 as Custom, will trick type legalization into
945  // promoting v8i8 FP_TO_UINT into FP_TO_SINT. When the v8i16 FP_TO_SINT is
946  // split again based on the input type, this will cause an AssertSExt i16 to
947  // be emitted instead of an AssertZExt. This will allow packssdw followed by
948  // packuswb to be used to truncate to v8i8. This is necessary since packusdw
949  // isn't available until sse4.1.
950  setOperationAction(ISD::FP_TO_SINT, MVT::v8i16, Custom);
951 
952  setOperationAction(ISD::SINT_TO_FP, MVT::v4i32, Legal);
953  setOperationAction(ISD::SINT_TO_FP, MVT::v2i32, Custom);
954 
955  setOperationAction(ISD::UINT_TO_FP, MVT::v2i32, Custom);
956 
957  // Fast v2f32 UINT_TO_FP( v2i32 ) custom conversion.
958  setOperationAction(ISD::UINT_TO_FP, MVT::v2f32, Custom);
959 
960  setOperationAction(ISD::FP_EXTEND, MVT::v2f32, Custom);
961  setOperationAction(ISD::FP_ROUND, MVT::v2f32, Custom);
962 
963  for (MVT VT : MVT::fp_vector_valuetypes())
964  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v2f32, Legal);
965 
966  // We want to legalize this to an f64 load rather than an i64 load on
967  // 64-bit targets and two 32-bit loads on a 32-bit target. Similar for
968  // store.
969  setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
970  setOperationAction(ISD::LOAD, MVT::v2i32, Custom);
971  setOperationAction(ISD::LOAD, MVT::v4i16, Custom);
972  setOperationAction(ISD::LOAD, MVT::v8i8, Custom);
973  setOperationAction(ISD::STORE, MVT::v2f32, Custom);
974  setOperationAction(ISD::STORE, MVT::v2i32, Custom);
975  setOperationAction(ISD::STORE, MVT::v4i16, Custom);
976  setOperationAction(ISD::STORE, MVT::v8i8, Custom);
977 
978  setOperationAction(ISD::BITCAST, MVT::v2i32, Custom);
979  setOperationAction(ISD::BITCAST, MVT::v4i16, Custom);
980  setOperationAction(ISD::BITCAST, MVT::v8i8, Custom);
981  if (!Subtarget.hasAVX512())
982  setOperationAction(ISD::BITCAST, MVT::v16i1, Custom);
983 
984  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v2i64, Custom);
985  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i32, Custom);
986  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v8i16, Custom);
987 
988  if (ExperimentalVectorWideningLegalization) {
989  setOperationAction(ISD::SIGN_EXTEND, MVT::v4i64, Custom);
990 
991  setOperationAction(ISD::TRUNCATE, MVT::v2i8, Custom);
992  setOperationAction(ISD::TRUNCATE, MVT::v2i16, Custom);
993  setOperationAction(ISD::TRUNCATE, MVT::v2i32, Custom);
994  setOperationAction(ISD::TRUNCATE, MVT::v4i8, Custom);
995  setOperationAction(ISD::TRUNCATE, MVT::v4i16, Custom);
996  setOperationAction(ISD::TRUNCATE, MVT::v8i8, Custom);
997  } else {
998  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v4i64, Custom);
999  }
1000 
1001  // In the customized shift lowering, the legal v4i32/v2i64 cases
1002  // in AVX2 will be recognized.
1003  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1004  setOperationAction(ISD::SRL, VT, Custom);
1005  setOperationAction(ISD::SHL, VT, Custom);
1006  setOperationAction(ISD::SRA, VT, Custom);
1007  }
1008 
1009  setOperationAction(ISD::ROTL, MVT::v4i32, Custom);
1010  setOperationAction(ISD::ROTL, MVT::v8i16, Custom);
1011 
1012  // With AVX512, expanding (and promoting the shifts) is better.
1013  if (!Subtarget.hasAVX512())
1014  setOperationAction(ISD::ROTL, MVT::v16i8, Custom);
1015  }
1016 
1017  if (!Subtarget.useSoftFloat() && Subtarget.hasSSSE3()) {
1018  setOperationAction(ISD::ABS, MVT::v16i8, Legal);
1019  setOperationAction(ISD::ABS, MVT::v8i16, Legal);
1020  setOperationAction(ISD::ABS, MVT::v4i32, Legal);
1021  setOperationAction(ISD::BITREVERSE, MVT::v16i8, Custom);
1022  setOperationAction(ISD::CTLZ, MVT::v16i8, Custom);
1023  setOperationAction(ISD::CTLZ, MVT::v8i16, Custom);
1024  setOperationAction(ISD::CTLZ, MVT::v4i32, Custom);
1025  setOperationAction(ISD::CTLZ, MVT::v2i64, Custom);
1026 
1027  // These might be better off as horizontal vector ops.
1028  setOperationAction(ISD::ADD, MVT::i16, Custom);
1029  setOperationAction(ISD::ADD, MVT::i32, Custom);
1030  setOperationAction(ISD::SUB, MVT::i16, Custom);
1031  setOperationAction(ISD::SUB, MVT::i32, Custom);
1032  }
1033 
1034  if (!Subtarget.useSoftFloat() && Subtarget.hasSSE41()) {
1035  for (MVT RoundedTy : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
1036  setOperationAction(ISD::FFLOOR, RoundedTy, Legal);
1037  setOperationAction(ISD::FCEIL, RoundedTy, Legal);
1038  setOperationAction(ISD::FTRUNC, RoundedTy, Legal);
1039  setOperationAction(ISD::FRINT, RoundedTy, Legal);
1040  setOperationAction(ISD::FNEARBYINT, RoundedTy, Legal);
1041  }
1042 
1043  setOperationAction(ISD::SMAX, MVT::v16i8, Legal);
1044  setOperationAction(ISD::SMAX, MVT::v4i32, Legal);
1045  setOperationAction(ISD::UMAX, MVT::v8i16, Legal);
1046  setOperationAction(ISD::UMAX, MVT::v4i32, Legal);
1047  setOperationAction(ISD::SMIN, MVT::v16i8, Legal);
1048  setOperationAction(ISD::SMIN, MVT::v4i32, Legal);
1049  setOperationAction(ISD::UMIN, MVT::v8i16, Legal);
1050  setOperationAction(ISD::UMIN, MVT::v4i32, Legal);
1051 
1052  // FIXME: Do we need to handle scalar-to-vector here?
1053  setOperationAction(ISD::MUL, MVT::v4i32, Legal);
1054 
1055  // We directly match byte blends in the backend as they match the VSELECT
1056  // condition form.
1057  setOperationAction(ISD::VSELECT, MVT::v16i8, Legal);
1058 
1059  // SSE41 brings specific instructions for doing vector sign extend even in
1060  // cases where we don't have SRA.
1061  for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1062  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Legal);
1063  setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Legal);
1064  }
1065 
1066  if (!ExperimentalVectorWideningLegalization) {
1067  // Avoid narrow result types when widening. The legal types are listed
1068  // in the next loop.
1069  for (MVT VT : MVT::integer_vector_valuetypes()) {
1070  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i8, Custom);
1071  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i16, Custom);
1072  setLoadExtAction(ISD::SEXTLOAD, VT, MVT::v2i32, Custom);
1073  }
1074  }
1075 
1076  // SSE41 also has vector sign/zero extending loads, PMOV[SZ]X
1077  for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1078  setLoadExtAction(LoadExtOp, MVT::v8i16, MVT::v8i8, Legal);
1079  setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i8, Legal);
1080  if (!ExperimentalVectorWideningLegalization)
1081  setLoadExtAction(LoadExtOp, MVT::v2i32, MVT::v2i8, Legal);
1082  setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i8, Legal);
1083  setLoadExtAction(LoadExtOp, MVT::v4i32, MVT::v4i16, Legal);
1084  setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i16, Legal);
1085  setLoadExtAction(LoadExtOp, MVT::v2i64, MVT::v2i32, Legal);
1086  }
1087 
1088  // i8 vectors are custom because the source register and source
1089  // source memory operand types are not the same width.
1090  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v16i8, Custom);
1091  }
1092 
1093  if (!Subtarget.useSoftFloat() && Subtarget.hasXOP()) {
1094  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1095  MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1096  setOperationAction(ISD::ROTL, VT, Custom);
1097 
1098  // XOP can efficiently perform BITREVERSE with VPPERM.
1099  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 })
1100  setOperationAction(ISD::BITREVERSE, VT, Custom);
1101 
1102  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1103  MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 })
1104  setOperationAction(ISD::BITREVERSE, VT, Custom);
1105  }
1106 
1107  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX()) {
1108  bool HasInt256 = Subtarget.hasInt256();
1109 
1110  addRegisterClass(MVT::v32i8, Subtarget.hasVLX() ? &X86::VR256XRegClass
1111  : &X86::VR256RegClass);
1112  addRegisterClass(MVT::v16i16, Subtarget.hasVLX() ? &X86::VR256XRegClass
1113  : &X86::VR256RegClass);
1114  addRegisterClass(MVT::v8i32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1115  : &X86::VR256RegClass);
1116  addRegisterClass(MVT::v8f32, Subtarget.hasVLX() ? &X86::VR256XRegClass
1117  : &X86::VR256RegClass);
1118  addRegisterClass(MVT::v4i64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1119  : &X86::VR256RegClass);
1120  addRegisterClass(MVT::v4f64, Subtarget.hasVLX() ? &X86::VR256XRegClass
1121  : &X86::VR256RegClass);
1122 
1123  for (auto VT : { MVT::v8f32, MVT::v4f64 }) {
1124  setOperationAction(ISD::FFLOOR, VT, Legal);
1125  setOperationAction(ISD::FCEIL, VT, Legal);
1126  setOperationAction(ISD::FTRUNC, VT, Legal);
1127  setOperationAction(ISD::FRINT, VT, Legal);
1128  setOperationAction(ISD::FNEARBYINT, VT, Legal);
1129  setOperationAction(ISD::FNEG, VT, Custom);
1130  setOperationAction(ISD::FABS, VT, Custom);
1131  setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1132  }
1133 
1134  // (fp_to_int:v8i16 (v8f32 ..)) requires the result type to be promoted
1135  // even though v8i16 is a legal type.
1136  setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i16, MVT::v8i32);
1137  setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i16, MVT::v8i32);
1138  setOperationAction(ISD::FP_TO_SINT, MVT::v8i32, Legal);
1139 
1140  setOperationAction(ISD::SINT_TO_FP, MVT::v8i32, Legal);
1141  setOperationAction(ISD::FP_ROUND, MVT::v4f32, Legal);
1142 
1143  if (!Subtarget.hasAVX512())
1144  setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
1145 
1146  for (MVT VT : MVT::fp_vector_valuetypes())
1147  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v4f32, Legal);
1148 
1149  // In the customized shift lowering, the legal v8i32/v4i64 cases
1150  // in AVX2 will be recognized.
1151  for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1152  setOperationAction(ISD::SRL, VT, Custom);
1153  setOperationAction(ISD::SHL, VT, Custom);
1154  setOperationAction(ISD::SRA, VT, Custom);
1155  }
1156 
1157  // These types need custom splitting if their input is a 128-bit vector.
1158  setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1159  setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1160  setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1161  setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1162 
1163  setOperationAction(ISD::ROTL, MVT::v8i32, Custom);
1164  setOperationAction(ISD::ROTL, MVT::v16i16, Custom);
1165 
1166  // With BWI, expanding (and promoting the shifts) is the better.
1167  if (!Subtarget.hasBWI())
1168  setOperationAction(ISD::ROTL, MVT::v32i8, Custom);
1169 
1170  setOperationAction(ISD::SELECT, MVT::v4f64, Custom);
1171  setOperationAction(ISD::SELECT, MVT::v4i64, Custom);
1172  setOperationAction(ISD::SELECT, MVT::v8i32, Custom);
1173  setOperationAction(ISD::SELECT, MVT::v16i16, Custom);
1174  setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
1175  setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
1176 
1177  for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1178  setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1179  setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1180  setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1181  }
1182 
1183  setOperationAction(ISD::TRUNCATE, MVT::v16i8, Custom);
1184  setOperationAction(ISD::TRUNCATE, MVT::v8i16, Custom);
1185  setOperationAction(ISD::TRUNCATE, MVT::v4i32, Custom);
1186  setOperationAction(ISD::BITREVERSE, MVT::v32i8, Custom);
1187 
1188  for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1189  setOperationAction(ISD::SETCC, VT, Custom);
1190  setOperationAction(ISD::CTPOP, VT, Custom);
1191  setOperationAction(ISD::CTLZ, VT, Custom);
1192 
1193  // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1194  // setcc all the way to isel and prefer SETGT in some isel patterns.
1195  setCondCodeAction(ISD::SETLT, VT, Custom);
1196  setCondCodeAction(ISD::SETLE, VT, Custom);
1197  }
1198 
1199  if (Subtarget.hasAnyFMA()) {
1200  for (auto VT : { MVT::f32, MVT::f64, MVT::v4f32, MVT::v8f32,
1201  MVT::v2f64, MVT::v4f64 })
1202  setOperationAction(ISD::FMA, VT, Legal);
1203  }
1204 
1205  for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1206  setOperationAction(ISD::ADD, VT, HasInt256 ? Legal : Custom);
1207  setOperationAction(ISD::SUB, VT, HasInt256 ? Legal : Custom);
1208  }
1209 
1210  setOperationAction(ISD::MUL, MVT::v4i64, Custom);
1211  setOperationAction(ISD::MUL, MVT::v8i32, HasInt256 ? Legal : Custom);
1212  setOperationAction(ISD::MUL, MVT::v16i16, HasInt256 ? Legal : Custom);
1213  setOperationAction(ISD::MUL, MVT::v32i8, Custom);
1214 
1215  setOperationAction(ISD::MULHU, MVT::v8i32, Custom);
1216  setOperationAction(ISD::MULHS, MVT::v8i32, Custom);
1217  setOperationAction(ISD::MULHU, MVT::v16i16, HasInt256 ? Legal : Custom);
1218  setOperationAction(ISD::MULHS, MVT::v16i16, HasInt256 ? Legal : Custom);
1219  setOperationAction(ISD::MULHU, MVT::v32i8, Custom);
1220  setOperationAction(ISD::MULHS, MVT::v32i8, Custom);
1221 
1222  setOperationAction(ISD::ABS, MVT::v4i64, Custom);
1223  setOperationAction(ISD::SMAX, MVT::v4i64, Custom);
1224  setOperationAction(ISD::UMAX, MVT::v4i64, Custom);
1225  setOperationAction(ISD::SMIN, MVT::v4i64, Custom);
1226  setOperationAction(ISD::UMIN, MVT::v4i64, Custom);
1227 
1228  setOperationAction(ISD::UADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1229  setOperationAction(ISD::SADDSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1230  setOperationAction(ISD::USUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1231  setOperationAction(ISD::SSUBSAT, MVT::v32i8, HasInt256 ? Legal : Custom);
1232  setOperationAction(ISD::UADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1233  setOperationAction(ISD::SADDSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1234  setOperationAction(ISD::USUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1235  setOperationAction(ISD::SSUBSAT, MVT::v16i16, HasInt256 ? Legal : Custom);
1236 
1237  for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32 }) {
1238  setOperationAction(ISD::ABS, VT, HasInt256 ? Legal : Custom);
1239  setOperationAction(ISD::SMAX, VT, HasInt256 ? Legal : Custom);
1240  setOperationAction(ISD::UMAX, VT, HasInt256 ? Legal : Custom);
1241  setOperationAction(ISD::SMIN, VT, HasInt256 ? Legal : Custom);
1242  setOperationAction(ISD::UMIN, VT, HasInt256 ? Legal : Custom);
1243  }
1244 
1245  for (auto VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64}) {
1246  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1247  setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1248  }
1249 
1250  if (HasInt256) {
1251  // The custom lowering for UINT_TO_FP for v8i32 becomes interesting
1252  // when we have a 256bit-wide blend with immediate.
1253  setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Custom);
1254 
1255  // AVX2 also has wider vector sign/zero extending loads, VPMOV[SZ]X
1256  for (auto LoadExtOp : { ISD::SEXTLOAD, ISD::ZEXTLOAD }) {
1257  setLoadExtAction(LoadExtOp, MVT::v16i16, MVT::v16i8, Legal);
1258  setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i8, Legal);
1259  setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i8, Legal);
1260  setLoadExtAction(LoadExtOp, MVT::v8i32, MVT::v8i16, Legal);
1261  setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i16, Legal);
1262  setLoadExtAction(LoadExtOp, MVT::v4i64, MVT::v4i32, Legal);
1263  }
1264  }
1265 
1266  for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1267  MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 }) {
1268  setOperationAction(ISD::MLOAD, VT, Legal);
1269  setOperationAction(ISD::MSTORE, VT, Legal);
1270  }
1271 
1272  // Extract subvector is special because the value type
1273  // (result) is 128-bit but the source is 256-bit wide.
1274  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64,
1275  MVT::v4f32, MVT::v2f64 }) {
1276  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1277  }
1278 
1279  // Custom lower several nodes for 256-bit types.
1280  for (MVT VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1281  MVT::v8f32, MVT::v4f64 }) {
1282  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1283  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1284  setOperationAction(ISD::VSELECT, VT, Custom);
1285  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1286  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1287  setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1288  setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1289  setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
1290  }
1291 
1292  if (HasInt256)
1293  setOperationAction(ISD::VSELECT, MVT::v32i8, Legal);
1294 
1295  if (HasInt256) {
1296  // Custom legalize 2x32 to get a little better code.
1297  setOperationAction(ISD::MGATHER, MVT::v2f32, Custom);
1298  setOperationAction(ISD::MGATHER, MVT::v2i32, Custom);
1299 
1300  for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1301  MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1302  setOperationAction(ISD::MGATHER, VT, Custom);
1303  }
1304  }
1305 
1306  // This block controls legalization of the mask vector sizes that are
1307  // available with AVX512. 512-bit vectors are in a separate block controlled
1308  // by useAVX512Regs.
1309  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1310  addRegisterClass(MVT::v1i1, &X86::VK1RegClass);
1311  addRegisterClass(MVT::v2i1, &X86::VK2RegClass);
1312  addRegisterClass(MVT::v4i1, &X86::VK4RegClass);
1313  addRegisterClass(MVT::v8i1, &X86::VK8RegClass);
1314  addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
1315 
1316  setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
1317  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
1318  setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
1319 
1320  setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v8i1, MVT::v8i32);
1321  setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v8i1, MVT::v8i32);
1322  setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v4i1, MVT::v4i32);
1323  setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v4i1, MVT::v4i32);
1324  setOperationAction(ISD::FP_TO_SINT, MVT::v2i1, Custom);
1325  setOperationAction(ISD::FP_TO_UINT, MVT::v2i1, Custom);
1326 
1327  // There is no byte sized k-register load or store without AVX512DQ.
1328  if (!Subtarget.hasDQI()) {
1329  setOperationAction(ISD::LOAD, MVT::v1i1, Custom);
1330  setOperationAction(ISD::LOAD, MVT::v2i1, Custom);
1331  setOperationAction(ISD::LOAD, MVT::v4i1, Custom);
1332  setOperationAction(ISD::LOAD, MVT::v8i1, Custom);
1333 
1334  setOperationAction(ISD::STORE, MVT::v1i1, Custom);
1335  setOperationAction(ISD::STORE, MVT::v2i1, Custom);
1336  setOperationAction(ISD::STORE, MVT::v4i1, Custom);
1337  setOperationAction(ISD::STORE, MVT::v8i1, Custom);
1338  }
1339 
1340  // Extends of v16i1/v8i1/v4i1/v2i1 to 128-bit vectors.
1341  for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64 }) {
1342  setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
1343  setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
1344  setOperationAction(ISD::ANY_EXTEND, VT, Custom);
1345  }
1346 
1347  for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
1348  setOperationAction(ISD::ADD, VT, Custom);
1349  setOperationAction(ISD::SUB, VT, Custom);
1350  setOperationAction(ISD::MUL, VT, Custom);
1351  setOperationAction(ISD::SETCC, VT, Custom);
1352  setOperationAction(ISD::SELECT, VT, Custom);
1353  setOperationAction(ISD::TRUNCATE, VT, Custom);
1354  setOperationAction(ISD::UADDSAT, VT, Custom);
1355  setOperationAction(ISD::SADDSAT, VT, Custom);
1356  setOperationAction(ISD::USUBSAT, VT, Custom);
1357  setOperationAction(ISD::SSUBSAT, VT, Custom);
1358 
1359  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1360  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1361  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1362  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1363  setOperationAction(ISD::VSELECT, VT, Expand);
1364  }
1365 
1366  setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i1, Custom);
1367  setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i1, Custom);
1368  setOperationAction(ISD::CONCAT_VECTORS, MVT::v4i1, Custom);
1369  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v2i1, Custom);
1370  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v4i1, Custom);
1371  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v8i1, Custom);
1372  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v16i1, Custom);
1373  for (auto VT : { MVT::v1i1, MVT::v2i1, MVT::v4i1, MVT::v8i1 })
1374  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1375  }
1376 
1377  // This block controls legalization for 512-bit operations with 32/64 bit
1378  // elements. 512-bits can be disabled based on prefer-vector-width and
1379  // required-vector-width function attributes.
1380  if (!Subtarget.useSoftFloat() && Subtarget.useAVX512Regs()) {
1381  addRegisterClass(MVT::v16i32, &X86::VR512RegClass);
1382  addRegisterClass(MVT::v16f32, &X86::VR512RegClass);
1383  addRegisterClass(MVT::v8i64, &X86::VR512RegClass);
1384  addRegisterClass(MVT::v8f64, &X86::VR512RegClass);
1385 
1386  for (MVT VT : MVT::fp_vector_valuetypes())
1387  setLoadExtAction(ISD::EXTLOAD, VT, MVT::v8f32, Legal);
1388 
1389  for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1390  setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i8, Legal);
1391  setLoadExtAction(ExtType, MVT::v16i32, MVT::v16i16, Legal);
1392  setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i8, Legal);
1393  setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i16, Legal);
1394  setLoadExtAction(ExtType, MVT::v8i64, MVT::v8i32, Legal);
1395  }
1396 
1397  for (MVT VT : { MVT::v16f32, MVT::v8f64 }) {
1398  setOperationAction(ISD::FNEG, VT, Custom);
1399  setOperationAction(ISD::FABS, VT, Custom);
1400  setOperationAction(ISD::FMA, VT, Legal);
1401  setOperationAction(ISD::FCOPYSIGN, VT, Custom);
1402  }
1403 
1404  setOperationAction(ISD::FP_TO_SINT, MVT::v16i32, Legal);
1405  setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v16i16, MVT::v16i32);
1406  setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v16i8, MVT::v16i32);
1407  setOperationPromotedToType(ISD::FP_TO_SINT, MVT::v16i1, MVT::v16i32);
1408  setOperationAction(ISD::FP_TO_UINT, MVT::v16i32, Legal);
1409  setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v16i1, MVT::v16i32);
1410  setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v16i8, MVT::v16i32);
1411  setOperationPromotedToType(ISD::FP_TO_UINT, MVT::v16i16, MVT::v16i32);
1412  setOperationAction(ISD::SINT_TO_FP, MVT::v16i32, Legal);
1413  setOperationAction(ISD::UINT_TO_FP, MVT::v16i32, Legal);
1414 
1415  setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
1416  setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
1417  setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
1418  setTruncStoreAction(MVT::v16i32, MVT::v16i8, Legal);
1419  setTruncStoreAction(MVT::v16i32, MVT::v16i16, Legal);
1420 
1421  if (!Subtarget.hasVLX()) {
1422  // With 512-bit vectors and no VLX, we prefer to widen MLOAD/MSTORE
1423  // to 512-bit rather than use the AVX2 instructions so that we can use
1424  // k-masks.
1425  for (auto VT : {MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1426  MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64}) {
1427  setOperationAction(ISD::MLOAD, VT, Custom);
1428  setOperationAction(ISD::MSTORE, VT, Custom);
1429  }
1430  }
1431 
1432  setOperationAction(ISD::TRUNCATE, MVT::v8i32, Custom);
1433  setOperationAction(ISD::TRUNCATE, MVT::v16i16, Custom);
1434  setOperationAction(ISD::ZERO_EXTEND, MVT::v16i32, Custom);
1435  setOperationAction(ISD::ZERO_EXTEND, MVT::v8i64, Custom);
1436  setOperationAction(ISD::ANY_EXTEND, MVT::v16i32, Custom);
1437  setOperationAction(ISD::ANY_EXTEND, MVT::v8i64, Custom);
1438  setOperationAction(ISD::SIGN_EXTEND, MVT::v16i32, Custom);
1439  setOperationAction(ISD::SIGN_EXTEND, MVT::v8i64, Custom);
1440 
1441  if (ExperimentalVectorWideningLegalization) {
1442  // Need to custom widen this if we don't have AVX512BW.
1443  setOperationAction(ISD::ANY_EXTEND, MVT::v8i8, Custom);
1444  setOperationAction(ISD::ZERO_EXTEND, MVT::v8i8, Custom);
1445  setOperationAction(ISD::SIGN_EXTEND, MVT::v8i8, Custom);
1446  }
1447 
1448  for (auto VT : { MVT::v16f32, MVT::v8f64 }) {
1449  setOperationAction(ISD::FFLOOR, VT, Legal);
1450  setOperationAction(ISD::FCEIL, VT, Legal);
1451  setOperationAction(ISD::FTRUNC, VT, Legal);
1452  setOperationAction(ISD::FRINT, VT, Legal);
1453  setOperationAction(ISD::FNEARBYINT, VT, Legal);
1454  }
1455 
1456  // Without BWI we need to use custom lowering to handle MVT::v64i8 input.
1457  for (auto VT : {MVT::v16i32, MVT::v8i64, MVT::v64i8}) {
1458  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, VT, Custom);
1459  setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, VT, Custom);
1460  }
1461 
1462  setOperationAction(ISD::CONCAT_VECTORS, MVT::v8f64, Custom);
1463  setOperationAction(ISD::CONCAT_VECTORS, MVT::v8i64, Custom);
1464  setOperationAction(ISD::CONCAT_VECTORS, MVT::v16f32, Custom);
1465  setOperationAction(ISD::CONCAT_VECTORS, MVT::v16i32, Custom);
1466 
1467  setOperationAction(ISD::MUL, MVT::v8i64, Custom);
1468  setOperationAction(ISD::MUL, MVT::v16i32, Legal);
1469 
1470  setOperationAction(ISD::MULHU, MVT::v16i32, Custom);
1471  setOperationAction(ISD::MULHS, MVT::v16i32, Custom);
1472 
1473  setOperationAction(ISD::SELECT, MVT::v8f64, Custom);
1474  setOperationAction(ISD::SELECT, MVT::v8i64, Custom);
1475  setOperationAction(ISD::SELECT, MVT::v16i32, Custom);
1476  setOperationAction(ISD::SELECT, MVT::v32i16, Custom);
1477  setOperationAction(ISD::SELECT, MVT::v64i8, Custom);
1478  setOperationAction(ISD::SELECT, MVT::v16f32, Custom);
1479 
1480  for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1481  setOperationAction(ISD::SMAX, VT, Legal);
1482  setOperationAction(ISD::UMAX, VT, Legal);
1483  setOperationAction(ISD::SMIN, VT, Legal);
1484  setOperationAction(ISD::UMIN, VT, Legal);
1485  setOperationAction(ISD::ABS, VT, Legal);
1486  setOperationAction(ISD::SRL, VT, Custom);
1487  setOperationAction(ISD::SHL, VT, Custom);
1488  setOperationAction(ISD::SRA, VT, Custom);
1489  setOperationAction(ISD::CTPOP, VT, Custom);
1490  setOperationAction(ISD::ROTL, VT, Custom);
1491  setOperationAction(ISD::ROTR, VT, Custom);
1492  setOperationAction(ISD::SETCC, VT, Custom);
1493 
1494  // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1495  // setcc all the way to isel and prefer SETGT in some isel patterns.
1496  setCondCodeAction(ISD::SETLT, VT, Custom);
1497  setCondCodeAction(ISD::SETLE, VT, Custom);
1498  }
1499 
1500  if (Subtarget.hasDQI()) {
1501  setOperationAction(ISD::SINT_TO_FP, MVT::v8i64, Legal);
1502  setOperationAction(ISD::UINT_TO_FP, MVT::v8i64, Legal);
1503  setOperationAction(ISD::FP_TO_SINT, MVT::v8i64, Legal);
1504  setOperationAction(ISD::FP_TO_UINT, MVT::v8i64, Legal);
1505 
1506  setOperationAction(ISD::MUL, MVT::v8i64, Legal);
1507  }
1508 
1509  if (Subtarget.hasCDI()) {
1510  // NonVLX sub-targets extend 128/256 vectors to use the 512 version.
1511  for (auto VT : { MVT::v16i32, MVT::v8i64} ) {
1512  setOperationAction(ISD::CTLZ, VT, Legal);
1513  }
1514  } // Subtarget.hasCDI()
1515 
1516  if (Subtarget.hasVPOPCNTDQ()) {
1517  for (auto VT : { MVT::v16i32, MVT::v8i64 })
1518  setOperationAction(ISD::CTPOP, VT, Legal);
1519  }
1520 
1521  // Extract subvector is special because the value type
1522  // (result) is 256-bit but the source is 512-bit wide.
1523  // 128-bit was made Legal under AVX1.
1524  for (auto VT : { MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64,
1525  MVT::v8f32, MVT::v4f64 })
1526  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Legal);
1527 
1528  for (auto VT : { MVT::v16i32, MVT::v8i64, MVT::v16f32, MVT::v8f64 }) {
1529  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1530  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1531  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1532  setOperationAction(ISD::VSELECT, VT, Custom);
1533  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1534  setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
1535  setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
1536  setOperationAction(ISD::MLOAD, VT, Legal);
1537  setOperationAction(ISD::MSTORE, VT, Legal);
1538  setOperationAction(ISD::MGATHER, VT, Custom);
1539  setOperationAction(ISD::MSCATTER, VT, Custom);
1540  }
1541  // Need to custom split v32i16/v64i8 bitcasts.
1542  if (!Subtarget.hasBWI()) {
1543  setOperationAction(ISD::BITCAST, MVT::v32i16, Custom);
1544  setOperationAction(ISD::BITCAST, MVT::v64i8, Custom);
1545  }
1546 
1547  if (Subtarget.hasVBMI2()) {
1548  for (auto VT : { MVT::v16i32, MVT::v8i64 }) {
1549  setOperationAction(ISD::FSHL, VT, Custom);
1550  setOperationAction(ISD::FSHR, VT, Custom);
1551  }
1552  }
1553  }// has AVX-512
1554 
1555  // This block controls legalization for operations that don't have
1556  // pre-AVX512 equivalents. Without VLX we use 512-bit operations for
1557  // narrower widths.
1558  if (!Subtarget.useSoftFloat() && Subtarget.hasAVX512()) {
1559  // These operations are handled on non-VLX by artificially widening in
1560  // isel patterns.
1561  // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
1562 
1563  setOperationAction(ISD::FP_TO_UINT, MVT::v8i32, Legal);
1564  setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Legal);
1565  setOperationAction(ISD::FP_TO_UINT, MVT::v2i32, Custom);
1566  setOperationAction(ISD::UINT_TO_FP, MVT::v8i32, Legal);
1567  setOperationAction(ISD::UINT_TO_FP, MVT::v4i32, Legal);
1568 
1569  for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
1570  setOperationAction(ISD::SMAX, VT, Legal);
1571  setOperationAction(ISD::UMAX, VT, Legal);
1572  setOperationAction(ISD::SMIN, VT, Legal);
1573  setOperationAction(ISD::UMIN, VT, Legal);
1574  setOperationAction(ISD::ABS, VT, Legal);
1575  }
1576 
1577  for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
1578  setOperationAction(ISD::ROTL, VT, Custom);
1579  setOperationAction(ISD::ROTR, VT, Custom);
1580  }
1581 
1582  // Custom legalize 2x32 to get a little better code.
1583  setOperationAction(ISD::MSCATTER, MVT::v2f32, Custom);
1584  setOperationAction(ISD::MSCATTER, MVT::v2i32, Custom);
1585 
1586  for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64,
1587  MVT::v4f32, MVT::v8f32, MVT::v2f64, MVT::v4f64 })
1588  setOperationAction(ISD::MSCATTER, VT, Custom);
1589 
1590  if (Subtarget.hasDQI()) {
1591  for (auto VT : { MVT::v2i64, MVT::v4i64 }) {
1592  setOperationAction(ISD::SINT_TO_FP, VT, Legal);
1593  setOperationAction(ISD::UINT_TO_FP, VT, Legal);
1594  setOperationAction(ISD::FP_TO_SINT, VT, Legal);
1595  setOperationAction(ISD::FP_TO_UINT, VT, Legal);
1596 
1597  setOperationAction(ISD::MUL, VT, Legal);
1598  }
1599  }
1600 
1601  if (Subtarget.hasCDI()) {
1602  for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 }) {
1603  setOperationAction(ISD::CTLZ, VT, Legal);
1604  }
1605  } // Subtarget.hasCDI()
1606 
1607  if (Subtarget.hasVPOPCNTDQ()) {
1608  for (auto VT : { MVT::v4i32, MVT::v8i32, MVT::v2i64, MVT::v4i64 })
1609  setOperationAction(ISD::CTPOP, VT, Legal);
1610  }
1611  }
1612 
1613  // This block control legalization of v32i1/v64i1 which are available with
1614  // AVX512BW. 512-bit v32i16 and v64i8 vector legalization is controlled with
1615  // useBWIRegs.
1616  if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
1617  addRegisterClass(MVT::v32i1, &X86::VK32RegClass);
1618  addRegisterClass(MVT::v64i1, &X86::VK64RegClass);
1619 
1620  for (auto VT : { MVT::v32i1, MVT::v64i1 }) {
1621  setOperationAction(ISD::ADD, VT, Custom);
1622  setOperationAction(ISD::SUB, VT, Custom);
1623  setOperationAction(ISD::MUL, VT, Custom);
1624  setOperationAction(ISD::VSELECT, VT, Expand);
1625  setOperationAction(ISD::UADDSAT, VT, Custom);
1626  setOperationAction(ISD::SADDSAT, VT, Custom);
1627  setOperationAction(ISD::USUBSAT, VT, Custom);
1628  setOperationAction(ISD::SSUBSAT, VT, Custom);
1629 
1630  setOperationAction(ISD::TRUNCATE, VT, Custom);
1631  setOperationAction(ISD::SETCC, VT, Custom);
1632  setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
1633  setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
1634  setOperationAction(ISD::SELECT, VT, Custom);
1635  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1636  setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
1637  }
1638 
1639  setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i1, Custom);
1640  setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i1, Custom);
1641  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i1, Custom);
1642  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i1, Custom);
1643  for (auto VT : { MVT::v16i1, MVT::v32i1 })
1644  setOperationAction(ISD::EXTRACT_SUBVECTOR, VT, Custom);
1645 
1646  // Extends from v32i1 masks to 256-bit vectors.
1647  setOperationAction(ISD::SIGN_EXTEND, MVT::v32i8, Custom);
1648  setOperationAction(ISD::ZERO_EXTEND, MVT::v32i8, Custom);
1649  setOperationAction(ISD::ANY_EXTEND, MVT::v32i8, Custom);
1650  }
1651 
1652  // This block controls legalization for v32i16 and v64i8. 512-bits can be
1653  // disabled based on prefer-vector-width and required-vector-width function
1654  // attributes.
1655  if (!Subtarget.useSoftFloat() && Subtarget.useBWIRegs()) {
1656  addRegisterClass(MVT::v32i16, &X86::VR512RegClass);
1657  addRegisterClass(MVT::v64i8, &X86::VR512RegClass);
1658 
1659  // Extends from v64i1 masks to 512-bit vectors.
1660  setOperationAction(ISD::SIGN_EXTEND, MVT::v64i8, Custom);
1661  setOperationAction(ISD::ZERO_EXTEND, MVT::v64i8, Custom);
1662  setOperationAction(ISD::ANY_EXTEND, MVT::v64i8, Custom);
1663 
1664  setOperationAction(ISD::MUL, MVT::v32i16, Legal);
1665  setOperationAction(ISD::MUL, MVT::v64i8, Custom);
1666  setOperationAction(ISD::MULHS, MVT::v32i16, Legal);
1667  setOperationAction(ISD::MULHU, MVT::v32i16, Legal);
1668  setOperationAction(ISD::MULHS, MVT::v64i8, Custom);
1669  setOperationAction(ISD::MULHU, MVT::v64i8, Custom);
1670  setOperationAction(ISD::CONCAT_VECTORS, MVT::v32i16, Custom);
1671  setOperationAction(ISD::CONCAT_VECTORS, MVT::v64i8, Custom);
1672  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v32i16, Legal);
1673  setOperationAction(ISD::INSERT_SUBVECTOR, MVT::v64i8, Legal);
1674  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v32i16, Custom);
1675  setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v64i8, Custom);
1676  setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v32i16, Custom);
1677  setOperationAction(ISD::SCALAR_TO_VECTOR, MVT::v64i8, Custom);
1678  setOperationAction(ISD::SIGN_EXTEND, MVT::v32i16, Custom);
1679  setOperationAction(ISD::ZERO_EXTEND, MVT::v32i16, Custom);
1680  setOperationAction(ISD::ANY_EXTEND, MVT::v32i16, Custom);
1681  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v32i16, Custom);
1682  setOperationAction(ISD::VECTOR_SHUFFLE, MVT::v64i8, Custom);
1683  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v32i16, Custom);
1684  setOperationAction(ISD::INSERT_VECTOR_ELT, MVT::v64i8, Custom);
1685  setOperationAction(ISD::TRUNCATE, MVT::v32i8, Custom);
1686  setOperationAction(ISD::BITREVERSE, MVT::v64i8, Custom);
1687 
1688  setOperationAction(ISD::SIGN_EXTEND_VECTOR_INREG, MVT::v32i16, Custom);
1689  setOperationAction(ISD::ZERO_EXTEND_VECTOR_INREG, MVT::v32i16, Custom);
1690 
1691  setTruncStoreAction(MVT::v32i16, MVT::v32i8, Legal);
1692 
1693  for (auto VT : { MVT::v64i8, MVT::v32i16 }) {
1694  setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
1695  setOperationAction(ISD::VSELECT, VT, Custom);
1696  setOperationAction(ISD::ABS, VT, Legal);
1697  setOperationAction(ISD::SRL, VT, Custom);
1698  setOperationAction(ISD::SHL, VT, Custom);
1699  setOperationAction(ISD::SRA, VT, Custom);
1700  setOperationAction(ISD::MLOAD, VT, Legal);
1701  setOperationAction(ISD::MSTORE, VT, Legal);
1702  setOperationAction(ISD::CTPOP, VT, Custom);
1703  setOperationAction(ISD::CTLZ, VT, Custom);
1704  setOperationAction(ISD::SMAX, VT, Legal);
1705  setOperationAction(ISD::UMAX, VT, Legal);
1706  setOperationAction(ISD::SMIN, VT, Legal);
1707  setOperationAction(ISD::UMIN, VT, Legal);
1708  setOperationAction(ISD::SETCC, VT, Custom);
1709  setOperationAction(ISD::UADDSAT, VT, Legal);
1710  setOperationAction(ISD::SADDSAT, VT, Legal);
1711  setOperationAction(ISD::USUBSAT, VT, Legal);
1712  setOperationAction(ISD::SSUBSAT, VT, Legal);
1713 
1714  // The condition codes aren't legal in SSE/AVX and under AVX512 we use
1715  // setcc all the way to isel and prefer SETGT in some isel patterns.
1716  setCondCodeAction(ISD::SETLT, VT, Custom);
1717  setCondCodeAction(ISD::SETLE, VT, Custom);
1718  }
1719 
1720  for (auto ExtType : {ISD::ZEXTLOAD, ISD::SEXTLOAD}) {
1721  setLoadExtAction(ExtType, MVT::v32i16, MVT::v32i8, Legal);
1722  }
1723 
1724  if (Subtarget.hasBITALG()) {
1725  for (auto VT : { MVT::v64i8, MVT::v32i16 })
1726  setOperationAction(ISD::CTPOP, VT, Legal);
1727  }
1728 
1729  if (Subtarget.hasVBMI2()) {
1730  setOperationAction(ISD::FSHL, MVT::v32i16, Custom);
1731  setOperationAction(ISD::FSHR, MVT::v32i16, Custom);
1732  }
1733  }
1734 
1735  if (!Subtarget.useSoftFloat() && Subtarget.hasBWI()) {
1736  for (auto VT : { MVT::v32i8, MVT::v16i8, MVT::v16i16, MVT::v8i16 }) {
1737  setOperationAction(ISD::MLOAD, VT, Subtarget.hasVLX() ? Legal : Custom);
1738  setOperationAction(ISD::MSTORE, VT, Subtarget.hasVLX() ? Legal : Custom);
1739  }
1740 
1741  // These operations are handled on non-VLX by artificially widening in
1742  // isel patterns.
1743  // TODO: Custom widen in lowering on non-VLX and drop the isel patterns?
1744 
1745  if (Subtarget.hasBITALG()) {
1746  for (auto VT : { MVT::v16i8, MVT::v32i8, MVT::v8i16, MVT::v16i16 })
1747  setOperationAction(ISD::CTPOP, VT, Legal);
1748  }
1749  }
1750 
1751  if (!Subtarget.useSoftFloat() && Subtarget.hasVLX()) {
1752  setTruncStoreAction(MVT::v4i64, MVT::v4i8, Legal);
1753  setTruncStoreAction(MVT::v4i64, MVT::v4i16, Legal);
1754  setTruncStoreAction(MVT::v4i64, MVT::v4i32, Legal);
1755  setTruncStoreAction(MVT::v8i32, MVT::v8i8, Legal);
1756  setTruncStoreAction(MVT::v8i32, MVT::v8i16, Legal);
1757 
1758  setTruncStoreAction(MVT::v2i64, MVT::v2i8, Legal);
1759  setTruncStoreAction(MVT::v2i64, MVT::v2i16, Legal);
1760  setTruncStoreAction(MVT::v2i64, MVT::v2i32, Legal);
1761  setTruncStoreAction(MVT::v4i32, MVT::v4i8, Legal);
1762  setTruncStoreAction(MVT::v4i32, MVT::v4i16, Legal);
1763 
1764  if (Subtarget.hasDQI()) {
1765  // Fast v2f32 SINT_TO_FP( v2i64 ) custom conversion.
1766  // v2f32 UINT_TO_FP is already custom under SSE2.
1767  setOperationAction(ISD::SINT_TO_FP, MVT::v2f32, Custom);
1768  assert(isOperationCustom(ISD::UINT_TO_FP, MVT::v2f32) &&
1769  "Unexpected operation action!");
1770  // v2i64 FP_TO_S/UINT(v2f32) custom conversion.
1771  setOperationAction(ISD::FP_TO_SINT, MVT::v2f32, Custom);
1772  setOperationAction(ISD::FP_TO_UINT, MVT::v2f32, Custom);
1773  }
1774 
1775  if (Subtarget.hasBWI()) {
1776  setTruncStoreAction(MVT::v16i16, MVT::v16i8, Legal);
1777  setTruncStoreAction(MVT::v8i16, MVT::v8i8, Legal);
1778  }
1779 
1780  if (Subtarget.hasVBMI2()) {
1781  // TODO: Make these legal even without VLX?
1782  for (auto VT : { MVT::v8i16, MVT::v4i32, MVT::v2i64,
1783  MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
1784  setOperationAction(ISD::FSHL, VT, Custom);
1785  setOperationAction(ISD::FSHR, VT, Custom);
1786  }
1787  }
1788  }
1789 
1790  // We want to custom lower some of our intrinsics.
1791  setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
1792  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
1793  setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
1794  if (!Subtarget.is64Bit()) {
1795  setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::i64, Custom);
1796  }
1797 
1798  // Only custom-lower 64-bit SADDO and friends on 64-bit because we don't
1799  // handle type legalization for these operations here.
1800  //
1801  // FIXME: We really should do custom legalization for addition and
1802  // subtraction on x86-32 once PR3203 is fixed. We really can't do much better
1803  // than generic legalization for 64-bit multiplication-with-overflow, though.
1804  for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
1805  if (VT == MVT::i64 && !Subtarget.is64Bit())
1806  continue;
1807  // Add/Sub/Mul with overflow operations are custom lowered.
1808  setOperationAction(ISD::SADDO, VT, Custom);
1809  setOperationAction(ISD::UADDO, VT, Custom);
1810  setOperationAction(ISD::SSUBO, VT, Custom);
1811  setOperationAction(ISD::USUBO, VT, Custom);
1812  setOperationAction(ISD::SMULO, VT, Custom);
1813  setOperationAction(ISD::UMULO, VT, Custom);
1814 
1815  // Support carry in as value rather than glue.
1816  setOperationAction(ISD::ADDCARRY, VT, Custom);
1817  setOperationAction(ISD::SUBCARRY, VT, Custom);
1818  setOperationAction(ISD::SETCCCARRY, VT, Custom);
1819  }
1820 
1821  if (!Subtarget.is64Bit()) {
1822  // These libcalls are not available in 32-bit.
1823  setLibcallName(RTLIB::SHL_I128, nullptr);
1824  setLibcallName(RTLIB::SRL_I128, nullptr);
1825  setLibcallName(RTLIB::SRA_I128, nullptr);
1826  setLibcallName(RTLIB::MUL_I128, nullptr);
1827  }
1828 
1829  // Combine sin / cos into _sincos_stret if it is available.
1830  if (getLibcallName(RTLIB::SINCOS_STRET_F32) != nullptr &&
1831  getLibcallName(RTLIB::SINCOS_STRET_F64) != nullptr) {
1832  setOperationAction(ISD::FSINCOS, MVT::f64, Custom);
1833  setOperationAction(ISD::FSINCOS, MVT::f32, Custom);
1834  }
1835 
1836  if (Subtarget.isTargetWin64()) {
1837  setOperationAction(ISD::SDIV, MVT::i128, Custom);
1838  setOperationAction(ISD::UDIV, MVT::i128, Custom);
1839  setOperationAction(ISD::SREM, MVT::i128, Custom);
1840  setOperationAction(ISD::UREM, MVT::i128, Custom);
1841  setOperationAction(ISD::SDIVREM, MVT::i128, Custom);
1842  setOperationAction(ISD::UDIVREM, MVT::i128, Custom);
1843  }
1844 
1845  // On 32 bit MSVC, `fmodf(f32)` is not defined - only `fmod(f64)`
1846  // is. We should promote the value to 64-bits to solve this.
1847  // This is what the CRT headers do - `fmodf` is an inline header
1848  // function casting to f64 and calling `fmod`.
1849  if (Subtarget.is32Bit() && (Subtarget.isTargetKnownWindowsMSVC() ||
1850  Subtarget.isTargetWindowsItanium()))
1851  for (ISD::NodeType Op :
1852  {ISD::FCEIL, ISD::FCOS, ISD::FEXP, ISD::FFLOOR, ISD::FREM, ISD::FLOG,
1853  ISD::FLOG10, ISD::FPOW, ISD::FSIN})
1854  if (isOperationExpand(Op, MVT::f32))
1855  setOperationAction(Op, MVT::f32, Promote);
1856 
1857  // We have target-specific dag combine patterns for the following nodes:
1858  setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
1859  setTargetDAGCombine(ISD::SCALAR_TO_VECTOR);
1860  setTargetDAGCombine(ISD::EXTRACT_VECTOR_ELT);
1861  setTargetDAGCombine(ISD::CONCAT_VECTORS);
1862  setTargetDAGCombine(ISD::INSERT_SUBVECTOR);
1863  setTargetDAGCombine(ISD::EXTRACT_SUBVECTOR);
1864  setTargetDAGCombine(ISD::BITCAST);
1865  setTargetDAGCombine(ISD::VSELECT);
1866  setTargetDAGCombine(ISD::SELECT);
1867  setTargetDAGCombine(ISD::SHL);
1868  setTargetDAGCombine(ISD::SRA);
1869  setTargetDAGCombine(ISD::SRL);
1870  setTargetDAGCombine(ISD::OR);
1871  setTargetDAGCombine(ISD::AND);
1872  setTargetDAGCombine(ISD::ADD);
1873  setTargetDAGCombine(ISD::FADD);
1874  setTargetDAGCombine(ISD::FSUB);
1875  setTargetDAGCombine(ISD::FNEG);
1876  setTargetDAGCombine(ISD::FMA);
1877  setTargetDAGCombine(ISD::FMINNUM);
1878  setTargetDAGCombine(ISD::FMAXNUM);
1879  setTargetDAGCombine(ISD::SUB);
1880  setTargetDAGCombine(ISD::LOAD);
1881  setTargetDAGCombine(ISD::MLOAD);
1882  setTargetDAGCombine(ISD::STORE);
1883  setTargetDAGCombine(ISD::MSTORE);
1884  setTargetDAGCombine(ISD::TRUNCATE);
1885  setTargetDAGCombine(ISD::ZERO_EXTEND);
1886  setTargetDAGCombine(ISD::ANY_EXTEND);
1887  setTargetDAGCombine(ISD::SIGN_EXTEND);
1888  setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1889  setTargetDAGCombine(ISD::ANY_EXTEND_VECTOR_INREG);
1890  setTargetDAGCombine(ISD::ZERO_EXTEND_VECTOR_INREG);
1891  setTargetDAGCombine(ISD::SINT_TO_FP);
1892  setTargetDAGCombine(ISD::UINT_TO_FP);
1893  setTargetDAGCombine(ISD::SETCC);
1894  setTargetDAGCombine(ISD::MUL);
1895  setTargetDAGCombine(ISD::XOR);
1896  setTargetDAGCombine(ISD::MSCATTER);
1897  setTargetDAGCombine(ISD::MGATHER);
1898 
1899  computeRegisterProperties(Subtarget.getRegisterInfo());
1900 
1901  MaxStoresPerMemset = 16; // For @llvm.memset -> sequence of stores
1902  MaxStoresPerMemsetOptSize = 8;
1903  MaxStoresPerMemcpy = 8; // For @llvm.memcpy -> sequence of stores
1904  MaxStoresPerMemcpyOptSize = 4;
1905  MaxStoresPerMemmove = 8; // For @llvm.memmove -> sequence of stores
1906  MaxStoresPerMemmoveOptSize = 4;
1907 
1908  // TODO: These control memcmp expansion in CGP and could be raised higher, but
1909  // that needs to benchmarked and balanced with the potential use of vector
1910  // load/store types (PR33329, PR33914).
1911  MaxLoadsPerMemcmp = 2;
1912  MaxLoadsPerMemcmpOptSize = 2;
1913 
1914  // Set loop alignment to 2^ExperimentalPrefLoopAlignment bytes (default: 2^4).
1915  setPrefLoopAlignment(ExperimentalPrefLoopAlignment);
1916 
1917  // An out-of-order CPU can speculatively execute past a predictable branch,
1918  // but a conditional move could be stalled by an expensive earlier operation.
1919  PredictableSelectIsExpensive = Subtarget.getSchedModel().isOutOfOrder();
1920  EnableExtLdPromotion = true;
1921  setPrefFunctionAlignment(4); // 2^4 bytes.
1922 
1923  verifyIntrinsicTables();
1924 }
1925 
1926 // This has so far only been implemented for 64-bit MachO.
1927 bool X86TargetLowering::useLoadStackGuardNode() const {
1928  return Subtarget.isTargetMachO() && Subtarget.is64Bit();
1929 }
1930 
1931 bool X86TargetLowering::useStackGuardXorFP() const {
1932  // Currently only MSVC CRTs XOR the frame pointer into the stack guard value.
1933  return Subtarget.getTargetTriple().isOSMSVCRT();
1934 }
1935 
1936 SDValue X86TargetLowering::emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
1937  const SDLoc &DL) const {
1938  EVT PtrTy = getPointerTy(DAG.getDataLayout());
1939  unsigned XorOp = Subtarget.is64Bit() ? X86::XOR64_FP : X86::XOR32_FP;
1940  MachineSDNode *Node = DAG.getMachineNode(XorOp, DL, PtrTy, Val);
1941  return SDValue(Node, 0);
1942 }
1943 
1944 TargetLoweringBase::LegalizeTypeAction
1945 X86TargetLowering::getPreferredVectorAction(MVT VT) const {
1946  if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
1947  return TypeSplitVector;
1948 
1949  if (ExperimentalVectorWideningLegalization &&
1950  VT.getVectorNumElements() != 1 &&
1951  VT.getVectorElementType() != MVT::i1)
1952  return TypeWidenVector;
1953 
1954  return TargetLoweringBase::getPreferredVectorAction(VT);
1955 }
1956 
1957 MVT X86TargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
1958  CallingConv::ID CC,
1959  EVT VT) const {
1960  if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
1961  return MVT::v32i8;
1962  return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
1963 }
1964 
1965 unsigned X86TargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
1966  CallingConv::ID CC,
1967  EVT VT) const {
1968  if (VT == MVT::v32i1 && Subtarget.hasAVX512() && !Subtarget.hasBWI())
1969  return 1;
1970  return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
1971 }
1972 
1973 EVT X86TargetLowering::getSetCCResultType(const DataLayout &DL,
1974  LLVMContext& Context,
1975  EVT VT) const {
1976  if (!VT.isVector())
1977  return MVT::i8;
1978 
1979  if (Subtarget.hasAVX512()) {
1980  const unsigned NumElts = VT.getVectorNumElements();
1981 
1982  // Figure out what this type will be legalized to.
1983  EVT LegalVT = VT;
1984  while (getTypeAction(Context, LegalVT) != TypeLegal)
1985  LegalVT = getTypeToTransformTo(Context, LegalVT);
1986 
1987  // If we got a 512-bit vector then we'll definitely have a vXi1 compare.
1988  if (LegalVT.getSimpleVT().is512BitVector())
1989  return EVT::getVectorVT(Context, MVT::i1, NumElts);
1990 
1991  if (LegalVT.getSimpleVT().isVector() && Subtarget.hasVLX()) {
1992  // If we legalized to less than a 512-bit vector, then we will use a vXi1
1993  // compare for vXi32/vXi64 for sure. If we have BWI we will also support
1994  // vXi16/vXi8.
1995  MVT EltVT = LegalVT.getSimpleVT().getVectorElementType();
1996  if (Subtarget.hasBWI() || EltVT.getSizeInBits() >= 32)
1997  return EVT::getVectorVT(Context, MVT::i1, NumElts);
1998  }
1999  }
2000 
2001  return VT.changeVectorElementTypeToInteger();
2002 }
2003 
2004 /// Helper for getByValTypeAlignment to determine
2005 /// the desired ByVal argument alignment.
2006 static void getMaxByValAlign(Type *Ty, unsigned &MaxAlign) {
2007  if (MaxAlign == 16)
2008  return;
2009  if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
2010  if (VTy->getBitWidth() == 128)
2011  MaxAlign = 16;
2012  } else if (ArrayType *ATy = dyn_cast<ArrayType>(Ty)) {
2013  unsigned EltAlign = 0;
2014  getMaxByValAlign(ATy->getElementType(), EltAlign);
2015  if (EltAlign > MaxAlign)
2016  MaxAlign = EltAlign;
2017  } else if (StructType *STy = dyn_cast<StructType>(Ty)) {
2018  for (auto *EltTy : STy->elements()) {
2019  unsigned EltAlign = 0;
2020  getMaxByValAlign(EltTy, EltAlign);
2021  if (EltAlign > MaxAlign)
2022  MaxAlign = EltAlign;
2023  if (MaxAlign == 16)
2024  break;
2025  }
2026  }
2027 }
2028 
2029 /// Return the desired alignment for ByVal aggregate
2030 /// function arguments in the caller parameter area. For X86, aggregates
2031 /// that contain SSE vectors are placed at 16-byte boundaries while the rest
2032 /// are at 4-byte boundaries.
2033 unsigned X86TargetLowering::getByValTypeAlignment(Type *Ty,
2034  const DataLayout &DL) const {
2035  if (Subtarget.is64Bit()) {
2036  // Max of 8 and alignment of type.
2037  unsigned TyAlign = DL.getABITypeAlignment(Ty);
2038  if (TyAlign > 8)
2039  return TyAlign;
2040  return 8;
2041  }
2042 
2043  unsigned Align = 4;
2044  if (Subtarget.hasSSE1())
2045  getMaxByValAlign(Ty, Align);
2046  return Align;
2047 }
2048 
2049 /// Returns the target specific optimal type for load
2050 /// and store operations as a result of memset, memcpy, and memmove
2051 /// lowering. If DstAlign is zero that means it's safe to destination
2052 /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
2053 /// means there isn't a need to check it against alignment requirement,
2054 /// probably because the source does not need to be loaded. If 'IsMemset' is
2055 /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that
2056 /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy
2057 /// source is constant so it does not need to be loaded.
2058 /// It returns EVT::Other if the type should be determined using generic
2059 /// target-independent logic.
2060 /// For vector ops we check that the overall size isn't larger than our
2061 /// preferred vector width.
2062 EVT X86TargetLowering::getOptimalMemOpType(
2063  uint64_t Size, unsigned DstAlign, unsigned SrcAlign, bool IsMemset,
2064  bool ZeroMemset, bool MemcpyStrSrc,
2065  const AttributeList &FuncAttributes) const {
2066  if (!FuncAttributes.hasFnAttribute(Attribute::NoImplicitFloat)) {
2067  if (Size >= 16 && (!Subtarget.isUnalignedMem16Slow() ||
2068  ((DstAlign == 0 || DstAlign >= 16) &&
2069  (SrcAlign == 0 || SrcAlign >= 16)))) {
2070  // FIXME: Check if unaligned 32-byte accesses are slow.
2071  if (Size >= 32 && Subtarget.hasAVX() &&
2072  (Subtarget.getPreferVectorWidth() >= 256)) {
2073  // Although this isn't a well-supported type for AVX1, we'll let
2074  // legalization and shuffle lowering produce the optimal codegen. If we
2075  // choose an optimal type with a vector element larger than a byte,
2076  // getMemsetStores() may create an intermediate splat (using an integer
2077  // multiply) before we splat as a vector.
2078  return MVT::v32i8;
2079  }
2080  if (Subtarget.hasSSE2() && (Subtarget.getPreferVectorWidth() >= 128))
2081  return MVT::v16i8;
2082  // TODO: Can SSE1 handle a byte vector?
2083  // If we have SSE1 registers we should be able to use them.
2084  if (Subtarget.hasSSE1() && (Subtarget.is64Bit() || Subtarget.hasX87()) &&
2085  (Subtarget.getPreferVectorWidth() >= 128))
2086  return MVT::v4f32;
2087  } else if ((!IsMemset || ZeroMemset) && !MemcpyStrSrc && Size >= 8 &&
2088  !Subtarget.is64Bit() && Subtarget.hasSSE2()) {
2089  // Do not use f64 to lower memcpy if source is string constant. It's
2090  // better to use i32 to avoid the loads.
2091  // Also, do not use f64 to lower memset unless this is a memset of zeros.
2092  // The gymnastics of splatting a byte value into an XMM register and then
2093  // only using 8-byte stores (because this is a CPU with slow unaligned
2094  // 16-byte accesses) makes that a loser.
2095  return MVT::f64;
2096  }
2097  }
2098  // This is a compromise. If we reach here, unaligned accesses may be slow on
2099  // this target. However, creating smaller, aligned accesses could be even
2100  // slower and would certainly be a lot more code.
2101  if (Subtarget.is64Bit() && Size >= 8)
2102  return MVT::i64;
2103  return MVT::i32;
2104 }
2105 
2106 bool X86TargetLowering::isSafeMemOpType(MVT VT) const {
2107  if (VT == MVT::f32)
2108  return X86ScalarSSEf32;
2109  else if (VT == MVT::f64)
2110  return X86ScalarSSEf64;
2111  return true;
2112 }
2113 
2114 bool
2115 X86TargetLowering::allowsMisalignedMemoryAccesses(EVT VT,
2116  unsigned,
2117  unsigned,
2118  bool *Fast) const {
2119  if (Fast) {
2120  switch (VT.getSizeInBits()) {
2121  default:
2122  // 8-byte and under are always assumed to be fast.
2123  *Fast = true;
2124  break;
2125  case 128:
2126  *Fast = !Subtarget.isUnalignedMem16Slow();
2127  break;
2128  case 256:
2129  *Fast = !Subtarget.isUnalignedMem32Slow();
2130  break;
2131  // TODO: What about AVX-512 (512-bit) accesses?
2132  }
2133  }
2134  // Misaligned accesses of any size are always allowed.
2135  return true;
2136 }
2137 
2138 /// Return the entry encoding for a jump table in the
2139 /// current function. The returned value is a member of the
2140 /// MachineJumpTableInfo::JTEntryKind enum.
2141 unsigned X86TargetLowering::getJumpTableEncoding() const {
2142  // In GOT pic mode, each entry in the jump table is emitted as a @GOTOFF
2143  // symbol.
2144  if (isPositionIndependent() && Subtarget.isPICStyleGOT())
2145  return MachineJumpTableInfo::EK_Custom32;
2146 
2147  // Otherwise, use the normal jump table encoding heuristics.
2148  return TargetLowering::getJumpTableEncoding();
2149 }
2150 
2151 bool X86TargetLowering::useSoftFloat() const {
2152  return Subtarget.useSoftFloat();
2153 }
2154 
2155 void X86TargetLowering::markLibCallAttributes(MachineFunction *MF, unsigned CC,
2156  ArgListTy &Args) const {
2157 
2158  // Only relabel X86-32 for C / Stdcall CCs.
2159  if (Subtarget.is64Bit())
2160  return;
2161  if (CC != CallingConv::C && CC != CallingConv::X86_StdCall)
2162  return;
2163  unsigned ParamRegs = 0;
2164  if (auto *M = MF->getFunction().getParent())
2165  ParamRegs = M->getNumberRegisterParameters();
2166 
2167  // Mark the first N int arguments as having reg
2168  for (unsigned Idx = 0; Idx < Args.size(); Idx++) {
2169  Type *T = Args[Idx].Ty;
2170  if (T->isIntOrPtrTy())
2171  if (MF->getDataLayout().getTypeAllocSize(T) <= 8) {
2172  unsigned numRegs = 1;
2173  if (MF->getDataLayout().getTypeAllocSize(T) > 4)
2174  numRegs = 2;
2175  if (ParamRegs < numRegs)
2176  return;
2177  ParamRegs -= numRegs;
2178  Args[Idx].IsInReg = true;
2179  }
2180  }
2181 }
2182 
2183 const MCExpr *
2184 X86TargetLowering::LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
2185  const MachineBasicBlock *MBB,
2186  unsigned uid,MCContext &Ctx) const{
2187  assert(isPositionIndependent() && Subtarget.isPICStyleGOT());
2188  // In 32-bit ELF systems, our jump table entries are formed with @GOTOFF
2189  // entries.
2190  return MCSymbolRefExpr::create(MBB->getSymbol(),
2191  MCSymbolRefExpr::VK_GOTOFF, Ctx);
2192 }
2193 
2194 /// Returns relocation base for the given PIC jumptable.
2195 SDValue X86TargetLowering::getPICJumpTableRelocBase(SDValue Table,
2196  SelectionDAG &DAG) const {
2197  if (!Subtarget.is64Bit())
2198  // This doesn't have SDLoc associated with it, but is not really the
2199  // same as a Register.
2200  return DAG.getNode(X86ISD::GlobalBaseReg, SDLoc(),
2201  getPointerTy(DAG.getDataLayout()));
2202  return Table;
2203 }
2204 
2205 /// This returns the relocation base for the given PIC jumptable,
2206 /// the same as getPICJumpTableRelocBase, but as an MCExpr.
2207 const MCExpr *X86TargetLowering::
2208 getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI,
2209  MCContext &Ctx) const {
2210  // X86-64 uses RIP relative addressing based on the jump table label.
2211  if (Subtarget.isPICStyleRIPRel())
2212  return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
2213 
2214  // Otherwise, the reference is relative to the PIC base.
2215  return MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx);
2216 }
2217 
2218 std::pair<const TargetRegisterClass *, uint8_t>
2219 X86TargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
2220  MVT VT) const {
2221  const TargetRegisterClass *RRC = nullptr;
2222  uint8_t Cost = 1;
2223  switch (VT.SimpleTy) {
2224  default:
2225  return TargetLowering::findRepresentativeClass(TRI, VT);
2226  case MVT::i8: case MVT::i16: case MVT::i32: case MVT::i64:
2227  RRC = Subtarget.is64Bit() ? &X86::GR64RegClass : &X86::GR32RegClass;
2228  break;
2229  case MVT::x86mmx:
2230  RRC = &X86::VR64RegClass;
2231  break;
2232  case MVT::f32: case MVT::f64:
2233  case MVT::v16i8: case MVT::v8i16: case MVT::v4i32: case MVT::v2i64:
2234  case MVT::v4f32: case MVT::v2f64:
2235  case MVT::v32i8: case MVT::v16i16: case MVT::v8i32: case MVT::v4i64:
2236  case MVT::v8f32: case MVT::v4f64:
2237  case MVT::v64i8: case MVT::v32i16: case MVT::v16i32: case MVT::v8i64:
2238  case MVT::v16f32: case MVT::v8f64:
2239  RRC = &X86::VR128XRegClass;
2240  break;
2241  }
2242  return std::make_pair(RRC, Cost);
2243 }
2244 
2245 unsigned X86TargetLowering::getAddressSpace() const {
2246  if (Subtarget.is64Bit())
2247  return (getTargetMachine().getCodeModel() == CodeModel::Kernel) ? 256 : 257;
2248  return 256;
2249 }
2250 
2251 static bool hasStackGuardSlotTLS(const Triple &TargetTriple) {
2252  return TargetTriple.isOSGlibc() || TargetTriple.isOSFuchsia() ||
2253  (TargetTriple.isAndroid() && !TargetTriple.isAndroidVersionLT(17));
2254 }
2255 
2256 static Constant* SegmentOffset(IRBuilder<> &IRB,
2257  unsigned Offset, unsigned AddressSpace) {
2258  return ConstantExpr::getIntToPtr(
2259  ConstantInt::get(Type::getInt32Ty(IRB.getContext()), Offset),
2260  Type::getInt8PtrTy(IRB.getContext())->getPointerTo(AddressSpace));
2261 }
2262 
2263 Value *X86TargetLowering::getIRStackGuard(IRBuilder<> &IRB) const {
2264  // glibc, bionic, and Fuchsia have a special slot for the stack guard in
2265  // tcbhead_t; use it instead of the usual global variable (see
2266  // sysdeps/{i386,x86_64}/nptl/tls.h)
2267  if (hasStackGuardSlotTLS(Subtarget.getTargetTriple())) {
2268  if (Subtarget.isTargetFuchsia()) {
2269  // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
2270  return SegmentOffset(IRB, 0x10, getAddressSpace());
2271  } else {
2272  // %fs:0x28, unless we're using a Kernel code model, in which case
2273  // it's %gs:0x28. gs:0x14 on i386.
2274  unsigned Offset = (Subtarget.is64Bit()) ? 0x28 : 0x14;
2275  return SegmentOffset(IRB, Offset, getAddressSpace());
2276  }
2277  }
2278 
2279  return TargetLowering::getIRStackGuard(IRB);
2280 }
2281 
2282 void X86TargetLowering::insertSSPDeclarations(Module &M) const {
2283  // MSVC CRT provides functionalities for stack protection.
2284  if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
2285  Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
2286  // MSVC CRT has a global variable holding security cookie.
2287  M.getOrInsertGlobal("__security_cookie",
2288  Type::getInt8PtrTy(M.getContext()));
2289 
2290  // MSVC CRT has a function to validate security cookie.
2291  FunctionCallee SecurityCheckCookie = M.getOrInsertFunction(
2292  "__security_check_cookie", Type::getVoidTy(M.getContext()),
2293  Type::getInt8PtrTy(M.getContext()));
2294  if (Function *F = dyn_cast<Function>(SecurityCheckCookie.getCallee())) {
2295  F->setCallingConv(CallingConv::X86_FastCall);
2296  F->addAttribute(1, Attribute::AttrKind::InReg);
2297  }
2298  return;
2299  }
2300  // glibc, bionic, and Fuchsia have a special slot for the stack guard.
2301  if (hasStackGuardSlotTLS(Subtarget.getTargetTriple()))
2302  return;
2303  TargetLowering::insertSSPDeclarations(M);
2304 }
2305 
2306 Value *X86TargetLowering::getSDagStackGuard(const Module &M) const {
2307  // MSVC CRT has a global variable holding security cookie.
2308  if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
2309  Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
2310  return M.getGlobalVariable("__security_cookie");
2311  }
2312  return TargetLowering::getSDagStackGuard(M);
2313 }
2314 
2315 Function *X86TargetLowering::getSSPStackGuardCheck(const Module &M) const {
2316  // MSVC CRT has a function to validate security cookie.
2317  if (Subtarget.getTargetTriple().isWindowsMSVCEnvironment() ||
2318  Subtarget.getTargetTriple().isWindowsItaniumEnvironment()) {
2319  return M.getFunction("__security_check_cookie");
2320  }
2321  return TargetLowering::getSSPStackGuardCheck(M);
2322 }
2323 
2324 Value *X86TargetLowering::getSafeStackPointerLocation(IRBuilder<> &IRB) const {
2325  if (Subtarget.getTargetTriple().isOSContiki())
2326  return getDefaultSafeStackPointerLocation(IRB, false);
2327 
2328  // Android provides a fixed TLS slot for the SafeStack pointer. See the
2329  // definition of TLS_SLOT_SAFESTACK in
2330  // https://android.googlesource.com/platform/bionic/+/master/libc/private/bionic_tls.h
2331  if (Subtarget.isTargetAndroid()) {
2332  // %fs:0x48, unless we're using a Kernel code model, in which case it's %gs:
2333  // %gs:0x24 on i386
2334  unsigned Offset = (Subtarget.is64Bit()) ? 0x48 : 0x24;
2335  return SegmentOffset(IRB, Offset, getAddressSpace());
2336  }
2337 
2338  // Fuchsia is similar.
2339  if (Subtarget.isTargetFuchsia()) {
2340  // <zircon/tls.h> defines ZX_TLS_UNSAFE_SP_OFFSET with this value.
2341  return SegmentOffset(IRB, 0x18, getAddressSpace());
2342  }
2343 
2344  return TargetLowering::getSafeStackPointerLocation(IRB);
2345 }
2346 
2347 bool X86TargetLowering::isNoopAddrSpaceCast(unsigned SrcAS,
2348  unsigned DestAS) const {
2349  assert(SrcAS != DestAS && "Expected different address spaces!");
2350 
2351  return SrcAS < 256 && DestAS < 256;
2352 }
2353 
2354 //===----------------------------------------------------------------------===//
2355 // Return Value Calling Convention Implementation
2356 //===----------------------------------------------------------------------===//
2357 
2358 bool X86TargetLowering::CanLowerReturn(
2359  CallingConv::ID CallConv, MachineFunction &MF, bool isVarArg,
2360  const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2361  SmallVector<CCValAssign, 16> RVLocs;
2362  CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
2363  return CCInfo.CheckReturn(Outs, RetCC_X86);
2364 }
2365 
2366 const MCPhysReg *X86TargetLowering::getScratchRegisters(CallingConv::ID) const {
2367  static const MCPhysReg ScratchRegs[] = { X86::R11, 0 };
2368  return ScratchRegs;
2369 }
2370 
2371 /// Lowers masks values (v*i1) to the local register values
2372 /// \returns DAG node after lowering to register type
2373 static SDValue lowerMasksToReg(const SDValue &ValArg, const EVT &ValLoc,
2374  const SDLoc &Dl, SelectionDAG &DAG) {
2375  EVT ValVT = ValArg.getValueType();
2376 
2377  if (ValVT == MVT::v1i1)
2378  return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Dl, ValLoc, ValArg,
2379  DAG.getIntPtrConstant(0, Dl));
2380 
2381  if ((ValVT == MVT::v8i1 && (ValLoc == MVT::i8 || ValLoc == MVT::i32)) ||
2382  (ValVT == MVT::v16i1 && (ValLoc == MVT::i16 || ValLoc == MVT::i32))) {
2383  // Two stage lowering might be required
2384  // bitcast: v8i1 -> i8 / v16i1 -> i16
2385  // anyextend: i8 -> i32 / i16 -> i32
2386  EVT TempValLoc = ValVT == MVT::v8i1 ? MVT::i8 : MVT::i16;
2387  SDValue ValToCopy = DAG.getBitcast(TempValLoc, ValArg);
2388  if (ValLoc == MVT::i32)
2389  ValToCopy = DAG.getNode(ISD::ANY_EXTEND, Dl, ValLoc, ValToCopy);
2390  return ValToCopy;
2391  }
2392 
2393  if ((ValVT == MVT::v32i1 && ValLoc == MVT::i32) ||
2394  (ValVT == MVT::v64i1 && ValLoc == MVT::i64)) {
2395  // One stage lowering is required
2396  // bitcast: v32i1 -> i32 / v64i1 -> i64
2397  return DAG.getBitcast(ValLoc, ValArg);
2398  }
2399 
2400  return DAG.getNode(ISD::ANY_EXTEND, Dl, ValLoc, ValArg);
2401 }
2402 
2403 /// Breaks v64i1 value into two registers and adds the new node to the DAG
2404 static void Passv64i1ArgInRegs(
2405  const SDLoc &Dl, SelectionDAG &DAG, SDValue Chain, SDValue &Arg,
2406  SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, CCValAssign &VA,
2407  CCValAssign &NextVA, const X86Subtarget &Subtarget) {
2408  assert(Subtarget.hasBWI() && "Expected AVX512BW target!");
2409  assert(Subtarget.is32Bit() && "Expecting 32 bit target");
2410  assert(Arg.getValueType() == MVT::i64 && "Expecting 64 bit value");
2411  assert(VA.isRegLoc() && NextVA.isRegLoc() &&
2412  "The value should reside in two registers");
2413 
2414  // Before splitting the value we cast it to i64
2415  Arg = DAG.getBitcast(MVT::i64, Arg);
2416 
2417  // Splitting the value into two i32 types
2418  SDValue Lo, Hi;
2419  Lo = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg,
2420  DAG.getConstant(0, Dl, MVT::i32));
2421  Hi = DAG.getNode(ISD::EXTRACT_ELEMENT, Dl, MVT::i32, Arg,
2422  DAG.getConstant(1, Dl, MVT::i32));
2423 
2424  // Attach the two i32 types into corresponding registers
2425  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Lo));
2426  RegsToPass.push_back(std::make_pair(NextVA.getLocReg(), Hi));
2427 }
2428 
2429 SDValue
2430 X86TargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2431  bool isVarArg,
2432  const SmallVectorImpl<ISD::OutputArg> &Outs,
2433  const SmallVectorImpl<SDValue> &OutVals,
2434  const SDLoc &dl, SelectionDAG &DAG) const {
2435  MachineFunction &MF = DAG.getMachineFunction();
2436  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
2437 
2438  // In some cases we need to disable registers from the default CSR list.
2439  // For example, when they are used for argument passing.
2440  bool ShouldDisableCalleeSavedRegister =
2441  CallConv == CallingConv::X86_RegCall ||
2442  MF.getFunction().hasFnAttribute("no_caller_saved_registers");
2443 
2444  if (CallConv == CallingConv::X86_INTR && !Outs.empty())
2445  report_fatal_error("X86 interrupts may not return any value");
2446 
2447  SmallVector<CCValAssign, 16> RVLocs;
2448  CCState CCInfo(CallConv, isVarArg, MF, RVLocs, *DAG.getContext());
2449  CCInfo.AnalyzeReturn(Outs, RetCC_X86);
2450 
2451  SDValue Flag;
2452  SmallVector<SDValue, 6> RetOps;
2453  RetOps.push_back(Chain); // Operand #0 = Chain (updated below)
2454  // Operand #1 = Bytes To Pop
2455  RetOps.push_back(DAG.getTargetConstant(FuncInfo->getBytesToPopOnReturn(), dl,
2456  MVT::i32));
2457 
2458  // Copy the result values into the output registers.
2459  for (unsigned I = 0, OutsIndex = 0, E = RVLocs.size(); I != E;
2460  ++I, ++OutsIndex) {
2461  CCValAssign &VA = RVLocs[I];
2462  assert(VA.isRegLoc() && "Can only return in registers!");
2463 
2464  // Add the register to the CalleeSaveDisableRegs list.
2465  if (ShouldDisableCalleeSavedRegister)
2466  MF.getRegInfo().disableCalleeSavedRegister(VA.getLocReg());
2467 
2468  SDValue ValToCopy = OutVals[OutsIndex];
2469  EVT ValVT = ValToCopy.getValueType();
2470 
2471  // Promote values to the appropriate types.
2472  if (VA.getLocInfo() == CCValAssign::SExt)
2473  ValToCopy = DAG.getNode(ISD::SIGN_EXTEND, dl, VA.getLocVT(), ValToCopy);
2474  else if (VA.getLocInfo() == CCValAssign::ZExt)
2475  ValToCopy = DAG.getNode(ISD::ZERO_EXTEND, dl, VA.getLocVT(), ValToCopy);
2476  else if (VA.getLocInfo() == CCValAssign::AExt) {
2477  if (ValVT.isVector() && ValVT.getVectorElementType() == MVT::i1)
2478  ValToCopy = lowerMasksToReg(ValToCopy, VA.getLocVT(), dl, DAG);
2479  else
2480  ValToCopy = DAG.getNode(ISD::ANY_EXTEND, dl, VA.getLocVT(), ValToCopy);
2481  }
2482  else if (VA.getLocInfo() == CCValAssign::BCvt)
2483  ValToCopy = DAG.getBitcast(VA.getLocVT(), ValToCopy);
2484 
2485  assert(VA.getLocInfo() != CCValAssign::FPExt &&
2486  "Unexpected FP-extend for return value.");
2487 
2488  // If this is x86-64, and we disabled SSE, we can't return FP values,
2489  // or SSE or MMX vectors.
2490  if ((ValVT == MVT::f32 || ValVT == MVT::f64 ||
2491  VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) &&
2492  (Subtarget.is64Bit() && !Subtarget.hasSSE1())) {
2493  errorUnsupported(DAG, dl, "SSE register return with SSE disabled");
2494  VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
2495  } else if (ValVT == MVT::f64 &&
2496  (Subtarget.is64Bit() && !Subtarget.hasSSE2())) {
2497  // Likewise we can't return F64 values with SSE1 only. gcc does so, but
2498  // llvm-gcc has never done it right and no one has noticed, so this
2499  // should be OK for now.
2500  errorUnsupported(DAG, dl, "SSE2 register return with SSE2 disabled");
2501  VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
2502  }
2503 
2504  // Returns in ST0/ST1 are handled specially: these are pushed as operands to
2505  // the RET instruction and handled by the FP Stackifier.
2506  if (VA.getLocReg() == X86::FP0 ||
2507  VA.getLocReg() == X86::FP1) {
2508  // If this is a copy from an xmm register to ST(0), use an FPExtend to
2509  // change the value to the FP stack register class.
2510  if (isScalarFPTypeInSSEReg(VA.getValVT()))
2511  ValToCopy = DAG.getNode(ISD::FP_EXTEND, dl, MVT::f80, ValToCopy);
2512  RetOps.push_back(ValToCopy);
2513  // Don't emit a copytoreg.
2514  continue;
2515  }
2516 
2517  // 64-bit vector (MMX) values are returned in XMM0 / XMM1 except for v1i64
2518  // which is returned in RAX / RDX.
2519  if (Subtarget.is64Bit()) {
2520  if (ValVT == MVT::x86mmx) {
2521  if (VA.getLocReg() == X86::XMM0 || VA.getLocReg() == X86::XMM1) {
2522  ValToCopy = DAG.getBitcast(MVT::i64, ValToCopy);
2523  ValToCopy = DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, MVT::v2i64,
2524  ValToCopy);
2525  // If we don't have SSE2 available, convert to v4f32 so the generated
2526  // register is legal.
2527  if (!Subtarget.hasSSE2())
2528  ValToCopy = DAG.getBitcast(MVT::v4f32, ValToCopy);
2529  }
2530  }
2531  }
2532 
2533  SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
2534 
2535  if (VA.needsCustom()) {
2536  assert(VA.getValVT() == MVT::v64i1 &&
2537  "Currently the only custom case is when we split v64i1 to 2 regs");
2538 
2539  Passv64i1ArgInRegs(dl, DAG, Chain, ValToCopy, RegsToPass, VA, RVLocs[++I],
2540  Subtarget);
2541 
2542  assert(2 == RegsToPass.size() &&
2543  "Expecting two registers after Pass64BitArgInRegs");
2544 
2545  // Add the second register to the CalleeSaveDisableRegs list.
2546  if (ShouldDisableCalleeSavedRegister)
2547  MF.getRegInfo().disableCalleeSavedRegister(RVLocs[I].getLocReg());
2548  } else {
2549  RegsToPass.push_back(std::make_pair(VA.getLocReg(), ValToCopy));
2550  }
2551 
2552  // Add nodes to the DAG and add the values into the RetOps list
2553  for (auto &Reg : RegsToPass) {
2554  Chain = DAG.getCopyToReg(Chain, dl, Reg.first, Reg.second, Flag);
2555  Flag = Chain.getValue(1);
2556  RetOps.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
2557  }
2558  }
2559 
2560  // Swift calling convention does not require we copy the sret argument
2561  // into %rax/%eax for the return, and SRetReturnReg is not set for Swift.
2562 
2563  // All x86 ABIs require that for returning structs by value we copy
2564  // the sret argument into %rax/%eax (depending on ABI) for the return.
2565  // We saved the argument into a virtual register in the entry block,
2566  // so now we copy the value out and into %rax/%eax.
2567  //
2568  // Checking Function.hasStructRetAttr() here is insufficient because the IR
2569  // may not have an explicit sret argument. If FuncInfo.CanLowerReturn is
2570  // false, then an sret argument may be implicitly inserted in the SelDAG. In
2571  // either case FuncInfo->setSRetReturnReg() will have been called.
2572  if (unsigned SRetReg = FuncInfo->getSRetReturnReg()) {
2573  // When we have both sret and another return value, we should use the
2574  // original Chain stored in RetOps[0], instead of the current Chain updated
2575  // in the above loop. If we only have sret, RetOps[0] equals to Chain.
2576 
2577  // For the case of sret and another return value, we have
2578  // Chain_0 at the function entry
2579  // Chain_1 = getCopyToReg(Chain_0) in the above loop
2580  // If we use Chain_1 in getCopyFromReg, we will have
2581  // Val = getCopyFromReg(Chain_1)
2582  // Chain_2 = getCopyToReg(Chain_1, Val) from below
2583 
2584  // getCopyToReg(Chain_0) will be glued together with
2585  // getCopyToReg(Chain_1, Val) into Unit A, getCopyFromReg(Chain_1) will be
2586  // in Unit B, and we will have cyclic dependency between Unit A and Unit B:
2587  // Data dependency from Unit B to Unit A due to usage of Val in
2588  // getCopyToReg(Chain_1, Val)
2589  // Chain dependency from Unit A to Unit B
2590 
2591  // So here, we use RetOps[0] (i.e Chain_0) for getCopyFromReg.
2592  SDValue Val = DAG.getCopyFromReg(RetOps[0], dl, SRetReg,
2593  getPointerTy(MF.getDataLayout()));
2594 
2595  unsigned RetValReg
2596  = (Subtarget.is64Bit() && !Subtarget.isTarget64BitILP32()) ?
2597  X86::RAX : X86::EAX;
2598  Chain = DAG.getCopyToReg(Chain, dl, RetValReg, Val, Flag);
2599  Flag = Chain.getValue(1);
2600 
2601  // RAX/EAX now acts like a return value.
2602  RetOps.push_back(
2603  DAG.getRegister(RetValReg, getPointerTy(DAG.getDataLayout())));
2604 
2605  // Add the returned register to the CalleeSaveDisableRegs list.
2606  if (ShouldDisableCalleeSavedRegister)
2607  MF.getRegInfo().disableCalleeSavedRegister(RetValReg);
2608  }
2609 
2610  const X86RegisterInfo *TRI = Subtarget.getRegisterInfo();
2611  const MCPhysReg *I =
2612  TRI->getCalleeSavedRegsViaCopy(&DAG.getMachineFunction());
2613  if (I) {
2614  for (; *I; ++I) {
2615  if (X86::GR64RegClass.contains(*I))
2616  RetOps.push_back(DAG.getRegister(*I, MVT::i64));
2617  else
2618  llvm_unreachable("Unexpected register class in CSRsViaCopy!");
2619  }
2620  }
2621 
2622  RetOps[0] = Chain; // Update chain.
2623 
2624  // Add the flag if we have it.
2625  if (Flag.getNode())
2626  RetOps.push_back(Flag);
2627 
2628  X86ISD::NodeType opcode = X86ISD::RET_FLAG;
2629  if (CallConv == CallingConv::X86_INTR)
2630  opcode = X86ISD::IRET;
2631  return DAG.getNode(opcode, dl, MVT::Other, RetOps);
2632 }
2633 
2634 bool X86TargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
2635  if (N->getNumValues() != 1 || !N->hasNUsesOfValue(1, 0))
2636  return false;
2637 
2638  SDValue TCChain = Chain;
2639  SDNode *Copy = *N->use_begin();
2640  if (Copy->getOpcode() == ISD::CopyToReg) {
2641  // If the copy has a glue operand, we conservatively assume it isn't safe to
2642  // perform a tail call.
2643  if (Copy->getOperand(Copy->getNumOperands()-1).getValueType() == MVT::Glue)
2644  return false;
2645  TCChain = Copy->getOperand(0);
2646  } else if (Copy->getOpcode() != ISD::FP_EXTEND)
2647  return false;
2648 
2649  bool HasRet = false;
2650  for (SDNode::use_iterator UI = Copy->use_begin(), UE = Copy->use_end();
2651  UI != UE; ++UI) {
2652  if (UI->getOpcode() != X86ISD::RET_FLAG)
2653  return false;
2654  // If we are returning more than one value, we can definitely
2655  // not make a tail call see PR19530
2656  if (UI->getNumOperands() > 4)
2657  return false;
2658  if (UI->getNumOperands() == 4 &&
2659  UI->getOperand(UI->getNumOperands()-1).getValueType() != MVT::Glue)
2660  return false;
2661  HasRet = true;
2662  }
2663 
2664  if (!HasRet)
2665  return false;
2666 
2667  Chain = TCChain;
2668  return true;
2669 }
2670 
2671 EVT X86TargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
2672  ISD::NodeType ExtendKind) const {
2673  MVT ReturnMVT = MVT::i32;
2674 
2675  bool Darwin = Subtarget.getTargetTriple().isOSDarwin();
2676  if (VT == MVT::i1 || (!Darwin && (VT == MVT::i8 || VT == MVT::i16))) {
2677  // The ABI does not require i1, i8 or i16 to be extended.
2678  //
2679  // On Darwin, there is code in the wild relying on Clang's old behaviour of
2680  // always extending i8/i16 return values, so keep doing that for now.
2681  // (PR26665).
2682  ReturnMVT = MVT::i8;
2683  }
2684 
2685  EVT MinVT = getRegisterType(Context, ReturnMVT);
2686  return VT.bitsLT(MinVT) ? MinVT : VT;
2687 }
2688 
2689 /// Reads two 32 bit registers and creates a 64 bit mask value.
2690 /// \param VA The current 32 bit value that need to be assigned.
2691 /// \param NextVA The next 32 bit value that need to be assigned.
2692 /// \param Root The parent DAG node.
2693 /// \param [in,out] InFlag Represents SDvalue in the parent DAG node for
2694 /// glue purposes. In the case the DAG is already using
2695 /// physical register instead of virtual, we should glue
2696 /// our new SDValue to InFlag SDvalue.
2697 /// \return a new SDvalue of size 64bit.
2698 static SDValue getv64i1Argument(CCValAssign &VA, CCValAssign &NextVA,
2699  SDValue &Root, SelectionDAG &DAG,
2700  const SDLoc &Dl, const X86Subtarget &Subtarget,
2701  SDValue *InFlag = nullptr) {
2702  assert((Subtarget.hasBWI()) && "Expected AVX512BW target!");
2703  assert(Subtarget.is32Bit() && "Expecting 32 bit target");
2704  assert(VA.getValVT() == MVT::v64i1 &&
2705  "Expecting first location of 64 bit width type");
2706  assert(NextVA.getValVT() == VA.getValVT() &&
2707  "The locations should have the same type");
2708  assert(VA.isRegLoc() && NextVA.isRegLoc() &&
2709  "The values should reside in two registers");
2710 
2711  SDValue Lo, Hi;
2712  SDValue ArgValueLo, ArgValueHi;
2713 
2714  MachineFunction &MF = DAG.getMachineFunction();
2715  const TargetRegisterClass *RC = &X86::GR32RegClass;
2716 
2717  // Read a 32 bit value from the registers.
2718  if (nullptr == InFlag) {
2719  // When no physical register is present,
2720  // create an intermediate virtual register.
2721  unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
2722  ArgValueLo = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);
2723  Reg = MF.addLiveIn(NextVA.getLocReg(), RC);
2724  ArgValueHi = DAG.getCopyFromReg(Root, Dl, Reg, MVT::i32);
2725  } else {
2726  // When a physical register is available read the value from it and glue
2727  // the reads together.
2728  ArgValueLo =
2729  DAG.getCopyFromReg(Root, Dl, VA.getLocReg(), MVT::i32, *InFlag);
2730  *InFlag = ArgValueLo.getValue(2);
2731  ArgValueHi =
2732  DAG.getCopyFromReg(Root, Dl, NextVA.getLocReg(), MVT::i32, *InFlag);
2733  *InFlag = ArgValueHi.getValue(2);
2734  }
2735 
2736  // Convert the i32 type into v32i1 type.
2737  Lo = DAG.getBitcast(MVT::v32i1, ArgValueLo);
2738 
2739  // Convert the i32 type into v32i1 type.
2740  Hi = DAG.getBitcast(MVT::v32i1, ArgValueHi);
2741 
2742  // Concatenate the two values together.
2743  return DAG.getNode(ISD::CONCAT_VECTORS, Dl, MVT::v64i1, Lo, Hi);
2744 }
2745 
2746 /// The function will lower a register of various sizes (8/16/32/64)
2747 /// to a mask value of the expected size (v8i1/v16i1/v32i1/v64i1)
2748 /// \returns a DAG node contains the operand after lowering to mask type.
2749 static SDValue lowerRegToMasks(const SDValue &ValArg, const EVT &ValVT,
2750  const EVT &ValLoc, const SDLoc &Dl,
2751  SelectionDAG &DAG) {
2752  SDValue ValReturned = ValArg;
2753 
2754  if (ValVT == MVT::v1i1)
2755  return DAG.getNode(ISD::SCALAR_TO_VECTOR, Dl, MVT::v1i1, ValReturned);
2756 
2757  if (ValVT == MVT::v64i1) {
2758  // In 32 bit machine, this case is handled by getv64i1Argument
2759  assert(ValLoc == MVT::i64 && "Expecting only i64 locations");
2760  // In 64 bit machine, There is no need to truncate the value only bitcast
2761  } else {
2762  MVT maskLen;
2763  switch (ValVT.getSimpleVT().SimpleTy) {
2764  case MVT::v8i1:
2765  maskLen = MVT::i8;
2766  break;
2767  case MVT::v16i1:
2768  maskLen = MVT::i16;
2769  break;
2770  case MVT::v32i1:
2771  maskLen = MVT::i32;
2772  break;
2773  default:
2774  llvm_unreachable("Expecting a vector of i1 types");
2775  }
2776 
2777  ValReturned = DAG.getNode(ISD::TRUNCATE, Dl, maskLen, ValReturned);
2778  }
2779  return DAG.getBitcast(ValVT, ValReturned);
2780 }
2781 
2782 /// Lower the result values of a call into the
2783 /// appropriate copies out of appropriate physical registers.
2784 ///
2785 SDValue X86TargetLowering::LowerCallResult(
2786  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool isVarArg,
2787  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
2788  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
2789  uint32_t *RegMask) const {
2790 
2791  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2792  // Assign locations to each value returned by this call.
2793  SmallVector<CCValAssign, 16> RVLocs;
2794  bool Is64Bit = Subtarget.is64Bit();
2795  CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
2796  *DAG.getContext());
2797  CCInfo.AnalyzeCallResult(Ins, RetCC_X86);
2798 
2799  // Copy all of the result registers out of their specified physreg.
2800  for (unsigned I = 0, InsIndex = 0, E = RVLocs.size(); I != E;
2801  ++I, ++InsIndex) {
2802  CCValAssign &VA = RVLocs[I];
2803  EVT CopyVT = VA.getLocVT();
2804 
2805  // In some calling conventions we need to remove the used registers
2806  // from the register mask.
2807  if (RegMask) {
2808  for (MCSubRegIterator SubRegs(VA.getLocReg(), TRI, /*IncludeSelf=*/true);
2809  SubRegs.isValid(); ++SubRegs)
2810  RegMask[*SubRegs / 32] &= ~(1u << (*SubRegs % 32));
2811  }
2812 
2813  // If this is x86-64, and we disabled SSE, we can't return FP values
2814  if ((CopyVT == MVT::f32 || CopyVT == MVT::f64 || CopyVT == MVT::f128) &&
2815  ((Is64Bit || Ins[InsIndex].Flags.isInReg()) && !Subtarget.hasSSE1())) {
2816  errorUnsupported(DAG, dl, "SSE register return with SSE disabled");
2817  VA.convertToReg(X86::FP0); // Set reg to FP0, avoid hitting asserts.
2818  }
2819 
2820  // If we prefer to use the value in xmm registers, copy it out as f80 and
2821  // use a truncate to move it from fp stack reg to xmm reg.
2822  bool RoundAfterCopy = false;
2823  if ((VA.getLocReg() == X86::FP0 || VA.getLocReg() == X86::FP1) &&
2824  isScalarFPTypeInSSEReg(VA.getValVT())) {
2825  if (!Subtarget.hasX87())
2826  report_fatal_error("X87 register return with X87 disabled");
2827  CopyVT = MVT::f80;
2828  RoundAfterCopy = (CopyVT != VA.getLocVT());
2829  }
2830 
2831  SDValue Val;
2832  if (VA.needsCustom()) {
2833  assert(VA.getValVT() == MVT::v64i1 &&
2834  "Currently the only custom case is when we split v64i1 to 2 regs");
2835  Val =
2836  getv64i1Argument(VA, RVLocs[++I], Chain, DAG, dl, Subtarget, &InFlag);
2837  } else {
2838  Chain = DAG.getCopyFromReg(Chain, dl, VA.getLocReg(), CopyVT, InFlag)
2839  .getValue(1);
2840  Val = Chain.getValue(0);
2841  InFlag = Chain.getValue(2);
2842  }
2843 
2844  if (RoundAfterCopy)
2845  Val = DAG.getNode(ISD::FP_ROUND, dl, VA.getValVT(), Val,
2846  // This truncation won't change the value.
2847  DAG.getIntPtrConstant(1, dl));
2848 
2849  if (VA.isExtInLoc() && (VA.getValVT().getScalarType() == MVT::i1)) {
2850  if (VA.getValVT().isVector() &&
2851  ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
2852  (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
2853  // promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
2854  Val = lowerRegToMasks(Val, VA.getValVT(), VA.getLocVT(), dl, DAG);
2855  } else
2856  Val = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val);
2857  }
2858 
2859  InVals.push_back(Val);
2860  }
2861 
2862  return Chain;
2863 }
2864 
2865 //===----------------------------------------------------------------------===//
2866 // C & StdCall & Fast Calling Convention implementation
2867 //===----------------------------------------------------------------------===//
2868 // StdCall calling convention seems to be standard for many Windows' API
2869 // routines and around. It differs from C calling convention just a little:
2870 // callee should clean up the stack, not caller. Symbols should be also
2871 // decorated in some fancy way :) It doesn't support any vector arguments.
2872 // For info on fast calling convention see Fast Calling Convention (tail call)
2873 // implementation LowerX86_32FastCCCallTo.
2874 
2875 /// CallIsStructReturn - Determines whether a call uses struct return
2876 /// semantics.
2877 enum StructReturnType {
2878  NotStructReturn,
2879  RegStructReturn,
2880  StackStructReturn
2881 };
2882 static StructReturnType
2883 callIsStructReturn(ArrayRef<ISD::OutputArg> Outs, bool IsMCU) {
2884  if (Outs.empty())
2885  return NotStructReturn;
2886 
2887  const ISD::ArgFlagsTy &Flags = Outs[0].Flags;
2888  if (!Flags.isSRet())
2889  return NotStructReturn;
2890  if (Flags.isInReg() || IsMCU)
2891  return RegStructReturn;
2892  return StackStructReturn;
2893 }
2894 
2895 /// Determines whether a function uses struct return semantics.
2896 static StructReturnType
2897 argsAreStructReturn(ArrayRef<ISD::InputArg> Ins, bool IsMCU) {
2898  if (Ins.empty())
2899  return NotStructReturn;
2900 
2901  const ISD::ArgFlagsTy &Flags = Ins[0].Flags;
2902  if (!Flags.isSRet())
2903  return NotStructReturn;
2904  if (Flags.isInReg() || IsMCU)
2905  return RegStructReturn;
2906  return StackStructReturn;
2907 }
2908 
2909 /// Make a copy of an aggregate at address specified by "Src" to address
2910 /// "Dst" with size and alignment information specified by the specific
2911 /// parameter attribute. The copy will be passed as a byval function parameter.
2912 static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
2913  SDValue Chain, ISD::ArgFlagsTy Flags,
2914  SelectionDAG &DAG, const SDLoc &dl) {
2915  SDValue SizeNode = DAG.getConstant(Flags.getByValSize(), dl, MVT::i32);
2916 
2917  return DAG.getMemcpy(Chain, dl, Dst, Src, SizeNode, Flags.getByValAlign(),
2918  /*isVolatile*/false, /*AlwaysInline=*/true,
2919  /*isTailCall*/false,
2920  MachinePointerInfo(), MachinePointerInfo());
2921 }
2922 
2923 /// Return true if the calling convention is one that we can guarantee TCO for.
2924 static bool canGuaranteeTCO(CallingConv::ID CC) {
2925  return (CC == CallingConv::Fast || CC == CallingConv::GHC ||
2926  CC == CallingConv::X86_RegCall || CC == CallingConv::HiPE ||
2927  CC == CallingConv::HHVM);
2928 }
2929 
2930 /// Return true if we might ever do TCO for calls with this calling convention.
2931 static bool mayTailCallThisCC(CallingConv::ID CC) {
2932  switch (CC) {
2933  // C calling conventions:
2934  case CallingConv::C:
2935  case CallingConv::Win64:
2936  case CallingConv::X86_64_SysV:
2937  // Callee pop conventions:
2938  case CallingConv::X86_ThisCall:
2939  case CallingConv::X86_StdCall:
2940  case CallingConv::X86_VectorCall:
2941  case CallingConv::X86_FastCall:
2942  // Swift:
2943  case CallingConv::Swift:
2944  return true;
2945  default:
2946  return canGuaranteeTCO(CC);
2947  }
2948 }
2949 
2950 /// Return true if the function is being made into a tailcall target by
2951 /// changing its ABI.
2952 static bool shouldGuaranteeTCO(CallingConv::ID CC, bool GuaranteedTailCallOpt) {
2953  return GuaranteedTailCallOpt && canGuaranteeTCO(CC);
2954 }
2955 
2956 bool X86TargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
2957  auto Attr =
2958  CI->getParent()->getParent()->getFnAttribute("disable-tail-calls");
2959  if (!CI->isTailCall() || Attr.getValueAsString() == "true")
2960  return false;
2961 
2962  ImmutableCallSite CS(CI);
2963  CallingConv::ID CalleeCC = CS.getCallingConv();
2964  if (!mayTailCallThisCC(CalleeCC))
2965  return false;
2966 
2967  return true;
2968 }
2969 
2970 SDValue
2971 X86TargetLowering::LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
2972  const SmallVectorImpl<ISD::InputArg> &Ins,
2973  const SDLoc &dl, SelectionDAG &DAG,
2974  const CCValAssign &VA,
2975  MachineFrameInfo &MFI, unsigned i) const {
2976  // Create the nodes corresponding to a load from this parameter slot.
2977  ISD::ArgFlagsTy Flags = Ins[i].Flags;
2978  bool AlwaysUseMutable = shouldGuaranteeTCO(
2979  CallConv, DAG.getTarget().Options.GuaranteedTailCallOpt);
2980  bool isImmutable = !AlwaysUseMutable && !Flags.isByVal();
2981  EVT ValVT;
2982  MVT PtrVT = getPointerTy(DAG.getDataLayout());
2983 
2984  // If value is passed by pointer we have address passed instead of the value
2985  // itself. No need to extend if the mask value and location share the same
2986  // absolute size.
2987  bool ExtendedInMem =
2988  VA.isExtInLoc() && VA.getValVT().getScalarType() == MVT::i1 &&
2989  VA.getValVT().getSizeInBits() != VA.getLocVT().getSizeInBits();
2990 
2991  if (VA.getLocInfo() == CCValAssign::Indirect || ExtendedInMem)
2992  ValVT = VA.getLocVT();
2993  else
2994  ValVT = VA.getValVT();
2995 
2996  // FIXME: For now, all byval parameter objects are marked mutable. This can be
2997  // changed with more analysis.
2998  // In case of tail call optimization mark all arguments mutable. Since they
2999  // could be overwritten by lowering of arguments in case of a tail call.
3000  if (Flags.isByVal()) {
3001  unsigned Bytes = Flags.getByValSize();
3002  if (Bytes == 0) Bytes = 1; // Don't create zero-sized stack objects.
3003 
3004  // FIXME: For now, all byval parameter objects are marked as aliasing. This
3005  // can be improved with deeper analysis.
3006  int FI = MFI.CreateFixedObject(Bytes, VA.getLocMemOffset(), isImmutable,
3007  /*isAliased=*/true);
3008  return DAG.getFrameIndex(FI, PtrVT);
3009  }
3010 
3011  // This is an argument in memory. We might be able to perform copy elision.
3012  // If the argument is passed directly in memory without any extension, then we
3013  // can perform copy elision. Large vector types, for example, may be passed
3014  // indirectly by pointer.
3015  if (Flags.isCopyElisionCandidate() &&
3016  VA.getLocInfo() != CCValAssign::Indirect && !ExtendedInMem) {
3017  EVT ArgVT = Ins[i].ArgVT;
3018  SDValue PartAddr;
3019  if (Ins[i].PartOffset == 0) {
3020  // If this is a one-part value or the first part of a multi-part value,
3021  // create a stack object for the entire argument value type and return a
3022  // load from our portion of it. This assumes that if the first part of an
3023  // argument is in memory, the rest will also be in memory.
3024  int FI = MFI.CreateFixedObject(ArgVT.getStoreSize(), VA.getLocMemOffset(),
3025  /*Immutable=*/false);
3026  PartAddr = DAG.getFrameIndex(FI, PtrVT);
3027  return DAG.getLoad(
3028  ValVT, dl, Chain, PartAddr,
3029  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
3030  } else {
3031  // This is not the first piece of an argument in memory. See if there is
3032  // already a fixed stack object including this offset. If so, assume it
3033  // was created by the PartOffset == 0 branch above and create a load from
3034  // the appropriate offset into it.
3035  int64_t PartBegin = VA.getLocMemOffset();
3036  int64_t PartEnd = PartBegin + ValVT.getSizeInBits() / 8;
3037  int FI = MFI.getObjectIndexBegin();
3038  for (; MFI.isFixedObjectIndex(FI); ++FI) {
3039  int64_t ObjBegin = MFI.getObjectOffset(FI);
3040  int64_t ObjEnd = ObjBegin + MFI.getObjectSize(FI);
3041  if (ObjBegin <= PartBegin && PartEnd <= ObjEnd)
3042  break;
3043  }
3044  if (MFI.isFixedObjectIndex(FI)) {
3045  SDValue Addr =
3046  DAG.getNode(ISD::ADD, dl, PtrVT, DAG.getFrameIndex(FI, PtrVT),
3047  DAG.getIntPtrConstant(Ins[i].PartOffset, dl));
3048  return DAG.getLoad(
3049  ValVT, dl, Chain, Addr,
3050  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI,
3051  Ins[i].PartOffset));
3052  }
3053  }
3054  }
3055 
3056  int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
3057  VA.getLocMemOffset(), isImmutable);
3058 
3059  // Set SExt or ZExt flag.
3060  if (VA.getLocInfo() == CCValAssign::ZExt) {
3061  MFI.setObjectZExt(FI, true);
3062  } else if (VA.getLocInfo() == CCValAssign::SExt) {
3063  MFI.setObjectSExt(FI, true);
3064  }
3065 
3066  SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
3067  SDValue Val = DAG.getLoad(
3068  ValVT, dl, Chain, FIN,
3069  MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI));
3070  return ExtendedInMem
3071  ? (VA.getValVT().isVector()
3072  ? DAG.getNode(ISD::SCALAR_TO_VECTOR, dl, VA.getValVT(), Val)
3073  : DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), Val))
3074  : Val;
3075 }
3076 
3077 // FIXME: Get this from tablegen.
3078 static ArrayRef<MCPhysReg> get64BitArgumentGPRs(CallingConv::ID CallConv,
3079  const X86Subtarget &Subtarget) {
3080  assert(Subtarget.is64Bit());
3081 
3082  if (Subtarget.isCallingConvWin64(CallConv)) {
3083  static const MCPhysReg GPR64ArgRegsWin64[] = {
3084  X86::RCX, X86::RDX, X86::R8, X86::R9
3085  };
3086  return makeArrayRef(std::begin(GPR64ArgRegsWin64), std::end(GPR64ArgRegsWin64));
3087  }
3088 
3089  static const MCPhysReg GPR64ArgRegs64Bit[] = {
3090  X86::RDI, X86::RSI, X86::RDX, X86::RCX, X86::R8, X86::R9
3091  };
3092  return makeArrayRef(std::begin(GPR64ArgRegs64Bit), std::end(GPR64ArgRegs64Bit));
3093 }
3094 
3095 // FIXME: Get this from tablegen.
3096 static ArrayRef<MCPhysReg> get64BitArgumentXMMs(MachineFunction &MF,
3097  CallingConv::ID CallConv,
3098  const X86Subtarget &Subtarget) {
3099  assert(Subtarget.is64Bit());
3100  if (Subtarget.isCallingConvWin64(CallConv)) {
3101  // The XMM registers which might contain var arg parameters are shadowed
3102  // in their paired GPR. So we only need to save the GPR to their home
3103  // slots.
3104  // TODO: __vectorcall will change this.
3105  return None;
3106  }
3107 
3108  const Function &F = MF.getFunction();
3109  bool NoImplicitFloatOps = F.hasFnAttribute(Attribute::NoImplicitFloat);
3110  bool isSoftFloat = Subtarget.useSoftFloat();
3111  assert(!(isSoftFloat && NoImplicitFloatOps) &&
3112  "SSE register cannot be used when SSE is disabled!");
3113  if (isSoftFloat || NoImplicitFloatOps || !Subtarget.hasSSE1())
3114  // Kernel mode asks for SSE to be disabled, so there are no XMM argument
3115  // registers.
3116  return None;
3117 
3118  static const MCPhysReg XMMArgRegs64Bit[] = {
3119  X86::XMM0, X86::XMM1, X86::XMM2, X86::XMM3,
3120  X86::XMM4, X86::XMM5, X86::XMM6, X86::XMM7
3121  };
3122  return makeArrayRef(std::begin(XMMArgRegs64Bit), std::end(XMMArgRegs64Bit));
3123 }
3124 
3125 #ifndef NDEBUG
3126 static bool isSortedByValueNo(ArrayRef<CCValAssign> ArgLocs) {
3127  return std::is_sorted(ArgLocs.begin(), ArgLocs.end(),
3128  [](const CCValAssign &A, const CCValAssign &B) -> bool {
3129  return A.getValNo() < B.getValNo();
3130  });
3131 }
3132 #endif
3133 
3134 SDValue X86TargetLowering::LowerFormalArguments(
3135  SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
3136  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
3137  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
3138  MachineFunction &MF = DAG.getMachineFunction();
3139  X86MachineFunctionInfo *FuncInfo = MF.getInfo<X86MachineFunctionInfo>();
3140  const TargetFrameLowering &TFI = *Subtarget.getFrameLowering();
3141 
3142  const Function &F = MF.getFunction();
3143  if (F.hasExternalLinkage() && Subtarget.isTargetCygMing() &&
3144  F.getName() == "main")
3145  FuncInfo->setForceFramePointer(true);
3146 
3147  MachineFrameInfo &MFI = MF.getFrameInfo();
3148  bool Is64Bit = Subtarget.is64Bit();
3149  bool IsWin64 = Subtarget.isCallingConvWin64(CallConv);
3150 
3151  assert(
3152  !(isVarArg && canGuaranteeTCO(CallConv)) &&
3153  "Var args not supported with calling conv' regcall, fastcc, ghc or hipe");
3154 
3155  // Assign locations to all of the incoming arguments.
3156  SmallVector<CCValAssign, 16> ArgLocs;
3157  CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
3158 
3159  // Allocate shadow area for Win64.
3160  if (IsWin64)
3161  CCInfo.AllocateStack(32, 8);
3162 
3163  CCInfo.AnalyzeArguments(Ins, CC_X86);
3164 
3165  // In vectorcall calling convention a second pass is required for the HVA
3166  // types.
3167  if (CallingConv::X86_VectorCall == CallConv) {
3168  CCInfo.AnalyzeArgumentsSecondPass(Ins, CC_X86);
3169  }
3170 
3171  // The next loop assumes that the locations are in the same order of the
3172  // input arguments.
3173  assert(isSortedByValueNo(ArgLocs) &&
3174  "Argument Location list must be sorted before lowering");
3175 
3176  SDValue ArgValue;
3177  for (unsigned I = 0, InsIndex = 0, E = ArgLocs.size(); I != E;
3178  ++I, ++InsIndex) {
3179  assert(InsIndex < Ins.size() && "Invalid Ins index");
3180  CCValAssign &VA = ArgLocs[I];
3181 
3182  if (VA.isRegLoc()) {
3183  EVT RegVT = VA.getLocVT();
3184  if (VA.needsCustom()) {
3185  assert(
3186  VA.getValVT() == MVT::v64i1 &&
3187  "Currently the only custom case is when we split v64i1 to 2 regs");
3188 
3189  // v64i1 values, in regcall calling convention, that are
3190  // compiled to 32 bit arch, are split up into two registers.
3191  ArgValue =
3192  getv64i1Argument(VA, ArgLocs[++I], Chain, DAG, dl, Subtarget);
3193  } else {
3194  const TargetRegisterClass *RC;
3195  if (RegVT == MVT::i8)
3196  RC = &X86::GR8RegClass;
3197  else if (RegVT == MVT::i16)
3198  RC = &X86::GR16RegClass;
3199  else if (RegVT == MVT::i32)
3200  RC = &X86::GR32RegClass;
3201  else if (Is64Bit && RegVT == MVT::i64)
3202  RC = &X86::GR64RegClass;
3203  else if (RegVT == MVT::f32)
3204  RC = Subtarget.hasAVX512() ? &X86::FR32XRegClass : &X86::FR32RegClass;
3205  else if (RegVT == MVT::f64)
3206  RC = Subtarget.hasAVX512() ? &X86::FR64XRegClass : &X86::FR64RegClass;
3207  else if (RegVT == MVT::f80)
3208  RC = &X86::RFP80RegClass;
3209  else if (RegVT == MVT::f128)
3210  RC = &X86::VR128RegClass;
3211  else if (RegVT.is512BitVector())
3212  RC = &X86::VR512RegClass;
3213  else if (RegVT.is256BitVector())
3214  RC = Subtarget.hasVLX() ? &X86::VR256XRegClass : &X86::VR256RegClass;
3215  else if (RegVT.is128BitVector())
3216  RC = Subtarget.hasVLX() ? &X86::VR128XRegClass : &X86::VR128RegClass;
3217  else if (RegVT == MVT::x86mmx)
3218  RC = &X86::VR64RegClass;
3219  else if (RegVT == MVT::v1i1)
3220  RC = &X86::VK1RegClass;
3221  else if (RegVT == MVT::v8i1)
3222  RC = &X86::VK8RegClass;
3223  else if (RegVT == MVT::v16i1)
3224  RC = &X86::VK16RegClass;
3225  else if (RegVT == MVT::v32i1)
3226  RC = &X86::VK32RegClass;
3227  else if (RegVT == MVT::v64i1)
3228  RC = &X86::VK64RegClass;
3229  else
3230  llvm_unreachable("Unknown argument type!");
3231 
3232  unsigned Reg = MF.addLiveIn(VA.getLocReg(), RC);
3233  ArgValue = DAG.getCopyFromReg(Chain, dl, Reg, RegVT);
3234  }
3235 
3236  // If this is an 8 or 16-bit value, it is really passed promoted to 32
3237  // bits. Insert an assert[sz]ext to capture this, then truncate to the
3238  // right size.
3239  if (VA.getLocInfo() == CCValAssign::SExt)
3240  ArgValue = DAG.getNode(ISD::AssertSext, dl, RegVT, ArgValue,
3241  DAG.getValueType(VA.getValVT()));
3242  else if (VA.getLocInfo() == CCValAssign::ZExt)
3243  ArgValue = DAG.getNode(ISD::AssertZext, dl, RegVT, ArgValue,
3244  DAG.getValueType(VA.getValVT()));
3245  else if (VA.getLocInfo() == CCValAssign::BCvt)
3246  ArgValue = DAG.getBitcast(VA.getValVT(), ArgValue);
3247 
3248  if (VA.isExtInLoc()) {
3249  // Handle MMX values passed in XMM regs.
3250  if (RegVT.isVector() && VA.getValVT().getScalarType() != MVT::i1)
3251  ArgValue = DAG.getNode(X86ISD::MOVDQ2Q, dl, VA.getValVT(), ArgValue);
3252  else if (VA.getValVT().isVector() &&
3253  VA.getValVT().getScalarType() == MVT::i1 &&
3254  ((VA.getLocVT() == MVT::i64) || (VA.getLocVT() == MVT::i32) ||
3255  (VA.getLocVT() == MVT::i16) || (VA.getLocVT() == MVT::i8))) {
3256  // Promoting a mask type (v*i1) into a register of type i64/i32/i16/i8
3257  ArgValue = lowerRegToMasks(ArgValue, VA.getValVT(), RegVT, dl, DAG);
3258  } else
3259  ArgValue = DAG.getNode(ISD::TRUNCATE, dl, VA.getValVT(), ArgValue);
3260  }
3261  } else {
3262  assert(VA.isMemLoc());
3263  ArgValue =
3264  LowerMemArgument(Chain, CallConv, Ins, dl, DAG, VA, MFI, InsIndex);
3265  }
3266 
3267  // If value is passed via pointer - do a load.
3268  if (VA.getLocInfo() == CCValAssign::Indirect && !Ins[I].Flags.isByVal())
3269  ArgValue =
3270  DAG.getLoad(VA.getValVT(), dl, Chain, ArgValue, MachinePointerInfo());
3271 
3272  InVals.push_back(ArgValue);
3273  }
3274 
3275  for (unsigned I = 0, E = Ins.size(); I != E; ++I) {
3276  // Swift calling convention does not require we copy the sret argument
3277  // into %rax/%eax for the return. We don't set SRetReturnReg for Swift.
3278  if (CallConv == CallingConv::Swift)
3279  continue;
3280 
3281  // All x86 ABIs require that for returning structs by value we copy the
3282  // sret argument into %rax/%eax (depending on ABI) for the return. Save
3283  // the argument into a virtual register so that we can access it from the
3284  // return points.
3285  if (Ins[I].Flags.isSRet()) {
3286  unsigned Reg = FuncInfo->getSRetReturnReg();
3287  if (!Reg) {
3288  MVT PtrTy = getPointerTy(DAG.getDataLayout());
3289  Reg = MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrTy));
3290  FuncInfo->setSRetReturnReg(Reg);
3291  }
3292  SDValue Copy = DAG.getCopyToReg(DAG.getEntryNode(), dl, Reg, InVals[I]);
3293  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, Copy, Chain);
3294  break;
3295  }
3296  }
3297 
3298  unsigned StackSize = CCInfo.getNextStackOffset();
3299  // Align stack specially for tail calls.
3300  if (shouldGuaranteeTCO(CallConv,
3301  MF.getTarget().Options.GuaranteedTailCallOpt))
3302  StackSize = GetAlignedArgumentStackSize(StackSize, DAG);
3303 
3304  // If the function takes variable number of arguments, make a frame index for
3305  // the start of the first vararg value... for expansion of llvm.va_start. We
3306  // can skip this if there are no va_start calls.
3307  if (MFI.hasVAStart() &&
3308  (Is64Bit || (CallConv != CallingConv::X86_FastCall &&
3309  CallConv != CallingConv::X86_ThisCall))) {
3310  FuncInfo->setVarArgsFrameIndex(MFI.CreateFixedObject(1, StackSize, true));
3311  }
3312 
3313  // Figure out if XMM registers are in use.
3314  assert(!(Subtarget.useSoftFloat() &&
3315  F.hasFnAttribute(Attribute::NoImplicitFloat)) &&
3316  "SSE register cannot be used when SSE is disabled!");
3317 
3318  // 64-bit calling conventions support varargs and register parameters, so we
3319  // have to do extra work to spill them in the prologue.
3320  if (Is64Bit && isVarArg && MFI.hasVAStart()) {
3321  // Find the first unallocated argument registers.
3322  ArrayRef<MCPhysReg> ArgGPRs = get64BitArgumentGPRs(CallConv, Subtarget);
3323  ArrayRef<MCPhysReg> ArgXMMs = get64BitArgumentXMMs(MF, CallConv, Subtarget);
3324  unsigned NumIntRegs = CCInfo.getFirstUnallocated(ArgGPRs);
3325  unsigned NumXMMRegs = CCInfo.getFirstUnallocated(ArgXMMs);
3326  assert(!(NumXMMRegs && !Subtarget.hasSSE1()) &&
3327  "SSE register cannot be used when SSE is disabled!");
3328 
3329  // Gather all the live in physical registers.
3330  SmallVector<SDValue, 6> LiveGPRs;
3331  SmallVector<SDValue, 8> LiveXMMRegs;
3332  SDValue ALVal;
3333  for (MCPhysReg Reg : ArgGPRs.slice(NumIntRegs)) {
3334  unsigned GPR = MF.addLiveIn(Reg, &X86::GR64RegClass);
3335  LiveGPRs.push_back(
3336  DAG.getCopyFromReg(Chain, dl, GPR, MVT::i64));
3337  }
3338  if (!ArgXMMs.empty()) {
3339  unsigned AL = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
3340  ALVal = DAG.getCopyFromReg(Chain, dl, AL, MVT::i8);
3341  for (MCPhysReg Reg : ArgXMMs.slice(NumXMMRegs)) {
3342  unsigned XMMReg = MF.addLiveIn(Reg, &X86::VR128RegClass);
3343  LiveXMMRegs.push_back(
3344  DAG.getCopyFromReg(Chain, dl, XMMReg, MVT::v4f32));
3345  }
3346  }
3347 
3348  if (IsWin64) {
3349  // Get to the caller-allocated home save location. Add 8 to account
3350  // for the return address.
3351  int HomeOffset = TFI.getOffsetOfLocalArea() + 8;
3352  FuncInfo->setRegSaveFrameIndex(
3353  MFI.CreateFixedObject(1, NumIntRegs * 8 + HomeOffset, false));
3354  // Fixup to set vararg frame on shadow area (4 x i64).
3355  if (NumIntRegs < 4)
3356  FuncInfo->setVarArgsFrameIndex(FuncInfo->getRegSaveFrameIndex());
3357  } else {
3358  // For X86-64, if there are vararg parameters that are passed via
3359  // registers, then we must store them to their spots on the stack so
3360  // they may be loaded by dereferencing the result of va_next.
3361  FuncInfo->setVarArgsGPOffset(NumIntRegs * 8);
3362  FuncInfo->setVarArgsFPOffset(ArgGPRs.size() * 8 + NumXMMRegs * 16);
3363  FuncInfo->setRegSaveFrameIndex(MFI.CreateStackObject(
3364  ArgGPRs.size() * 8 + ArgXMMs.size() * 16, 16, false));
3365  }
3366 
3367  // Store the integer parameter registers.
3368  SmallVector<SDValue, 8> MemOps;
3369  SDValue RSFIN = DAG.getFrameIndex(FuncInfo->getRegSaveFrameIndex(),
3370  getPointerTy(DAG.getDataLayout()));
3371  unsigned Offset = FuncInfo->getVarArgsGPOffset();
3372  for (SDValue Val : LiveGPRs) {
3373  SDValue FIN = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
3374  RSFIN, DAG.getIntPtrConstant(Offset, dl));
3375  SDValue Store =
3376  DAG.getStore(Val.getValue(1), dl, Val, FIN,
3377  MachinePointerInfo::getFixedStack(
3378  DAG.getMachineFunction(),
3379  FuncInfo->getRegSaveFrameIndex(), Offset));
3380  MemOps.push_back(Store);
3381  Offset += 8;
3382  }
3383 
3384  if (!ArgXMMs.empty() && NumXMMRegs != ArgXMMs.size()) {
3385  // Now store the XMM (fp + vector) parameter registers.
3386  SmallVector<SDValue, 12> SaveXMMOps;
3387  SaveXMMOps.push_back(Chain);
3388  SaveXMMOps.push_back(ALVal);
3389  SaveXMMOps.push_back(DAG.getIntPtrConstant(
3390  FuncInfo->getRegSaveFrameIndex(), dl));
3391  SaveXMMOps.push_back(DAG.getIntPtrConstant(
3392  FuncInfo->getVarArgsFPOffset(), dl));
3393  SaveXMMOps.insert(SaveXMMOps.end(), LiveXMMRegs.begin(),
3394  LiveXMMRegs.end());
3395  MemOps.push_back(DAG.getNode(X86ISD::VASTART_SAVE_XMM_REGS, dl,
3396  MVT::Other, SaveXMMOps));
3397  }
3398 
3399  if (!MemOps.empty())
3400  Chain = DAG.getNode(ISD::TokenFactor, dl, MVT::Other, MemOps);
3401  }
3402 
3403  if (isVarArg && MFI.hasMustTailInVarArgFunc()) {
3404  // Find the largest legal vector type.
3405  MVT VecVT = MVT::Other;
3406  // FIXME: Only some x86_32 calling conventions support AVX512.
3407  if (Subtarget.hasAVX512() &&
3408  (Is64Bit || (CallConv == CallingConv::X86_VectorCall ||
3409  CallConv == CallingConv::Intel_OCL_BI)))
3410  VecVT = MVT::v16f32;
3411  else if (Subtarget.hasAVX())
3412  VecVT = MVT::v8f32;
3413  else if (Subtarget.hasSSE2())
3414  VecVT = MVT::v4f32;
3415 
3416  // We forward some GPRs and some vector types.
3417  SmallVector<MVT, 2> RegParmTypes;
3418  MVT IntVT = Is64Bit ? MVT::i64 : MVT::i32;
3419  RegParmTypes.push_back(IntVT);
3420  if (VecVT != MVT::Other)
3421  RegParmTypes.push_back(VecVT);
3422 
3423  // Compute the set of forwarded registers. The rest are scratch.
3424  SmallVectorImpl<ForwardedRegister> &Forwards =
3425  FuncInfo->getForwardedMustTailRegParms();
3426  CCInfo.analyzeMustTailForwardedRegisters(Forwards, RegParmTypes, CC_X86);
3427 
3428  // Conservatively forward AL on x86_64, since it might be used for varargs.
3429  if (Is64Bit && !CCInfo.isAllocated(X86::AL)) {
3430  unsigned ALVReg = MF.addLiveIn(X86::AL, &X86::GR8RegClass);
3431  Forwards.push_back(ForwardedRegister(ALVReg, X86::AL, MVT::i8));
3432  }
3433 
3434  // Copy all forwards from physical to virtual registers.
3435  for (ForwardedRegister &FR : Forwards) {
3436  // FIXME: Can we use a less constrained schedule?
3437  SDValue RegVal = DAG.getCopyFromReg(Chain, dl, FR.VReg, FR.VT);
3438  FR.VReg = MF.getRegInfo().createVirtualRegister(getRegClassFor(FR.VT));
3439  Chain = DAG.getCopyToReg(Chain, dl, FR.VReg, RegVal);
3440  }
3441  }
3442 
3443  // Some CCs need callee pop.
3444  if (X86::isCalleePop(CallConv, Is64Bit, isVarArg,
3445  MF.getTarget().Options.GuaranteedTailCallOpt)) {
3446  FuncInfo->setBytesToPopOnReturn(StackSize); // Callee pops everything.
3447  } else if (CallConv == CallingConv::X86_INTR && Ins.size() == 2) {
3448  // X86 interrupts must pop the error code (and the alignment padding) if
3449  // present.
3450  FuncInfo->setBytesToPopOnReturn(Is64Bit ? 16 : 4);
3451  } else {
3452  FuncInfo->setBytesToPopOnReturn(0); // Callee pops nothing.
3453  // If this is an sret function, the return should pop the hidden pointer.
3454  if (!Is64Bit && !canGuaranteeTCO(CallConv) &&
3455  !Subtarget.getTargetTriple().isOSMSVCRT() &&
3456  argsAreStructReturn(Ins, Subtarget.isTargetMCU()) == StackStructReturn)
3457  FuncInfo->setBytesToPopOnReturn(4);
3458  }
3459 
3460  if (!Is64Bit) {
3461  // RegSaveFrameIndex is X86-64 only.
3462  FuncInfo->setRegSaveFrameIndex(0xAAAAAAA);
3463  if (CallConv == CallingConv::X86_FastCall ||
3464  CallConv == CallingConv::X86_ThisCall)
3465  // fastcc functions can't have varargs.
3466  FuncInfo->setVarArgsFrameIndex(0xAAAAAAA);
3467  }
3468 
3469  FuncInfo->setArgumentStackSize(StackSize);
3470 
3471  if (WinEHFuncInfo *EHInfo = MF.getWinEHFuncInfo()) {
3472  EHPersonality Personality = classifyEHPersonality(F.getPersonalityFn());
3473  if (Personality == EHPersonality::CoreCLR) {
3474  assert(Is64Bit);
3475  // TODO: Add a mechanism to frame lowering that will allow us to indicate
3476  // that we'd prefer this slot be allocated towards the bottom of the frame
3477  // (i.e. near the stack pointer after allocating the frame). Every
3478  // funclet needs a copy of this slot in its (mostly empty) frame, and the
3479  // offset from the bottom of this and each funclet's frame must be the
3480  // same, so the size of funclets' (mostly empty) frames is dictated by
3481  // how far this slot is from the bottom (since they allocate just enough
3482  // space to accommodate holding this slot at the correct offset).
3483  int PSPSymFI = MFI.CreateStackObject(8, 8, /*isSS=*/false);
3484  EHInfo->PSPSymFrameIdx = PSPSymFI;
3485  }
3486  }
3487 
3488  if (CallConv == CallingConv::X86_RegCall ||
3489  F.hasFnAttribute("no_caller_saved_registers")) {
3490  MachineRegisterInfo &MRI = MF.getRegInfo();
3491  for (std::pair<unsigned, unsigned> Pair : MRI.liveins())
3492  MRI.disableCalleeSavedRegister(Pair.first);
3493  }
3494 
3495  return Chain;
3496 }
3497 
3498 SDValue X86TargetLowering::LowerMemOpCallTo(SDValue Chain, SDValue StackPtr,
3499  SDValue Arg, const SDLoc &dl,
3500  SelectionDAG &DAG,
3501  const CCValAssign &VA,
3502  ISD::ArgFlagsTy Flags) const {
3503  unsigned LocMemOffset = VA.getLocMemOffset();
3504  SDValue PtrOff = DAG.getIntPtrConstant(LocMemOffset, dl);
3505  PtrOff = DAG.getNode(ISD::ADD, dl, getPointerTy(DAG.getDataLayout()),
3506  StackPtr, PtrOff);
3507  if (Flags.isByVal())
3508  return CreateCopyOfByValArgument(Arg, PtrOff, Chain, Flags, DAG, dl);
3509 
3510  return DAG.getStore(
3511  Chain, dl, Arg, PtrOff,
3512  MachinePointerInfo::getStack(DAG.getMachineFunction(), LocMemOffset));
3513 }
3514 
3515 /// Emit a load of return address if tail call
3516 /// optimization is performed and it is required.
3517 SDValue X86TargetLowering::EmitTailCallLoadRetAddr(
3518  SelectionDAG &DAG, SDValue &OutRetAddr, SDValue Chain, bool IsTailCall,
3519  bool Is64Bit, int FPDiff, const SDLoc &dl) const {
3520  // Adjust the Return address stack slot.
3521  EVT VT = getPointerTy(DAG.getDataLayout());
3522  OutRetAddr = getReturnAddressFrameIndex(DAG);
3523 
3524  // Load the "old" Return address.
3525  OutRetAddr = DAG.getLoad(VT, dl, Chain, OutRetAddr, MachinePointerInfo());
3526  return SDValue(OutRetAddr.getNode(), 1);
3527 }
3528 
3529 /// Emit a store of the return address if tail call
3530 /// optimization is performed and it is required (FPDiff!=0).
3531 static SDValue EmitTailCallStoreRetAddr(SelectionDAG &DAG, MachineFunction &MF,
3532  SDValue Chain, SDValue RetAddrFrIdx,
3533  EVT PtrVT, unsigned SlotSize,
3534  int FPDiff, const SDLoc &dl) {
3535  // Store the return address to the appropriate stack slot.
3536  if (!FPDiff) return Chain;
3537  // Calculate the new stack slot for the return address.
3538  int NewReturnAddrFI =
3539  MF.getFrameInfo().CreateFixedObject(SlotSize, (int64_t)FPDiff - SlotSize,
3540  false);
3541  SDValue NewRetAddrFrIdx = DAG.getFrameIndex(NewReturnAddrFI, PtrVT);
3542  Chain = DAG.getStore(Chain, dl, RetAddrFrIdx, NewRetAddrFrIdx,
3543  MachinePointerInfo::getFixedStack(
3544  DAG.getMachineFunction(), NewReturnAddrFI));
3545  return Chain;
3546 }
3547 
3548 /// Returns a vector_shuffle mask for an movs{s|d}, movd
3549 /// operation of specified width.
3550 static SDValue getMOVL(SelectionDAG &DAG, const SDLoc &dl, MVT VT, SDValue V1,
3551  SDValue V2) {
3552  unsigned NumElems = VT.getVectorNumElements();
3553  SmallVector<int, 8> Mask;
3554  Mask.push_back(NumElems);
3555  for (unsigned i = 1; i != NumElems; ++i)
3556  Mask.push_back(i);
3557  return DAG.getVectorShuffle(VT, dl, V1, V2, Mask);
3558 }
3559 
3560 SDValue
3561 X86TargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
3562  SmallVectorImpl<SDValue> &InVals) const {
3563  SelectionDAG &DAG = CLI.DAG;
3564  SDLoc &dl = CLI.DL;
3565  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
3566  SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;