LLVM 17.0.0git
X86ISelLowering.h
Go to the documentation of this file.
1//===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that X86 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
15#define LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
16
19
20namespace llvm {
21 class X86Subtarget;
22 class X86TargetMachine;
23
24 namespace X86ISD {
25 // X86 Specific DAG Nodes
26 enum NodeType : unsigned {
27 // Start the numbering where the builtin ops leave off.
29
30 /// Bit scan forward.
32 /// Bit scan reverse.
34
35 /// X86 funnel/double shift i16 instructions. These correspond to
36 /// X86::SHLDW and X86::SHRDW instructions which have different amt
37 /// modulo rules to generic funnel shifts.
38 /// NOTE: The operand order matches ISD::FSHL/FSHR not SHLD/SHRD.
41
42 /// Bitwise logical AND of floating point values. This corresponds
43 /// to X86::ANDPS or X86::ANDPD.
45
46 /// Bitwise logical OR of floating point values. This corresponds
47 /// to X86::ORPS or X86::ORPD.
49
50 /// Bitwise logical XOR of floating point values. This corresponds
51 /// to X86::XORPS or X86::XORPD.
53
54 /// Bitwise logical ANDNOT of floating point values. This
55 /// corresponds to X86::ANDNPS or X86::ANDNPD.
57
58 /// These operations represent an abstract X86 call
59 /// instruction, which includes a bunch of information. In particular the
60 /// operands of these node are:
61 ///
62 /// #0 - The incoming token chain
63 /// #1 - The callee
64 /// #2 - The number of arg bytes the caller pushes on the stack.
65 /// #3 - The number of arg bytes the callee pops off the stack.
66 /// #4 - The value to pass in AL/AX/EAX (optional)
67 /// #5 - The value to pass in DL/DX/EDX (optional)
68 ///
69 /// The result values of these nodes are:
70 ///
71 /// #0 - The outgoing token chain
72 /// #1 - The first register result value (optional)
73 /// #2 - The second register result value (optional)
74 ///
76
77 /// Same as call except it adds the NoTrack prefix.
79
80 // Pseudo for a OBJC call that gets emitted together with a special
81 // marker instruction.
83
84 /// X86 compare and logical compare instructions.
89
90 /// X86 bit-test instructions.
92
93 /// X86 SetCC. Operand 0 is condition code, and operand 1 is the EFLAGS
94 /// operand, usually produced by a CMP instruction.
96
97 /// X86 Select
99
100 // Same as SETCC except it's materialized with a sbb and the value is all
101 // one's or all zero's.
102 SETCC_CARRY, // R = carry_bit ? ~0 : 0
103
104 /// X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
105 /// Operands are two FP values to compare; result is a mask of
106 /// 0s or 1s. Generally DTRT for C/C++ with NaNs.
108
109 /// X86 FP SETCC, similar to above, but with output as an i1 mask and
110 /// and a version with SAE.
113
114 /// X86 conditional moves. Operand 0 and operand 1 are the two values
115 /// to select from. Operand 2 is the condition code, and operand 3 is the
116 /// flag operand produced by a CMP or TEST instruction.
118
119 /// X86 conditional branches. Operand 0 is the chain operand, operand 1
120 /// is the block to branch if condition is true, operand 2 is the
121 /// condition code, and operand 3 is the flag operand produced by a CMP
122 /// or TEST instruction.
124
125 /// BRIND node with NoTrack prefix. Operand 0 is the chain operand and
126 /// operand 1 is the target address.
128
129 /// Return with a glue operand. Operand 0 is the chain operand, operand
130 /// 1 is the number of bytes of stack to pop.
132
133 /// Return from interrupt. Operand 0 is the number of bytes to pop.
135
136 /// Repeat fill, corresponds to X86::REP_STOSx.
138
139 /// Repeat move, corresponds to X86::REP_MOVSx.
141
142 /// On Darwin, this node represents the result of the popl
143 /// at function entry, used for PIC code.
145
146 /// A wrapper node for TargetConstantPool, TargetJumpTable,
147 /// TargetExternalSymbol, TargetGlobalAddress, TargetGlobalTLSAddress,
148 /// MCSymbol and TargetBlockAddress.
150
151 /// Special wrapper used under X86-64 PIC mode for RIP
152 /// relative displacements.
154
155 /// Copies a 64-bit value from an MMX vector to the low word
156 /// of an XMM vector, with the high word zero filled.
158
159 /// Copies a 64-bit value from the low word of an XMM vector
160 /// to an MMX vector.
162
163 /// Copies a 32-bit value from the low word of a MMX
164 /// vector to a GPR.
166
167 /// Copies a GPR into the low 32-bit word of a MMX vector
168 /// and zero out the high word.
170
171 /// Extract an 8-bit value from a vector and zero extend it to
172 /// i32, corresponds to X86::PEXTRB.
174
175 /// Extract a 16-bit value from a vector and zero extend it to
176 /// i32, corresponds to X86::PEXTRW.
178
179 /// Insert any element of a 4 x float vector into any element
180 /// of a destination 4 x floatvector.
182
183 /// Insert the lower 8-bits of a 32-bit value to a vector,
184 /// corresponds to X86::PINSRB.
186
187 /// Insert the lower 16-bits of a 32-bit value to a vector,
188 /// corresponds to X86::PINSRW.
190
191 /// Shuffle 16 8-bit values within a vector.
193
194 /// Compute Sum of Absolute Differences.
196 /// Compute Double Block Packed Sum-Absolute-Differences
198
199 /// Bitwise Logical AND NOT of Packed FP values.
201
202 /// Blend where the selector is an immediate.
204
205 /// Dynamic (non-constant condition) vector blend where only the sign bits
206 /// of the condition elements are used. This is used to enforce that the
207 /// condition mask is not valid for generic VSELECT optimizations. This
208 /// is also used to implement the intrinsics.
209 /// Operands are in VSELECT order: MASK, TRUE, FALSE
211
212 /// Combined add and sub on an FP vector.
214
215 // FP vector ops with rounding mode.
235
236 // FP vector get exponent.
241 // Extract Normalized Mantissas.
246 // FP Scale.
251
252 /// Integer horizontal add/sub.
255
256 /// Floating point horizontal add/sub.
259
260 // Detect Conflicts Within a Vector
262
263 /// Floating point max and min.
266
267 /// Commutative FMIN and FMAX.
270
271 /// Scalar intrinsic floating point max and min.
274
275 /// Floating point reciprocal-sqrt and reciprocal approximation.
276 /// Note that these typically require refinement
277 /// in order to obtain suitable precision.
280
281 // AVX-512 reciprocal approximations with a little more precision.
286
287 // Thread Local Storage.
289
290 // Thread Local Storage. A call to get the start address
291 // of the TLS block for the current module.
293
294 // Thread Local Storage. When calling to an OS provided
295 // thunk at the address from an earlier relocation.
297
298 // Exception Handling helpers.
300
301 // SjLj exception handling setjmp.
303
304 // SjLj exception handling longjmp.
306
307 // SjLj exception handling dispatch.
309
310 /// Tail call return. See X86TargetLowering::LowerCall for
311 /// the list of operands.
313
314 // Vector move to low scalar and zero higher vector elements.
316
317 // Vector integer truncate.
319 // Vector integer truncate with unsigned/signed saturation.
322
323 // Masked version of the above. Used when less than a 128-bit result is
324 // produced since the mask only applies to the lower elements and can't
325 // be represented by a select.
326 // SRC, PASSTHRU, MASK
330
331 // Vector FP extend.
336
337 // Vector FP round.
342
343 // Masked version of above. Used for v2f64->v4f32.
344 // SRC, PASSTHRU, MASK
346
347 // 128-bit vector logical left / right shift
350
351 // Vector shift elements
355
356 // Vector variable shift
360
361 // Vector shift elements by immediate
365
366 // Shifts of mask registers.
369
370 // Bit rotate by immediate
373
374 // Vector packed double/float comparison.
376
377 // Vector integer comparisons.
380
381 // v8i16 Horizontal minimum and position.
383
385
386 /// Vector comparison generating mask bits for fp and
387 /// integer signed and unsigned data types.
389 // Vector mask comparison generating mask bits for FP values.
391 // Vector mask comparison with SAE for FP values.
393
394 // Arithmetic operations with FLAGS results.
404
405 // Bit field extract.
408
409 // Zero High Bits Starting with Specified Bit Position.
411
412 // Parallel extract and deposit.
415
416 // X86-specific multiply by immediate.
418
419 // Vector sign bit extraction.
421
422 // Vector bitwise comparisons.
424
425 // Vector packed fp sign bitwise comparisons.
427
428 // OR/AND test for masks.
431
432 // ADD for masks.
434
435 // Several flavors of instructions with vector shuffle behaviors.
436 // Saturated signed/unnsigned packing.
439 // Intra-lane alignr.
441 // AVX512 inter-lane alignr.
447 // VBMI2 Concat & Shift.
452 // Shuffle Packed Values at 128-bit granularity.
468
469 // Variable Permute (VPERM).
470 // Res = VPERMV MaskV, V0
472
473 // 3-op Variable Permute (VPERMT2).
474 // Res = VPERMV3 V0, MaskV, V1
476
477 // Bitwise ternary logic.
479 // Fix Up Special Packed Float32/64 values.
484 // Range Restriction Calculation For Packed Pairs of Float32/64 values.
489 // Reduce - Perform Reduction Transformation on scalar\packed FP.
494 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.
495 // Also used by the legacy (V)ROUND intrinsics where we mask out the
496 // scaling part of the immediate.
501 // Tests Types Of a FP Values for packed types.
503 // Tests Types Of a FP Values for scalar types.
505
506 // Broadcast (splat) scalar or element 0 of a vector. If the operand is
507 // a vector, this node may change the vector length as part of the splat.
509 // Broadcast mask to vector.
511
512 /// SSE4A Extraction and Insertion.
515
516 // XOP arithmetic/logical shifts.
519 // XOP signed/unsigned integer comparisons.
522 // XOP packed permute bytes.
524 // XOP two source permutation.
526
527 // Vector multiply packed unsigned doubleword integers.
529 // Vector multiply packed signed doubleword integers.
531 // Vector Multiply Packed UnsignedIntegers with Round and Scale.
533
534 // Multiply and Add Packed Integers.
537
538 // AVX512IFMA multiply and add.
539 // NOTE: These are different than the instruction and perform
540 // op0 x op1 + op2.
543
544 // VNNI
549
550 // FMA nodes.
551 // We use the target independent ISD::FMA for the non-inverted case.
557
558 // FMA with rounding mode.
565
566 // AVX512-FP16 complex addition and multiplication.
571
576
581
586
593
594 // Compress and expand.
597
598 // Bits shuffle
600
601 // Convert Unsigned/Integer to Floating-Point Value with rounding mode.
608
609 // Vector float/double to signed/unsigned integer.
614 // Scalar float/double to signed/unsigned integer.
619
620 // Vector float/double to signed/unsigned integer with truncation.
625 // Scalar float/double to signed/unsigned integer with truncation.
630
631 // Vector signed/unsigned integer to float/double.
634
635 // Masked versions of above. Used for v2f64->v4f32.
636 // SRC, PASSTHRU, MASK
643
644 // Vector float to bfloat16.
645 // Convert TWO packed single data to one packed BF16 data
647 // Convert packed single data to packed BF16 data
649 // Masked version of above.
650 // SRC, PASSTHRU, MASK
652
653 // Dot product of BF16 pairs to accumulated into
654 // packed single precision.
656
657 // A stack checking function call. On Windows it's _chkstk call.
659
660 // For allocating variable amounts of stack space when using
661 // segmented stacks. Check if the current stacklet has enough space, and
662 // falls back to heap allocation if not.
664
665 // For allocating stack space when using stack clash protector.
666 // Allocation is performed by block, and each block is probed.
668
669 // Memory barriers.
671
672 // Get a random integer and indicate whether it is valid in CF.
674
675 // Get a NIST SP800-90B & C compliant random integer and
676 // indicate whether it is valid in CF.
678
679 // Protection keys
680 // RDPKRU - Operand 0 is chain. Operand 1 is value for ECX.
681 // WRPKRU - Operand 0 is chain. Operand 1 is value for EDX. Operand 2 is
682 // value for ECX.
685
686 // SSE42 string comparisons.
687 // These nodes produce 3 results, index, mask, and flags. X86ISelDAGToDAG
688 // will emit one or two instructions based on which results are used. If
689 // flags and index/mask this allows us to use a single instruction since
690 // we won't have to pick and opcode for flags. Instead we can rely on the
691 // DAG to CSE everything and decide at isel.
694
695 // Test if in transactional execution.
697
698 // ERI instructions.
709
710 // Conversions between float and half-float.
715
716 // Masked version of above.
717 // SRC, RND, PASSTHRU, MASK
720
721 // Galois Field Arithmetic Instructions
725
726 // LWP insert record.
728
729 // User level wait
732
733 // Enqueue Stores Instructions
736
737 // For avx512-vp2intersect
739
740 // User level interrupts - testui
742
743 // Perform an FP80 add after changing precision control in FPCW.
745
746 /// X86 strict FP compare instructions.
749
750 // Vector packed double/float comparison.
752
753 /// Vector comparison generating mask bits for fp and
754 /// integer signed and unsigned data types.
756
757 // Vector float/double to signed/unsigned integer with truncation.
760
761 // Vector FP extend.
763
764 // Vector FP round.
766
767 // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.
768 // Also used by the legacy (V)ROUND intrinsics where we mask out the
769 // scaling part of the immediate.
771
772 // Vector signed/unsigned integer to float/double.
775
776 // Strict FMA nodes.
780
781 // Conversions between float and half-float.
784
785 // Perform an FP80 add after changing precision control in FPCW.
787
788 // WARNING: Only add nodes here if they are strict FP nodes. Non-memory and
789 // non-strict FP nodes should be above FIRST_TARGET_STRICTFP_OPCODE.
790
791 // Compare and swap.
796
797 /// LOCK-prefixed arithmetic read-modify-write instructions.
798 /// EFLAGS, OUTCHAIN = LADD(INCHAIN, PTR, RHS)
810
811 /// RAO arithmetic instructions.
812 /// OUTCHAIN = AADD(INCHAIN, PTR, RHS)
817
818 // Load, scalar_to_vector, and zero extend.
820
821 // extract_vector_elt, store.
823
824 // scalar broadcast from memory.
826
827 // subvector broadcast from memory.
829
830 // Store FP control word into i16 memory.
832
833 // Load FP control word from i16 memory.
835
836 /// This instruction implements FP_TO_SINT with the
837 /// integer destination in memory and a FP reg source. This corresponds
838 /// to the X86::FIST*m instructions and the rounding mode change stuff. It
839 /// has two inputs (token chain and address) and two outputs (int value
840 /// and token chain). Memory VT specifies the type to store to.
842
843 /// This instruction implements SINT_TO_FP with the
844 /// integer source in memory and FP reg result. This corresponds to the
845 /// X86::FILD*m instructions. It has two inputs (token chain and address)
846 /// and two outputs (FP value and token chain). The integer source type is
847 /// specified by the memory VT.
849
850 /// This instruction implements a fp->int store from FP stack
851 /// slots. This corresponds to the fist instruction. It takes a
852 /// chain operand, value to store, address, and glue. The memory VT
853 /// specifies the type to store as.
855
856 /// This instruction implements an extending load to FP stack slots.
857 /// This corresponds to the X86::FLD32m / X86::FLD64m. It takes a chain
858 /// operand, and ptr to load from. The memory VT specifies the type to
859 /// load from.
861
862 /// This instruction implements a truncating store from FP stack
863 /// slots. This corresponds to the X86::FST32m / X86::FST64m. It takes a
864 /// chain operand, value to store, address, and glue. The memory VT
865 /// specifies the type to store as.
867
868 /// These instructions grab the address of the next argument
869 /// from a va_list. (reads and modifies the va_list in memory)
872
873 // Vector truncating store with unsigned/signed saturation
876 // Vector truncating masked store with unsigned/signed saturation
879
880 // X86 specific gather and scatter
883
884 // Key locker nodes that produce flags.
893
894 /// Compare and Add if Condition is Met. Compare value in operand 2 with
895 /// value in memory of operand 1. If condition of operand 4 is met, add
896 /// value operand 3 to m32 and write new value in operand 1. Operand 2 is
897 /// always updated with the original value from operand 1.
899
900 // Save xmm argument registers to the stack, according to %al. An operator
901 // is needed so that this can be expanded with control flow.
903
904 // WARNING: Do not add anything in the end unless you want the node to
905 // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all
906 // opcodes will be thought as target memory ops!
907 };
908 } // end namespace X86ISD
909
910 namespace X86 {
911 /// Current rounding mode is represented in bits 11:10 of FPSR. These
912 /// values are same as corresponding constants for rounding mode used
913 /// in glibc.
915 rmToNearest = 0, // FE_TONEAREST
916 rmDownward = 1 << 10, // FE_DOWNWARD
917 rmUpward = 2 << 10, // FE_UPWARD
918 rmTowardZero = 3 << 10, // FE_TOWARDZERO
919 rmMask = 3 << 10 // Bit mask selecting rounding mode
920 };
921 }
922
923 /// Define some predicates that are used for node matching.
924 namespace X86 {
925 /// Returns true if Elt is a constant zero or floating point constant +0.0.
926 bool isZeroNode(SDValue Elt);
927
928 /// Returns true of the given offset can be
929 /// fit into displacement field of the instruction.
931 bool hasSymbolicDisplacement);
932
933 /// Determines whether the callee is required to pop its
934 /// own arguments. Callee pop is necessary to support tail calls.
935 bool isCalleePop(CallingConv::ID CallingConv,
936 bool is64Bit, bool IsVarArg, bool GuaranteeTCO);
937
938 /// If Op is a constant whose elements are all the same constant or
939 /// undefined, return true and return the constant value in \p SplatVal.
940 /// If we have undef bits that don't cover an entire element, we treat these
941 /// as zero if AllowPartialUndefs is set, else we fail and return false.
942 bool isConstantSplat(SDValue Op, APInt &SplatVal,
943 bool AllowPartialUndefs = true);
944
945 /// Check if Op is a load operation that could be folded into some other x86
946 /// instruction as a memory operand. Example: vpaddd (%rdi), %xmm0, %xmm0.
947 bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget,
948 bool AssumeSingleUse = false);
949
950 /// Check if Op is a load operation that could be folded into a vector splat
951 /// instruction as a memory operand. Example: vbroadcastss 16(%rdi), %xmm2.
952 bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT,
953 const X86Subtarget &Subtarget,
954 bool AssumeSingleUse = false);
955
956 /// Check if Op is a value that could be used to fold a store into some
957 /// other x86 instruction as a memory operand. Ex: pextrb $0, %xmm0, (%rdi).
958 bool mayFoldIntoStore(SDValue Op);
959
960 /// Check if Op is an operation that could be folded into a zero extend x86
961 /// instruction.
962 bool mayFoldIntoZeroExtend(SDValue Op);
963 } // end namespace X86
964
965 //===--------------------------------------------------------------------===//
966 // X86 Implementation of the TargetLowering interface
967 class X86TargetLowering final : public TargetLowering {
968 public:
969 explicit X86TargetLowering(const X86TargetMachine &TM,
970 const X86Subtarget &STI);
971
972 unsigned getJumpTableEncoding() const override;
973 bool useSoftFloat() const override;
974
975 void markLibCallAttributes(MachineFunction *MF, unsigned CC,
976 ArgListTy &Args) const override;
977
978 MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override {
979 return MVT::i8;
980 }
981
982 const MCExpr *
984 const MachineBasicBlock *MBB, unsigned uid,
985 MCContext &Ctx) const override;
986
987 /// Returns relocation base for the given PIC jumptable.
989 SelectionDAG &DAG) const override;
990 const MCExpr *
992 unsigned JTI, MCContext &Ctx) const override;
993
994 /// Return the desired alignment for ByVal aggregate
995 /// function arguments in the caller parameter area. For X86, aggregates
996 /// that contains are placed at 16-byte boundaries while the rest are at
997 /// 4-byte boundaries.
999 const DataLayout &DL) const override;
1000
1001 EVT getOptimalMemOpType(const MemOp &Op,
1002 const AttributeList &FuncAttributes) const override;
1003
1004 /// Returns true if it's safe to use load / store of the
1005 /// specified type to expand memcpy / memset inline. This is mostly true
1006 /// for all types except for some special cases. For example, on X86
1007 /// targets without SSE2 f64 load / store are done with fldl / fstpl which
1008 /// also does type conversion. Note the specified type doesn't have to be
1009 /// legal as the hook is used before type legalization.
1010 bool isSafeMemOpType(MVT VT) const override;
1011
1012 bool isMemoryAccessFast(EVT VT, Align Alignment) const;
1013
1014 /// Returns true if the target allows unaligned memory accesses of the
1015 /// specified type. Returns whether it is "fast" in the last argument.
1016 bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment,
1018 unsigned *Fast) const override;
1019
1020 /// This function returns true if the memory access is aligned or if the
1021 /// target allows this specific unaligned memory access. If the access is
1022 /// allowed, the optional final parameter returns a relative speed of the
1023 /// access (as defined by the target).
1024 bool allowsMemoryAccess(
1025 LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace,
1026 Align Alignment,
1028 unsigned *Fast = nullptr) const override;
1029
1031 const MachineMemOperand &MMO,
1032 unsigned *Fast) const {
1033 return allowsMemoryAccess(Context, DL, VT, MMO.getAddrSpace(),
1034 MMO.getAlign(), MMO.getFlags(), Fast);
1035 }
1036
1037 /// Provide custom lowering hooks for some operations.
1038 ///
1039 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
1040
1041 /// Replace the results of node with an illegal result
1042 /// type with new values built out of custom code.
1043 ///
1045 SelectionDAG &DAG) const override;
1046
1047 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
1048
1049 bool preferABDSToABSWithNSW(EVT VT) const override;
1050
1051 /// Return true if the target has native support for
1052 /// the specified value type and it is 'desirable' to use the type for the
1053 /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
1054 /// instruction encodings are longer and some i16 instructions are slow.
1055 bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;
1056
1057 /// Return true if the target has native support for the
1058 /// specified value type and it is 'desirable' to use the type. e.g. On x86
1059 /// i16 is legal, but undesirable since i16 instruction encodings are longer
1060 /// and some i16 instructions are slow.
1061 bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
1062
1063 /// Return prefered fold type, Abs if this is a vector, AddAnd if its an
1064 /// integer, None otherwise.
1067 const SDNode *SETCC0,
1068 const SDNode *SETCC1) const override;
1069
1070 /// Return the newly negated expression if the cost is not expensive and
1071 /// set the cost in \p Cost to indicate that if it is cheaper or neutral to
1072 /// do the negation.
1074 bool LegalOperations, bool ForCodeSize,
1076 unsigned Depth) const override;
1077
1080 MachineBasicBlock *MBB) const override;
1081
1082 /// This method returns the name of a target specific DAG node.
1083 const char *getTargetNodeName(unsigned Opcode) const override;
1084
1085 /// Do not merge vector stores after legalization because that may conflict
1086 /// with x86-specific store splitting optimizations.
1087 bool mergeStoresAfterLegalization(EVT MemVT) const override {
1088 return !MemVT.isVector();
1089 }
1090
1091 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
1092 const MachineFunction &MF) const override;
1093
1094 bool isCheapToSpeculateCttz(Type *Ty) const override;
1095
1096 bool isCheapToSpeculateCtlz(Type *Ty) const override;
1097
1098 bool isCtlzFast() const override;
1099
1100 bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override {
1101 // If the pair to store is a mixture of float and int values, we will
1102 // save two bitwise instructions and one float-to-int instruction and
1103 // increase one store instruction. There is potentially a more
1104 // significant benefit because it avoids the float->int domain switch
1105 // for input value. So It is more likely a win.
1106 if ((LTy.isFloatingPoint() && HTy.isInteger()) ||
1107 (LTy.isInteger() && HTy.isFloatingPoint()))
1108 return true;
1109 // If the pair only contains int values, we will save two bitwise
1110 // instructions and increase one store instruction (costing one more
1111 // store buffer). Since the benefit is more blurred so we leave
1112 // such pair out until we get testcase to prove it is a win.
1113 return false;
1114 }
1115
1116 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
1117
1118 bool hasAndNotCompare(SDValue Y) const override;
1119
1120 bool hasAndNot(SDValue Y) const override;
1121
1122 bool hasBitTest(SDValue X, SDValue Y) const override;
1123
1126 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
1127 SelectionDAG &DAG) const override;
1128
1129 bool preferScalarizeSplat(SDNode *N) const override;
1130
1132 CombineLevel Level) const override;
1133
1134 bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override;
1135
1136 bool
1138 unsigned KeptBits) const override {
1139 // For vectors, we don't have a preference..
1140 if (XVT.isVector())
1141 return false;
1142
1143 auto VTIsOk = [](EVT VT) -> bool {
1144 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
1145 VT == MVT::i64;
1146 };
1147
1148 // We are ok with KeptBitsVT being byte/word/dword, what MOVS supports.
1149 // XVT will be larger than KeptBitsVT.
1150 MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);
1151 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
1152 }
1153
1156 unsigned ExpansionFactor) const override;
1157
1158 bool shouldSplatInsEltVarIndex(EVT VT) const override;
1159
1160 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override {
1161 // Converting to sat variants holds little benefit on X86 as we will just
1162 // need to saturate the value back using fp arithmatic.
1163 return Op != ISD::FP_TO_UINT_SAT && isOperationLegalOrCustom(Op, VT);
1164 }
1165
1166 bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
1167 return VT.isScalarInteger();
1168 }
1169
1170 /// Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
1171 MVT hasFastEqualityCompare(unsigned NumBits) const override;
1172
1173 /// Return the value type to use for ISD::SETCC.
1175 EVT VT) const override;
1176
1178 const APInt &DemandedElts,
1179 TargetLoweringOpt &TLO) const override;
1180
1181 /// Determine which of the bits specified in Mask are known to be either
1182 /// zero or one and return them in the KnownZero/KnownOne bitsets.
1184 KnownBits &Known,
1185 const APInt &DemandedElts,
1186 const SelectionDAG &DAG,
1187 unsigned Depth = 0) const override;
1188
1189 /// Determine the number of bits in the operation that are sign bits.
1191 const APInt &DemandedElts,
1192 const SelectionDAG &DAG,
1193 unsigned Depth) const override;
1194
1196 const APInt &DemandedElts,
1197 APInt &KnownUndef,
1198 APInt &KnownZero,
1199 TargetLoweringOpt &TLO,
1200 unsigned Depth) const override;
1201
1203 const APInt &DemandedElts,
1204 unsigned MaskIndex,
1205 TargetLoweringOpt &TLO,
1206 unsigned Depth) const;
1207
1209 const APInt &DemandedBits,
1210 const APInt &DemandedElts,
1211 KnownBits &Known,
1212 TargetLoweringOpt &TLO,
1213 unsigned Depth) const override;
1214
1216 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
1217 SelectionDAG &DAG, unsigned Depth) const override;
1218
1220 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1221 bool PoisonOnly, unsigned Depth) const override;
1222
1224 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1225 bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override;
1226
1227 bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts,
1228 APInt &UndefElts, const SelectionDAG &DAG,
1229 unsigned Depth) const override;
1230
1231 bool isTargetCanonicalConstantNode(SDValue Op) const override {
1232 // Peek through bitcasts/extracts/inserts to see if we have a broadcast
1233 // vector from memory.
1234 while (Op.getOpcode() == ISD::BITCAST ||
1235 Op.getOpcode() == ISD::EXTRACT_SUBVECTOR ||
1236 (Op.getOpcode() == ISD::INSERT_SUBVECTOR &&
1237 Op.getOperand(0).isUndef()))
1238 Op = Op.getOperand(Op.getOpcode() == ISD::INSERT_SUBVECTOR ? 1 : 0);
1239
1240 return Op.getOpcode() == X86ISD::VBROADCAST_LOAD ||
1242 }
1243
1244 const Constant *getTargetConstantFromLoad(LoadSDNode *LD) const override;
1245
1246 SDValue unwrapAddress(SDValue N) const override;
1247
1249
1250 bool ExpandInlineAsm(CallInst *CI) const override;
1251
1252 ConstraintType getConstraintType(StringRef Constraint) const override;
1253
1254 /// Examine constraint string and operand type and determine a weight value.
1255 /// The operand object must already have been set up with the operand type.
1257 getSingleConstraintMatchWeight(AsmOperandInfo &info,
1258 const char *constraint) const override;
1259
1260 const char *LowerXConstraint(EVT ConstraintVT) const override;
1261
1262 /// Lower the specified operand into the Ops vector. If it is invalid, don't
1263 /// add anything to Ops. If hasMemory is true it means one of the asm
1264 /// constraint of the inline asm instruction being processed is 'm'.
1266 std::string &Constraint,
1267 std::vector<SDValue> &Ops,
1268 SelectionDAG &DAG) const override;
1269
1270 unsigned
1271 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
1272 if (ConstraintCode == "v")
1274 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
1275 }
1276
1277 /// Handle Lowering flag assembly outputs.
1279 const SDLoc &DL,
1280 const AsmOperandInfo &Constraint,
1281 SelectionDAG &DAG) const override;
1282
1283 /// Given a physical register constraint
1284 /// (e.g. {edx}), return the register number and the register class for the
1285 /// register. This should only be used for C_Register constraints. On
1286 /// error, this returns a register number of 0.
1287 std::pair<unsigned, const TargetRegisterClass *>
1289 StringRef Constraint, MVT VT) const override;
1290
1291 /// Return true if the addressing mode represented
1292 /// by AM is legal for this target, for a load/store of the specified type.
1293 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
1294 Type *Ty, unsigned AS,
1295 Instruction *I = nullptr) const override;
1296
1297 /// Return true if the specified immediate is legal
1298 /// icmp immediate, that is the target has icmp instructions which can
1299 /// compare a register against the immediate without having to materialize
1300 /// the immediate into a register.
1301 bool isLegalICmpImmediate(int64_t Imm) const override;
1302
1303 /// Return true if the specified immediate is legal
1304 /// add immediate, that is the target has add instructions which can
1305 /// add a register and the immediate without having to materialize
1306 /// the immediate into a register.
1307 bool isLegalAddImmediate(int64_t Imm) const override;
1308
1309 bool isLegalStoreImmediate(int64_t Imm) const override;
1310
1311 /// This is used to enable splatted operand transforms for vector shifts
1312 /// and vector funnel shifts.
1313 bool isVectorShiftByScalarCheap(Type *Ty) const override;
1314
1315 /// Add x86-specific opcodes to the default list.
1316 bool isBinOp(unsigned Opcode) const override;
1317
1318 /// Returns true if the opcode is a commutative binary operation.
1319 bool isCommutativeBinOp(unsigned Opcode) const override;
1320
1321 /// Return true if it's free to truncate a value of
1322 /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
1323 /// register EAX to i16 by referencing its sub-register AX.
1324 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
1325 bool isTruncateFree(EVT VT1, EVT VT2) const override;
1326
1327 bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;
1328
1329 /// Return true if any actual instruction that defines a
1330 /// value of type Ty1 implicit zero-extends the value to Ty2 in the result
1331 /// register. This does not necessarily include registers defined in
1332 /// unknown ways, such as incoming arguments, or copies from unknown
1333 /// virtual registers. Also, if isTruncateFree(Ty2, Ty1) is true, this
1334 /// does not necessarily apply to truncate instructions. e.g. on x86-64,
1335 /// all instructions that define 32-bit values implicit zero-extend the
1336 /// result out to 64 bits.
1337 bool isZExtFree(Type *Ty1, Type *Ty2) const override;
1338 bool isZExtFree(EVT VT1, EVT VT2) const override;
1339 bool isZExtFree(SDValue Val, EVT VT2) const override;
1340
1342 SmallVectorImpl<Use *> &Ops) const override;
1343 bool shouldConvertPhiType(Type *From, Type *To) const override;
1344
1345 /// Return true if folding a vector load into ExtVal (a sign, zero, or any
1346 /// extend node) is profitable.
1347 bool isVectorLoadExtDesirable(SDValue) const override;
1348
1349 /// Return true if an FMA operation is faster than a pair of fmul and fadd
1350 /// instructions. fmuladd intrinsics will be expanded to FMAs when this
1351 /// method returns true, otherwise fmuladd is expanded to fmul + fadd.
1353 EVT VT) const override;
1354
1355 /// Return true if it's profitable to narrow operations of type SrcVT to
1356 /// DestVT. e.g. on x86, it's profitable to narrow from i32 to i8 but not
1357 /// from i32 to i16.
1358 bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override;
1359
1360 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
1361 EVT VT) const override;
1362
1363 /// Given an intrinsic, checks if on the target the intrinsic will need to map
1364 /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
1365 /// true and stores the intrinsic information into the IntrinsicInfo that was
1366 /// passed to the function.
1367 bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
1368 MachineFunction &MF,
1369 unsigned Intrinsic) const override;
1370
1371 /// Returns true if the target can instruction select the
1372 /// specified FP immediate natively. If false, the legalizer will
1373 /// materialize the FP immediate as a load from a constant pool.
1374 bool isFPImmLegal(const APFloat &Imm, EVT VT,
1375 bool ForCodeSize) const override;
1376
1377 /// Targets can use this to indicate that they only support *some*
1378 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
1379 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to
1380 /// be legal.
1381 bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
1382
1383 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
1384 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
1385 /// constant pool entry.
1386 bool isVectorClearMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
1387
1388 /// Returns true if lowering to a jump table is allowed.
1389 bool areJTsAllowed(const Function *Fn) const override;
1390
1392 EVT ConditionVT) const override;
1393
1394 /// If true, then instruction selection should
1395 /// seek to shrink the FP constant of the specified type to a smaller type
1396 /// in order to save space and / or reduce runtime.
1397 bool ShouldShrinkFPConstant(EVT VT) const override;
1398
1399 /// Return true if we believe it is correct and profitable to reduce the
1400 /// load node to a smaller type.
1402 EVT NewVT) const override;
1403
1404 /// Return true if the specified scalar FP type is computed in an SSE
1405 /// register, not on the X87 floating point stack.
1406 bool isScalarFPTypeInSSEReg(EVT VT) const;
1407
1408 /// Returns true if it is beneficial to convert a load of a constant
1409 /// to just the constant itself.
1411 Type *Ty) const override;
1412
1413 bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override;
1414
1415 bool convertSelectOfConstantsToMath(EVT VT) const override;
1416
1417 bool decomposeMulByConstant(LLVMContext &Context, EVT VT,
1418 SDValue C) const override;
1419
1420 /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
1421 /// with this index.
1422 bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
1423 unsigned Index) const override;
1424
1425 /// Scalar ops always have equal or better analysis/performance/power than
1426 /// the vector equivalent, so this always makes sense if the scalar op is
1427 /// supported.
1428 bool shouldScalarizeBinop(SDValue) const override;
1429
1430 /// Extract of a scalar FP value from index 0 of a vector is free.
1431 bool isExtractVecEltCheap(EVT VT, unsigned Index) const override {
1432 EVT EltVT = VT.getScalarType();
1433 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
1434 }
1435
1436 /// Overflow nodes should get combined/lowered to optimal instructions
1437 /// (they should allow eliminating explicit compares by getting flags from
1438 /// math ops).
1439 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
1440 bool MathUsed) const override;
1441
1442 bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem,
1443 unsigned AddrSpace) const override {
1444 // If we can replace more than 2 scalar stores, there will be a reduction
1445 // in instructions even after we add a vector constant load.
1446 return IsZero || NumElem > 2;
1447 }
1448
1449 bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
1450 const SelectionDAG &DAG,
1451 const MachineMemOperand &MMO) const override;
1452
1453 /// Intel processors have a unified instruction and data cache
1454 const char * getClearCacheBuiltinName() const override {
1455 return nullptr; // nothing to do, move along.
1456 }
1457
1458 Register getRegisterByName(const char* RegName, LLT VT,
1459 const MachineFunction &MF) const override;
1460
1461 /// If a physical register, this returns the register that receives the
1462 /// exception address on entry to an EH pad.
1463 Register
1464 getExceptionPointerRegister(const Constant *PersonalityFn) const override;
1465
1466 /// If a physical register, this returns the register that receives the
1467 /// exception typeid on entry to a landing pad.
1468 Register
1469 getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
1470
1471 bool needsFixedCatchObjects() const override;
1472
1473 /// This method returns a target specific FastISel object,
1474 /// or null if the target does not support "fast" ISel.
1476 const TargetLibraryInfo *libInfo) const override;
1477
1478 /// If the target has a standard location for the stack protector cookie,
1479 /// returns the address of that location. Otherwise, returns nullptr.
1480 Value *getIRStackGuard(IRBuilderBase &IRB) const override;
1481
1482 bool useLoadStackGuardNode() const override;
1483 bool useStackGuardXorFP() const override;
1484 void insertSSPDeclarations(Module &M) const override;
1485 Value *getSDagStackGuard(const Module &M) const override;
1486 Function *getSSPStackGuardCheck(const Module &M) const override;
1488 const SDLoc &DL) const override;
1489
1490
1491 /// Return true if the target stores SafeStack pointer at a fixed offset in
1492 /// some non-standard address space, and populates the address space and
1493 /// offset as appropriate.
1494 Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const override;
1495
1496 std::pair<SDValue, SDValue> BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL,
1497 SDValue Chain, SDValue Pointer,
1498 MachinePointerInfo PtrInfo,
1499 Align Alignment,
1500 SelectionDAG &DAG) const;
1501
1502 /// Customize the preferred legalization strategy for certain types.
1504
1505 bool softPromoteHalfType() const override { return true; }
1506
1508 EVT VT) const override;
1509
1512 EVT VT) const override;
1513
1515 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
1516 unsigned &NumIntermediates, MVT &RegisterVT) const override;
1517
1518 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
1519
1520 bool supportSwiftError() const override;
1521
1522 bool supportKCFIBundles() const override { return true; }
1523
1526 const TargetInstrInfo *TII) const override;
1527
1528 bool hasStackProbeSymbol(const MachineFunction &MF) const override;
1529 bool hasInlineStackProbe(const MachineFunction &MF) const override;
1530 StringRef getStackProbeSymbolName(const MachineFunction &MF) const override;
1531
1532 unsigned getStackProbeSize(const MachineFunction &MF) const;
1533
1534 bool hasVectorBlend() const override { return true; }
1535
1536 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
1537
1539 unsigned OpNo) const override;
1540
1541 /// Lower interleaved load(s) into target specific
1542 /// instructions/intrinsics.
1545 ArrayRef<unsigned> Indices,
1546 unsigned Factor) const override;
1547
1548 /// Lower interleaved store(s) into target specific
1549 /// instructions/intrinsics.
1551 unsigned Factor) const override;
1552
1555 const override;
1556
1557 Align getPrefLoopAlignment(MachineLoop *ML) const override;
1558
1559 EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override {
1560 if (VT == MVT::f80)
1561 return EVT::getIntegerVT(Context, 96);
1563 }
1564
1565 protected:
1566 std::pair<const TargetRegisterClass *, uint8_t>
1568 MVT VT) const override;
1569
1570 private:
1571 /// Keep a reference to the X86Subtarget around so that we can
1572 /// make the right decision when generating code for different targets.
1573 const X86Subtarget &Subtarget;
1574
1575 /// A list of legal FP immediates.
1576 std::vector<APFloat> LegalFPImmediates;
1577
1578 /// Indicate that this x86 target can instruction
1579 /// select the specified FP immediate natively.
1580 void addLegalFPImmediate(const APFloat& Imm) {
1581 LegalFPImmediates.push_back(Imm);
1582 }
1583
1584 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
1585 CallingConv::ID CallConv, bool isVarArg,
1586 const SmallVectorImpl<ISD::InputArg> &Ins,
1587 const SDLoc &dl, SelectionDAG &DAG,
1588 SmallVectorImpl<SDValue> &InVals,
1589 uint32_t *RegMask) const;
1590 SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
1591 const SmallVectorImpl<ISD::InputArg> &ArgInfo,
1592 const SDLoc &dl, SelectionDAG &DAG,
1593 const CCValAssign &VA, MachineFrameInfo &MFI,
1594 unsigned i) const;
1595 SDValue LowerMemOpCallTo(SDValue Chain, SDValue StackPtr, SDValue Arg,
1596 const SDLoc &dl, SelectionDAG &DAG,
1597 const CCValAssign &VA,
1598 ISD::ArgFlagsTy Flags, bool isByval) const;
1599
1600 // Call lowering helpers.
1601
1602 /// Check whether the call is eligible for tail call optimization. Targets
1603 /// that want to do tail call optimization should implement this function.
1604 bool IsEligibleForTailCallOptimization(
1605 SDValue Callee, CallingConv::ID CalleeCC, bool IsCalleeStackStructRet,
1606 bool isVarArg, Type *RetTy, const SmallVectorImpl<ISD::OutputArg> &Outs,
1607 const SmallVectorImpl<SDValue> &OutVals,
1608 const SmallVectorImpl<ISD::InputArg> &Ins, SelectionDAG &DAG) const;
1609 SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
1610 SDValue Chain, bool IsTailCall,
1611 bool Is64Bit, int FPDiff,
1612 const SDLoc &dl) const;
1613
1614 unsigned GetAlignedArgumentStackSize(unsigned StackSize,
1615 SelectionDAG &DAG) const;
1616
1617 unsigned getAddressSpace() const;
1618
1619 SDValue FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG, bool IsSigned,
1620 SDValue &Chain) const;
1621 SDValue LRINT_LLRINTHelper(SDNode *N, SelectionDAG &DAG) const;
1622
1623 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1624 SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
1625 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1626 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1627
1628 unsigned getGlobalWrapperKind(const GlobalValue *GV = nullptr,
1629 const unsigned char OpFlags = 0) const;
1630 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1631 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1632 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1633 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1634 SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
1635
1636 /// Creates target global address or external symbol nodes for calls or
1637 /// other uses.
1638 SDValue LowerGlobalOrExternal(SDValue Op, SelectionDAG &DAG,
1639 bool ForCall) const;
1640
1641 SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1642 SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1643 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1644 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1645 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
1646 SDValue LowerLRINT_LLRINT(SDValue Op, SelectionDAG &DAG) const;
1647 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1648 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
1649 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
1650 SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
1651 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1652 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1653 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1654 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1655 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1656 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1657 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1658 SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1659 SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
1660 SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1661 SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1662 SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
1663 SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1664 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1665 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1666 SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
1667 SDValue LowerWin64_FP_TO_INT128(SDValue Op, SelectionDAG &DAG,
1668 SDValue &Chain) const;
1669 SDValue LowerWin64_INT128_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1670 SDValue LowerGC_TRANSITION(SDValue Op, SelectionDAG &DAG) const;
1671 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1672 SDValue lowerFaddFsub(SDValue Op, SelectionDAG &DAG) const;
1673 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1674 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
1675 SDValue LowerFP_TO_BF16(SDValue Op, SelectionDAG &DAG) const;
1676
1677 SDValue
1678 LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1679 const SmallVectorImpl<ISD::InputArg> &Ins,
1680 const SDLoc &dl, SelectionDAG &DAG,
1681 SmallVectorImpl<SDValue> &InVals) const override;
1682 SDValue LowerCall(CallLoweringInfo &CLI,
1683 SmallVectorImpl<SDValue> &InVals) const override;
1684
1685 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1686 const SmallVectorImpl<ISD::OutputArg> &Outs,
1687 const SmallVectorImpl<SDValue> &OutVals,
1688 const SDLoc &dl, SelectionDAG &DAG) const override;
1689
1690 bool supportSplitCSR(MachineFunction *MF) const override {
1691 return MF->getFunction().getCallingConv() == CallingConv::CXX_FAST_TLS &&
1692 MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
1693 }
1694 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
1695 void insertCopiesSplitCSR(
1696 MachineBasicBlock *Entry,
1697 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
1698
1699 bool splitValueIntoRegisterParts(
1700 SelectionDAG & DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
1701 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC)
1702 const override;
1703
1704 SDValue joinRegisterPartsIntoValue(
1705 SelectionDAG & DAG, const SDLoc &DL, const SDValue *Parts,
1706 unsigned NumParts, MVT PartVT, EVT ValueVT,
1707 std::optional<CallingConv::ID> CC) const override;
1708
1709 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
1710
1711 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
1712
1713 EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
1714 ISD::NodeType ExtendKind) const override;
1715
1716 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
1717 bool isVarArg,
1718 const SmallVectorImpl<ISD::OutputArg> &Outs,
1719 LLVMContext &Context) const override;
1720
1721 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
1722 ArrayRef<MCPhysReg> getRoundingControlRegisters() const override;
1723
1725 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
1727 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
1729 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
1731 shouldExpandLogicAtomicRMWInIR(AtomicRMWInst *AI) const;
1732 void emitBitTestAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1733 void emitCmpArithAtomicRMWIntrinsic(AtomicRMWInst *AI) const override;
1734
1735 LoadInst *
1736 lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
1737
1738 bool lowerAtomicStoreAsStoreSDNode(const StoreInst &SI) const override;
1739 bool lowerAtomicLoadAsLoadSDNode(const LoadInst &LI) const override;
1740
1741 bool needsCmpXchgNb(Type *MemType) const;
1742
1743 template<typename T> bool isSoftFP16(T VT) const;
1744
1745 void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
1746 MachineBasicBlock *DispatchBB, int FI) const;
1747
1748 // Utility function to emit the low-level va_arg code for X86-64.
1749 MachineBasicBlock *
1750 EmitVAARGWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const;
1751
1752 /// Utility function to emit the xmm reg save portion of va_start.
1753 MachineBasicBlock *EmitLoweredCascadedSelect(MachineInstr &MI1,
1754 MachineInstr &MI2,
1755 MachineBasicBlock *BB) const;
1756
1757 MachineBasicBlock *EmitLoweredSelect(MachineInstr &I,
1758 MachineBasicBlock *BB) const;
1759
1760 MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
1761 MachineBasicBlock *BB) const;
1762
1763 MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr &MI,
1764 MachineBasicBlock *BB) const;
1765
1766 MachineBasicBlock *EmitLoweredProbedAlloca(MachineInstr &MI,
1767 MachineBasicBlock *BB) const;
1768
1769 MachineBasicBlock *EmitLoweredTLSAddr(MachineInstr &MI,
1770 MachineBasicBlock *BB) const;
1771
1772 MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
1773 MachineBasicBlock *BB) const;
1774
1775 MachineBasicBlock *EmitLoweredIndirectThunk(MachineInstr &MI,
1776 MachineBasicBlock *BB) const;
1777
1778 MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
1779 MachineBasicBlock *MBB) const;
1780
1781 void emitSetJmpShadowStackFix(MachineInstr &MI,
1782 MachineBasicBlock *MBB) const;
1783
1784 MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
1785 MachineBasicBlock *MBB) const;
1786
1787 MachineBasicBlock *emitLongJmpShadowStackFix(MachineInstr &MI,
1788 MachineBasicBlock *MBB) const;
1789
1790 MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr &MI,
1791 MachineBasicBlock *MBB) const;
1792
1793 /// Emit flags for the given setcc condition and operands. Also returns the
1794 /// corresponding X86 condition code constant in X86CC.
1795 SDValue emitFlagsForSetcc(SDValue Op0, SDValue Op1, ISD::CondCode CC,
1796 const SDLoc &dl, SelectionDAG &DAG,
1797 SDValue &X86CC) const;
1798
1799 /// Check if replacement of SQRT with RSQRT should be disabled.
1800 bool isFsqrtCheap(SDValue Op, SelectionDAG &DAG) const override;
1801
1802 /// Use rsqrt* to speed up sqrt calculations.
1803 SDValue getSqrtEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
1804 int &RefinementSteps, bool &UseOneConstNR,
1805 bool Reciprocal) const override;
1806
1807 /// Use rcp* to speed up fdiv calculations.
1808 SDValue getRecipEstimate(SDValue Op, SelectionDAG &DAG, int Enabled,
1809 int &RefinementSteps) const override;
1810
1811 /// Reassociate floating point divisions into multiply by reciprocal.
1812 unsigned combineRepeatedFPDivisors() const override;
1813
1814 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
1815 SmallVectorImpl<SDNode *> &Created) const override;
1816 };
1817
1818 namespace X86 {
1819 FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
1820 const TargetLibraryInfo *libInfo);
1821 } // end namespace X86
1822
1823 // X86 specific Gather/Scatter nodes.
1824 // The class has the same order of operands as MaskedGatherScatterSDNode for
1825 // convenience.
1827 public:
1828 // This is a intended as a utility and should never be directly created.
1831
1832 const SDValue &getBasePtr() const { return getOperand(3); }
1833 const SDValue &getIndex() const { return getOperand(4); }
1834 const SDValue &getMask() const { return getOperand(2); }
1835 const SDValue &getScale() const { return getOperand(5); }
1836
1837 static bool classof(const SDNode *N) {
1838 return N->getOpcode() == X86ISD::MGATHER ||
1839 N->getOpcode() == X86ISD::MSCATTER;
1840 }
1841 };
1842
1844 public:
1845 const SDValue &getPassThru() const { return getOperand(1); }
1846
1847 static bool classof(const SDNode *N) {
1848 return N->getOpcode() == X86ISD::MGATHER;
1849 }
1850 };
1851
1853 public:
1854 const SDValue &getValue() const { return getOperand(1); }
1855
1856 static bool classof(const SDNode *N) {
1857 return N->getOpcode() == X86ISD::MSCATTER;
1858 }
1859 };
1860
1861 /// Generate unpacklo/unpackhi shuffle mask.
1862 void createUnpackShuffleMask(EVT VT, SmallVectorImpl<int> &Mask, bool Lo,
1863 bool Unary);
1864
1865 /// Similar to unpacklo/unpackhi, but without the 128-bit lane limitation
1866 /// imposed by AVX and specific to the unary pattern. Example:
1867 /// v8iX Lo --> <0, 0, 1, 1, 2, 2, 3, 3>
1868 /// v8iX Hi --> <4, 4, 5, 5, 6, 6, 7, 7>
1869 void createSplat2ShuffleMask(MVT VT, SmallVectorImpl<int> &Mask, bool Lo);
1870
1871} // end namespace llvm
1872
1873#endif // LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Function Alias Analysis Results
BlockVerifier::State From
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
return RetTy
uint64_t Addr
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define RegName(no)
lazy value info
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
LLVMContext & Context
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
const char LLVMTargetMachineRef TM
This file describes how to lower LLVM code to machine code.
@ Flags
Definition: TextStubV5.cpp:93
static bool is64Bit(const char *name)
Class for arbitrary precision integers.
Definition: APInt.h:75
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:177
This class is used to represent ISD::LOAD nodes.
Context object for machine code objects.
Definition: MCContext.h:76
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:35
Machine Value Type.
static MVT getIntegerVT(unsigned BitWidth)
Instructions::iterator instr_iterator
Representation of each machine instruction.
Definition: MachineInstr.h:68
A description of a memory reference used in the backend.
unsigned getAddrSpace() const
Flags
Flags values. These may be or'd together.
Flags getFlags() const
Return the raw flags of the source value,.
Align getAlign() const
Return the minimum known alignment in bytes of the actual memory reference.
This SDNode is used for target intrinsics that touch memory and need an associated MachineMemOperand.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const SDValue & getOperand(unsigned Num) const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
An instruction for storing to memory.
Definition: Instructions.h:301
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
AndOrSETCCFoldKind
Enum of different potentially desirable ways to fold (and/or (setcc ...), (setcc ....
NegatibleCost
Enum that specifies when a float negation is beneficial.
std::vector< ArgListEntry > ArgListTy
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual bool isTargetCanonicalConstantNode(SDValue Op) const
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
const SDValue & getPassThru() const
static bool classof(const SDNode *N)
const SDValue & getBasePtr() const
const SDValue & getScale() const
static bool classof(const SDNode *N)
const SDValue & getIndex() const
const SDValue & getValue() const
static bool classof(const SDNode *N)
bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override
Overflow nodes should get combined/lowered to optimal instructions (they should allow eliminating exp...
Align getPrefLoopAlignment(MachineLoop *ML) const override
Return the preferred loop alignment.
std::pair< const TargetRegisterClass *, uint8_t > findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const override
Return the largest legal super-reg register class of the register class for the specified type and it...
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue getPICJumpTableRelocBase(SDValue Table, SelectionDAG &DAG) const override
Returns relocation base for the given PIC jumptable.
bool preferABDSToABSWithNSW(EVT VT) const override
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
unsigned getJumpTableEncoding() const override
Return the entry encoding for a jump table in the current function.
std::pair< SDValue, SDValue > BuildFILD(EVT DstVT, EVT SrcVT, const SDLoc &DL, SDValue Chain, SDValue Pointer, MachinePointerInfo PtrInfo, Align Alignment, SelectionDAG &DAG) const
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool SimplifyDemandedVectorEltsForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded vector elements, returning true on success...
bool isMemoryAccessFast(EVT VT, Align Alignment) const
SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag, const SDLoc &DL, const AsmOperandInfo &Constraint, SelectionDAG &DAG) const override
Handle Lowering flag assembly outputs.
bool supportKCFIBundles() const override
Return true if the target supports kcfi operand bundles.
const char * LowerXConstraint(EVT ConstraintVT) const override
Try to replace an X constraint, which matches anything, with another that has more specific requireme...
SDValue SimplifyMultipleUseDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth) const override
More limited version of SimplifyDemandedBits that can be used to "look through" ops that don't contri...
bool isSplatValueForTargetNode(SDValue Op, const APInt &DemandedElts, APInt &UndefElts, const SelectionDAG &DAG, unsigned Depth) const override
Return true if vector Op has the same value across all DemandedElts, indicating any elements which ma...
bool convertSelectOfConstantsToMath(EVT VT) const override
Return true if a select of constants (select Cond, C1, C2) should be transformed into simple math ops...
ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint letter, return the type of constraint for this target.
bool hasVectorBlend() const override
Return true if the target has a vector blend instruction.
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
bool useSoftFloat() const override
bool isVectorShiftByScalarCheap(Type *Ty) const override
This is used to enable splatted operand transforms for vector shifts and vector funnel shifts.
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
bool isLegalStoreImmediate(int64_t Imm) const override
Return true if the specified immediate is legal for the value input of a store instruction.
SDValue getNegatedExpression(SDValue Op, SelectionDAG &DAG, bool LegalOperations, bool ForCodeSize, NegatibleCost &Cost, unsigned Depth) const override
Return the newly negated expression if the cost is not expensive and set the cost in Cost to indicate...
bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
const MCExpr * getPICJumpTableRelocBaseExpr(const MachineFunction *MF, unsigned JTI, MCContext &Ctx) const override
This returns the relocation base for the given PIC jumptable, the same as getPICJumpTableRelocBase,...
bool isCtlzFast() const override
Return true if ctlz instruction is fast.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool isSafeMemOpType(MVT VT) const override
Returns true if it's safe to use load / store of the specified type to expand memcpy / memset inline.
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool isNarrowingProfitable(EVT SrcVT, EVT DestVT) const override
Return true if it's profitable to narrow operations of type SrcVT to DestVT.
MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
bool storeOfVectorConstantIsCheap(bool IsZero, EVT MemVT, unsigned NumElem, unsigned AddrSpace) const override
Return true if it is expected to be cheaper to do a store of vector constant with the given size and ...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool shouldSplatInsEltVarIndex(EVT VT) const override
Return true if inserting a scalar into a variable element of an undef vector is more efficiently hand...
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Return true if sinking I's operands to the same basic block as I is profitable, e....
bool isInlineAsmTargetBranch(const SmallVectorImpl< StringRef > &AsmStrs, unsigned OpNo) const override
On x86, return true if the operand with index OpNo is a CALL or JUMP instruction, which can use eithe...
MVT hasFastEqualityCompare(unsigned NumBits) const override
Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
It returns EVT::Other if the type should be determined using generic target-independent logic.
bool SimplifyDemandedVectorEltsForTargetShuffle(SDValue Op, const APInt &DemandedElts, unsigned MaskIndex, TargetLoweringOpt &TLO, unsigned Depth) const
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool hasInlineStackProbe(const MachineFunction &MF) const override
Returns true if stack probing through inline assembly is requested.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
bool ExpandInlineAsm(CallInst *CI) const override
This hook allows the target to expand an inline asm call to be explicit llvm code if it wants to.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if we believe it is correct and profitable to reduce the load node to a smaller type.
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
ConstraintWeight getSingleConstraintMatchWeight(AsmOperandInfo &info, const char *constraint) const override
Examine constraint string and operand type and determine a weight value.
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, const MachineMemOperand &MMO, unsigned *Fast) const
bool preferScalarizeSplat(SDNode *N) const override
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, Align Alignment, MachineMemOperand::Flags Flags, unsigned *Fast) const override
Returns true if the target allows unaligned memory accesses of the specified type.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT, const SelectionDAG &DAG, const MachineMemOperand &MMO) const override
Return true if the following transform is beneficial: fold (conv (load x)) -> (load (conv*)x) On arch...
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const override
Return true if it is profitable to convert a select of FP constants into a constant pool load whose a...
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower interleaved load(s) into target specific instructions/intrinsics.
StringRef getStackProbeSymbolName(const MachineFunction &MF) const override
Returns the name of the symbol used to emit stack probes or the empty string if not applicable.
SDValue expandIndirectJTBranch(const SDLoc &dl, SDValue Value, SDValue Addr, SelectionDAG &DAG) const override
Expands target specific indirect branch for the case of JumpTable expanasion.
uint64_t getByValTypeAlignment(Type *Ty, const DataLayout &DL) const override
Return the desired alignment for ByVal aggregate function arguments in the caller parameter area.
bool hasBitTest(SDValue X, SDValue Y) const override
Return true if the target has a bit-test instruction: (X & (1 << Y)) ==/!= 0 This knowledge can be us...
const char * getClearCacheBuiltinName() const override
Intel processors have a unified instruction and data cache.
bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override
Return true if a truncation from FromTy to ToTy is permitted when deciding whether a call is in tail ...
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
bool isShuffleMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Targets can use this to indicate that they only support some VECTOR_SHUFFLE operations,...
bool useStackGuardXorFP() const override
If this function returns true, stack protection checks should XOR the frame pointer (or whichever poi...
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine the number of bits in the operation that are sign bits.
bool shouldScalarizeBinop(SDValue) const override
Scalar ops always have equal or better analysis/performance/power than the vector equivalent,...
void markLibCallAttributes(MachineFunction *MF, unsigned CC, ArgListTy &Args) const override
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type Ty1 to type Ty2.
Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const override
Return true if the target stores SafeStack pointer at a fixed offset in some non-standard address spa...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool areJTsAllowed(const Function *Fn) const override
Returns true if lowering to a jump table is allowed.
bool isCommutativeBinOp(unsigned Opcode) const override
Returns true if the opcode is a commutative binary operation.
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
MVT getPreferredSwitchConditionType(LLVMContext &Context, EVT ConditionVT) const override
Returns preferred type for switch condition.
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower interleaved store(s) into target specific instructions/intrinsics.
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
bool convertSetCCLogicToBitwiseLogic(EVT VT) const override
Use bitwise logic to make pairs of compares more efficient.
bool isVectorClearMaskLegal(ArrayRef< int > Mask, EVT VT) const override
Similar to isShuffleMaskLegal.
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Customize the preferred legalization strategy for certain types.
bool shouldConvertPhiType(Type *From, Type *To) const override
Given a set in interconnected phis of type 'From' that are loaded/stored or bitcast to type 'To',...
bool hasStackProbeSymbol(const MachineFunction &MF) const override
Returns true if stack probing through a function call is requested.
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type Ty1 implicit zero-extends the valu...
bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
This function returns true if the memory access is aligned or if the target allows this specific unal...
bool isTargetCanonicalConstantNode(SDValue Op) const override
Returns true if the given Opc is considered a canonical constant for the target, which should not be ...
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
bool softPromoteHalfType() const override
SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val, const SDLoc &DL) const override
bool mergeStoresAfterLegalization(EVT MemVT) const override
Do not merge vector stores after legalization because that may conflict with x86-specific store split...
TargetLowering::AndOrSETCCFoldKind isDesirableToCombineLogicOpOfSETCC(const SDNode *LogicOp, const SDNode *SETCC0, const SDNode *SETCC1) const override
Return prefered fold type, Abs if this is a vector, AddAnd if its an integer, None otherwise.
bool shouldFoldMaskToVariableShiftPair(SDValue Y) const override
There are two ways to clear extreme bits (either low or high): Mask: x & (-1 << y) (the instcombine c...
SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool isBinOp(unsigned Opcode) const override
Add x86-specific opcodes to the default list.
bool isGuaranteedNotToBeUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, unsigned Depth) const override
Return true if this function can prove that Op is never poison and, if PoisonOnly is false,...
bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override
Return true if the target has native support for the specified value type and it is 'desirable' to us...
SDValue unwrapAddress(SDValue N) const override
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the value type to use for ISD::SETCC.
bool isVectorLoadExtDesirable(SDValue) const override
Return true if folding a vector load into ExtVal (a sign, zero, or any extend node) is profitable.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
const Constant * getTargetConstantFromLoad(LoadSDNode *LD) const override
This method returns the constant pool value that will be loaded by LD.
EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const override
For types supported by the target, this is an identity function.
bool canCreateUndefOrPoisonForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const override
Return true if Op can create undef or poison from non-undef & non-poison operands.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
const MCExpr * LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB, unsigned uid, MCContext &Ctx) const override
unsigned getStackProbeSize(const MachineFunction &MF) const
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
Replace the results of node with an illegal result type with new values built out of custom code.
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool needsFixedCatchObjects() const override
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Extract of a scalar FP value from index 0 of a vector is free.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
@ CXX_FAST_TLS
Used for access functions.
Definition: CallingConv.h:72
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:40
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition: ISDOpcodes.h:558
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:898
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1324
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition: ISDOpcodes.h:572
@ FP_TO_UINT_SAT
Definition: ISDOpcodes.h:852
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:1336
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
Definition: ISDOpcodes.h:1330
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1447
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1427
@ X86
Windows x64, Windows Itanium (IA-64)
@ FST
This instruction implements a truncating store from FP stack slots.
@ REP_MOVS
Repeat move, corresponds to X86::REP_MOVSx.
@ CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ FMAX
Floating point max and min.
@ BT
X86 bit-test instructions.
@ HADD
Integer horizontal add/sub.
@ MOVQ2DQ
Copies a 64-bit value from an MMX vector to the low word of an XMM vector, with the high word zero fi...
@ BLENDI
Blend where the selector is an immediate.
@ CMP
X86 compare and logical compare instructions.
@ BLENDV
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
@ ADDSUB
Combined add and sub on an FP vector.
@ RET_GLUE
Return with a glue operand.
@ STRICT_FCMP
X86 strict FP compare instructions.
@ STRICT_CMPM
Vector comparison generating mask bits for fp and integer signed and unsigned data types.
@ FHADD
Floating point horizontal add/sub.
@ FMAXS
Scalar intrinsic floating point max and min.
@ BSR
Bit scan reverse.
@ IRET
Return from interrupt. Operand 0 is the number of bytes to pop.
@ SETCC
X86 SetCC.
@ NT_BRIND
BRIND node with NoTrack prefix.
@ SELECTS
X86 Select.
@ FSETCCM
X86 FP SETCC, similar to above, but with output as an i1 mask and and a version with SAE.
@ PEXTRB
Extract an 8-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRB.
@ FXOR
Bitwise logical XOR of floating point values.
@ BRCOND
X86 conditional branches.
@ FSETCC
X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
@ PINSRB
Insert the lower 8-bits of a 32-bit value to a vector, corresponds to X86::PINSRB.
@ REP_STOS
Repeat fill, corresponds to X86::REP_STOSx.
@ INSERTPS
Insert any element of a 4 x float vector into any element of a destination 4 x floatvector.
@ PSHUFB
Shuffle 16 8-bit values within a vector.
@ PEXTRW
Extract a 16-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRW.
@ CALL
These operations represent an abstract X86 call instruction, which includes a bunch of information.
@ AADD
RAO arithmetic instructions.
@ FANDN
Bitwise logical ANDNOT of floating point values.
@ GlobalBaseReg
On Darwin, this node represents the result of the popl at function entry, used for PIC code.
@ FMAXC
Commutative FMIN and FMAX.
@ EXTRQI
SSE4A Extraction and Insertion.
@ FLD
This instruction implements an extending load to FP stack slots.
@ TC_RETURN
Tail call return.
@ PSADBW
Compute Sum of Absolute Differences.
@ FOR
Bitwise logical OR of floating point values.
@ FIST
This instruction implements a fp->int store from FP stack slots.
@ FP_TO_INT_IN_MEM
This instruction implements FP_TO_SINT with the integer destination in memory and a FP reg source.
@ LADD
LOCK-prefixed arithmetic read-modify-write instructions.
@ DBPSADBW
Compute Double Block Packed Sum-Absolute-Differences.
@ MMX_MOVW2D
Copies a GPR into the low 32-bit word of a MMX vector and zero out the high word.
@ Wrapper
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress,...
@ PINSRW
Insert the lower 16-bits of a 32-bit value to a vector, corresponds to X86::PINSRW.
@ CMPCCXADD
Compare and Add if Condition is Met.
@ NT_CALL
Same as call except it adds the NoTrack prefix.
@ MMX_MOVD2W
Copies a 32-bit value from the low word of a MMX vector to a GPR.
@ FILD
This instruction implements SINT_TO_FP with the integer source in memory and FP reg result.
@ MOVDQ2Q
Copies a 64-bit value from the low word of an XMM vector to an MMX vector.
@ ANDNP
Bitwise Logical AND NOT of Packed FP values.
@ BSF
Bit scan forward.
@ VAARG_64
These instructions grab the address of the next argument from a va_list.
@ FAND
Bitwise logical AND of floating point values.
@ CMOV
X86 conditional moves.
@ WrapperRIP
Special wrapper used under X86-64 PIC mode for RIP relative displacements.
@ FSHL
X86 funnel/double shift i16 instructions.
@ FRSQRT
Floating point reciprocal-sqrt and reciprocal approximation.
RoundingMode
Current rounding mode is represented in bits 11:10 of FPSR.
bool mayFoldLoadIntoBroadcastFromMem(SDValue Op, MVT EltVT, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into a vector splat instruction as a memory oper...
bool isZeroNode(SDValue Elt)
Returns true if Elt is a constant zero or floating point constant +0.0.
bool mayFoldIntoZeroExtend(SDValue Op)
Check if Op is an operation that could be folded into a zero extend x86 instruction.
bool mayFoldIntoStore(SDValue Op)
Check if Op is a value that could be used to fold a store into some other x86 instruction as a memory...
bool isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool GuaranteeTCO)
Determines whether the callee is required to pop its own arguments.
bool mayFoldLoad(SDValue Op, const X86Subtarget &Subtarget, bool AssumeSingleUse=false)
Check if Op is a load operation that could be folded into some other x86 instruction as a memory oper...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement)
Returns true of the given offset can be fit into displacement field of the instruction.
bool isConstantSplat(SDValue Op, APInt &SplatVal, bool AllowPartialUndefs)
If Op is a constant whose elements are all the same constant or undefined, return true and return the...
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:406
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:21
AddressSpace
Definition: NVPTXBaseInfo.h:21
void createUnpackShuffleMask(EVT VT, SmallVectorImpl< int > &Mask, bool Lo, bool Unary)
Generate unpacklo/unpackhi shuffle mask.
void createSplat2ShuffleMask(MVT VT, SmallVectorImpl< int > &Mask, bool Lo)
Similar to unpacklo/unpackhi, but without the 128-bit lane limitation imposed by AVX and specific to ...
CombineLevel
Definition: DAGCombine.h:15
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:139
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition: ValueTypes.h:64
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:160
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition: ValueTypes.h:306
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:149
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
This class contains a discriminated union of information about pointers in memory operands,...