LLVM  7.0.0svn
X86ISelLowering.h
Go to the documentation of this file.
1 //===-- X86ISelLowering.h - X86 DAG Lowering Interface ----------*- C++ -*-===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the interfaces that X86 uses to lower LLVM code into a
11 // selection DAG.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #ifndef LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
16 #define LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
17 
22 
23 namespace llvm {
24  class X86Subtarget;
25  class X86TargetMachine;
26 
27  namespace X86ISD {
28  // X86 Specific DAG Nodes
29  enum NodeType : unsigned {
30  // Start the numbering where the builtin ops leave off.
32 
33  /// Bit scan forward.
34  BSF,
35  /// Bit scan reverse.
36  BSR,
37 
38  /// Double shift instructions. These correspond to
39  /// X86::SHLDxx and X86::SHRDxx instructions.
42 
43  /// Bitwise logical AND of floating point values. This corresponds
44  /// to X86::ANDPS or X86::ANDPD.
46 
47  /// Bitwise logical OR of floating point values. This corresponds
48  /// to X86::ORPS or X86::ORPD.
49  FOR,
50 
51  /// Bitwise logical XOR of floating point values. This corresponds
52  /// to X86::XORPS or X86::XORPD.
54 
55  /// Bitwise logical ANDNOT of floating point values. This
56  /// corresponds to X86::ANDNPS or X86::ANDNPD.
58 
59  /// These operations represent an abstract X86 call
60  /// instruction, which includes a bunch of information. In particular the
61  /// operands of these node are:
62  ///
63  /// #0 - The incoming token chain
64  /// #1 - The callee
65  /// #2 - The number of arg bytes the caller pushes on the stack.
66  /// #3 - The number of arg bytes the callee pops off the stack.
67  /// #4 - The value to pass in AL/AX/EAX (optional)
68  /// #5 - The value to pass in DL/DX/EDX (optional)
69  ///
70  /// The result values of these nodes are:
71  ///
72  /// #0 - The outgoing token chain
73  /// #1 - The first register result value (optional)
74  /// #2 - The second register result value (optional)
75  ///
77 
78  /// Same as call except it adds the NoTrack prefix.
80 
81  /// This operation implements the lowering for readcyclecounter.
83 
84  /// X86 Read Time-Stamp Counter and Processor ID.
86 
87  /// X86 Read Performance Monitoring Counters.
89 
90  /// X86 compare and logical compare instructions.
92 
93  /// X86 bit-test instructions.
94  BT,
95 
96  /// X86 SetCC. Operand 0 is condition code, and operand 1 is the EFLAGS
97  /// operand, usually produced by a CMP instruction.
99 
100  /// X86 Select
102 
103  // Same as SETCC except it's materialized with a sbb and the value is all
104  // one's or all zero's.
105  SETCC_CARRY, // R = carry_bit ? ~0 : 0
106 
107  /// X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
108  /// Operands are two FP values to compare; result is a mask of
109  /// 0s or 1s. Generally DTRT for C/C++ with NaNs.
111 
112  /// X86 FP SETCC, similar to above, but with output as an i1 mask and
113  /// with optional rounding mode.
115 
116  /// X86 conditional moves. Operand 0 and operand 1 are the two values
117  /// to select from. Operand 2 is the condition code, and operand 3 is the
118  /// flag operand produced by a CMP or TEST instruction. It also writes a
119  /// flag result.
121 
122  /// X86 conditional branches. Operand 0 is the chain operand, operand 1
123  /// is the block to branch if condition is true, operand 2 is the
124  /// condition code, and operand 3 is the flag operand produced by a CMP
125  /// or TEST instruction.
127 
128  /// BRIND node with NoTrack prefix. Operand 0 is the chain operand and
129  /// operand 1 is the target address.
131 
132  /// Return with a flag operand. Operand 0 is the chain operand, operand
133  /// 1 is the number of bytes of stack to pop.
135 
136  /// Return from interrupt. Operand 0 is the number of bytes to pop.
138 
139  /// Repeat fill, corresponds to X86::REP_STOSx.
141 
142  /// Repeat move, corresponds to X86::REP_MOVSx.
144 
145  /// On Darwin, this node represents the result of the popl
146  /// at function entry, used for PIC code.
148 
149  /// A wrapper node for TargetConstantPool, TargetJumpTable,
150  /// TargetExternalSymbol, TargetGlobalAddress, TargetGlobalTLSAddress,
151  /// MCSymbol and TargetBlockAddress.
153 
154  /// Special wrapper used under X86-64 PIC mode for RIP
155  /// relative displacements.
157 
158  /// Copies a 64-bit value from the low word of an XMM vector
159  /// to an MMX vector.
161 
162  /// Copies a 32-bit value from the low word of a MMX
163  /// vector to a GPR.
165 
166  /// Copies a GPR into the low 32-bit word of a MMX vector
167  /// and zero out the high word.
169 
170  /// Extract an 8-bit value from a vector and zero extend it to
171  /// i32, corresponds to X86::PEXTRB.
173 
174  /// Extract a 16-bit value from a vector and zero extend it to
175  /// i32, corresponds to X86::PEXTRW.
177 
178  /// Insert any element of a 4 x float vector into any element
179  /// of a destination 4 x floatvector.
181 
182  /// Insert the lower 8-bits of a 32-bit value to a vector,
183  /// corresponds to X86::PINSRB.
185 
186  /// Insert the lower 16-bits of a 32-bit value to a vector,
187  /// corresponds to X86::PINSRW.
189 
190  /// Shuffle 16 8-bit values within a vector.
192 
193  /// Compute Sum of Absolute Differences.
195  /// Compute Double Block Packed Sum-Absolute-Differences
197 
198  /// Bitwise Logical AND NOT of Packed FP values.
200 
201  /// Blend where the selector is an immediate.
203 
204  /// Dynamic (non-constant condition) vector blend where only the sign bits
205  /// of the condition elements are used. This is used to enforce that the
206  /// condition mask is not valid for generic VSELECT optimizations.
208 
209  /// Combined add and sub on an FP vector.
211 
212  // FP vector ops with rounding mode.
220 
221  // FP vector get exponent.
223  // Extract Normalized Mantissas.
225  // FP Scale.
228 
229  // Integer add/sub with unsigned saturation.
232 
233  // Integer add/sub with signed saturation.
236 
237  // Unsigned Integer average.
239 
240  /// Integer horizontal add/sub.
243 
244  /// Floating point horizontal add/sub.
247 
248  // Detect Conflicts Within a Vector
250 
251  /// Floating point max and min.
253 
254  /// Commutative FMIN and FMAX.
256 
257  /// Scalar intrinsic floating point max and min.
259 
260  /// Floating point reciprocal-sqrt and reciprocal approximation.
261  /// Note that these typically require refinement
262  /// in order to obtain suitable precision.
264 
265  // AVX-512 reciprocal approximations with a little more precision.
267 
268  // Thread Local Storage.
270 
271  // Thread Local Storage. A call to get the start address
272  // of the TLS block for the current module.
274 
275  // Thread Local Storage. When calling to an OS provided
276  // thunk at the address from an earlier relocation.
278 
279  // Exception Handling helpers.
281 
282  // SjLj exception handling setjmp.
284 
285  // SjLj exception handling longjmp.
287 
288  // SjLj exception handling dispatch.
290 
291  /// Tail call return. See X86TargetLowering::LowerCall for
292  /// the list of operands.
294 
295  // Vector move to low scalar and zero higher vector elements.
297 
298  // Vector integer zero-extend.
300  // Vector integer signed-extend.
302 
303  // Vector integer truncate.
305  // Vector integer truncate with unsigned/signed saturation.
307 
308  // Vector FP extend.
310 
311  // Vector FP round.
313 
314  // 128-bit vector logical left / right shift
316 
317  // Vector shift elements
319 
320  // Vector variable shift right arithmetic.
321  // Unlike ISD::SRA, in case shift count greater then element size
322  // use sign bit to fill destination data element.
324 
325  // Vector shift elements by immediate
327 
328  // Shifts of mask registers.
330 
331  // Bit rotate by immediate
333 
334  // Vector packed double/float comparison.
336 
337  // Vector integer comparisons.
339 
340  // v8i16 Horizontal minimum and position.
342 
344 
345  /// Vector comparison generating mask bits for fp and
346  /// integer signed and unsigned data types.
349  // Vector comparison with rounding mode for FP values
351 
352  // Arithmetic operations with FLAGS results.
354  INC, DEC, OR, XOR, AND,
355 
356  // Bit field extract.
358 
359  // LOW, HI, FLAGS = umul LHS, RHS.
361 
362  // 8-bit SMUL/UMUL - AX, FLAGS = smul8/umul8 AL, RHS.
364 
365  // 8-bit divrem that zero-extend the high result (AH).
368 
369  // X86-specific multiply by immediate.
371 
372  // Vector sign bit extraction.
374 
375  // Vector bitwise comparisons.
377 
378  // Vector packed fp sign bitwise comparisons.
380 
381  // OR/AND test for masks.
384 
385  // ADD for masks.
387 
388  // Several flavors of instructions with vector shuffle behaviors.
389  // Saturated signed/unnsigned packing.
392  // Intra-lane alignr.
394  // AVX512 inter-lane alignr.
400  // VBMI2 Concat & Shift.
405  //Shuffle Packed Values at 128-bit granularity.
422 
423  // Variable Permute (VPERM).
424  // Res = VPERMV MaskV, V0
426 
427  // 3-op Variable Permute (VPERMT2).
428  // Res = VPERMV3 V0, MaskV, V1
430 
431  // 3-op Variable Permute overwriting the index (VPERMI2).
432  // Res = VPERMIV3 V0, MaskV, V1
434 
435  // Bitwise ternary logic.
437  // Fix Up Special Packed Float32/64 values.
440  // Range Restriction Calculation For Packed Pairs of Float32/64 values.
442  // Reduce - Perform Reduction Transformation on scalar\packed FP.
444  // RndScale - Round FP Values To Include A Given Number Of Fraction Bits.
445  // Also used by the legacy (V)ROUND intrinsics where we mask out the
446  // scaling part of the immediate.
448  // Tests Types Of a FP Values for packed types.
450  // Tests Types Of a FP Values for scalar types.
452 
453  // Broadcast scalar to vector.
455  // Broadcast mask to vector.
457  // Broadcast subvector to vector.
459 
460  /// SSE4A Extraction and Insertion.
462 
463  // XOP arithmetic/logical shifts.
465  // XOP signed/unsigned integer comparisons.
467  // XOP packed permute bytes.
469  // XOP two source permutation.
471 
472  // Vector multiply packed unsigned doubleword integers.
474  // Vector multiply packed signed doubleword integers.
476  // Vector Multiply Packed UnsignedIntegers with Round and Scale.
478 
479  // Multiply and Add Packed Integers.
481 
482  // AVX512IFMA multiply and add.
483  // NOTE: These are different than the instruction and perform
484  // op0 x op1 + op2.
486 
487  // VNNI
492 
493  // FMA nodes.
494  // We use the target independent ISD::FMA for the non-inverted case.
500 
501  // FMA with rounding mode.
508 
509  // FMA4 specific scalar intrinsics bits that zero the non-scalar bits.
511 
512  // Scalar intrinsic FMA.
517 
518  // Scalar intrinsic FMA with rounding mode.
519  // Two versions, passthru bits on op1 or op3.
524 
525  // Compress and expand.
528 
529  // Bits shuffle
531 
532  // Convert Unsigned/Integer to Floating-Point Value with rounding mode.
535 
536  // Vector float/double to signed/unsigned integer.
538  // Scalar float/double to signed/unsigned integer.
540 
541  // Vector float/double to signed/unsigned integer with truncation.
543  // Scalar float/double to signed/unsigned integer with truncation.
545 
546  // Vector signed/unsigned integer to float/double.
548 
549  // Save xmm argument registers to the stack, according to %al. An operator
550  // is needed so that this can be expanded with control flow.
552 
553  // Windows's _chkstk call to do stack probing.
555 
556  // For allocating variable amounts of stack space when using
557  // segmented stacks. Check if the current stacklet has enough space, and
558  // falls back to heap allocation if not.
560 
561  // Memory barriers.
564 
565  // Store FP status word into i16 register.
567 
568  // Store contents of %ah into %eflags.
570 
571  // Get a random integer and indicate whether it is valid in CF.
573 
574  // Get a NIST SP800-90B & C compliant random integer and
575  // indicate whether it is valid in CF.
577 
578  // SSE42 string comparisons.
581 
582  // Test if in transactional execution.
584 
585  // ERI instructions.
587 
588  // Conversions between float and half-float.
590 
591  // Galois Field Arithmetic Instructions
593 
594  // LWP insert record.
596 
597  // Compare and swap.
603 
604  /// LOCK-prefixed arithmetic read-modify-write instructions.
605  /// EFLAGS, OUTCHAIN = LADD(INCHAIN, PTR, RHS)
607 
608  // Load, scalar_to_vector, and zero extend.
610 
611  // Store FP control world into i16 memory.
613 
614  /// This instruction implements FP_TO_SINT with the
615  /// integer destination in memory and a FP reg source. This corresponds
616  /// to the X86::FIST*m instructions and the rounding mode change stuff. It
617  /// has two inputs (token chain and address) and two outputs (int value
618  /// and token chain).
622 
623  /// This instruction implements SINT_TO_FP with the
624  /// integer source in memory and FP reg result. This corresponds to the
625  /// X86::FILD*m instructions. It has three inputs (token chain, address,
626  /// and source type) and two outputs (FP value and token chain). FILD_FLAG
627  /// also produces a flag).
630 
631  /// This instruction implements an extending load to FP stack slots.
632  /// This corresponds to the X86::FLD32m / X86::FLD64m. It takes a chain
633  /// operand, ptr to load from, and a ValueType node indicating the type
634  /// to load to.
636 
637  /// This instruction implements a truncating store to FP stack
638  /// slots. This corresponds to the X86::FST32m / X86::FST64m. It takes a
639  /// chain operand, value to store, address, and a ValueType to store it
640  /// as.
642 
643  /// This instruction grabs the address of the next argument
644  /// from a va_list. (reads and modifies the va_list in memory)
646 
647  // Vector truncating store with unsigned/signed saturation
649  // Vector truncating masked store with unsigned/signed saturation
651 
652  // X86 specific gather and scatter
654 
655  // WARNING: Do not add anything in the end unless you want the node to
656  // have memop! In fact, starting from FIRST_TARGET_MEMORY_OPCODE all
657  // opcodes will be thought as target memory ops!
658  };
659  } // end namespace X86ISD
660 
661  /// Define some predicates that are used for node matching.
662  namespace X86 {
663  /// Returns true if Elt is a constant zero or floating point constant +0.0.
664  bool isZeroNode(SDValue Elt);
665 
666  /// Returns true of the given offset can be
667  /// fit into displacement field of the instruction.
669  bool hasSymbolicDisplacement = true);
670 
671  /// Determines whether the callee is required to pop its
672  /// own arguments. Callee pop is necessary to support tail calls.
673  bool isCalleePop(CallingConv::ID CallingConv,
674  bool is64Bit, bool IsVarArg, bool GuaranteeTCO);
675 
676  } // end namespace X86
677 
678  //===--------------------------------------------------------------------===//
679  // X86 Implementation of the TargetLowering interface
680  class X86TargetLowering final : public TargetLowering {
681  public:
682  explicit X86TargetLowering(const X86TargetMachine &TM,
683  const X86Subtarget &STI);
684 
685  unsigned getJumpTableEncoding() const override;
686  bool useSoftFloat() const override;
687 
688  void markLibCallAttributes(MachineFunction *MF, unsigned CC,
689  ArgListTy &Args) const override;
690 
691  MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override {
692  return MVT::i8;
693  }
694 
695  const MCExpr *
696  LowerCustomJumpTableEntry(const MachineJumpTableInfo *MJTI,
697  const MachineBasicBlock *MBB, unsigned uid,
698  MCContext &Ctx) const override;
699 
700  /// Returns relocation base for the given PIC jumptable.
701  SDValue getPICJumpTableRelocBase(SDValue Table,
702  SelectionDAG &DAG) const override;
703  const MCExpr *
704  getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
705  unsigned JTI, MCContext &Ctx) const override;
706 
707  /// Return the desired alignment for ByVal aggregate
708  /// function arguments in the caller parameter area. For X86, aggregates
709  /// that contains are placed at 16-byte boundaries while the rest are at
710  /// 4-byte boundaries.
711  unsigned getByValTypeAlignment(Type *Ty,
712  const DataLayout &DL) const override;
713 
714  /// Returns the target specific optimal type for load
715  /// and store operations as a result of memset, memcpy, and memmove
716  /// lowering. If DstAlign is zero that means it's safe to destination
717  /// alignment can satisfy any constraint. Similarly if SrcAlign is zero it
718  /// means there isn't a need to check it against alignment requirement,
719  /// probably because the source does not need to be loaded. If 'IsMemset' is
720  /// true, that means it's expanding a memset. If 'ZeroMemset' is true, that
721  /// means it's a memset of zero. 'MemcpyStrSrc' indicates whether the memcpy
722  /// source is constant so it does not need to be loaded.
723  /// It returns EVT::Other if the type should be determined using generic
724  /// target-independent logic.
725  EVT getOptimalMemOpType(uint64_t Size, unsigned DstAlign, unsigned SrcAlign,
726  bool IsMemset, bool ZeroMemset, bool MemcpyStrSrc,
727  MachineFunction &MF) const override;
728 
729  /// Returns true if it's safe to use load / store of the
730  /// specified type to expand memcpy / memset inline. This is mostly true
731  /// for all types except for some special cases. For example, on X86
732  /// targets without SSE2 f64 load / store are done with fldl / fstpl which
733  /// also does type conversion. Note the specified type doesn't have to be
734  /// legal as the hook is used before type legalization.
735  bool isSafeMemOpType(MVT VT) const override;
736 
737  /// Returns true if the target allows unaligned memory accesses of the
738  /// specified type. Returns whether it is "fast" in the last argument.
739  bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AS, unsigned Align,
740  bool *Fast) const override;
741 
742  /// Provide custom lowering hooks for some operations.
743  ///
744  SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
745 
746  /// Places new result values for the node in Results (their number
747  /// and types must exactly match those of the original return values of
748  /// the node), or leaves Results empty, which indicates that the node is not
749  /// to be custom lowered after all.
750  void LowerOperationWrapper(SDNode *N,
752  SelectionDAG &DAG) const override;
753 
754  /// Replace the results of node with an illegal result
755  /// type with new values built out of custom code.
756  ///
757  void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue>&Results,
758  SelectionDAG &DAG) const override;
759 
760  SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
761 
762  // Return true if it is profitable to combine a BUILD_VECTOR with a
763  // stride-pattern to a shuffle and a truncate.
764  // Example of such a combine:
765  // v4i32 build_vector((extract_elt V, 1),
766  // (extract_elt V, 3),
767  // (extract_elt V, 5),
768  // (extract_elt V, 7))
769  // -->
770  // v4i32 truncate (bitcast (shuffle<1,u,3,u,4,u,5,u,6,u,7,u> V, u) to
771  // v4i64)
772  bool isDesirableToCombineBuildVectorToShuffleTruncate(
773  ArrayRef<int> ShuffleMask, EVT SrcVT, EVT TruncVT) const override;
774 
775  /// Return true if the target has native support for
776  /// the specified value type and it is 'desirable' to use the type for the
777  /// given node type. e.g. On x86 i16 is legal, but undesirable since i16
778  /// instruction encodings are longer and some i16 instructions are slow.
779  bool isTypeDesirableForOp(unsigned Opc, EVT VT) const override;
780 
781  /// Return true if the target has native support for the
782  /// specified value type and it is 'desirable' to use the type. e.g. On x86
783  /// i16 is legal, but undesirable since i16 instruction encodings are longer
784  /// and some i16 instructions are slow.
785  bool IsDesirableToPromoteOp(SDValue Op, EVT &PVT) const override;
786 
788  EmitInstrWithCustomInserter(MachineInstr &MI,
789  MachineBasicBlock *MBB) const override;
790 
791  /// This method returns the name of a target specific DAG node.
792  const char *getTargetNodeName(unsigned Opcode) const override;
793 
794  bool mergeStoresAfterLegalization() const override { return true; }
795 
796  bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
797  const SelectionDAG &DAG) const override;
798 
799  bool isCheapToSpeculateCttz() const override;
800 
801  bool isCheapToSpeculateCtlz() const override;
802 
803  bool isCtlzFast() const override;
804 
805  bool hasBitPreservingFPLogic(EVT VT) const override {
806  return VT == MVT::f32 || VT == MVT::f64 || VT.isVector();
807  }
808 
809  bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override {
810  // If the pair to store is a mixture of float and int values, we will
811  // save two bitwise instructions and one float-to-int instruction and
812  // increase one store instruction. There is potentially a more
813  // significant benefit because it avoids the float->int domain switch
814  // for input value. So It is more likely a win.
815  if ((LTy.isFloatingPoint() && HTy.isInteger()) ||
816  (LTy.isInteger() && HTy.isFloatingPoint()))
817  return true;
818  // If the pair only contains int values, we will save two bitwise
819  // instructions and increase one store instruction (costing one more
820  // store buffer). Since the benefit is more blurred so we leave
821  // such pair out until we get testcase to prove it is a win.
822  return false;
823  }
824 
825  bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
826 
827  bool hasAndNotCompare(SDValue Y) const override;
828 
829  bool convertSetCCLogicToBitwiseLogic(EVT VT) const override {
830  return VT.isScalarInteger();
831  }
832 
833  /// Vector-sized comparisons are fast using PCMPEQ + PMOVMSK or PTEST.
834  MVT hasFastEqualityCompare(unsigned NumBits) const override;
835 
836  /// Allow multiple load pairs per block for smaller and faster code.
837  unsigned getMemcmpEqZeroLoadsPerBlock() const override {
838  return 2;
839  }
840 
841  /// Return the value type to use for ISD::SETCC.
842  EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
843  EVT VT) const override;
844 
845  bool targetShrinkDemandedConstant(SDValue Op, const APInt &Demanded,
846  TargetLoweringOpt &TLO) const override;
847 
848  /// Determine which of the bits specified in Mask are known to be either
849  /// zero or one and return them in the KnownZero/KnownOne bitsets.
850  void computeKnownBitsForTargetNode(const SDValue Op,
851  KnownBits &Known,
852  const APInt &DemandedElts,
853  const SelectionDAG &DAG,
854  unsigned Depth = 0) const override;
855 
856  /// Determine the number of bits in the operation that are sign bits.
857  unsigned ComputeNumSignBitsForTargetNode(SDValue Op,
858  const APInt &DemandedElts,
859  const SelectionDAG &DAG,
860  unsigned Depth) const override;
861 
862  SDValue unwrapAddress(SDValue N) const override;
863 
864  bool isGAPlusOffset(SDNode *N, const GlobalValue* &GA,
865  int64_t &Offset) const override;
866 
867  SDValue getReturnAddressFrameIndex(SelectionDAG &DAG) const;
868 
869  bool ExpandInlineAsm(CallInst *CI) const override;
870 
871  ConstraintType getConstraintType(StringRef Constraint) const override;
872 
873  /// Examine constraint string and operand type and determine a weight value.
874  /// The operand object must already have been set up with the operand type.
876  getSingleConstraintMatchWeight(AsmOperandInfo &info,
877  const char *constraint) const override;
878 
879  const char *LowerXConstraint(EVT ConstraintVT) const override;
880 
881  /// Lower the specified operand into the Ops vector. If it is invalid, don't
882  /// add anything to Ops. If hasMemory is true it means one of the asm
883  /// constraint of the inline asm instruction being processed is 'm'.
884  void LowerAsmOperandForConstraint(SDValue Op,
885  std::string &Constraint,
886  std::vector<SDValue> &Ops,
887  SelectionDAG &DAG) const override;
888 
889  unsigned
890  getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
891  if (ConstraintCode == "i")
893  else if (ConstraintCode == "o")
895  else if (ConstraintCode == "v")
897  else if (ConstraintCode == "X")
899  return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
900  }
901 
902  /// Given a physical register constraint
903  /// (e.g. {edx}), return the register number and the register class for the
904  /// register. This should only be used for C_Register constraints. On
905  /// error, this returns a register number of 0.
906  std::pair<unsigned, const TargetRegisterClass *>
907  getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
908  StringRef Constraint, MVT VT) const override;
909 
910  /// Return true if the addressing mode represented
911  /// by AM is legal for this target, for a load/store of the specified type.
912  bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
913  Type *Ty, unsigned AS,
914  Instruction *I = nullptr) const override;
915 
916  /// Return true if the specified immediate is legal
917  /// icmp immediate, that is the target has icmp instructions which can
918  /// compare a register against the immediate without having to materialize
919  /// the immediate into a register.
920  bool isLegalICmpImmediate(int64_t Imm) const override;
921 
922  /// Return true if the specified immediate is legal
923  /// add immediate, that is the target has add instructions which can
924  /// add a register and the immediate without having to materialize
925  /// the immediate into a register.
926  bool isLegalAddImmediate(int64_t Imm) const override;
927 
928  /// \brief Return the cost of the scaling factor used in the addressing
929  /// mode represented by AM for this target, for a load/store
930  /// of the specified type.
931  /// If the AM is supported, the return value must be >= 0.
932  /// If the AM is not supported, it returns a negative value.
933  int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM, Type *Ty,
934  unsigned AS) const override;
935 
936  bool isVectorShiftByScalarCheap(Type *Ty) const override;
937 
938  /// Return true if it's free to truncate a value of
939  /// type Ty1 to type Ty2. e.g. On x86 it's free to truncate a i32 value in
940  /// register EAX to i16 by referencing its sub-register AX.
941  bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
942  bool isTruncateFree(EVT VT1, EVT VT2) const override;
943 
944  bool allowTruncateForTailCall(Type *Ty1, Type *Ty2) const override;
945 
946  /// Return true if any actual instruction that defines a
947  /// value of type Ty1 implicit zero-extends the value to Ty2 in the result
948  /// register. This does not necessarily include registers defined in
949  /// unknown ways, such as incoming arguments, or copies from unknown
950  /// virtual registers. Also, if isTruncateFree(Ty2, Ty1) is true, this
951  /// does not necessarily apply to truncate instructions. e.g. on x86-64,
952  /// all instructions that define 32-bit values implicit zero-extend the
953  /// result out to 64 bits.
954  bool isZExtFree(Type *Ty1, Type *Ty2) const override;
955  bool isZExtFree(EVT VT1, EVT VT2) const override;
956  bool isZExtFree(SDValue Val, EVT VT2) const override;
957 
958  /// Return true if folding a vector load into ExtVal (a sign, zero, or any
959  /// extend node) is profitable.
960  bool isVectorLoadExtDesirable(SDValue) const override;
961 
962  /// Return true if an FMA operation is faster than a pair of fmul and fadd
963  /// instructions. fmuladd intrinsics will be expanded to FMAs when this
964  /// method returns true, otherwise fmuladd is expanded to fmul + fadd.
965  bool isFMAFasterThanFMulAndFAdd(EVT VT) const override;
966 
967  /// Return true if it's profitable to narrow
968  /// operations of type VT1 to VT2. e.g. on x86, it's profitable to narrow
969  /// from i32 to i8 but not from i32 to i16.
970  bool isNarrowingProfitable(EVT VT1, EVT VT2) const override;
971 
972  /// Given an intrinsic, checks if on the target the intrinsic will need to map
973  /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
974  /// true and stores the intrinsic information into the IntrinsicInfo that was
975  /// passed to the function.
976  bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
977  MachineFunction &MF,
978  unsigned Intrinsic) const override;
979 
980  /// Returns true if the target can instruction select the
981  /// specified FP immediate natively. If false, the legalizer will
982  /// materialize the FP immediate as a load from a constant pool.
983  bool isFPImmLegal(const APFloat &Imm, EVT VT) const override;
984 
985  /// Targets can use this to indicate that they only support *some*
986  /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
987  /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to
988  /// be legal.
989  bool isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const override;
990 
991  /// Similar to isShuffleMaskLegal. This is used by Targets can use this to
992  /// indicate if there is a suitable VECTOR_SHUFFLE that can be used to
993  /// replace a VAND with a constant pool entry.
994  bool isVectorClearMaskLegal(const SmallVectorImpl<int> &Mask,
995  EVT VT) const override;
996 
997  /// Returns true if lowering to a jump table is allowed.
998  bool areJTsAllowed(const Function *Fn) const override;
999 
1000  /// If true, then instruction selection should
1001  /// seek to shrink the FP constant of the specified type to a smaller type
1002  /// in order to save space and / or reduce runtime.
1003  bool ShouldShrinkFPConstant(EVT VT) const override {
1004  // Don't shrink FP constpool if SSE2 is available since cvtss2sd is more
1005  // expensive than a straight movsd. On the other hand, it's important to
1006  // shrink long double fp constant since fldt is very slow.
1007  return !X86ScalarSSEf64 || VT == MVT::f80;
1008  }
1009 
1010  /// Return true if we believe it is correct and profitable to reduce the
1011  /// load node to a smaller type.
1012  bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy,
1013  EVT NewVT) const override;
1014 
1015  /// Return true if the specified scalar FP type is computed in an SSE
1016  /// register, not on the X87 floating point stack.
1017  bool isScalarFPTypeInSSEReg(EVT VT) const {
1018  return (VT == MVT::f64 && X86ScalarSSEf64) || // f64 is when SSE2
1019  (VT == MVT::f32 && X86ScalarSSEf32); // f32 is when SSE1
1020  }
1021 
1022  /// \brief Returns true if it is beneficial to convert a load of a constant
1023  /// to just the constant itself.
1024  bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
1025  Type *Ty) const override;
1026 
1027  bool convertSelectOfConstantsToMath(EVT VT) const override;
1028 
1029  /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
1030  /// with this index.
1031  bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
1032  unsigned Index) const override;
1033 
1034  bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem,
1035  unsigned AddrSpace) const override {
1036  // If we can replace more than 2 scalar stores, there will be a reduction
1037  // in instructions even after we add a vector constant load.
1038  return NumElem > 2;
1039  }
1040 
1041  bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT) const override;
1042 
1043  /// Intel processors have a unified instruction and data cache
1044  const char * getClearCacheBuiltinName() const override {
1045  return nullptr; // nothing to do, move along.
1046  }
1047 
1048  unsigned getRegisterByName(const char* RegName, EVT VT,
1049  SelectionDAG &DAG) const override;
1050 
1051  /// If a physical register, this returns the register that receives the
1052  /// exception address on entry to an EH pad.
1053  unsigned
1054  getExceptionPointerRegister(const Constant *PersonalityFn) const override;
1055 
1056  /// If a physical register, this returns the register that receives the
1057  /// exception typeid on entry to a landing pad.
1058  unsigned
1059  getExceptionSelectorRegister(const Constant *PersonalityFn) const override;
1060 
1061  virtual bool needsFixedCatchObjects() const override;
1062 
1063  /// This method returns a target specific FastISel object,
1064  /// or null if the target does not support "fast" ISel.
1066  const TargetLibraryInfo *libInfo) const override;
1067 
1068  /// If the target has a standard location for the stack protector cookie,
1069  /// returns the address of that location. Otherwise, returns nullptr.
1070  Value *getIRStackGuard(IRBuilder<> &IRB) const override;
1071 
1072  bool useLoadStackGuardNode() const override;
1073  bool useStackGuardXorFP() const override;
1074  void insertSSPDeclarations(Module &M) const override;
1075  Value *getSDagStackGuard(const Module &M) const override;
1076  Value *getSSPStackGuardCheck(const Module &M) const override;
1077  SDValue emitStackGuardXorFP(SelectionDAG &DAG, SDValue Val,
1078  const SDLoc &DL) const override;
1079 
1080 
1081  /// Return true if the target stores SafeStack pointer at a fixed offset in
1082  /// some non-standard address space, and populates the address space and
1083  /// offset as appropriate.
1084  Value *getSafeStackPointerLocation(IRBuilder<> &IRB) const override;
1085 
1086  SDValue BuildFILD(SDValue Op, EVT SrcVT, SDValue Chain, SDValue StackSlot,
1087  SelectionDAG &DAG) const;
1088 
1089  bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const override;
1090 
1091  /// \brief Customize the preferred legalization strategy for certain types.
1092  LegalizeTypeAction getPreferredVectorAction(EVT VT) const override;
1093 
1094  MVT getRegisterTypeForCallingConv(MVT VT) const override;
1095 
1096  MVT getRegisterTypeForCallingConv(LLVMContext &Context,
1097  EVT VT) const override;
1098 
1099  unsigned getNumRegistersForCallingConv(LLVMContext &Context,
1100  EVT VT) const override;
1101 
1102  bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
1103 
1104  bool supportSwiftError() const override;
1105 
1106  StringRef getStackProbeSymbolName(MachineFunction &MF) const override;
1107 
1108  bool hasVectorBlend() const override { return true; }
1109 
1110  unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
1111 
1112  /// \brief Lower interleaved load(s) into target specific
1113  /// instructions/intrinsics.
1114  bool lowerInterleavedLoad(LoadInst *LI,
1116  ArrayRef<unsigned> Indices,
1117  unsigned Factor) const override;
1118 
1119  /// \brief Lower interleaved store(s) into target specific
1120  /// instructions/intrinsics.
1121  bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
1122  unsigned Factor) const override;
1123 
1124  SDValue expandIndirectJTBranch(const SDLoc& dl, SDValue Value,
1125  SDValue Addr, SelectionDAG &DAG)
1126  const override;
1127 
1128  protected:
1129  std::pair<const TargetRegisterClass *, uint8_t>
1130  findRepresentativeClass(const TargetRegisterInfo *TRI,
1131  MVT VT) const override;
1132 
1133  private:
1134  /// Keep a reference to the X86Subtarget around so that we can
1135  /// make the right decision when generating code for different targets.
1136  const X86Subtarget &Subtarget;
1137 
1138  /// Select between SSE or x87 floating point ops.
1139  /// When SSE is available, use it for f32 operations.
1140  /// When SSE2 is available, use it for f64 operations.
1141  bool X86ScalarSSEf32;
1142  bool X86ScalarSSEf64;
1143 
1144  /// A list of legal FP immediates.
1145  std::vector<APFloat> LegalFPImmediates;
1146 
1147  /// Indicate that this x86 target can instruction
1148  /// select the specified FP immediate natively.
1149  void addLegalFPImmediate(const APFloat& Imm) {
1150  LegalFPImmediates.push_back(Imm);
1151  }
1152 
1153  SDValue LowerCallResult(SDValue Chain, SDValue InFlag,
1154  CallingConv::ID CallConv, bool isVarArg,
1156  const SDLoc &dl, SelectionDAG &DAG,
1157  SmallVectorImpl<SDValue> &InVals,
1158  uint32_t *RegMask) const;
1159  SDValue LowerMemArgument(SDValue Chain, CallingConv::ID CallConv,
1160  const SmallVectorImpl<ISD::InputArg> &ArgInfo,
1161  const SDLoc &dl, SelectionDAG &DAG,
1162  const CCValAssign &VA, MachineFrameInfo &MFI,
1163  unsigned i) const;
1165  const SDLoc &dl, SelectionDAG &DAG,
1166  const CCValAssign &VA,
1167  ISD::ArgFlagsTy Flags) const;
1168 
1169  // Call lowering helpers.
1170 
1171  /// Check whether the call is eligible for tail call optimization. Targets
1172  /// that want to do tail call optimization should implement this function.
1173  bool IsEligibleForTailCallOptimization(SDValue Callee,
1174  CallingConv::ID CalleeCC,
1175  bool isVarArg,
1176  bool isCalleeStructRet,
1177  bool isCallerStructRet,
1178  Type *RetTy,
1179  const SmallVectorImpl<ISD::OutputArg> &Outs,
1180  const SmallVectorImpl<SDValue> &OutVals,
1181  const SmallVectorImpl<ISD::InputArg> &Ins,
1182  SelectionDAG& DAG) const;
1183  SDValue EmitTailCallLoadRetAddr(SelectionDAG &DAG, SDValue &OutRetAddr,
1184  SDValue Chain, bool IsTailCall,
1185  bool Is64Bit, int FPDiff,
1186  const SDLoc &dl) const;
1187 
1188  unsigned GetAlignedArgumentStackSize(unsigned StackSize,
1189  SelectionDAG &DAG) const;
1190 
1191  unsigned getAddressSpace(void) const;
1192 
1193  std::pair<SDValue,SDValue> FP_TO_INTHelper(SDValue Op, SelectionDAG &DAG,
1194  bool isSigned,
1195  bool isReplace) const;
1196 
1197  SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1198  SDValue LowerVSELECT(SDValue Op, SelectionDAG &DAG) const;
1201 
1202  unsigned getGlobalWrapperKind(const GlobalValue *GV = nullptr,
1203  const unsigned char OpFlags = 0) const;
1204  SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1205  SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1206  SDValue LowerGlobalAddress(const GlobalValue *GV, const SDLoc &dl,
1207  int64_t Offset, SelectionDAG &DAG) const;
1208  SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1209  SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1210  SDValue LowerExternalSymbol(SDValue Op, SelectionDAG &DAG) const;
1211 
1212  SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1213  SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1214  SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1215  SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1216  SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1217  SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
1218  SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
1219  SDValue LowerBRCOND(SDValue Op, SelectionDAG &DAG) const;
1220  SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1222  SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1223  SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1224  SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1225  SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1226  SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1227  SDValue LowerFRAME_TO_ARGS_OFFSET(SDValue Op, SelectionDAG &DAG) const;
1228  SDValue LowerEH_RETURN(SDValue Op, SelectionDAG &DAG) const;
1229  SDValue lowerEH_SJLJ_SETJMP(SDValue Op, SelectionDAG &DAG) const;
1230  SDValue lowerEH_SJLJ_LONGJMP(SDValue Op, SelectionDAG &DAG) const;
1231  SDValue lowerEH_SJLJ_SETUP_DISPATCH(SDValue Op, SelectionDAG &DAG) const;
1232  SDValue LowerINIT_TRAMPOLINE(SDValue Op, SelectionDAG &DAG) const;
1233  SDValue LowerFLT_ROUNDS_(SDValue Op, SelectionDAG &DAG) const;
1234  SDValue LowerWin64_i128OP(SDValue Op, SelectionDAG &DAG) const;
1235  SDValue LowerGC_TRANSITION_START(SDValue Op, SelectionDAG &DAG) const;
1236  SDValue LowerGC_TRANSITION_END(SDValue Op, SelectionDAG &DAG) const;
1237  SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1238 
1239  SDValue
1240  LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1241  const SmallVectorImpl<ISD::InputArg> &Ins,
1242  const SDLoc &dl, SelectionDAG &DAG,
1243  SmallVectorImpl<SDValue> &InVals) const override;
1244  SDValue LowerCall(CallLoweringInfo &CLI,
1245  SmallVectorImpl<SDValue> &InVals) const override;
1246 
1247  SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1248  const SmallVectorImpl<ISD::OutputArg> &Outs,
1249  const SmallVectorImpl<SDValue> &OutVals,
1250  const SDLoc &dl, SelectionDAG &DAG) const override;
1251 
1252  bool supportSplitCSR(MachineFunction *MF) const override {
1254  MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
1255  }
1256  void initializeSplitCSR(MachineBasicBlock *Entry) const override;
1257  void insertCopiesSplitCSR(
1258  MachineBasicBlock *Entry,
1259  const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
1260 
1261  bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
1262 
1263  bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
1264 
1265  EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
1266  ISD::NodeType ExtendKind) const override;
1267 
1268  bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
1269  bool isVarArg,
1270  const SmallVectorImpl<ISD::OutputArg> &Outs,
1271  LLVMContext &Context) const override;
1272 
1273  const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
1274 
1276  shouldExpandAtomicLoadInIR(LoadInst *SI) const override;
1277  bool shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
1279  shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
1280 
1281  LoadInst *
1282  lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *AI) const override;
1283 
1284  bool needsCmpXchgNb(Type *MemType) const;
1285 
1286  void SetupEntryBlockForSjLj(MachineInstr &MI, MachineBasicBlock *MBB,
1287  MachineBasicBlock *DispatchBB, int FI) const;
1288 
1289  // Utility function to emit the low-level va_arg code for X86-64.
1291  EmitVAARG64WithCustomInserter(MachineInstr &MI,
1292  MachineBasicBlock *MBB) const;
1293 
1294  /// Utility function to emit the xmm reg save portion of va_start.
1296  EmitVAStartSaveXMMRegsWithCustomInserter(MachineInstr &BInstr,
1297  MachineBasicBlock *BB) const;
1298 
1299  MachineBasicBlock *EmitLoweredCascadedSelect(MachineInstr &MI1,
1300  MachineInstr &MI2,
1301  MachineBasicBlock *BB) const;
1302 
1303  MachineBasicBlock *EmitLoweredSelect(MachineInstr &I,
1304  MachineBasicBlock *BB) const;
1305 
1306  MachineBasicBlock *EmitLoweredAtomicFP(MachineInstr &I,
1307  MachineBasicBlock *BB) const;
1308 
1309  MachineBasicBlock *EmitLoweredCatchRet(MachineInstr &MI,
1310  MachineBasicBlock *BB) const;
1311 
1312  MachineBasicBlock *EmitLoweredCatchPad(MachineInstr &MI,
1313  MachineBasicBlock *BB) const;
1314 
1315  MachineBasicBlock *EmitLoweredSegAlloca(MachineInstr &MI,
1316  MachineBasicBlock *BB) const;
1317 
1318  MachineBasicBlock *EmitLoweredTLSAddr(MachineInstr &MI,
1319  MachineBasicBlock *BB) const;
1320 
1321  MachineBasicBlock *EmitLoweredTLSCall(MachineInstr &MI,
1322  MachineBasicBlock *BB) const;
1323 
1324  MachineBasicBlock *EmitLoweredRetpoline(MachineInstr &MI,
1325  MachineBasicBlock *BB) const;
1326 
1327  MachineBasicBlock *emitEHSjLjSetJmp(MachineInstr &MI,
1328  MachineBasicBlock *MBB) const;
1329 
1330  MachineBasicBlock *emitEHSjLjLongJmp(MachineInstr &MI,
1331  MachineBasicBlock *MBB) const;
1332 
1333  MachineBasicBlock *emitFMA3Instr(MachineInstr &MI,
1334  MachineBasicBlock *MBB) const;
1335 
1336  MachineBasicBlock *EmitSjLjDispatchBlock(MachineInstr &MI,
1337  MachineBasicBlock *MBB) const;
1338 
1339  /// Emit nodes that will be selected as "test Op0,Op0", or something
1340  /// equivalent, for use with the given x86 condition code.
1341  SDValue EmitTest(SDValue Op0, unsigned X86CC, const SDLoc &dl,
1342  SelectionDAG &DAG) const;
1343 
1344  /// Emit nodes that will be selected as "cmp Op0,Op1", or something
1345  /// equivalent, for use with the given x86 condition code.
1346  SDValue EmitCmp(SDValue Op0, SDValue Op1, unsigned X86CC, const SDLoc &dl,
1347  SelectionDAG &DAG) const;
1348 
1349  /// Convert a comparison if required by the subtarget.
1350  SDValue ConvertCmpIfNecessary(SDValue Cmp, SelectionDAG &DAG) const;
1351 
1352  /// Check if replacement of SQRT with RSQRT should be disabled.
1353  bool isFsqrtCheap(SDValue Operand, SelectionDAG &DAG) const override;
1354 
1355  /// Use rsqrt* to speed up sqrt calculations.
1356  SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1357  int &RefinementSteps, bool &UseOneConstNR,
1358  bool Reciprocal) const override;
1359 
1360  /// Use rcp* to speed up fdiv calculations.
1361  SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1362  int &RefinementSteps) const override;
1363 
1364  /// Reassociate floating point divisions into multiply by reciprocal.
1365  unsigned combineRepeatedFPDivisors() const override;
1366  };
1367 
1368  namespace X86 {
1370  const TargetLibraryInfo *libInfo);
1371  } // end namespace X86
1372 
1373  // Base class for all X86 non-masked store operations.
1374  class X86StoreSDNode : public MemSDNode {
1375  public:
1376  X86StoreSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1377  SDVTList VTs, EVT MemVT,
1378  MachineMemOperand *MMO)
1379  :MemSDNode(Opcode, Order, dl, VTs, MemVT, MMO) {}
1380  const SDValue &getValue() const { return getOperand(1); }
1381  const SDValue &getBasePtr() const { return getOperand(2); }
1382 
1383  static bool classof(const SDNode *N) {
1384  return N->getOpcode() == X86ISD::VTRUNCSTORES ||
1386  }
1387  };
1388 
1389  // Base class for all X86 masked store operations.
1390  // The class has the same order of operands as MaskedStoreSDNode for
1391  // convenience.
1393  public:
1394  X86MaskedStoreSDNode(unsigned Opcode, unsigned Order,
1395  const DebugLoc &dl, SDVTList VTs, EVT MemVT,
1396  MachineMemOperand *MMO)
1397  : MemSDNode(Opcode, Order, dl, VTs, MemVT, MMO) {}
1398 
1399  const SDValue &getBasePtr() const { return getOperand(1); }
1400  const SDValue &getMask() const { return getOperand(2); }
1401  const SDValue &getValue() const { return getOperand(3); }
1402 
1403  static bool classof(const SDNode *N) {
1404  return N->getOpcode() == X86ISD::VMTRUNCSTORES ||
1406  }
1407  };
1408 
1409  // X86 Truncating Store with Signed saturation.
1411  public:
1412  TruncSStoreSDNode(unsigned Order, const DebugLoc &dl,
1413  SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
1414  : X86StoreSDNode(X86ISD::VTRUNCSTORES, Order, dl, VTs, MemVT, MMO) {}
1415 
1416  static bool classof(const SDNode *N) {
1417  return N->getOpcode() == X86ISD::VTRUNCSTORES;
1418  }
1419  };
1420 
1421  // X86 Truncating Store with Unsigned saturation.
1423  public:
1424  TruncUSStoreSDNode(unsigned Order, const DebugLoc &dl,
1425  SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
1426  : X86StoreSDNode(X86ISD::VTRUNCSTOREUS, Order, dl, VTs, MemVT, MMO) {}
1427 
1428  static bool classof(const SDNode *N) {
1429  return N->getOpcode() == X86ISD::VTRUNCSTOREUS;
1430  }
1431  };
1432 
1433  // X86 Truncating Masked Store with Signed saturation.
1435  public:
1436  MaskedTruncSStoreSDNode(unsigned Order,
1437  const DebugLoc &dl, SDVTList VTs, EVT MemVT,
1438  MachineMemOperand *MMO)
1439  : X86MaskedStoreSDNode(X86ISD::VMTRUNCSTORES, Order, dl, VTs, MemVT, MMO) {}
1440 
1441  static bool classof(const SDNode *N) {
1442  return N->getOpcode() == X86ISD::VMTRUNCSTORES;
1443  }
1444  };
1445 
1446  // X86 Truncating Masked Store with Unsigned saturation.
1448  public:
1450  const DebugLoc &dl, SDVTList VTs, EVT MemVT,
1451  MachineMemOperand *MMO)
1452  : X86MaskedStoreSDNode(X86ISD::VMTRUNCSTOREUS, Order, dl, VTs, MemVT, MMO) {}
1453 
1454  static bool classof(const SDNode *N) {
1455  return N->getOpcode() == X86ISD::VMTRUNCSTOREUS;
1456  }
1457  };
1458 
1459  // X86 specific Gather/Scatter nodes.
1460  // The class has the same order of operands as MaskedGatherScatterSDNode for
1461  // convenience.
1463  public:
1464  X86MaskedGatherScatterSDNode(unsigned Opc, unsigned Order,
1465  const DebugLoc &dl, SDVTList VTs, EVT MemVT,
1466  MachineMemOperand *MMO)
1467  : MemSDNode(Opc, Order, dl, VTs, MemVT, MMO) {}
1468 
1469  const SDValue &getBasePtr() const { return getOperand(3); }
1470  const SDValue &getIndex() const { return getOperand(4); }
1471  const SDValue &getMask() const { return getOperand(2); }
1472  const SDValue &getValue() const { return getOperand(1); }
1473  const SDValue &getScale() const { return getOperand(5); }
1474 
1475  static bool classof(const SDNode *N) {
1476  return N->getOpcode() == X86ISD::MGATHER ||
1477  N->getOpcode() == X86ISD::MSCATTER;
1478  }
1479  };
1480 
1482  public:
1483  X86MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
1484  EVT MemVT, MachineMemOperand *MMO)
1485  : X86MaskedGatherScatterSDNode(X86ISD::MGATHER, Order, dl, VTs, MemVT,
1486  MMO) {}
1487 
1488  static bool classof(const SDNode *N) {
1489  return N->getOpcode() == X86ISD::MGATHER;
1490  }
1491  };
1492 
1494  public:
1495  X86MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
1496  EVT MemVT, MachineMemOperand *MMO)
1497  : X86MaskedGatherScatterSDNode(X86ISD::MSCATTER, Order, dl, VTs, MemVT,
1498  MMO) {}
1499 
1500  static bool classof(const SDNode *N) {
1501  return N->getOpcode() == X86ISD::MSCATTER;
1502  }
1503  };
1504 
1505  /// Generate unpacklo/unpackhi shuffle mask.
1506  template <typename T = int>
1508  bool Unary) {
1509  assert(Mask.empty() && "Expected an empty shuffle mask vector");
1510  int NumElts = VT.getVectorNumElements();
1511  int NumEltsInLane = 128 / VT.getScalarSizeInBits();
1512  for (int i = 0; i < NumElts; ++i) {
1513  unsigned LaneStart = (i / NumEltsInLane) * NumEltsInLane;
1514  int Pos = (i % NumEltsInLane) / 2 + LaneStart;
1515  Pos += (Unary ? 0 : NumElts * (i % 2));
1516  Pos += (Lo ? 0 : NumEltsInLane / 2);
1517  Mask.push_back(Pos);
1518  }
1519  }
1520 
1521  /// Helper function to scale a shuffle or target shuffle mask, replacing each
1522  /// mask index with the scaled sequential indices for an equivalent narrowed
1523  /// mask. This is the reverse process to canWidenShuffleElements, but can
1524  /// always succeed.
1525  template <typename T>
1527  SmallVectorImpl<T> &ScaledMask) {
1528  assert(0 < Scale && "Unexpected scaling factor");
1529  int NumElts = Mask.size();
1530  ScaledMask.assign(static_cast<size_t>(NumElts * Scale), -1);
1531 
1532  for (int i = 0; i != NumElts; ++i) {
1533  int M = Mask[i];
1534 
1535  // Repeat sentinel values in every mask element.
1536  if (M < 0) {
1537  for (int s = 0; s != Scale; ++s)
1538  ScaledMask[(Scale * i) + s] = M;
1539  continue;
1540  }
1541 
1542  // Scale mask element and increment across each mask element.
1543  for (int s = 0; s != Scale; ++s)
1544  ScaledMask[(Scale * i) + s] = (Scale * M) + s;
1545  }
1546  }
1547 } // end namespace llvm
1548 
1549 #endif // LLVM_LIB_TARGET_X86_X86ISELLOWERING_H
static SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG)
const SDValue & getIndex() const
Double shift instructions.
static SDValue LowerCallResult(SDValue Chain, SDValue InFlag, const SmallVectorImpl< CCValAssign > &RVLocs, const SDLoc &dl, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals)
LowerCallResult - Lower the result values of a call into the appropriate copies out of appropriate ph...
TruncUSStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:836
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
Vector comparison generating mask bits for fp and integer signed and unsigned data types...
Repeat move, corresponds to X86::REP_MOVSx.
void createUnpackShuffleMask(MVT VT, SmallVectorImpl< T > &Mask, bool Lo, bool Unary)
Generate unpacklo/unpackhi shuffle mask.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
LLVMContext & Context
static bool classof(const SDNode *N)
Return with a flag operand.
const SDValue & getBasePtr() const
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
const SDValue & getScale() const
Tail call return.
Compute Double Block Packed Sum-Absolute-Differences.
A Module instance is used to store all the information related to an LLVM module. ...
Definition: Module.h:63
bool mergeStoresAfterLegalization() const override
Allow store merging after legalization in addition to before legalization.
static bool classof(const SDNode *N)
Same as call except it adds the NoTrack prefix.
static void LowerMemOpCallTo(SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg, SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64, bool isTailCall, bool isVector, SmallVectorImpl< SDValue > &MemOpChains, SmallVectorImpl< TailCallArgumentInfo > &TailCallArguments, const SDLoc &dl)
LowerMemOpCallTo - Store the argument to the stack or remember it in case of tail calls...
bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const override
Return true if it is cheaper to split the store of a merged int val from a pair of smaller values int...
X86 conditional moves.
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:146
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned getVectorNumElements() const
Function Alias Analysis Results
This instruction constructs a fixed permutation of two input vectors.
const SDValue & getValue() const
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:302
A debug info location.
Definition: DebugLoc.h:34
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:141
static bool classof(const SDNode *N)
SSE4A Extraction and Insertion.
static bool classof(const SDNode *N)
An instruction for reading from memory.
Definition: Instructions.h:164
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:677
Bitwise logical ANDNOT of floating point values.
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
bool isCalleePop(CallingConv::ID CallingConv, bool is64Bit, bool IsVarArg, bool GuaranteeTCO)
Determines whether the callee is required to pop its own arguments.
This operation implements the lowering for readcyclecounter.
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:39
X86MaskedGatherScatterSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...
X86 compare and logical compare instructions.
MaskedTruncUSStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition: ValueTypes.h:136
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
Extract an 8-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRB.
A description of a memory reference used in the backend.
X86StoreSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
Bitwise Logical AND NOT of Packed FP values.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
Base class for the full range of assembler expressions which are needed for parsing.
Definition: MCExpr.h:36
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:677
This instruction implements SINT_TO_FP with the integer source in memory and FP reg result...
static SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG, const SparcTargetLowering &TLI, const SparcSubtarget *Subtarget)
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Integer horizontal add/sub.
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
Copies a 64-bit value from the low word of an XMM vector to an MMX vector.
void assign(size_type NumElts, const T &Elt)
Definition: SmallVector.h:425
Context object for machine code objects.
Definition: MCContext.h:63
static SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG)
Copies a 32-bit value from the low word of a MMX vector to a GPR.
This is a fast-path instruction selection class that generates poor code and doesn&#39;t support illegal ...
Definition: FastISel.h:67
X86 FP SETCC, similar to above, but with output as an i1 mask and with optional rounding mode...
Return from interrupt. Operand 0 is the number of bytes to pop.
This contains information for each constraint that we are lowering.
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory)...
Definition: APInt.h:33
An instruction for storing to memory.
Definition: Instructions.h:306
static bool classof(const SDNode *N)
X86 FP SETCC, implemented with CMP{cc}SS/CMP{cc}SD.
const SDValue & getBasePtr() const
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const
Floating point horizontal add/sub.
unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override
amdgpu Simplify well known AMD library false Value * Callee
Bitwise logical XOR of floating point values.
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
static bool classof(const SDNode *N)
const SDValue & getMask() const
Machine Value Type.
static SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG)
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:46
This instruction implements an extending load to FP stack slots.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:69
Insert any element of a 4 x float vector into any element of a destination 4 x floatvector.
unsigned getScalarSizeInBits() const
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:149
This is an important base class in LLVM.
Definition: Constant.h:42
Repeat fill, corresponds to X86::REP_STOSx.
static bool is64Bit(const char *name)
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:893
bool storeOfVectorConstantIsCheap(EVT MemVT, unsigned NumElem, unsigned AddrSpace) const override
Return true if it is expected to be cheaper to do a store of a non-zero vector constant with the give...
X86 conditional branches.
Insert the lower 16-bits of a 32-bit value to a vector, corresponds to X86::PINSRW.
Commutative FMIN and FMAX.
static SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG, const SparcSubtarget *Subtarget)
On Darwin, this node represents the result of the popl at function entry, used for PIC code...
bool convertSetCCLogicToBitwiseLogic(EVT VT) const override
Use bitwise logic to make pairs of compares more efficient.
static SDValue LowerUINT_TO_FP(SDValue Op, SelectionDAG &DAG, const SparcTargetLowering &TLI, bool hasHardQuad)
const SDValue & getValue() const
static SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG, const SparcSubtarget *Subtarget)
lazy value info
std::vector< ArgListEntry > ArgListTy
Extended Value Type.
Definition: ValueTypes.h:34
const AMDGPUAS & AS
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This structure contains all information that is necessary for lowering calls.
These operations represent an abstract X86 call instruction, which includes a bunch of information...
Floating point max and min.
TruncSStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
bool isScalarFPTypeInSSEReg(EVT VT) const
Return true if the specified scalar FP type is computed in an SSE register, not on the X87 floating p...
Copies a GPR into the low 32-bit word of a MMX vector and zero out the high word. ...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:194
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:212
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
Provides information about what library functions are available for the current target.
X86 Read Time-Stamp Counter and Processor ID.
CCValAssign - Represent assignment of one arg/retval to a location.
AddressSpace
Definition: NVPTXBaseInfo.h:22
unsigned getMemcmpEqZeroLoadsPerBlock() const override
Allow multiple load pairs per block for smaller and faster code.
X86MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
This is an abstract virtual class for memory operations.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Bit scan reverse.
Floating point reciprocal-sqrt and reciprocal approximation.
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:843
const SDValue & getValue() const
Represents one node in the SelectionDAG.
X86 bit-test instructions.
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:43
static bool Enabled
Definition: Statistic.cpp:51
const Function & getFunction() const
Return the LLVM function that this machine code represents.
static bool classof(const SDNode *N)
MaskedTruncSStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
Class for arbitrary precision integers.
Definition: APInt.h:69
static SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG)
static bool classof(const SDNode *N)
This instruction implements FP_TO_SINT with the integer destination in memory and a FP reg source...
Bit scan forward.
const char * getClearCacheBuiltinName() const override
Intel processors have a unified instruction and data cache.
amdgpu Simplify well known AMD library false Value Value * Arg
bool ShouldShrinkFPConstant(EVT VT) const override
If true, then instruction selection should seek to shrink the FP constant of the specified type to a ...
Representation of each machine instruction.
Definition: MachineInstr.h:60
static unsigned getScalingFactorCost(const TargetTransformInfo &TTI, const LSRUse &LU, const Formula &F, const Loop &L)
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:151
Insert the lower 8-bits of a 32-bit value to a vector, corresponds to X86::PINSRB.
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:61
A wrapper node for TargetConstantPool, TargetJumpTable, TargetExternalSymbol, TargetGlobalAddress, TargetGlobalTLSAddress, MCSymbol and TargetBlockAddress.
Bitwise logical AND of floating point values.
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
static bool classof(const SDNode *N)
X86MaskedStoreSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
LOCK-prefixed arithmetic read-modify-write instructions.
Extract a 16-bit value from a vector and zero extend it to i32, corresponds to X86::PEXTRW.
bool hasVectorBlend() const override
Return true if the target has a vector blend instruction.
Blend where the selector is an immediate.
X86MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT, MachineMemOperand *MMO)
This instruction implements a truncating store to FP stack slots.
Combined add and sub on an FP vector.
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
This instruction grabs the address of the next argument from a va_list.
LLVM Value Representation.
Definition: Value.h:73
Bitwise logical OR of floating point values.
Dynamic (non-constant condition) vector blend where only the sign bits of the condition elements are ...
X86 Read Performance Monitoring Counters.
constexpr char Size[]
Key for Kernel::Arg::Metadata::mSize.
const SDValue & getBasePtr() const
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:81
static SDValue LowerSINT_TO_FP(SDValue Op, SelectionDAG &DAG, const SparcTargetLowering &TLI, bool hasHardQuad)
IRTranslator LLVM IR MI
bool isZeroNode(SDValue Elt)
Returns true if Elt is a constant zero or floating point constant +0.0.
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
bool isOffsetSuitableForCodeModel(int64_t Offset, CodeModel::Model M, bool hasSymbolicDisplacement=true)
Returns true of the given offset can be fit into displacement field of the instruction.
bool hasBitPreservingFPLogic(EVT VT) const override
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
Compute Sum of Absolute Differences.
Scalar intrinsic floating point max and min.
MVT getScalarShiftAmountTy(const DataLayout &, EVT VT) const override
EVT is not used in-tree, but is used by out-of-tree target.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
void scaleShuffleMask(int Scale, ArrayRef< T > Mask, SmallVectorImpl< T > &ScaledMask)
Helper function to scale a shuffle or target shuffle mask, replacing each mask index with the scaled ...
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
BRIND node with NoTrack prefix.
Shuffle 16 8-bit values within a vector.
This file describes how to lower LLVM code to machine code.
Special wrapper used under X86-64 PIC mode for RIP relative displacements.