LLVM 19.0.0git
AArch64ISelLowering.h
Go to the documentation of this file.
1//==-- AArch64ISelLowering.h - AArch64 DAG Lowering Interface ----*- C++ -*-==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that AArch64 uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
15#define LLVM_LIB_TARGET_AARCH64_AARCH64ISELLOWERING_H
16
17#include "AArch64.h"
23#include "llvm/IR/CallingConv.h"
24#include "llvm/IR/Instruction.h"
25
26namespace llvm {
27
28namespace AArch64ISD {
29
30// For predicated nodes where the result is a vector, the operation is
31// controlled by a governing predicate and the inactive lanes are explicitly
32// defined with a value, please stick the following naming convention:
33//
34// _MERGE_OP<n> The result value is a vector with inactive lanes equal
35// to source operand OP<n>.
36//
37// _MERGE_ZERO The result value is a vector with inactive lanes
38// actively zeroed.
39//
40// _MERGE_PASSTHRU The result value is a vector with inactive lanes equal
41// to the last source operand which only purpose is being
42// a passthru value.
43//
44// For other cases where no explicit action is needed to set the inactive lanes,
45// or when the result is not a vector and it is needed or helpful to
46// distinguish a node from similar unpredicated nodes, use:
47//
48// _PRED
49//
50enum NodeType : unsigned {
52 WrapperLarge, // 4-instruction MOVZ/MOVK sequence for 64-bit addresses.
53 CALL, // Function call.
54
55 // Pseudo for a OBJC call that gets emitted together with a special `mov
56 // x29, x29` marker instruction.
58
59 CALL_BTI, // Function call followed by a BTI instruction.
60
62
68
69 // A call with the callee in x16, i.e. "blr x16".
71
72 // Produces the full sequence of instructions for getting the thread pointer
73 // offset of a variable into X0, using the TLSDesc model.
75 ADRP, // Page address of a TargetGlobalAddress operand.
76 ADR, // ADR
77 ADDlow, // Add the low 12 bits of a TargetGlobalAddress operand.
78 LOADgot, // Load from automatically generated descriptor (e.g. Global
79 // Offset Table, TLS record).
80 RET_GLUE, // Return with a glue operand. Operand 0 is the chain operand.
81 BRCOND, // Conditional branch instruction; "b.cond".
83 CSINV, // Conditional select invert.
84 CSNEG, // Conditional select negate.
85 CSINC, // Conditional select increment.
86
87 // Pointer to the thread's local storage area. Materialised from TPIDR_EL0 on
88 // ELF.
91 SBC, // adc, sbc instructions
92
93 // To avoid stack clash, allocation is performed by block and each block is
94 // probed.
96
97 // Predicated instructions where inactive lanes produce undefined results.
125
126 // Unpredicated vector instructions
128
130
131 // Predicated instructions with the result of inactive lanes provided by the
132 // last operand.
154
156
157 // Arithmetic instructions which write flags.
163
164 // Conditional compares. Operands: left,right,falsecc,cc,flags
168
169 // Floating point comparison
171
172 // Scalar-to-vector duplication
179
180 // Vector immedate moves
188
189 // Vector immediate ops
192
193 // Vector bitwise select: similar to ISD::VSELECT but not all bits within an
194 // element must be identical.
196
197 // Vector shuffles
209
210 // Vector shift by scalar
214
215 // Vector shift by scalar (again)
222
223 // Vector narrowing shift by immediate (bottom)
225
226 // Vector shift by constant and insert
229
230 // Vector comparisons
239
240 // Vector zero comparisons
251
252 // Round wide FP to narrow FP with inexact results to odd.
254
255 // Vector across-lanes addition
256 // Only the lower result lane is defined.
259
260 // Unsigned sum Long across Vector
263
264 // Add Pairwise of two vectors
266 // Add Long Pairwise
269
270 // udot/sdot instructions
273
274 // Vector across-lanes min/max
275 // Only the lower result lane is defined.
280
290
291 // Compare-and-branch
296
297 // Tail calls
299
300 // Custom prefetch handling
302
303 // {s|u}int to FP within a FP register.
306
307 /// Natural vector cast. ISD::BITCAST is not natural in the big-endian
308 /// world w.r.t vectors; which causes additional REV instructions to be
309 /// generated to compensate for the byte-swapping. But sometimes we do
310 /// need to re-interpret the data in SIMD vector registers in big-endian
311 /// mode without emitting such REV instructions.
313
314 MRS, // MRS, also sets the flags via a glue.
315
318
320
321 // Reciprocal estimates and steps.
326
331
337
338 // Floating-point reductions.
345
350
352
361
362 // Cast between vectors of the same element type but differ in length.
364
365 // Nodes to build an LD64B / ST64B 64-bit quantity out of i64, and vice versa
368
377
378 // Structured loads.
382
383 // Unsigned gather loads.
393
394 // Signed gather loads
402
403 // Unsigned gather loads.
411
412 // Signed gather loads.
420
421 // Non-temporal gather loads
425
426 // Contiguous masked store.
428
429 // Scatter store
439
440 // Non-temporal scatter store
443
444 // SME
447
448 // Asserts that a function argument (i32) is zero-extended to i8 by
449 // the caller
451
452 // 128-bit system register accesses
453 // lo64, hi64, chain = MRRS(chain, sysregname)
455 // chain = MSRR(chain, sysregname, lo64, hi64)
457
458 // Strict (exception-raising) floating point comparison
461
462 // SME ZA loads and stores
465
466 // NEON Load/Store with post-increment base updates
490
495
502
503 // Memory Operations
508};
509
510} // end namespace AArch64ISD
511
512namespace AArch64 {
513/// Possible values of current rounding mode, which is specified in bits
514/// 23:22 of FPCR.
516 RN = 0, // Round to Nearest
517 RP = 1, // Round towards Plus infinity
518 RM = 2, // Round towards Minus infinity
519 RZ = 3, // Round towards Zero
520 rmMask = 3 // Bit mask selecting rounding mode
522
523// Bit position of rounding mode bits in FPCR.
524const unsigned RoundingBitsPos = 22;
525
526// Registers used to pass function arguments.
529
530/// Maximum allowed number of unprobed bytes above SP at an ABI
531/// boundary.
532const unsigned StackProbeMaxUnprobedStack = 1024;
533
534/// Maximum number of iterations to unroll for a constant size probing loop.
535const unsigned StackProbeMaxLoopUnroll = 4;
536
537} // namespace AArch64
538
539class AArch64Subtarget;
540
542public:
543 explicit AArch64TargetLowering(const TargetMachine &TM,
544 const AArch64Subtarget &STI);
545
546 /// Control the following reassociation of operands: (op (op x, c1), y) -> (op
547 /// (op x, y), c1) where N0 is (op x, c1) and N1 is y.
549 SDValue N1) const override;
550
551 /// Selects the correct CCAssignFn for a given CallingConvention value.
552 CCAssignFn *CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const;
553
554 /// Selects the correct CCAssignFn for a given CallingConvention value.
556
557 /// Determine which of the bits specified in Mask are known to be either zero
558 /// or one and return them in the KnownZero/KnownOne bitsets.
560 const APInt &DemandedElts,
561 const SelectionDAG &DAG,
562 unsigned Depth = 0) const override;
563
565 const APInt &DemandedElts,
566 const SelectionDAG &DAG,
567 unsigned Depth) const override;
568
569 MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const override {
570 // Returning i64 unconditionally here (i.e. even for ILP32) means that the
571 // *DAG* representation of pointers will always be 64-bits. They will be
572 // truncated and extended when transferred to memory, but the 64-bit DAG
573 // allows us to use AArch64's addressing modes much more easily.
574 return MVT::getIntegerVT(64);
575 }
576
578 const APInt &DemandedElts,
579 TargetLoweringOpt &TLO) const override;
580
581 MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override;
582
583 /// Returns true if the target allows unaligned memory accesses of the
584 /// specified type.
586 EVT VT, unsigned AddrSpace = 0, Align Alignment = Align(1),
588 unsigned *Fast = nullptr) const override;
589 /// LLT variant.
590 bool allowsMisalignedMemoryAccesses(LLT Ty, unsigned AddrSpace,
591 Align Alignment,
593 unsigned *Fast = nullptr) const override;
594
595 /// Provide custom lowering hooks for some operations.
596 SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
597
598 const char *getTargetNodeName(unsigned Opcode) const override;
599
600 SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override;
601
602 /// This method returns a target specific FastISel object, or null if the
603 /// target does not support "fast" ISel.
605 const TargetLibraryInfo *libInfo) const override;
606
607 bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
608
609 bool isFPImmLegal(const APFloat &Imm, EVT VT,
610 bool ForCodeSize) const override;
611
612 /// Return true if the given shuffle mask can be codegen'd directly, or if it
613 /// should be stack expanded.
614 bool isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const override;
615
616 /// Similar to isShuffleMaskLegal. Return true is the given 'select with zero'
617 /// shuffle mask can be codegen'd directly.
618 bool isVectorClearMaskLegal(ArrayRef<int> M, EVT VT) const override;
619
620 /// Return the ISD::SETCC ValueType.
622 EVT VT) const override;
623
625
627 MachineBasicBlock *BB) const;
628
630 MachineBasicBlock *BB) const;
631
633 MachineBasicBlock *MBB) const;
634
635 MachineBasicBlock *EmitTileLoad(unsigned Opc, unsigned BaseReg,
637 MachineBasicBlock *BB) const;
639 MachineBasicBlock *EmitZAInstr(unsigned Opc, unsigned BaseReg,
641 bool HasTile) const;
643 unsigned Opcode, bool Op0IsDef) const;
645
648 MachineBasicBlock *MBB) const override;
649
650 bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I,
651 MachineFunction &MF,
652 unsigned Intrinsic) const override;
653
655 EVT NewVT) const override;
656
657 bool shouldRemoveRedundantExtend(SDValue Op) const override;
658
659 bool isTruncateFree(Type *Ty1, Type *Ty2) const override;
660 bool isTruncateFree(EVT VT1, EVT VT2) const override;
661
662 bool isProfitableToHoist(Instruction *I) const override;
663
664 bool isZExtFree(Type *Ty1, Type *Ty2) const override;
665 bool isZExtFree(EVT VT1, EVT VT2) const override;
666 bool isZExtFree(SDValue Val, EVT VT2) const override;
667
669 SmallVectorImpl<Use *> &Ops) const override;
670
672 Instruction *I, Loop *L, const TargetTransformInfo &TTI) const override;
673
674 bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override;
675
676 unsigned getMaxSupportedInterleaveFactor() const override { return 4; }
677
680 ArrayRef<unsigned> Indices,
681 unsigned Factor) const override;
683 unsigned Factor) const override;
684
686 LoadInst *LI) const override;
687
689 StoreInst *SI) const override;
690
691 bool isLegalAddImmediate(int64_t) const override;
692 bool isLegalICmpImmediate(int64_t) const override;
693
695 SDValue ConstNode) const override;
696
697 bool shouldConsiderGEPOffsetSplit() const override;
698
700 const AttributeList &FuncAttributes) const override;
701
703 const AttributeList &FuncAttributes) const override;
704
705 /// Return true if the addressing mode represented by AM is legal for this
706 /// target, for a load/store of the specified type.
707 bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,
708 unsigned AS,
709 Instruction *I = nullptr) const override;
710
711 int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset,
712 int64_t MaxOffset) const override;
713
714 /// Return true if an FMA operation is faster than a pair of fmul and fadd
715 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
716 /// returns true, otherwise fmuladd is expanded to fmul + fadd.
718 EVT VT) const override;
719 bool isFMAFasterThanFMulAndFAdd(const Function &F, Type *Ty) const override;
720
722 CodeGenOptLevel OptLevel) const override;
723
724 const MCPhysReg *getScratchRegisters(CallingConv::ID CC) const override;
726
727 /// Returns false if N is a bit extraction pattern of (X >> C) & Mask.
729 CombineLevel Level) const override;
730
731 bool isDesirableToPullExtFromShl(const MachineInstr &MI) const override {
732 return false;
733 }
734
735 /// Returns false if N is a bit extraction pattern of (X >> C) & Mask.
736 bool isDesirableToCommuteXorWithShift(const SDNode *N) const override;
737
738 /// Return true if it is profitable to fold a pair of shifts into a mask.
740 CombineLevel Level) const override;
741
742 bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode,
743 EVT VT) const override;
744
745 /// Returns true if it is beneficial to convert a load of a constant
746 /// to just the constant itself.
748 Type *Ty) const override;
749
750 /// Return true if EXTRACT_SUBVECTOR is cheap for this result type
751 /// with this index.
752 bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
753 unsigned Index) const override;
754
755 bool shouldFormOverflowOp(unsigned Opcode, EVT VT,
756 bool MathUsed) const override {
757 // Using overflow ops for overflow checks only should beneficial on
758 // AArch64.
759 return TargetLowering::shouldFormOverflowOp(Opcode, VT, true);
760 }
761
762 Value *emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr,
763 AtomicOrdering Ord) const override;
765 AtomicOrdering Ord) const override;
766
767 void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override;
768
769 bool isOpSuitableForLDPSTP(const Instruction *I) const;
770 bool isOpSuitableForLSE128(const Instruction *I) const;
771 bool isOpSuitableForRCPC3(const Instruction *I) const;
772 bool shouldInsertFencesForAtomic(const Instruction *I) const override;
773 bool
775
777 shouldExpandAtomicLoadInIR(LoadInst *LI) const override;
779 shouldExpandAtomicStoreInIR(StoreInst *SI) const override;
781 shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override;
782
785
786 bool useLoadStackGuardNode() const override;
788 getPreferredVectorAction(MVT VT) const override;
789
790 /// If the target has a standard location for the stack protector cookie,
791 /// returns the address of that location. Otherwise, returns nullptr.
792 Value *getIRStackGuard(IRBuilderBase &IRB) const override;
793
794 void insertSSPDeclarations(Module &M) const override;
795 Value *getSDagStackGuard(const Module &M) const override;
796 Function *getSSPStackGuardCheck(const Module &M) const override;
797
798 /// If the target has a standard location for the unsafe stack pointer,
799 /// returns the address of that location. Otherwise, returns nullptr.
800 Value *getSafeStackPointerLocation(IRBuilderBase &IRB) const override;
801
802 /// If a physical register, this returns the register that receives the
803 /// exception address on entry to an EH pad.
805 getExceptionPointerRegister(const Constant *PersonalityFn) const override {
806 // FIXME: This is a guess. Has this been defined yet?
807 return AArch64::X0;
808 }
809
810 /// If a physical register, this returns the register that receives the
811 /// exception typeid on entry to a landing pad.
813 getExceptionSelectorRegister(const Constant *PersonalityFn) const override {
814 // FIXME: This is a guess. Has this been defined yet?
815 return AArch64::X1;
816 }
817
818 bool isIntDivCheap(EVT VT, AttributeList Attr) const override;
819
820 bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT,
821 const MachineFunction &MF) const override {
822 // Do not merge to float value size (128 bytes) if no implicit
823 // float attribute is set.
824
825 bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
826
827 if (NoFloat)
828 return (MemVT.getSizeInBits() <= 64);
829 return true;
830 }
831
832 bool isCheapToSpeculateCttz(Type *) const override {
833 return true;
834 }
835
836 bool isCheapToSpeculateCtlz(Type *) const override {
837 return true;
838 }
839
840 bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override;
841
842 bool hasAndNotCompare(SDValue V) const override {
843 // We can use bics for any scalar.
844 return V.getValueType().isScalarInteger();
845 }
846
847 bool hasAndNot(SDValue Y) const override {
848 EVT VT = Y.getValueType();
849
850 if (!VT.isVector())
851 return hasAndNotCompare(Y);
852
853 TypeSize TS = VT.getSizeInBits();
854 // TODO: We should be able to use bic/bif too for SVE.
855 return !TS.isScalable() && TS.getFixedValue() >= 64; // vector 'bic'
856 }
857
860 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
861 SelectionDAG &DAG) const override;
862
865 unsigned ExpansionFactor) const override;
866
868 unsigned KeptBits) const override {
869 // For vectors, we don't have a preference..
870 if (XVT.isVector())
871 return false;
872
873 auto VTIsOk = [](EVT VT) -> bool {
874 return VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32 ||
875 VT == MVT::i64;
876 };
877
878 // We are ok with KeptBitsVT being byte/word/dword, what SXT supports.
879 // XVT will be larger than KeptBitsVT.
880 MVT KeptBitsVT = MVT::getIntegerVT(KeptBits);
881 return VTIsOk(XVT) && VTIsOk(KeptBitsVT);
882 }
883
884 bool preferIncOfAddToSubOfNot(EVT VT) const override;
885
886 bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override;
887
888 bool isComplexDeinterleavingSupported() const override;
890 ComplexDeinterleavingOperation Operation, Type *Ty) const override;
891
894 ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB,
895 Value *Accumulator = nullptr) const override;
896
897 bool supportSplitCSR(MachineFunction *MF) const override {
899 MF->getFunction().hasFnAttribute(Attribute::NoUnwind);
900 }
901 void initializeSplitCSR(MachineBasicBlock *Entry) const override;
903 MachineBasicBlock *Entry,
904 const SmallVectorImpl<MachineBasicBlock *> &Exits) const override;
905
906 bool supportSwiftError() const override {
907 return true;
908 }
909
910 bool supportKCFIBundles() const override { return true; }
911
914 const TargetInstrInfo *TII) const override;
915
916 /// Enable aggressive FMA fusion on targets that want it.
917 bool enableAggressiveFMAFusion(EVT VT) const override;
918
919 /// Returns the size of the platform's va_list object.
920 unsigned getVaListSizeInBits(const DataLayout &DL) const override;
921
922 /// Returns true if \p VecTy is a legal interleaved access type. This
923 /// function checks the vector element type and the overall width of the
924 /// vector.
926 bool &UseScalable) const;
927
928 /// Returns the number of interleaved accesses that will be generated when
929 /// lowering accesses of the given type.
930 unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL,
931 bool UseScalable) const;
932
934 const Instruction &I) const override;
935
937 Type *Ty, CallingConv::ID CallConv, bool isVarArg,
938 const DataLayout &DL) const override;
939
940 /// Used for exception handling on Win64.
941 bool needsFixedCatchObjects() const override;
942
943 bool fallBackToDAGISel(const Instruction &Inst) const override;
944
945 /// SVE code generation for fixed length vectors does not custom lower
946 /// BUILD_VECTOR. This makes BUILD_VECTOR legalisation a source of stores to
947 /// merge. However, merging them creates a BUILD_VECTOR that is just as
948 /// illegal as the original, thus leading to an infinite legalisation loop.
949 /// NOTE: Once BUILD_VECTOR is legal or can be custom lowered for all legal
950 /// vector types this override can be removed.
951 bool mergeStoresAfterLegalization(EVT VT) const override;
952
953 // If the platform/function should have a redzone, return the size in bytes.
954 unsigned getRedZoneSize(const Function &F) const {
955 if (F.hasFnAttribute(Attribute::NoRedZone))
956 return 0;
957 return 128;
958 }
959
960 bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const;
962
964 bool AllowUnknown = false) const override;
965
966 bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override;
967
968 bool shouldExpandCttzElements(EVT VT) const override;
969
970 /// If a change in streaming mode is required on entry to/return from a
971 /// function call it emits and returns the corresponding SMSTART or SMSTOP
972 /// node. \p Condition should be one of the enum values from
973 /// AArch64SME::ToggleCondition.
975 SDValue Chain, SDValue InGlue, unsigned Condition,
976 SDValue PStateSM = SDValue()) const;
977
978 bool isVScaleKnownToBeAPowerOfTwo() const override { return true; }
979
980 // Normally SVE is only used for byte size vectors that do not fit within a
981 // NEON vector. This changes when OverrideNEON is true, allowing SVE to be
982 // used for 64bit and 128bit vectors as well.
983 bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON = false) const;
984
985 // Follow NEON ABI rules even when using SVE for fixed length vectors.
987 EVT VT) const override;
990 EVT VT) const override;
993 EVT &IntermediateVT,
994 unsigned &NumIntermediates,
995 MVT &RegisterVT) const override;
996
997 /// True if stack clash protection is enabled for this functions.
998 bool hasInlineStackProbe(const MachineFunction &MF) const override;
999
1000private:
1001 /// Keep a pointer to the AArch64Subtarget around so that we can
1002 /// make the right decision when generating code for different targets.
1003 const AArch64Subtarget *Subtarget;
1004
1005 bool isExtFreeImpl(const Instruction *Ext) const override;
1006
1007 void addTypeForNEON(MVT VT);
1008 void addTypeForFixedLengthSVE(MVT VT, bool StreamingSVE);
1009 void addDRTypeForNEON(MVT VT);
1010 void addQRTypeForNEON(MVT VT);
1011
1012 unsigned allocateLazySaveBuffer(SDValue &Chain, const SDLoc &DL,
1013 SelectionDAG &DAG) const;
1014
1015 SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv,
1016 bool isVarArg,
1018 const SDLoc &DL, SelectionDAG &DAG,
1019 SmallVectorImpl<SDValue> &InVals) const override;
1020
1021 void AdjustInstrPostInstrSelection(MachineInstr &MI,
1022 SDNode *Node) const override;
1023
1024 SDValue LowerCall(CallLoweringInfo & /*CLI*/,
1025 SmallVectorImpl<SDValue> &InVals) const override;
1026
1027 SDValue LowerCallResult(SDValue Chain, SDValue InGlue,
1028 CallingConv::ID CallConv, bool isVarArg,
1029 const SmallVectorImpl<CCValAssign> &RVLocs,
1030 const SDLoc &DL, SelectionDAG &DAG,
1031 SmallVectorImpl<SDValue> &InVals, bool isThisReturn,
1032 SDValue ThisVal, bool RequiresSMChange) const;
1033
1034 SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
1035 SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
1036 SDValue LowerStore128(SDValue Op, SelectionDAG &DAG) const;
1037 SDValue LowerABS(SDValue Op, SelectionDAG &DAG) const;
1038
1039 SDValue LowerMGATHER(SDValue Op, SelectionDAG &DAG) const;
1040 SDValue LowerMSCATTER(SDValue Op, SelectionDAG &DAG) const;
1041
1042 SDValue LowerMLOAD(SDValue Op, SelectionDAG &DAG) const;
1043
1044 SDValue LowerINTRINSIC_W_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1045 SDValue LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const;
1046 SDValue LowerINTRINSIC_VOID(SDValue Op, SelectionDAG &DAG) const;
1047
1048 bool
1049 isEligibleForTailCallOptimization(const CallLoweringInfo &CLI) const;
1050
1051 /// Finds the incoming stack arguments which overlap the given fixed stack
1052 /// object and incorporates their load into the current chain. This prevents
1053 /// an upcoming store from clobbering the stack argument before it's used.
1054 SDValue addTokenForArgument(SDValue Chain, SelectionDAG &DAG,
1055 MachineFrameInfo &MFI, int ClobberedFI) const;
1056
1057 bool DoesCalleeRestoreStack(CallingConv::ID CallCC, bool TailCallOpt) const;
1058
1059 void saveVarArgRegisters(CCState &CCInfo, SelectionDAG &DAG, const SDLoc &DL,
1060 SDValue &Chain) const;
1061
1062 bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF,
1063 bool isVarArg,
1065 LLVMContext &Context) const override;
1066
1067 SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
1069 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1070 SelectionDAG &DAG) const override;
1071
1072 SDValue getTargetNode(GlobalAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
1073 unsigned Flag) const;
1074 SDValue getTargetNode(JumpTableSDNode *N, EVT Ty, SelectionDAG &DAG,
1075 unsigned Flag) const;
1076 SDValue getTargetNode(ConstantPoolSDNode *N, EVT Ty, SelectionDAG &DAG,
1077 unsigned Flag) const;
1078 SDValue getTargetNode(BlockAddressSDNode *N, EVT Ty, SelectionDAG &DAG,
1079 unsigned Flag) const;
1080 SDValue getTargetNode(ExternalSymbolSDNode *N, EVT Ty, SelectionDAG &DAG,
1081 unsigned Flag) const;
1082 template <class NodeTy>
1083 SDValue getGOT(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
1084 template <class NodeTy>
1085 SDValue getAddrLarge(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
1086 template <class NodeTy>
1087 SDValue getAddr(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
1088 template <class NodeTy>
1089 SDValue getAddrTiny(NodeTy *N, SelectionDAG &DAG, unsigned Flags = 0) const;
1090 SDValue LowerADDROFRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1091 SDValue LowerGlobalAddress(SDValue Op, SelectionDAG &DAG) const;
1092 SDValue LowerGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1093 SDValue LowerDarwinGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1094 SDValue LowerELFGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1095 SDValue LowerELFTLSLocalExec(const GlobalValue *GV, SDValue ThreadBase,
1096 const SDLoc &DL, SelectionDAG &DAG) const;
1097 SDValue LowerELFTLSDescCallSeq(SDValue SymAddr, const SDLoc &DL,
1098 SelectionDAG &DAG) const;
1099 SDValue LowerWindowsGlobalTLSAddress(SDValue Op, SelectionDAG &DAG) const;
1100 SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
1101 SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
1102 SDValue LowerBR_CC(SDValue Op, SelectionDAG &DAG) const;
1103 SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
1104 SDValue LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const;
1105 SDValue LowerSELECT_CC(ISD::CondCode CC, SDValue LHS, SDValue RHS,
1106 SDValue TVal, SDValue FVal, const SDLoc &dl,
1107 SelectionDAG &DAG) const;
1108 SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
1109 SDValue LowerBR_JT(SDValue Op, SelectionDAG &DAG) const;
1110 SDValue LowerConstantPool(SDValue Op, SelectionDAG &DAG) const;
1111 SDValue LowerBlockAddress(SDValue Op, SelectionDAG &DAG) const;
1112 SDValue LowerAAPCS_VASTART(SDValue Op, SelectionDAG &DAG) const;
1113 SDValue LowerDarwin_VASTART(SDValue Op, SelectionDAG &DAG) const;
1114 SDValue LowerWin64_VASTART(SDValue Op, SelectionDAG &DAG) const;
1115 SDValue LowerVASTART(SDValue Op, SelectionDAG &DAG) const;
1116 SDValue LowerVACOPY(SDValue Op, SelectionDAG &DAG) const;
1117 SDValue LowerVAARG(SDValue Op, SelectionDAG &DAG) const;
1118 SDValue LowerFRAMEADDR(SDValue Op, SelectionDAG &DAG) const;
1119 SDValue LowerSPONENTRY(SDValue Op, SelectionDAG &DAG) const;
1120 SDValue LowerRETURNADDR(SDValue Op, SelectionDAG &DAG) const;
1121 SDValue LowerGET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1122 SDValue LowerSET_ROUNDING(SDValue Op, SelectionDAG &DAG) const;
1123 SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1124 SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
1125 SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1126 SDValue LowerZERO_EXTEND_VECTOR_INREG(SDValue Op, SelectionDAG &DAG) const;
1127 SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
1128 SDValue LowerSPLAT_VECTOR(SDValue Op, SelectionDAG &DAG) const;
1129 SDValue LowerDUPQLane(SDValue Op, SelectionDAG &DAG) const;
1130 SDValue LowerToPredicatedOp(SDValue Op, SelectionDAG &DAG,
1131 unsigned NewOp) const;
1132 SDValue LowerToScalableOp(SDValue Op, SelectionDAG &DAG) const;
1133 SDValue LowerVECTOR_SPLICE(SDValue Op, SelectionDAG &DAG) const;
1134 SDValue LowerEXTRACT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
1135 SDValue LowerINSERT_SUBVECTOR(SDValue Op, SelectionDAG &DAG) const;
1136 SDValue LowerVECTOR_DEINTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
1137 SDValue LowerVECTOR_INTERLEAVE(SDValue Op, SelectionDAG &DAG) const;
1138 SDValue LowerDIV(SDValue Op, SelectionDAG &DAG) const;
1139 SDValue LowerMUL(SDValue Op, SelectionDAG &DAG) const;
1140 SDValue LowerVectorSRA_SRL_SHL(SDValue Op, SelectionDAG &DAG) const;
1141 SDValue LowerShiftParts(SDValue Op, SelectionDAG &DAG) const;
1142 SDValue LowerVSETCC(SDValue Op, SelectionDAG &DAG) const;
1143 SDValue LowerCTPOP_PARITY(SDValue Op, SelectionDAG &DAG) const;
1144 SDValue LowerCTTZ(SDValue Op, SelectionDAG &DAG) const;
1145 SDValue LowerBitreverse(SDValue Op, SelectionDAG &DAG) const;
1146 SDValue LowerMinMax(SDValue Op, SelectionDAG &DAG) const;
1147 SDValue LowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG) const;
1148 SDValue LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const;
1149 SDValue LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const;
1150 SDValue LowerVectorFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1151 SDValue LowerVectorFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
1152 SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const;
1153 SDValue LowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG) const;
1154 SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1155 SDValue LowerVectorINT_TO_FP(SDValue Op, SelectionDAG &DAG) const;
1156 SDValue LowerVectorOR(SDValue Op, SelectionDAG &DAG) const;
1157 SDValue LowerXOR(SDValue Op, SelectionDAG &DAG) const;
1158 SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
1159 SDValue LowerFSINCOS(SDValue Op, SelectionDAG &DAG) const;
1160 SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
1161 SDValue LowerVSCALE(SDValue Op, SelectionDAG &DAG) const;
1162 SDValue LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const;
1163 SDValue LowerVECREDUCE(SDValue Op, SelectionDAG &DAG) const;
1164 SDValue LowerATOMIC_LOAD_AND(SDValue Op, SelectionDAG &DAG) const;
1165 SDValue LowerWindowsDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1166 SDValue LowerInlineDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1167 SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
1168
1169 SDValue LowerAVG(SDValue Op, SelectionDAG &DAG, unsigned NewOp) const;
1170
1171 SDValue LowerFixedLengthVectorIntDivideToSVE(SDValue Op,
1172 SelectionDAG &DAG) const;
1173 SDValue LowerFixedLengthVectorIntExtendToSVE(SDValue Op,
1174 SelectionDAG &DAG) const;
1175 SDValue LowerFixedLengthVectorLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
1176 SDValue LowerFixedLengthVectorMLoadToSVE(SDValue Op, SelectionDAG &DAG) const;
1177 SDValue LowerVECREDUCE_SEQ_FADD(SDValue ScalarOp, SelectionDAG &DAG) const;
1178 SDValue LowerPredReductionToSVE(SDValue ScalarOp, SelectionDAG &DAG) const;
1179 SDValue LowerReductionToSVE(unsigned Opcode, SDValue ScalarOp,
1180 SelectionDAG &DAG) const;
1181 SDValue LowerFixedLengthVectorSelectToSVE(SDValue Op, SelectionDAG &DAG) const;
1182 SDValue LowerFixedLengthVectorSetccToSVE(SDValue Op, SelectionDAG &DAG) const;
1183 SDValue LowerFixedLengthVectorStoreToSVE(SDValue Op, SelectionDAG &DAG) const;
1184 SDValue LowerFixedLengthVectorMStoreToSVE(SDValue Op,
1185 SelectionDAG &DAG) const;
1186 SDValue LowerFixedLengthVectorTruncateToSVE(SDValue Op,
1187 SelectionDAG &DAG) const;
1188 SDValue LowerFixedLengthExtractVectorElt(SDValue Op, SelectionDAG &DAG) const;
1189 SDValue LowerFixedLengthInsertVectorElt(SDValue Op, SelectionDAG &DAG) const;
1190 SDValue LowerFixedLengthBitcastToSVE(SDValue Op, SelectionDAG &DAG) const;
1191 SDValue LowerFixedLengthConcatVectorsToSVE(SDValue Op,
1192 SelectionDAG &DAG) const;
1193 SDValue LowerFixedLengthFPExtendToSVE(SDValue Op, SelectionDAG &DAG) const;
1194 SDValue LowerFixedLengthFPRoundToSVE(SDValue Op, SelectionDAG &DAG) const;
1195 SDValue LowerFixedLengthIntToFPToSVE(SDValue Op, SelectionDAG &DAG) const;
1196 SDValue LowerFixedLengthFPToIntToSVE(SDValue Op, SelectionDAG &DAG) const;
1197 SDValue LowerFixedLengthVECTOR_SHUFFLEToSVE(SDValue Op,
1198 SelectionDAG &DAG) const;
1199
1200 SDValue BuildSDIVPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
1201 SmallVectorImpl<SDNode *> &Created) const override;
1202 SDValue BuildSREMPow2(SDNode *N, const APInt &Divisor, SelectionDAG &DAG,
1203 SmallVectorImpl<SDNode *> &Created) const override;
1204 SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1205 int &ExtraSteps, bool &UseOneConst,
1206 bool Reciprocal) const override;
1207 SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled,
1208 int &ExtraSteps) const override;
1209 SDValue getSqrtInputTest(SDValue Operand, SelectionDAG &DAG,
1210 const DenormalMode &Mode) const override;
1211 SDValue getSqrtResultForDenormInput(SDValue Operand,
1212 SelectionDAG &DAG) const override;
1213 unsigned combineRepeatedFPDivisors() const override;
1214
1215 ConstraintType getConstraintType(StringRef Constraint) const override;
1216 Register getRegisterByName(const char* RegName, LLT VT,
1217 const MachineFunction &MF) const override;
1218
1219 /// Examine constraint string and operand type and determine a weight value.
1220 /// The operand object must already have been set up with the operand type.
1222 getSingleConstraintMatchWeight(AsmOperandInfo &info,
1223 const char *constraint) const override;
1224
1225 std::pair<unsigned, const TargetRegisterClass *>
1226 getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
1227 StringRef Constraint, MVT VT) const override;
1228
1229 const char *LowerXConstraint(EVT ConstraintVT) const override;
1230
1231 void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint,
1232 std::vector<SDValue> &Ops,
1233 SelectionDAG &DAG) const override;
1234
1236 getInlineAsmMemConstraint(StringRef ConstraintCode) const override {
1237 if (ConstraintCode == "Q")
1239 // FIXME: clang has code for 'Ump', 'Utf', 'Usa', and 'Ush' but these are
1240 // followed by llvm_unreachable so we'll leave them unimplemented in
1241 // the backend for now.
1242 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
1243 }
1244
1245 /// Handle Lowering flag assembly outputs.
1246 SDValue LowerAsmOutputForConstraint(SDValue &Chain, SDValue &Flag,
1247 const SDLoc &DL,
1248 const AsmOperandInfo &Constraint,
1249 SelectionDAG &DAG) const override;
1250
1251 bool shouldExtendGSIndex(EVT VT, EVT &EltTy) const override;
1252 bool shouldRemoveExtendFromGSIndex(SDValue Extend, EVT DataVT) const override;
1253 bool isVectorLoadExtDesirable(SDValue ExtVal) const override;
1254 bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override;
1255 bool mayBeEmittedAsTailCall(const CallInst *CI) const override;
1256 bool getIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
1257 SDValue &Offset, SelectionDAG &DAG) const;
1258 bool getPreIndexedAddressParts(SDNode *N, SDValue &Base, SDValue &Offset,
1260 SelectionDAG &DAG) const override;
1261 bool getPostIndexedAddressParts(SDNode *N, SDNode *Op, SDValue &Base,
1262 SDValue &Offset, ISD::MemIndexedMode &AM,
1263 SelectionDAG &DAG) const override;
1264 bool isIndexingLegal(MachineInstr &MI, Register Base, Register Offset,
1265 bool IsPre, MachineRegisterInfo &MRI) const override;
1266
1267 void ReplaceNodeResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
1268 SelectionDAG &DAG) const override;
1269 void ReplaceBITCASTResults(SDNode *N, SmallVectorImpl<SDValue> &Results,
1270 SelectionDAG &DAG) const;
1271 void ReplaceExtractSubVectorResults(SDNode *N,
1272 SmallVectorImpl<SDValue> &Results,
1273 SelectionDAG &DAG) const;
1274
1275 bool shouldNormalizeToSelectSequence(LLVMContext &, EVT) const override;
1276
1277 void finalizeLowering(MachineFunction &MF) const override;
1278
1279 bool shouldLocalize(const MachineInstr &MI,
1280 const TargetTransformInfo *TTI) const override;
1281
1282 bool SimplifyDemandedBitsForTargetNode(SDValue Op,
1283 const APInt &OriginalDemandedBits,
1284 const APInt &OriginalDemandedElts,
1285 KnownBits &Known,
1286 TargetLoweringOpt &TLO,
1287 unsigned Depth) const override;
1288
1289 bool isTargetCanonicalConstantNode(SDValue Op) const override;
1290
1291 // With the exception of data-predicate transitions, no instructions are
1292 // required to cast between legal scalable vector types. However:
1293 // 1. Packed and unpacked types have different bit lengths, meaning BITCAST
1294 // is not universally useable.
1295 // 2. Most unpacked integer types are not legal and thus integer extends
1296 // cannot be used to convert between unpacked and packed types.
1297 // These can make "bitcasting" a multiphase process. REINTERPRET_CAST is used
1298 // to transition between unpacked and packed types of the same element type,
1299 // with BITCAST used otherwise.
1300 // This function does not handle predicate bitcasts.
1301 SDValue getSVESafeBitCast(EVT VT, SDValue Op, SelectionDAG &DAG) const;
1302
1303 // Returns the runtime value for PSTATE.SM by generating a call to
1304 // __arm_sme_state.
1305 SDValue getRuntimePStateSM(SelectionDAG &DAG, SDValue Chain, SDLoc DL,
1306 EVT VT) const;
1307
1308 bool preferScalarizeSplat(SDNode *N) const override;
1309
1310 unsigned getMinimumJumpTableEntries() const override;
1311
1312 bool softPromoteHalfType() const override { return true; }
1313};
1314
1315namespace AArch64 {
1316FastISel *createFastISel(FunctionLoweringInfo &funcInfo,
1317 const TargetLibraryInfo *libInfo);
1318} // end namespace AArch64
1319
1320} // end namespace llvm
1321
1322#endif
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
Function Alias Analysis Results
static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")
Analysis containing CSE Info
Definition: CSEInfo.cpp:27
uint64_t Addr
static GCMetadataPrinterRegistry::Add< ErlangGCPrinter > X("erlang", "erlang-compatible garbage collector")
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define RegName(no)
lazy value info
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
unsigned const TargetRegisterInfo * TRI
static GCMetadataPrinterRegistry::Add< OcamlGCMetadataPrinter > Y("ocaml", "ocaml 3.10-compatible collector")
PowerPC Reduce CR logical Operation
const char LLVMTargetMachineRef TM
static cl::opt< RegAllocEvictionAdvisorAnalysis::AdvisorMode > Mode("regalloc-enable-advisor", cl::Hidden, cl::init(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default), cl::desc("Enable regalloc advisor mode"), cl::values(clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Default, "default", "Default"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Release, "release", "precompiled"), clEnumValN(RegAllocEvictionAdvisorAnalysis::AdvisorMode::Development, "development", "for training")))
This file describes how to lower LLVM code to machine code.
Value * RHS
Value * LHS
bool isTruncateFree(Type *Ty1, Type *Ty2) const override
Return true if it's free to truncate a value of type FromTy to type ToTy.
bool supportSplitCSR(MachineFunction *MF) const override
Return true if the target supports that a subset of CSRs for the given machine function is handled ex...
bool shouldFoldSelectWithIdentityConstant(unsigned BinOpcode, EVT VT) const override
Return true if pulling a binary operation into a select with an identity constant is profitable.
bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize) const override
Returns true if the target can instruction select the specified FP immediate natively.
MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
void initializeSplitCSR(MachineBasicBlock *Entry) const override
Perform necessary initialization to handle a subset of CSRs explicitly via copies.
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth=0) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool hasAndNotCompare(SDValue V) const override
Return true if the target should transform: (X & Y) == Y —> (~X & Y) == 0 (X & Y) !...
Register getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool isShuffleMaskLegal(ArrayRef< int > M, EVT VT) const override
Return true if the given shuffle mask can be codegen'd directly, or if it should be stack expanded.
unsigned getVaListSizeInBits(const DataLayout &DL) const override
Returns the size of the platform's va_list object.
void insertCopiesSplitCSR(MachineBasicBlock *Entry, const SmallVectorImpl< MachineBasicBlock * > &Exits) const override
Insert explicit copies in entry and exit blocks.
int64_t getPreferredLargeGEPBaseOffset(int64_t MinOffset, int64_t MaxOffset) const override
Return the prefered common base offset.
bool shouldInsertTrailingFenceForAtomicStore(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert a trailing fence without reducing the ordering f...
bool shouldExpandCttzElements(EVT VT) const override
Return true if the @llvm.experimental.cttz.elts intrinsic should be expanded using generic code in Se...
MachineBasicBlock * EmitTileLoad(unsigned Opc, unsigned BaseReg, MachineInstr &MI, MachineBasicBlock *BB) const
unsigned getNumInterleavedAccesses(VectorType *VecTy, const DataLayout &DL, bool UseScalable) const
Returns the number of interleaved accesses that will be generated when lowering accesses of the given...
bool shouldConvertConstantLoadToIntImm(const APInt &Imm, Type *Ty) const override
Returns true if it is beneficial to convert a load of a constant to just the constant itself.
unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override
Certain targets require unusual breakdowns of certain types.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
Provide custom lowering hooks for some operations.
bool shouldConvertFpToSat(unsigned Op, EVT FPVT, EVT VT) const override
Should we generate fp_to_si_sat and fp_to_ui_sat from type FPVT to type VT from min(max(fptoi)) satur...
bool isIntDivCheap(EVT VT, AttributeList Attr) const override
Return true if integer divide is usually cheaper than a sequence of several shifts,...
bool shouldRemoveRedundantExtend(SDValue Op) const override
Return true (the default) if it is profitable to remove a sext_inreg(x) where the sext is redundant,...
CCAssignFn * CCAssignFnForReturn(CallingConv::ID CC) const
Selects the correct CCAssignFn for a given CallingConvention value.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ISD::SETCC ValueType.
bool optimizeExtendOrTruncateConversion(Instruction *I, Loop *L, const TargetTransformInfo &TTI) const override
Try to optimize extending or truncating conversion instructions (like zext, trunc,...
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo) const override
This method returns a target specific FastISel object, or null if the target does not support "fast" ...
CCAssignFn * CCAssignFnForCall(CallingConv::ID CC, bool IsVarArg) const
Selects the correct CCAssignFn for a given CallingConvention value.
MachineMemOperand::Flags getTargetMMOFlags(const Instruction &I) const override
This callback is used to inspect load/store instructions and add target-specific MachineMemOperand fl...
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this functions.
bool isLegalICmpImmediate(int64_t) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
EVT getOptimalMemOpType(const MemOp &Op, const AttributeList &FuncAttributes) const override
Returns the target specific optimal type for load and store operations as a result of memset,...
Value * emitStoreConditional(IRBuilderBase &Builder, Value *Val, Value *Addr, AtomicOrdering Ord) const override
Perform a store-conditional operation to Addr.
bool preferIncOfAddToSubOfNot(EVT VT) const override
These two forms are equivalent: sub y, (xor x, -1) add (add x, 1), y The variant with two add's is IR...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const override
Returns how the given (atomic) load should be expanded by the IR-level AtomicExpand pass.
MachineBasicBlock * EmitZAInstr(unsigned Opc, unsigned BaseReg, MachineInstr &MI, MachineBasicBlock *BB, bool HasTile) const
ShiftLegalizationStrategy preferredShiftLegalizationStrategy(SelectionDAG &DAG, SDNode *N, unsigned ExpansionFactor) const override
bool isOpSuitableForLSE128(const Instruction *I) const
bool lowerInterleavedLoad(LoadInst *LI, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor) const override
Lower an interleaved load into a ldN intrinsic.
bool isVScaleKnownToBeAPowerOfTwo() const override
Return true only if vscale must be a power of two.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool shouldSinkOperands(Instruction *I, SmallVectorImpl< Use * > &Ops) const override
Check if sinking I's operands to I's basic block is profitable, because the operands can be folded in...
bool fallBackToDAGISel(const Instruction &Inst) const override
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
getTgtMemIntrinsic - Represent NEON load and store intrinsics as MemIntrinsicNodes.
Function * getSSPStackGuardCheck(const Module &M) const override
If the target has a standard stack protection check function that performs validation and error handl...
bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const override
Try to convert math with an overflow comparison into the corresponding DAG node operation.
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
Value * createComplexDeinterleavingIR(IRBuilderBase &B, ComplexDeinterleavingOperation OperationType, ComplexDeinterleavingRotation Rotation, Value *InputA, Value *InputB, Value *Accumulator=nullptr) const override
Create the IR node for the given complex deinterleaving operation.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Returns true if the target allows unaligned memory accesses of the specified type.
unsigned getMaxSupportedInterleaveFactor() const override
Get the maximum supported factor for interleaved memory accesses.
bool isLegalInterleavedAccessType(VectorType *VecTy, const DataLayout &DL, bool &UseScalable) const
Returns true if VecTy is a legal interleaved access type.
void insertSSPDeclarations(Module &M) const override
Inserts necessary declarations for SSP (stack protection) purpose.
bool functionArgumentNeedsConsecutiveRegisters(Type *Ty, CallingConv::ID CallConv, bool isVarArg, const DataLayout &DL) const override
For some targets, an LLVM struct type must be broken down into multiple simple types,...
Value * emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy, Value *Addr, AtomicOrdering Ord) const override
Perform a load-linked operation on Addr, returning a "Value *" with the corresponding pointee type.
MachineBasicBlock * EmitLoweredCatchRet(MachineInstr &MI, MachineBasicBlock *BB) const
bool isComplexDeinterleavingSupported() const override
Does this target support complex deinterleaving.
bool isZExtFree(Type *Ty1, Type *Ty2) const override
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
EVT getAsmOperandValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const override
SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const
MachineBasicBlock * EmitZero(MachineInstr &MI, MachineBasicBlock *BB) const
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
bool useLoadStackGuardNode() const override
If this function returns true, SelectionDAGBuilder emits a LOAD_STACK_GUARD node when it is lowering ...
Value * getSafeStackPointerLocation(IRBuilderBase &IRB) const override
If the target has a standard location for the unsafe stack pointer, returns the address of that locat...
bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const override
Return if the target supports combining a chain like:
bool isProfitableToHoist(Instruction *I) const override
Check if it is profitable to hoist instruction in then/else to if.
bool isOpSuitableForRCPC3(const Instruction *I) const
bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy, EVT NewVT) const override
Return true if it is profitable to reduce a load to a smaller type.
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool isCheapToSpeculateCttz(Type *) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI, unsigned Factor) const override
Lower an interleaved store into a stN intrinsic.
unsigned getRedZoneSize(const Function &F) const
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
MachineBasicBlock * EmitZTInstr(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode, bool Op0IsDef) const
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
MachineBasicBlock * EmitFill(MachineInstr &MI, MachineBasicBlock *BB) const
bool isCheapToSpeculateCtlz(Type *) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
bool shouldInsertFencesForAtomic(const Instruction *I) const override
Whether AtomicExpandPass should automatically insert fences and reduce ordering for this atomic.
bool isReassocProfitable(SelectionDAG &DAG, SDValue N0, SDValue N1) const override
Control the following reassociation of operands: (op (op x, c1), y) -> (op (op x, y),...
TargetLoweringBase::AtomicExpansionKind shouldExpandAtomicStoreInIR(StoreInst *SI) const override
Returns how the given (atomic) store should be expanded by the IR-level AtomicExpand pass into.
MachineBasicBlock * EmitF128CSEL(MachineInstr &MI, MachineBasicBlock *BB) const
LLT getOptimalMemOpLLT(const MemOp &Op, const AttributeList &FuncAttributes) const override
LLT returning variant.
bool isDesirableToPullExtFromShl(const MachineInstr &MI) const override
GlobalISel - return true if it's profitable to perform the combine: shl ([sza]ext x),...
bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y, unsigned OldShiftOpcode, unsigned NewShiftOpcode, SelectionDAG &DAG) const override
Given the pattern (X & (C l>>/<< Y)) ==/!= 0 return true if it should be transformed into: ((X <</l>>...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
bool needsFixedCatchObjects() const override
Used for exception handling on Win64.
Register getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
bool lowerDeinterleaveIntrinsicToLoad(IntrinsicInst *DI, LoadInst *LI) const override
Lower a deinterleave intrinsic to a target specific load intrinsic.
unsigned getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) const override
Certain targets such as MIPS require that some types such as vectors are always broken down into scal...
Value * getIRStackGuard(IRBuilderBase &IRB) const override
If the target has a standard location for the stack protector cookie, returns the address of that loc...
bool targetShrinkDemandedConstant(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, TargetLoweringOpt &TLO) const override
bool generateFMAsInMachineCombiner(EVT VT, CodeGenOptLevel OptLevel) const override
bool isComplexDeinterleavingOperationSupported(ComplexDeinterleavingOperation Operation, Type *Ty) const override
Does this target support complex deinterleaving with the given operation and type.
bool hasPairedLoad(EVT LoadedType, Align &RequiredAligment) const override
Return true if the target supplies and combines to a paired load two loaded values of type LoadedType...
bool isOpSuitableForLDPSTP(const Instruction *I) const
bool shouldFoldConstantShiftPairToMask(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to fold a pair of shifts into a mask.
bool isLegalAddImmediate(int64_t) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool shouldConsiderGEPOffsetSplit() const override
bool shouldTransformSignedTruncationCheck(EVT XVT, unsigned KeptBits) const override
Should we tranform the IR-optimal check for whether given truncation down into KeptBits would be trun...
bool isVectorClearMaskLegal(ArrayRef< int > M, EVT VT) const override
Similar to isShuffleMaskLegal.
const MCPhysReg * getScratchRegisters(CallingConv::ID CC) const override
Returns a 0 terminated array of registers that can be safely used as scratch registers.
void emitAtomicCmpXchgNoStoreLLBalance(IRBuilderBase &Builder) const override
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return true if EXTRACT_SUBVECTOR is cheap for this result type with this index.
ArrayRef< MCPhysReg > getRoundingControlRegisters() const override
Returns a 0 terminated array of rounding control registers that can be attached into strict FP call.
MachineInstr * EmitKCFICheck(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator &MBBI, const TargetInstrInfo *TII) const override
bool isAllActivePredicate(SelectionDAG &DAG, SDValue N) const
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
bool isDesirableToCommuteXorWithShift(const SDNode *N) const override
Returns false if N is a bit extraction pattern of (X >> C) & Mask.
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Returns false if N is a bit extraction pattern of (X >> C) & Mask.
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
bool supportSwiftError() const override
Return true if the target supports swifterror attribute.
bool lowerInterleaveIntrinsicToStore(IntrinsicInst *II, StoreInst *SI) const override
Lower an interleave intrinsic to a target specific store intrinsic.
bool enableAggressiveFMAFusion(EVT VT) const override
Enable aggressive FMA fusion on targets that want it.
Value * getSDagStackGuard(const Module &M) const override
Return the variable that's previously inserted by insertSSPDeclarations, if any, otherwise return nul...
MVT getScalarShiftAmountTy(const DataLayout &DL, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
MachineBasicBlock * EmitDynamicProbedAlloc(MachineInstr &MI, MachineBasicBlock *MBB) const
SDValue changeStreamingMode(SelectionDAG &DAG, SDLoc DL, bool Enable, SDValue Chain, SDValue InGlue, unsigned Condition, SDValue PStateSM=SDValue()) const
If a change in streaming mode is required on entry to/return from a function call it emits and return...
bool shouldExpandGetActiveLaneMask(EVT VT, EVT OpVT) const override
Return true if the @llvm.get.active.lane.mask intrinsic should be expanded using generic code in Sele...
bool supportKCFIBundles() const override
Return true if the target supports kcfi operand bundles.
bool isMulAddWithConstProfitable(SDValue AddNode, SDValue ConstNode) const override
Return true if it may be profitable to transform (mul (add x, c1), c2) -> (add (mul x,...
bool useSVEForFixedLengthVectorVT(EVT VT, bool OverrideNEON=false) const
bool mergeStoresAfterLegalization(EVT VT) const override
SVE code generation for fixed length vectors does not custom lower BUILD_VECTOR.
Class for arbitrary precision integers.
Definition: APInt.h:76
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition: ArrayRef.h:41
An instruction that atomically checks whether a specified value is in a memory location,...
Definition: Instructions.h:539
an instruction that atomically reads a memory location, combines it with another value,...
Definition: Instructions.h:748
CCState - This class holds information needed while lowering arguments and return values.
This class represents a function call, abstracting a target machine's calling convention.
This is an important base class in LLVM.
Definition: Constant.h:41
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition: FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition: Function.h:262
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.cpp:669
Common base class shared among various IRBuilders.
Definition: IRBuilder.h:94
A wrapper class for inspecting calls to intrinsic functions.
Definition: IntrinsicInst.h:47
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
An instruction for reading from memory.
Definition: Instructions.h:184
Represents a single loop in the control flow graph.
Definition: LoopInfo.h:44
Machine Value Type.
static MVT getIntegerVT(unsigned BitWidth)
Instructions::iterator instr_iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
Function & getFunction()
Return the LLVM function that this machine code represents.
Representation of each machine instruction.
Definition: MachineInstr.h:69
Flags
Flags values. These may be or'd together.
A Module instance is used to store all the information related to an LLVM module.
Definition: Module.h:65
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
This instruction constructs a fixed permutation of two input vectors.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:586
An instruction for storing to memory.
Definition: Instructions.h:317
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT, bool MathUsed) const
Try to convert math with an overflow comparison into the corresponding DAG node operation.
ShiftLegalizationStrategy
Return the preferred strategy to legalize tihs SHIFT instruction, with ExpansionFactor being the recu...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:76
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This pass provides access to the codegen interfaces that are needed for IR-level transformations.
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
LLVM Value Representation.
Definition: Value.h:74
Base class of all SIMD vector types.
Definition: DerivedTypes.h:403
constexpr ScalarTy getFixedValue() const
Definition: TypeSize.h:187
constexpr bool isScalable() const
Returns whether the quantity is scaled by a runtime quantity (vscale).
Definition: TypeSize.h:171
@ NVCAST
Natural vector cast.
ArrayRef< MCPhysReg > getFPRArgRegs()
Rounding
Possible values of current rounding mode, which is specified in bits 23:22 of FPCR.
const unsigned StackProbeMaxLoopUnroll
Maximum number of iterations to unroll for a constant size probing loop.
const unsigned StackProbeMaxUnprobedStack
Maximum allowed number of unprobed bytes above SP at an ABI boundary.
const unsigned RoundingBitsPos
ArrayRef< MCPhysReg > getGPRArgRegs()
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
@ CXX_FAST_TLS
Used for access functions.
Definition: CallingConv.h:72
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
Definition: ISDOpcodes.h:1400
static const int FIRST_TARGET_MEMORY_OPCODE
FIRST_TARGET_MEMORY_OPCODE - Target-specific pre-isel operations which do not reference a specific me...
Definition: ISDOpcodes.h:1412
MemIndexedMode
MemIndexedMode enum - This enum defines the load / store indexed addressing modes.
Definition: ISDOpcodes.h:1472
static const int FIRST_TARGET_STRICTFP_OPCODE
FIRST_TARGET_STRICTFP_OPCODE - Target-specific pre-isel operations which cannot raise FP exceptions s...
Definition: ISDOpcodes.h:1406
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1523
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:1503
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
@ Offset
Definition: DWP.cpp:456
AddressSpace
Definition: NVPTXBaseInfo.h:21
CodeGenOptLevel
Code generation optimization level.
Definition: CodeGen.h:54
AtomicOrdering
Atomic ordering for LLVM's memory model.
TargetTransformInfo TTI
bool CCAssignFn(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State)
CCAssignFn - This function assigns a location for Val, updating State to reflect the change.
CombineLevel
Definition: DAGCombine.h:15
DWARFExpression::Operation Op
@ Enable
Enable colors.
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Represent subnormal handling kind for floating point instruction inputs and outputs.
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:358
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:167