34#define DEBUG_TYPE "bpf-lower"
38 cl::desc(
"Expand memcpy into load/store pairs in order"));
81 for (
auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
96 for (
auto VT : { MVT::i32, MVT::i64 }) {
175 unsigned CommonMaxStores =
195bool BPFTargetLowering::isTruncateFree(
Type *Ty1,
Type *Ty2)
const {
200 return NumBits1 > NumBits2;
203bool BPFTargetLowering::isTruncateFree(
EVT VT1,
EVT VT2)
const {
208 return NumBits1 > NumBits2;
211bool BPFTargetLowering::isZExtFree(
Type *Ty1,
Type *Ty2)
const {
216 return NumBits1 == 32 && NumBits2 == 64;
219bool BPFTargetLowering::isZExtFree(
EVT VT1,
EVT VT2)
const {
224 return NumBits1 == 32 && NumBits2 == 64;
227bool BPFTargetLowering::isZExtFree(
SDValue Val,
EVT VT2)
const {
232 if ((MT1 == MVT::i8 || MT1 == MVT::i16 || MT1 == MVT::i32) &&
233 (MT2 == MVT::i32 || MT2 == MVT::i64))
241 if (Constraint.
size() == 1) {
242 switch (Constraint[0]) {
253std::pair<unsigned, const TargetRegisterClass *>
257 if (Constraint.
size() == 1) {
259 switch (Constraint[0]) {
261 return std::make_pair(0U, &BPF::GPRRegClass);
264 return std::make_pair(0U, &BPF::GPR32RegClass);
274void BPFTargetLowering::ReplaceNodeResults(
288 Msg =
"unsupported atomic operation, please use 32/64 bit version";
290 Msg =
"unsupported atomic operation, please use 64 bit version";
301 switch (
Op.getOpcode()) {
305 return LowerBR_CC(
Op, DAG);
307 return LowerGlobalAddress(
Op, DAG);
309 return LowerSELECT_CC(
Op, DAG);
316#include "BPFGenCallingConv.inc"
318SDValue BPFTargetLowering::LowerFormalArguments(
336 CCInfo.AnalyzeFormalArguments(Ins,
getHasAlu32() ? CC_BPF32 : CC_BPF64);
338 bool HasMemArgs =
false;
339 for (
size_t I = 0;
I < ArgLocs.
size(); ++
I) {
340 auto &VA = ArgLocs[
I];
344 EVT RegVT = VA.getLocVT();
358 SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
359 RegInfo.addLiveIn(VA.getLocReg(), VReg);
387 fail(
DL, DAG,
"stack arguments are not supported");
389 fail(
DL, DAG,
"variadic functions are not supported");
391 fail(
DL, DAG,
"aggregate returns are not supported");
396const size_t BPFTargetLowering::MaxArgs = 5;
401 auto &Outs = CLI.
Outs;
426 CCInfo.AnalyzeCallOperands(Outs,
getHasAlu32() ? CC_BPF32 : CC_BPF64);
428 unsigned NumBytes = CCInfo.getStackSize();
430 if (Outs.size() > MaxArgs)
431 fail(CLI.
DL, DAG,
"too many arguments", Callee);
433 for (
auto &Arg : Outs) {
435 if (!
Flags.isByVal())
437 fail(CLI.
DL, DAG,
"pass by value not supported", Callee);
447 for (
size_t i = 0; i < std::min(ArgLocs.
size(), MaxArgs); ++i) {
480 for (
auto &Reg : RegsToPass) {
495 "' is not supported."));
506 for (
auto &Reg : RegsToPass)
523 return LowerCallResult(Chain, InGlue, CallConv, IsVarArg, Ins, CLI.
DL, DAG,
543 fail(
DL, DAG,
"aggregate returns are not supported");
544 return DAG.
getNode(Opc,
DL, MVT::Other, Chain);
548 CCInfo.AnalyzeReturn(Outs,
getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
554 for (
size_t i = 0; i != RVLocs.
size(); ++i) {
571 RetOps.push_back(Glue);
573 return DAG.
getNode(Opc,
DL, MVT::Other, RetOps);
576SDValue BPFTargetLowering::LowerCallResult(
586 if (
Ins.size() > 1) {
587 fail(
DL, DAG,
"only small returns supported");
593 CCInfo.AnalyzeCallResult(Ins,
getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
596 for (
auto &Val : RVLocs) {
598 Val.getValVT(), InGlue).
getValue(1);
658 return "BPFISD::RET_GLUE";
660 return "BPFISD::CALL";
662 return "BPFISD::SELECT_CC";
664 return "BPFISD::BR_CC";
666 return "BPFISD::Wrapper";
668 return "BPFISD::MEMCPY";
675 auto *
N = cast<GlobalAddressSDNode>(
Op);
676 if (
N->getOffset() != 0)
689 unsigned Reg,
bool isSigned)
const {
692 int RShiftOp =
isSigned ? BPF::SRA_ri : BPF::SRL_ri;
720BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(
MachineInstr &
MI,
743 ScratchReg =
MRI.createVirtualRegister(&BPF::GPRRegClass);
744 MIB.addReg(ScratchReg,
755 unsigned Opc =
MI.getOpcode();
756 bool isSelectRROp = (Opc == BPF::Select ||
757 Opc == BPF::Select_64_32 ||
758 Opc == BPF::Select_32 ||
759 Opc == BPF::Select_32_64);
761 bool isMemcpyOp = Opc == BPF::MEMCPY;
764 bool isSelectRIOp = (Opc == BPF::Select_Ri ||
765 Opc == BPF::Select_Ri_64_32 ||
766 Opc == BPF::Select_Ri_32 ||
767 Opc == BPF::Select_Ri_32_64);
769 if (!(isSelectRROp || isSelectRIOp || isMemcpyOp))
774 return EmitInstrWithCustomInserterMemcpy(
MI, BB);
776 bool is32BitCmp = (Opc == BPF::Select_32 ||
777 Opc == BPF::Select_32_64 ||
778 Opc == BPF::Select_Ri_32 ||
779 Opc == BPF::Select_Ri_32_64);
798 F->insert(
I, Copy0MBB);
799 F->insert(
I, Copy1MBB);
810 int CC =
MI.getOperand(3).getImm();
813#define SET_NEWCC(X, Y) \
815 if (is32BitCmp && HasJmp32) \
816 NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
818 NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
847 if (is32BitCmp && !HasJmp32)
848 LHS = EmitSubregExt(
MI, BB,
LHS, isSignedCmp);
853 if (is32BitCmp && !HasJmp32)
854 RHS = EmitSubregExt(
MI, BB,
RHS, isSignedCmp);
858 int64_t imm32 =
MI.getOperand(2).getImm();
860 if (!isInt<32>(imm32))
884 MI.eraseFromParent();
895 return (
getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
898bool BPFTargetLowering::isLegalAddressingMode(
const DataLayout &
DL,
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis Results
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val={})
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
unsigned getCommonMaxStoresPerMemFunc() const
bool getHasJmpExt() const
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
const TargetRegisterInfo * getRegisterInfo() const override
BPFTargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool getHasJmpExt() const
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
LLVM Basic Block Representation.
CCState - This class holds information needed while lowering arguments and return values.
CCValAssign - Represent assignment of one arg/retval to a location.
Register getLocReg() const
LocInfo getLocInfo() const
This class represents an Operation in the Expression.
A parsed version of the target data layout string in and methods for querying it.
Diagnostic information for unsupported feature in backend.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Type * getReturnType() const
Returns the type of the ret val.
This is an important class for using LLVM in a threaded context.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
static auto integer_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Wrapper class representing virtual and physical registers.
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
MachineFunction & getMachineFunction() const
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
LLVMContext * getContext() const
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StringRef - Represent a constant reference to a string, i.e.
constexpr size_t size() const
size - Get the string size.
TargetInstrInfo - Interface to description of machine instruction set.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
@ ZeroOrOneBooleanContent
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Primary interface to the complete machine description for the target machine.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
The instances of the Type class are immutable: once they are created, they are never changed.
bool isAggregateType() const
Return true if the type is an aggregate type.
bool isIntegerTy() const
True if this is an instance of IntegerType.
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
self_iterator getIterator()
A raw_ostream that writes to an std::string.
@ Fast
Attempts to make calls as fast as possible (e.g.
@ C
The default llvm calling convention, compatible with C.
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
@ BSWAP
Byte Swap and Counting operators.
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
@ SIGN_EXTEND
Conversion operators.
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
@ BR_CC
BR_CC - Conditional branch.
@ BRIND
BRIND - Indirect branch.
@ BR_JT
BR_JT - Jumptable branch.
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
@ Define
Register definition.
@ EarlyClobber
Register definition happens before uses.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
This is an optimization pass for GlobalISel generic memory operations.
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
detail::concat_range< ValueT, RangeTs... > concat(RangeTs &&... Ranges)
Concatenated range across two or more ranges.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
This struct is a compact representation of a valid (non-zero power of two) alignment.
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
void print(raw_ostream &OS) const
Implement operator<<.
bool isInteger() const
Return true if this is an integer or a vector integer type.
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals