LLVM  15.0.0git
BPFISelLowering.cpp
Go to the documentation of this file.
1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that BPF uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "BPFISelLowering.h"
15 #include "BPF.h"
16 #include "BPFSubtarget.h"
17 #include "BPFTargetMachine.h"
25 #include "llvm/IR/DiagnosticInfo.h"
27 #include "llvm/Support/Debug.h"
30 using namespace llvm;
31 
32 #define DEBUG_TYPE "bpf-lower"
33 
34 static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
35  cl::Hidden, cl::init(false),
36  cl::desc("Expand memcpy into load/store pairs in order"));
37 
38 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) {
40  DAG.getContext()->diagnose(
41  DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
42 }
43 
44 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg,
45  SDValue Val) {
47  std::string Str;
48  raw_string_ostream OS(Str);
49  OS << Msg;
50  Val->print(OS);
51  OS.flush();
52  DAG.getContext()->diagnose(
53  DiagnosticInfoUnsupported(MF.getFunction(), Str, DL.getDebugLoc()));
54 }
55 
57  const BPFSubtarget &STI)
58  : TargetLowering(TM) {
59 
60  // Set up the register classes.
61  addRegisterClass(MVT::i64, &BPF::GPRRegClass);
62  if (STI.getHasAlu32())
63  addRegisterClass(MVT::i32, &BPF::GPR32RegClass);
64 
65  // Compute derived properties from the register classes
67 
69 
74 
76 
80 
81  // Set unsupported atomic operations as Custom so
82  // we can emit better error messages than fatal error
83  // from selectiondag.
84  for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
85  if (VT == MVT::i32) {
86  if (STI.getHasAlu32())
87  continue;
88  } else {
90  }
91 
97  }
98 
99  for (auto VT : { MVT::i32, MVT::i64 }) {
100  if (VT == MVT::i32 && !STI.getHasAlu32())
101  continue;
102 
116 
120  }
121 
122  if (STI.getHasAlu32()) {
125  STI.getHasJmp32() ? Custom : Promote);
126  }
127 
132 
137 
138  // Extended load operations for i1 types must be promoted
139  for (MVT VT : MVT::integer_valuetypes()) {
143 
147  }
148 
150 
151  // Function alignments
154 
156  // LLVM generic code will try to expand memcpy into load/store pairs at this
157  // stage which is before quite a few IR optimization passes, therefore the
158  // loads and stores could potentially be moved apart from each other which
159  // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
160  // compilers.
161  //
162  // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
163  // of memcpy to later stage in IR optimization pipeline so those load/store
164  // pairs won't be touched and could be kept in order. Hence, we set
165  // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
166  // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
170  MaxLoadsPerMemcmp = 0;
171  } else {
172  // inline memcpy() for kernel to see explicit copy
173  unsigned CommonMaxStores =
175 
176  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
177  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
179  MaxLoadsPerMemcmp = MaxLoadsPerMemcmpOptSize = CommonMaxStores;
180  }
181 
182  // CPU/Feature control
183  HasAlu32 = STI.getHasAlu32();
184  HasJmp32 = STI.getHasJmp32();
185  HasJmpExt = STI.getHasJmpExt();
186 }
187 
189  return false;
190 }
191 
192 bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
193  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
194  return false;
195  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
196  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
197  return NumBits1 > NumBits2;
198 }
199 
200 bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
201  if (!VT1.isInteger() || !VT2.isInteger())
202  return false;
203  unsigned NumBits1 = VT1.getSizeInBits();
204  unsigned NumBits2 = VT2.getSizeInBits();
205  return NumBits1 > NumBits2;
206 }
207 
208 bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
209  if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
210  return false;
211  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
212  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
213  return NumBits1 == 32 && NumBits2 == 64;
214 }
215 
216 bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
217  if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
218  return false;
219  unsigned NumBits1 = VT1.getSizeInBits();
220  unsigned NumBits2 = VT2.getSizeInBits();
221  return NumBits1 == 32 && NumBits2 == 64;
222 }
223 
226  if (Constraint.size() == 1) {
227  switch (Constraint[0]) {
228  default:
229  break;
230  case 'w':
231  return C_RegisterClass;
232  }
233  }
234 
235  return TargetLowering::getConstraintType(Constraint);
236 }
237 
238 std::pair<unsigned, const TargetRegisterClass *>
240  StringRef Constraint,
241  MVT VT) const {
242  if (Constraint.size() == 1)
243  // GCC Constraint Letters
244  switch (Constraint[0]) {
245  case 'r': // GENERAL_REGS
246  return std::make_pair(0U, &BPF::GPRRegClass);
247  case 'w':
248  if (HasAlu32)
249  return std::make_pair(0U, &BPF::GPR32RegClass);
250  break;
251  default:
252  break;
253  }
254 
255  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
256 }
257 
258 void BPFTargetLowering::ReplaceNodeResults(
260  const char *err_msg;
261  uint32_t Opcode = N->getOpcode();
262  switch (Opcode) {
263  default:
264  report_fatal_error("Unhandled custom legalization");
267  case ISD::ATOMIC_LOAD_OR:
269  case ISD::ATOMIC_SWAP:
271  if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD)
272  err_msg = "Unsupported atomic operations, please use 32/64 bit version";
273  else
274  err_msg = "Unsupported atomic operations, please use 64 bit version";
275  break;
276  }
277 
278  SDLoc DL(N);
279  fail(DL, DAG, err_msg);
280 }
281 
283  switch (Op.getOpcode()) {
284  case ISD::BR_CC:
285  return LowerBR_CC(Op, DAG);
286  case ISD::GlobalAddress:
287  return LowerGlobalAddress(Op, DAG);
288  case ISD::SELECT_CC:
289  return LowerSELECT_CC(Op, DAG);
291  report_fatal_error("Unsupported dynamic stack allocation");
292  default:
293  llvm_unreachable("unimplemented operand");
294  }
295 }
296 
297 // Calling Convention Implementation
298 #include "BPFGenCallingConv.inc"
299 
300 SDValue BPFTargetLowering::LowerFormalArguments(
301  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
303  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
304  switch (CallConv) {
305  default:
306  report_fatal_error("Unsupported calling convention");
307  case CallingConv::C:
308  case CallingConv::Fast:
309  break;
310  }
311 
313  MachineRegisterInfo &RegInfo = MF.getRegInfo();
314 
315  // Assign locations to all of the incoming arguments.
317  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
318  CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64);
319 
320  for (auto &VA : ArgLocs) {
321  if (VA.isRegLoc()) {
322  // Arguments passed in registers
323  EVT RegVT = VA.getLocVT();
324  MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
325  switch (SimpleTy) {
326  default: {
327  errs() << "LowerFormalArguments Unhandled argument type: "
328  << RegVT.getEVTString() << '\n';
329  llvm_unreachable(nullptr);
330  }
331  case MVT::i32:
332  case MVT::i64:
333  Register VReg = RegInfo.createVirtualRegister(
334  SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
335  RegInfo.addLiveIn(VA.getLocReg(), VReg);
336  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
337 
338  // If this is an value that has been promoted to wider types, insert an
339  // assert[sz]ext to capture this, then truncate to the right size.
340  if (VA.getLocInfo() == CCValAssign::SExt)
341  ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue,
342  DAG.getValueType(VA.getValVT()));
343  else if (VA.getLocInfo() == CCValAssign::ZExt)
344  ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue,
345  DAG.getValueType(VA.getValVT()));
346 
347  if (VA.getLocInfo() != CCValAssign::Full)
348  ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue);
349 
350  InVals.push_back(ArgValue);
351 
352  break;
353  }
354  } else {
355  fail(DL, DAG, "defined with too many args");
356  InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT()));
357  }
358  }
359 
360  if (IsVarArg || MF.getFunction().hasStructRetAttr()) {
361  fail(DL, DAG, "functions with VarArgs or StructRet are not supported");
362  }
363 
364  return Chain;
365 }
366 
367 const unsigned BPFTargetLowering::MaxArgs = 5;
368 
369 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
370  SmallVectorImpl<SDValue> &InVals) const {
371  SelectionDAG &DAG = CLI.DAG;
372  auto &Outs = CLI.Outs;
373  auto &OutVals = CLI.OutVals;
374  auto &Ins = CLI.Ins;
375  SDValue Chain = CLI.Chain;
376  SDValue Callee = CLI.Callee;
377  bool &IsTailCall = CLI.IsTailCall;
378  CallingConv::ID CallConv = CLI.CallConv;
379  bool IsVarArg = CLI.IsVarArg;
381 
382  // BPF target does not support tail call optimization.
383  IsTailCall = false;
384 
385  switch (CallConv) {
386  default:
387  report_fatal_error("Unsupported calling convention");
388  case CallingConv::Fast:
389  case CallingConv::C:
390  break;
391  }
392 
393  // Analyze operands of the call, assigning locations to each operand.
395  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
396 
397  CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64);
398 
399  unsigned NumBytes = CCInfo.getNextStackOffset();
400 
401  if (Outs.size() > MaxArgs)
402  fail(CLI.DL, DAG, "too many args to ", Callee);
403 
404  for (auto &Arg : Outs) {
405  ISD::ArgFlagsTy Flags = Arg.Flags;
406  if (!Flags.isByVal())
407  continue;
408 
409  fail(CLI.DL, DAG, "pass by value not supported ", Callee);
410  }
411 
412  auto PtrVT = getPointerTy(MF.getDataLayout());
413  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
414 
415  SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
416 
417  // Walk arg assignments
418  for (unsigned i = 0,
419  e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs);
420  i != e; ++i) {
421  CCValAssign &VA = ArgLocs[i];
422  SDValue Arg = OutVals[i];
423 
424  // Promote the value if needed.
425  switch (VA.getLocInfo()) {
426  default:
427  llvm_unreachable("Unknown loc info");
428  case CCValAssign::Full:
429  break;
430  case CCValAssign::SExt:
431  Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg);
432  break;
433  case CCValAssign::ZExt:
434  Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg);
435  break;
436  case CCValAssign::AExt:
437  Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg);
438  break;
439  }
440 
441  // Push arguments into RegsToPass vector
442  if (VA.isRegLoc())
443  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
444  else
445  llvm_unreachable("call arg pass bug");
446  }
447 
448  SDValue InFlag;
449 
450  // Build a sequence of copy-to-reg nodes chained together with token chain and
451  // flag operands which copy the outgoing args into registers. The InFlag in
452  // necessary since all emitted instructions must be stuck together.
453  for (auto &Reg : RegsToPass) {
454  Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag);
455  InFlag = Chain.getValue(1);
456  }
457 
458  // If the callee is a GlobalAddress node (quite common, every direct call is)
459  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
460  // Likewise ExternalSymbol -> TargetExternalSymbol.
461  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
462  Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
463  G->getOffset(), 0);
464  } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
465  Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
466  fail(CLI.DL, DAG, Twine("A call to built-in function '"
467  + StringRef(E->getSymbol())
468  + "' is not supported."));
469  }
470 
471  // Returns a chain & a flag for retval copy to use.
472  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
474  Ops.push_back(Chain);
475  Ops.push_back(Callee);
476 
477  // Add argument registers to the end of the list so that they are
478  // known live into the call.
479  for (auto &Reg : RegsToPass)
480  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
481 
482  if (InFlag.getNode())
483  Ops.push_back(InFlag);
484 
485  Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops);
486  InFlag = Chain.getValue(1);
487 
488  // Create the CALLSEQ_END node.
489  Chain = DAG.getCALLSEQ_END(
490  Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true),
491  DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL);
492  InFlag = Chain.getValue(1);
493 
494  // Handle result values, copying them out of physregs into vregs that we
495  // return.
496  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG,
497  InVals);
498 }
499 
500 SDValue
501 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
502  bool IsVarArg,
504  const SmallVectorImpl<SDValue> &OutVals,
505  const SDLoc &DL, SelectionDAG &DAG) const {
506  unsigned Opc = BPFISD::RET_FLAG;
507 
508  // CCValAssign - represent the assignment of the return value to a location
511 
512  // CCState - Info about the registers and stack slot.
513  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
514 
515  if (MF.getFunction().getReturnType()->isAggregateType()) {
516  fail(DL, DAG, "only integer returns supported");
517  return DAG.getNode(Opc, DL, MVT::Other, Chain);
518  }
519 
520  // Analize return values.
521  CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
522 
523  SDValue Flag;
524  SmallVector<SDValue, 4> RetOps(1, Chain);
525 
526  // Copy the result values into the output registers.
527  for (unsigned i = 0; i != RVLocs.size(); ++i) {
528  CCValAssign &VA = RVLocs[i];
529  assert(VA.isRegLoc() && "Can only return in registers!");
530 
531  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
532 
533  // Guarantee that all emitted copies are stuck together,
534  // avoiding something bad.
535  Flag = Chain.getValue(1);
536  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
537  }
538 
539  RetOps[0] = Chain; // Update chain.
540 
541  // Add the flag if we have it.
542  if (Flag.getNode())
543  RetOps.push_back(Flag);
544 
545  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
546 }
547 
548 SDValue BPFTargetLowering::LowerCallResult(
549  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
551  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
552 
554  // Assign locations to each value returned by this call.
556  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
557 
558  if (Ins.size() >= 2) {
559  fail(DL, DAG, "only small returns supported");
560  for (unsigned i = 0, e = Ins.size(); i != e; ++i)
561  InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT));
562  return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1);
563  }
564 
565  CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
566 
567  // Copy all of the result registers out of their specified physreg.
568  for (auto &Val : RVLocs) {
569  Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(),
570  Val.getValVT(), InFlag).getValue(1);
571  InFlag = Chain.getValue(2);
572  InVals.push_back(Chain.getValue(0));
573  }
574 
575  return Chain;
576 }
577 
578 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
579  switch (CC) {
580  default:
581  break;
582  case ISD::SETULT:
583  case ISD::SETULE:
584  case ISD::SETLT:
585  case ISD::SETLE:
587  std::swap(LHS, RHS);
588  break;
589  }
590 }
591 
592 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
593  SDValue Chain = Op.getOperand(0);
594  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
595  SDValue LHS = Op.getOperand(2);
596  SDValue RHS = Op.getOperand(3);
597  SDValue Dest = Op.getOperand(4);
598  SDLoc DL(Op);
599 
600  if (!getHasJmpExt())
601  NegateCC(LHS, RHS, CC);
602 
603  return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
604  DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
605 }
606 
607 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
608  SDValue LHS = Op.getOperand(0);
609  SDValue RHS = Op.getOperand(1);
610  SDValue TrueV = Op.getOperand(2);
611  SDValue FalseV = Op.getOperand(3);
612  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
613  SDLoc DL(Op);
614 
615  if (!getHasJmpExt())
616  NegateCC(LHS, RHS, CC);
617 
618  SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
619  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
620  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
621 
622  return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops);
623 }
624 
625 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
626  switch ((BPFISD::NodeType)Opcode) {
628  break;
629  case BPFISD::RET_FLAG:
630  return "BPFISD::RET_FLAG";
631  case BPFISD::CALL:
632  return "BPFISD::CALL";
633  case BPFISD::SELECT_CC:
634  return "BPFISD::SELECT_CC";
635  case BPFISD::BR_CC:
636  return "BPFISD::BR_CC";
637  case BPFISD::Wrapper:
638  return "BPFISD::Wrapper";
639  case BPFISD::MEMCPY:
640  return "BPFISD::MEMCPY";
641  }
642  return nullptr;
643 }
644 
645 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
646  SelectionDAG &DAG) const {
647  auto N = cast<GlobalAddressSDNode>(Op);
648  assert(N->getOffset() == 0 && "Invalid offset for global address");
649 
650  SDLoc DL(Op);
651  const GlobalValue *GV = N->getGlobal();
652  SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64);
653 
654  return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA);
655 }
656 
657 unsigned
658 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
659  unsigned Reg, bool isSigned) const {
660  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
662  int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
663  MachineFunction *F = BB->getParent();
664  DebugLoc DL = MI.getDebugLoc();
665 
666  MachineRegisterInfo &RegInfo = F->getRegInfo();
667 
668  if (!isSigned) {
669  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
670  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
671  return PromotedReg0;
672  }
673  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
674  Register PromotedReg1 = RegInfo.createVirtualRegister(RC);
675  Register PromotedReg2 = RegInfo.createVirtualRegister(RC);
676  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
677  BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1)
678  .addReg(PromotedReg0).addImm(32);
679  BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2)
680  .addReg(PromotedReg1).addImm(32);
681 
682  return PromotedReg2;
683 }
684 
686 BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
688  const {
689  MachineFunction *MF = MI.getParent()->getParent();
691  MachineInstrBuilder MIB(*MF, MI);
692  unsigned ScratchReg;
693 
694  // This function does custom insertion during lowering BPFISD::MEMCPY which
695  // only has two register operands from memcpy semantics, the copy source
696  // address and the copy destination address.
697  //
698  // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
699  // a third scratch register to serve as the destination register of load and
700  // source register of store.
701  //
702  // The scratch register here is with the Define | Dead | EarlyClobber flags.
703  // The EarlyClobber flag has the semantic property that the operand it is
704  // attached to is clobbered before the rest of the inputs are read. Hence it
705  // must be unique among the operands to the instruction. The Define flag is
706  // needed to coerce the machine verifier that an Undef value isn't a problem
707  // as we anyway is loading memory into it. The Dead flag is needed as the
708  // value in scratch isn't supposed to be used by any other instruction.
709  ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass);
710  MIB.addReg(ScratchReg,
712 
713  return BB;
714 }
715 
718  MachineBasicBlock *BB) const {
719  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
720  DebugLoc DL = MI.getDebugLoc();
721  unsigned Opc = MI.getOpcode();
722  bool isSelectRROp = (Opc == BPF::Select ||
723  Opc == BPF::Select_64_32 ||
724  Opc == BPF::Select_32 ||
725  Opc == BPF::Select_32_64);
726 
727  bool isMemcpyOp = Opc == BPF::MEMCPY;
728 
729 #ifndef NDEBUG
730  bool isSelectRIOp = (Opc == BPF::Select_Ri ||
731  Opc == BPF::Select_Ri_64_32 ||
732  Opc == BPF::Select_Ri_32 ||
733  Opc == BPF::Select_Ri_32_64);
734 
735 
736  assert((isSelectRROp || isSelectRIOp || isMemcpyOp) &&
737  "Unexpected instr type to insert");
738 #endif
739 
740  if (isMemcpyOp)
741  return EmitInstrWithCustomInserterMemcpy(MI, BB);
742 
743  bool is32BitCmp = (Opc == BPF::Select_32 ||
744  Opc == BPF::Select_32_64 ||
745  Opc == BPF::Select_Ri_32 ||
746  Opc == BPF::Select_Ri_32_64);
747 
748  // To "insert" a SELECT instruction, we actually have to insert the diamond
749  // control-flow pattern. The incoming instruction knows the destination vreg
750  // to set, the condition code register to branch on, the true/false values to
751  // select between, and a branch opcode to use.
752  const BasicBlock *LLVM_BB = BB->getBasicBlock();
753  MachineFunction::iterator I = ++BB->getIterator();
754 
755  // ThisMBB:
756  // ...
757  // TrueVal = ...
758  // jmp_XX r1, r2 goto Copy1MBB
759  // fallthrough --> Copy0MBB
760  MachineBasicBlock *ThisMBB = BB;
761  MachineFunction *F = BB->getParent();
762  MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
763  MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB);
764 
765  F->insert(I, Copy0MBB);
766  F->insert(I, Copy1MBB);
767  // Update machine-CFG edges by transferring all successors of the current
768  // block to the new block which will contain the Phi node for the select.
769  Copy1MBB->splice(Copy1MBB->begin(), BB,
770  std::next(MachineBasicBlock::iterator(MI)), BB->end());
772  // Next, add the true and fallthrough blocks as its successors.
773  BB->addSuccessor(Copy0MBB);
774  BB->addSuccessor(Copy1MBB);
775 
776  // Insert Branch if Flag
777  int CC = MI.getOperand(3).getImm();
778  int NewCC;
779  switch (CC) {
780 #define SET_NEWCC(X, Y) \
781  case ISD::X: \
782  if (is32BitCmp && HasJmp32) \
783  NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
784  else \
785  NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
786  break
787  SET_NEWCC(SETGT, JSGT);
788  SET_NEWCC(SETUGT, JUGT);
789  SET_NEWCC(SETGE, JSGE);
790  SET_NEWCC(SETUGE, JUGE);
791  SET_NEWCC(SETEQ, JEQ);
792  SET_NEWCC(SETNE, JNE);
793  SET_NEWCC(SETLT, JSLT);
794  SET_NEWCC(SETULT, JULT);
795  SET_NEWCC(SETLE, JSLE);
796  SET_NEWCC(SETULE, JULE);
797  default:
798  report_fatal_error("unimplemented select CondCode " + Twine(CC));
799  }
800 
801  Register LHS = MI.getOperand(1).getReg();
802  bool isSignedCmp = (CC == ISD::SETGT ||
803  CC == ISD::SETGE ||
804  CC == ISD::SETLT ||
805  CC == ISD::SETLE);
806 
807  // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
808  // to be promoted, however if the 32-bit comparison operands are destination
809  // registers then they are implicitly zero-extended already, there is no
810  // need of explicit zero-extend sequence for them.
811  //
812  // We simply do extension for all situations in this method, but we will
813  // try to remove those unnecessary in BPFMIPeephole pass.
814  if (is32BitCmp && !HasJmp32)
815  LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
816 
817  if (isSelectRROp) {
818  Register RHS = MI.getOperand(2).getReg();
819 
820  if (is32BitCmp && !HasJmp32)
821  RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
822 
823  BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
824  } else {
825  int64_t imm32 = MI.getOperand(2).getImm();
826  // Check before we build J*_ri instruction.
827  assert (isInt<32>(imm32));
828  BuildMI(BB, DL, TII.get(NewCC))
829  .addReg(LHS).addImm(imm32).addMBB(Copy1MBB);
830  }
831 
832  // Copy0MBB:
833  // %FalseValue = ...
834  // # fallthrough to Copy1MBB
835  BB = Copy0MBB;
836 
837  // Update machine-CFG edges
838  BB->addSuccessor(Copy1MBB);
839 
840  // Copy1MBB:
841  // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
842  // ...
843  BB = Copy1MBB;
844  BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg())
845  .addReg(MI.getOperand(5).getReg())
846  .addMBB(Copy0MBB)
847  .addReg(MI.getOperand(4).getReg())
848  .addMBB(ThisMBB);
849 
850  MI.eraseFromParent(); // The pseudo instruction is gone now.
851  return BB;
852 }
853 
855  EVT VT) const {
856  return getHasAlu32() ? MVT::i32 : MVT::i64;
857 }
858 
860  EVT VT) const {
861  return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
862 }
863 
864 bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL,
865  const AddrMode &AM, Type *Ty,
866  unsigned AS,
867  Instruction *I) const {
868  // No global is ever allowed as a base.
869  if (AM.BaseGV)
870  return false;
871 
872  switch (AM.Scale) {
873  case 0: // "r+i" or just "i", depending on HasBaseReg.
874  break;
875  case 1:
876  if (!AM.HasBaseReg) // allow "r+i".
877  break;
878  return false; // disallow "r+r" or "r+r+i".
879  default:
880  return false;
881  }
882 
883  return true;
884 }
llvm::CCValAssign::getLocVT
MVT getLocVT() const
Definition: CallingConvLower.h:151
llvm::MachineRegisterInfo::addLiveIn
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Definition: MachineRegisterInfo.h:954
i
i
Definition: README.txt:29
llvm::ISD::SETUGE
@ SETUGE
Definition: ISDOpcodes.h:1424
llvm::TargetLoweringBase::MaxStoresPerMemsetOptSize
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3335
llvm::CCValAssign::ZExt
@ ZExt
Definition: CallingConvLower.h:36
ValueTypes.h
llvm::BPFTargetLowering::getTargetNodeName
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Definition: BPFISelLowering.cpp:625
llvm::SelectionDAG::getCALLSEQ_START
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:943
llvm::ISD::SETLE
@ SETLE
Definition: ISDOpcodes.h:1435
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:104
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:17
BPFSubtarget.h
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1090
llvm::CCValAssign::Full
@ Full
Definition: CallingConvLower.h:34
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:113
llvm::ISD::SETGT
@ SETGT
Definition: ISDOpcodes.h:1432
llvm::ISD::SETNE
@ SETNE
Definition: ISDOpcodes.h:1436
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:156
llvm::TargetLowering::ConstraintType
ConstraintType
Definition: TargetLowering.h:4419
llvm::ISD::BR_JT
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:991
llvm::DiagnosticInfoUnsupported
Diagnostic information for unsupported feature in backend.
Definition: DiagnosticInfo.h:1009
llvm::BPFSubtarget::getHasJmp32
bool getHasJmp32() const
Definition: BPFSubtarget.h:72
llvm::CCState
CCState - This class holds information needed while lowering arguments and return values.
Definition: CallingConvLower.h:189
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:750
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:50
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:151
llvm::ISD::BSWAP
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:700
llvm::ISD::DYNAMIC_STACKALLOC
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:976
llvm::BPFISD::MEMCPY
@ MEMCPY
Definition: BPFISelLowering.h:31
llvm::ISD::CTTZ_ZERO_UNDEF
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:708
llvm::SelectionDAG::getValueType
SDValue getValueType(EVT)
Definition: SelectionDAG.cpp:1798
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:632
llvm::BPFSubtarget::getRegisterInfo
const TargetRegisterInfo * getRegisterInfo() const override
Definition: BPFSubtarget.h:86
llvm::TargetLoweringBase::MaxStoresPerMemset
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
Definition: TargetLowering.h:3333
BPFISelLowering.h
llvm::BPFTargetLowering::getConstraintType
BPFTargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
Definition: BPFISelLowering.cpp:225
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1185
llvm::ISD::SETEQ
@ SETEQ
Definition: ISDOpcodes.h:1431
llvm::ISD::STACKRESTORE
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1057
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:9108
llvm::BPFTargetLowering::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFISelLowering.h:62
llvm::BPFTargetLowering::getHasAlu32
bool getHasAlu32() const
Definition: BPFISelLowering.h:60
ErrorHandling.h
llvm::X86Disassembler::Reg
Reg
All possible values of the reg field in the ModR/M byte.
Definition: X86DisassemblerDecoder.h:462
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:766
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:454
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:139
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:270
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:234
llvm::ISD::SETULE
@ SETULE
Definition: ISDOpcodes.h:1426
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::ISD::SHL_PARTS
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:749
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:736
llvm::ARMISD::MEMCPY
@ MEMCPY
Definition: ARMISelLowering.h:310
llvm::TargetLoweringBase::setMinFunctionAlignment
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
Definition: TargetLowering.h:2447
Results
Function Alias Analysis Results
Definition: AliasAnalysis.cpp:848
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:893
llvm::ISD::EXTLOAD
@ EXTLOAD
Definition: ISDOpcodes.h:1391
RHS
Value * RHS
Definition: X86PartialReduction.cpp:76
BPF.h
llvm::BPFISD::NodeType
NodeType
Definition: BPFISelLowering.h:24
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1628
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:462
llvm::ISD::ATOMIC_LOAD_OR
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1189
F
#define F(x, y, z)
Definition: MD5.cpp:55
llvm::ISD::BRCOND
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1001
MachineRegisterInfo.h
llvm::BPFTargetLowering::getRegForInlineAsmConstraint
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
Definition: BPFISelLowering.cpp:239
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:2061
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:55
llvm::RegState::EarlyClobber
@ EarlyClobber
Register definition happens before uses.
Definition: MachineInstrBuilder.h:54
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:33
llvm::ISD::BRIND
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:987
llvm::ISD::ROTL
@ ROTL
Definition: ISDOpcodes.h:694
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:186
llvm::MVT::integer_valuetypes
static auto integer_valuetypes()
Definition: MachineValueType.h:1461
llvm::BPFISD::BR_CC
@ BR_CC
Definition: BPFISelLowering.h:29
llvm::BPFISD::FIRST_NUMBER
@ FIRST_NUMBER
Definition: BPFISelLowering.h:25
LHS
Value * LHS
Definition: X86PartialReduction.cpp:75
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1008
llvm::BPFTargetLowering::getSetCCResultType
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
Definition: BPFISelLowering.cpp:854
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::TargetLowering::CallLoweringInfo::IsVarArg
bool IsVarArg
Definition: TargetLowering.h:4014
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:666
llvm::CCValAssign::AExt
@ AExt
Definition: CallingConvLower.h:37
llvm::ISD::GlobalAddress
@ GlobalAddress
Definition: ISDOpcodes.h:78
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:728
llvm::ExternalSymbolSDNode
Definition: SelectionDAGNodes.h:2207
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
llvm::CCValAssign
CCValAssign - Represent assignment of one arg/retval to a location.
Definition: CallingConvLower.h:31
llvm::BPFISD::SELECT_CC
@ SELECT_CC
Definition: BPFISelLowering.h:28
llvm::ISD::CTLZ
@ CTLZ
Definition: ISDOpcodes.h:702
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:146
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:220
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:713
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:763
llvm::ISD::ArgFlagsTy::isByVal
bool isByVal() const
Definition: TargetCallingConv.h:85
llvm::ISD::SETGE
@ SETGE
Definition: ISDOpcodes.h:1433
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::CCValAssign::getLocReg
Register getLocReg() const
Definition: CallingConvLower.h:148
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:781
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:34
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1449
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3412
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:45
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:769
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:329
llvm::BPFTargetLowering::BPFTargetLowering
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
Definition: BPFISelLowering.cpp:56
llvm::ISD::UDIVREM
@ UDIVREM
Definition: ISDOpcodes.h:256
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:125
llvm::TargetLoweringBase::addRegisterClass
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
Definition: TargetLowering.h:2284
llvm::TargetLoweringBase::MaxLoadsPerMemcmpOptSize
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3371
llvm::ISD::ATOMIC_LOAD_AND
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1187
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
llvm::Instruction
Definition: Instruction.h:42
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:147
llvm::BPFTargetLowering::EmitInstrWithCustomInserter
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: BPFISelLowering.cpp:717
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:143
llvm::raw_ostream::flush
void flush()
Definition: raw_ostream.h:187
llvm::TargetLoweringBase::MaxStoresPerMemcpy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
Definition: TargetLowering.h:3348
llvm::CCValAssign::getLocInfo
LocInfo getLocInfo() const
Definition: CallingConvLower.h:153
Align
uint64_t Align
Definition: ELFObjHandler.cpp:81
llvm::CCValAssign::isRegLoc
bool isRegLoc() const
Definition: CallingConvLower.h:143
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:94
llvm::ISD::SETUGT
@ SETUGT
Definition: ISDOpcodes.h:1423
llvm::SelectionDAG::getTargetGlobalAddress
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:698
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:191
llvm::TargetLowering::CallLoweringInfo::Outs
SmallVector< ISD::OutputArg, 32 > Outs
Definition: TargetLowering.h:4037
G
const DataFlowGraph & G
Definition: RDFGraph.cpp:200
llvm::TargetLoweringBase::setOperationAction
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
Definition: TargetLowering.h:2301
llvm::TargetLoweringBase::MaxStoresPerMemcpyOptSize
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3350
llvm::cl::opt< bool >
llvm::GlobalValue
Definition: GlobalValue.h:44
llvm::isInt< 32 >
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:373
llvm::BPFTargetLowering::LowerOperation
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Definition: BPFISelLowering.cpp:282
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:340
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:180
llvm::CCValAssign::SExt
@ SExt
Definition: CallingConvLower.h:35
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:66
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1176
llvm::TargetLowering::CallLoweringInfo::Chain
SDValue Chain
Definition: TargetLowering.h:4010
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::TargetLoweringBase::Promote
@ Promote
Definition: TargetLowering.h:197
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:776
llvm::TargetLowering::CallLoweringInfo::CallConv
CallingConv::ID CallConv
Definition: TargetLowering.h:4031
llvm::TargetLoweringBase::setStackPointerRegisterToSaveRestore
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
Definition: TargetLowering.h:2250
fail
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
Definition: BPFISelLowering.cpp:38
llvm::ISD::CTLZ_ZERO_UNDEF
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:709
I
#define I(x, y, z)
Definition: MD5.cpp:58
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:8838
llvm::TargetLoweringBase::setPrefFunctionAlignment
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
Definition: TargetLowering.h:2453
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:432
llvm::ISD::ATOMIC_LOAD_ADD
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1185
llvm::TargetLowering::CallLoweringInfo::DL
SDLoc DL
Definition: TargetLowering.h:4035
llvm::Function::hasStructRetAttr
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition: Function.h:645
llvm::ISD::ZEXTLOAD
@ ZEXTLOAD
Definition: ISDOpcodes.h:1391
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:171
llvm::MVT::i8
@ i8
Definition: MachineValueType.h:46
llvm::TargetLowering::CallLoweringInfo
This structure contains all information that is necessary for lowering calls.
Definition: TargetLowering.h:4009
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::BPFSubtarget::getSelectionDAGInfo
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
Definition: BPFSubtarget.h:83
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:77
llvm::ISD::MULHS
@ MULHS
Definition: ISDOpcodes.h:638
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:853
llvm::ISD::SETULT
@ SETULT
Definition: ISDOpcodes.h:1425
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::SDNode::print
void print(raw_ostream &OS, const SelectionDAG *G=nullptr) const
Definition: SelectionDAGDumper.cpp:1056
BPFExpandMemcpyInOrder
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1411
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::ISD::SRA_PARTS
@ SRA_PARTS
Definition: ISDOpcodes.h:750
NegateCC
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
Definition: BPFISelLowering.cpp:578
llvm::MachineFunction
Definition: MachineFunction.h:257
llvm::SelectionDAG::getCALLSEQ_END
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
Definition: SelectionDAG.h:955
llvm::CallingConv::C
@ C
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::TargetLowering::CallLoweringInfo::Ins
SmallVector< ISD::InputArg, 32 > Ins
Definition: TargetLowering.h:4039
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
llvm::TargetLowering::CallLoweringInfo::DAG
SelectionDAG & DAG
Definition: TargetLowering.h:4034
llvm::BPFSelectionDAGInfo::getCommonMaxStoresPerMemFunc
unsigned getCommonMaxStoresPerMemFunc() const
Definition: BPFSelectionDAGInfo.h:29
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:49
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:982
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:143
llvm::ISD::SREM
@ SREM
Definition: ISDOpcodes.h:244
llvm::ISD::UMUL_LOHI
@ UMUL_LOHI
Definition: ISDOpcodes.h:251
uint32_t
llvm::ISD::ArgFlagsTy
Definition: TargetCallingConv.h:27
llvm::TargetLoweringBase::MaxStoresPerMemmoveOptSize
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3385
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
AddrMode
AddrMode
Definition: MSP430Disassembler.cpp:142
llvm::BPFISD::CALL
@ CALL
Definition: BPFISelLowering.h:27
llvm::ISD::SMUL_LOHI
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
llvm::TargetLowering::getRegForInlineAsmConstraint
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Definition: TargetLowering.cpp:5103
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:78
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1184
llvm::ISD::SEXTLOAD
@ SEXTLOAD
Definition: ISDOpcodes.h:1391
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::TargetLowering::C_RegisterClass
@ C_RegisterClass
Definition: TargetLowering.h:4421
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
llvm::AMDGPU::SendMsg::Msg
const CustomOperand< const MCSubtargetInfo & > Msg[]
Definition: AMDGPUAsmUtils.cpp:39
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:186
SET_NEWCC
#define SET_NEWCC(X, Y)
llvm::EVT::getEVTString
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
Definition: ValueTypes.cpp:152
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:164
llvm::StringRef::size
constexpr LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:157
llvm::TargetLowering::CallLoweringInfo::IsTailCall
bool IsTailCall
Definition: TargetLowering.h:4025
llvm::ISD::SETLT
@ SETLT
Definition: ISDOpcodes.h:1434
llvm::LLVMContext::diagnose
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition: LLVMContext.cpp:243
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::BPFTargetLowering::getScalarShiftAmountTy
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
Definition: BPFISelLowering.cpp:859
llvm::TargetLoweringBase::setLoadExtAction
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
Definition: TargetLowering.h:2318
llvm::GlobalAddressSDNode
Definition: SelectionDAGNodes.h:1734
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:622
CallingConvLower.h
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:345
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::ISD::getSetCCSwappedOperands
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
Definition: SelectionDAG.cpp:491
DiagnosticInfo.h
llvm::BPFSubtarget::getHasAlu32
bool getHasAlu32() const
Definition: BPFSubtarget.h:73
llvm::TargetLoweringBase::Custom
@ Custom
Definition: TargetLowering.h:200
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::SelectionDAG::getTargetExternalSymbol
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.cpp:1829
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:48
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:137
llvm::TargetLoweringBase::ZeroOrOneBooleanContent
@ ZeroOrOneBooleanContent
Definition: TargetLowering.h:232
llvm::ISD::SRL_PARTS
@ SRL_PARTS
Definition: ISDOpcodes.h:751
llvm::BPFISD::Wrapper
@ Wrapper
Definition: BPFISelLowering.h:30
llvm::TargetLoweringBase::setBooleanContents
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
Definition: TargetLowering.h:2218
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:278
MachineInstrBuilder.h
llvm::TargetLoweringBase::Expand
@ Expand
Definition: TargetLowering.h:198
llvm::BPFTargetLowering::isOffsetFoldingLegal
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
Definition: BPFISelLowering.cpp:188
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
N
#define N
llvm::TargetLoweringBase::computeRegisterProperties
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
Definition: TargetLoweringBase.cpp:1277
llvm::ISD::CTTZ
@ CTTZ
Definition: ISDOpcodes.h:701
llvm::TargetLoweringBase::getRegClassFor
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
Definition: TargetLowering.h:891
llvm::MachineFunction::getDataLayout
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Definition: MachineFunction.cpp:285
llvm::MipsISD::Ins
@ Ins
Definition: MipsISelLowering.h:160
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:637
llvm::TargetLowering::CallLoweringInfo::OutVals
SmallVector< SDValue, 32 > OutVals
Definition: TargetLowering.h:4038
llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
Definition: MachineBasicBlock.cpp:886
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::TargetLowering::CallLoweringInfo::Callee
SDValue Callee
Definition: TargetLowering.h:4032
llvm::BPFSubtarget::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFSubtarget.h:71
llvm::MVT::i16
@ i16
Definition: MachineValueType.h:47
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:449
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:257
llvm::BPFISD::RET_FLAG
@ RET_FLAG
Definition: BPFISelLowering.h:26
llvm::ISD::SDIVREM
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
llvm::TargetLoweringBase::MaxLoadsPerMemcmp
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
Definition: TargetLowering.h:3369
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:405
llvm::ISD::ATOMIC_LOAD_XOR
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1190
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:760
raw_ostream.h
MachineFunction.h
DiagnosticPrinter.h
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::ISD::STACKSAVE
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1053
llvm::TargetLoweringBase::MaxStoresPerMemmove
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
Definition: TargetLowering.h:3383
llvm::ISD::ROTR
@ ROTR
Definition: ISDOpcodes.h:695
Debug.h
llvm::TargetLowering::getConstraintType
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
Definition: TargetLowering.cpp:4964
llvm::TargetLoweringBase::getPointerTy
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Definition: TargetLowering.h:354
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:703
llvm::Type::getPrimitiveSizeInBits
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:164
llvm::BPFSubtarget
Definition: BPFSubtarget.h:31
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:288
BPFTargetMachine.h
TargetLoweringObjectFileImpl.h