LLVM  14.0.0git
BPFISelLowering.cpp
Go to the documentation of this file.
1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that BPF uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "BPFISelLowering.h"
15 #include "BPF.h"
16 #include "BPFSubtarget.h"
17 #include "BPFTargetMachine.h"
25 #include "llvm/IR/DiagnosticInfo.h"
27 #include "llvm/Support/Debug.h"
30 using namespace llvm;
31 
32 #define DEBUG_TYPE "bpf-lower"
33 
34 static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
35  cl::Hidden, cl::init(false),
36  cl::desc("Expand memcpy into load/store pairs in order"));
37 
38 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) {
40  DAG.getContext()->diagnose(
41  DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
42 }
43 
44 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg,
45  SDValue Val) {
47  std::string Str;
48  raw_string_ostream OS(Str);
49  OS << Msg;
50  Val->print(OS);
51  OS.flush();
52  DAG.getContext()->diagnose(
53  DiagnosticInfoUnsupported(MF.getFunction(), Str, DL.getDebugLoc()));
54 }
55 
57  const BPFSubtarget &STI)
58  : TargetLowering(TM) {
59 
60  // Set up the register classes.
61  addRegisterClass(MVT::i64, &BPF::GPRRegClass);
62  if (STI.getHasAlu32())
63  addRegisterClass(MVT::i32, &BPF::GPR32RegClass);
64 
65  // Compute derived properties from the register classes
67 
69 
74 
76 
80 
81  // Set unsupported atomic operations as Custom so
82  // we can emit better error messages than fatal error
83  // from selectiondag.
84  for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
85  if (VT == MVT::i32) {
86  if (STI.getHasAlu32())
87  continue;
88  } else {
90  }
91 
97  }
98 
99  for (auto VT : { MVT::i32, MVT::i64 }) {
100  if (VT == MVT::i32 && !STI.getHasAlu32())
101  continue;
102 
117 
121  }
122 
123  if (STI.getHasAlu32()) {
126  STI.getHasJmp32() ? Custom : Promote);
127  }
128 
133 
138 
139  // Extended load operations for i1 types must be promoted
140  for (MVT VT : MVT::integer_valuetypes()) {
144 
148  }
149 
151 
152  // Function alignments
155 
157  // LLVM generic code will try to expand memcpy into load/store pairs at this
158  // stage which is before quite a few IR optimization passes, therefore the
159  // loads and stores could potentially be moved apart from each other which
160  // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
161  // compilers.
162  //
163  // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
164  // of memcpy to later stage in IR optimization pipeline so those load/store
165  // pairs won't be touched and could be kept in order. Hence, we set
166  // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
167  // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
171  } else {
172  // inline memcpy() for kernel to see explicit copy
173  unsigned CommonMaxStores =
175 
176  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
177  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
179  }
180 
181  // CPU/Feature control
182  HasAlu32 = STI.getHasAlu32();
183  HasJmp32 = STI.getHasJmp32();
184  HasJmpExt = STI.getHasJmpExt();
185 }
186 
188  return false;
189 }
190 
191 bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
192  if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
193  return false;
194  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
195  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
196  return NumBits1 > NumBits2;
197 }
198 
199 bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
200  if (!VT1.isInteger() || !VT2.isInteger())
201  return false;
202  unsigned NumBits1 = VT1.getSizeInBits();
203  unsigned NumBits2 = VT2.getSizeInBits();
204  return NumBits1 > NumBits2;
205 }
206 
207 bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
208  if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
209  return false;
210  unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
211  unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
212  return NumBits1 == 32 && NumBits2 == 64;
213 }
214 
215 bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
216  if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
217  return false;
218  unsigned NumBits1 = VT1.getSizeInBits();
219  unsigned NumBits2 = VT2.getSizeInBits();
220  return NumBits1 == 32 && NumBits2 == 64;
221 }
222 
225  if (Constraint.size() == 1) {
226  switch (Constraint[0]) {
227  default:
228  break;
229  case 'w':
230  return C_RegisterClass;
231  }
232  }
233 
234  return TargetLowering::getConstraintType(Constraint);
235 }
236 
237 std::pair<unsigned, const TargetRegisterClass *>
239  StringRef Constraint,
240  MVT VT) const {
241  if (Constraint.size() == 1)
242  // GCC Constraint Letters
243  switch (Constraint[0]) {
244  case 'r': // GENERAL_REGS
245  return std::make_pair(0U, &BPF::GPRRegClass);
246  case 'w':
247  if (HasAlu32)
248  return std::make_pair(0U, &BPF::GPR32RegClass);
249  break;
250  default:
251  break;
252  }
253 
254  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
255 }
256 
257 void BPFTargetLowering::ReplaceNodeResults(
259  const char *err_msg;
260  uint32_t Opcode = N->getOpcode();
261  switch (Opcode) {
262  default:
263  report_fatal_error("Unhandled custom legalization");
266  case ISD::ATOMIC_LOAD_OR:
268  case ISD::ATOMIC_SWAP:
270  if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD)
271  err_msg = "Unsupported atomic operations, please use 32/64 bit version";
272  else
273  err_msg = "Unsupported atomic operations, please use 64 bit version";
274  break;
275  }
276 
277  SDLoc DL(N);
278  fail(DL, DAG, err_msg);
279 }
280 
282  switch (Op.getOpcode()) {
283  case ISD::BR_CC:
284  return LowerBR_CC(Op, DAG);
285  case ISD::GlobalAddress:
286  return LowerGlobalAddress(Op, DAG);
287  case ISD::SELECT_CC:
288  return LowerSELECT_CC(Op, DAG);
290  report_fatal_error("Unsupported dynamic stack allocation");
291  default:
292  llvm_unreachable("unimplemented operand");
293  }
294 }
295 
296 // Calling Convention Implementation
297 #include "BPFGenCallingConv.inc"
298 
299 SDValue BPFTargetLowering::LowerFormalArguments(
300  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
302  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
303  switch (CallConv) {
304  default:
305  report_fatal_error("Unsupported calling convention");
306  case CallingConv::C:
307  case CallingConv::Fast:
308  break;
309  }
310 
312  MachineRegisterInfo &RegInfo = MF.getRegInfo();
313 
314  // Assign locations to all of the incoming arguments.
316  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
317  CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64);
318 
319  for (auto &VA : ArgLocs) {
320  if (VA.isRegLoc()) {
321  // Arguments passed in registers
322  EVT RegVT = VA.getLocVT();
323  MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
324  switch (SimpleTy) {
325  default: {
326  errs() << "LowerFormalArguments Unhandled argument type: "
327  << RegVT.getEVTString() << '\n';
328  llvm_unreachable(0);
329  }
330  case MVT::i32:
331  case MVT::i64:
332  Register VReg = RegInfo.createVirtualRegister(
333  SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
334  RegInfo.addLiveIn(VA.getLocReg(), VReg);
335  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
336 
337  // If this is an value that has been promoted to wider types, insert an
338  // assert[sz]ext to capture this, then truncate to the right size.
339  if (VA.getLocInfo() == CCValAssign::SExt)
340  ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue,
341  DAG.getValueType(VA.getValVT()));
342  else if (VA.getLocInfo() == CCValAssign::ZExt)
343  ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue,
344  DAG.getValueType(VA.getValVT()));
345 
346  if (VA.getLocInfo() != CCValAssign::Full)
347  ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue);
348 
349  InVals.push_back(ArgValue);
350 
351  break;
352  }
353  } else {
354  fail(DL, DAG, "defined with too many args");
355  InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT()));
356  }
357  }
358 
359  if (IsVarArg || MF.getFunction().hasStructRetAttr()) {
360  fail(DL, DAG, "functions with VarArgs or StructRet are not supported");
361  }
362 
363  return Chain;
364 }
365 
366 const unsigned BPFTargetLowering::MaxArgs = 5;
367 
368 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
369  SmallVectorImpl<SDValue> &InVals) const {
370  SelectionDAG &DAG = CLI.DAG;
371  auto &Outs = CLI.Outs;
372  auto &OutVals = CLI.OutVals;
373  auto &Ins = CLI.Ins;
374  SDValue Chain = CLI.Chain;
375  SDValue Callee = CLI.Callee;
376  bool &IsTailCall = CLI.IsTailCall;
377  CallingConv::ID CallConv = CLI.CallConv;
378  bool IsVarArg = CLI.IsVarArg;
380 
381  // BPF target does not support tail call optimization.
382  IsTailCall = false;
383 
384  switch (CallConv) {
385  default:
386  report_fatal_error("Unsupported calling convention");
387  case CallingConv::Fast:
388  case CallingConv::C:
389  break;
390  }
391 
392  // Analyze operands of the call, assigning locations to each operand.
394  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
395 
396  CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64);
397 
398  unsigned NumBytes = CCInfo.getNextStackOffset();
399 
400  if (Outs.size() > MaxArgs)
401  fail(CLI.DL, DAG, "too many args to ", Callee);
402 
403  for (auto &Arg : Outs) {
404  ISD::ArgFlagsTy Flags = Arg.Flags;
405  if (!Flags.isByVal())
406  continue;
407 
408  fail(CLI.DL, DAG, "pass by value not supported ", Callee);
409  }
410 
411  auto PtrVT = getPointerTy(MF.getDataLayout());
412  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
413 
414  SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
415 
416  // Walk arg assignments
417  for (unsigned i = 0,
418  e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs);
419  i != e; ++i) {
420  CCValAssign &VA = ArgLocs[i];
421  SDValue Arg = OutVals[i];
422 
423  // Promote the value if needed.
424  switch (VA.getLocInfo()) {
425  default:
426  llvm_unreachable("Unknown loc info");
427  case CCValAssign::Full:
428  break;
429  case CCValAssign::SExt:
430  Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg);
431  break;
432  case CCValAssign::ZExt:
433  Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg);
434  break;
435  case CCValAssign::AExt:
436  Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg);
437  break;
438  }
439 
440  // Push arguments into RegsToPass vector
441  if (VA.isRegLoc())
442  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
443  else
444  llvm_unreachable("call arg pass bug");
445  }
446 
447  SDValue InFlag;
448 
449  // Build a sequence of copy-to-reg nodes chained together with token chain and
450  // flag operands which copy the outgoing args into registers. The InFlag in
451  // necessary since all emitted instructions must be stuck together.
452  for (auto &Reg : RegsToPass) {
453  Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag);
454  InFlag = Chain.getValue(1);
455  }
456 
457  // If the callee is a GlobalAddress node (quite common, every direct call is)
458  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
459  // Likewise ExternalSymbol -> TargetExternalSymbol.
460  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
461  Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
462  G->getOffset(), 0);
463  } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
464  Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
465  fail(CLI.DL, DAG, Twine("A call to built-in function '"
466  + StringRef(E->getSymbol())
467  + "' is not supported."));
468  }
469 
470  // Returns a chain & a flag for retval copy to use.
471  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
473  Ops.push_back(Chain);
474  Ops.push_back(Callee);
475 
476  // Add argument registers to the end of the list so that they are
477  // known live into the call.
478  for (auto &Reg : RegsToPass)
479  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
480 
481  if (InFlag.getNode())
482  Ops.push_back(InFlag);
483 
484  Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops);
485  InFlag = Chain.getValue(1);
486 
487  // Create the CALLSEQ_END node.
488  Chain = DAG.getCALLSEQ_END(
489  Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true),
490  DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL);
491  InFlag = Chain.getValue(1);
492 
493  // Handle result values, copying them out of physregs into vregs that we
494  // return.
495  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG,
496  InVals);
497 }
498 
499 SDValue
500 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
501  bool IsVarArg,
503  const SmallVectorImpl<SDValue> &OutVals,
504  const SDLoc &DL, SelectionDAG &DAG) const {
505  unsigned Opc = BPFISD::RET_FLAG;
506 
507  // CCValAssign - represent the assignment of the return value to a location
510 
511  // CCState - Info about the registers and stack slot.
512  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
513 
514  if (MF.getFunction().getReturnType()->isAggregateType()) {
515  fail(DL, DAG, "only integer returns supported");
516  return DAG.getNode(Opc, DL, MVT::Other, Chain);
517  }
518 
519  // Analize return values.
520  CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
521 
522  SDValue Flag;
523  SmallVector<SDValue, 4> RetOps(1, Chain);
524 
525  // Copy the result values into the output registers.
526  for (unsigned i = 0; i != RVLocs.size(); ++i) {
527  CCValAssign &VA = RVLocs[i];
528  assert(VA.isRegLoc() && "Can only return in registers!");
529 
530  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
531 
532  // Guarantee that all emitted copies are stuck together,
533  // avoiding something bad.
534  Flag = Chain.getValue(1);
535  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
536  }
537 
538  RetOps[0] = Chain; // Update chain.
539 
540  // Add the flag if we have it.
541  if (Flag.getNode())
542  RetOps.push_back(Flag);
543 
544  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
545 }
546 
547 SDValue BPFTargetLowering::LowerCallResult(
548  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
550  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
551 
553  // Assign locations to each value returned by this call.
555  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
556 
557  if (Ins.size() >= 2) {
558  fail(DL, DAG, "only small returns supported");
559  for (unsigned i = 0, e = Ins.size(); i != e; ++i)
560  InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT));
561  return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1);
562  }
563 
564  CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
565 
566  // Copy all of the result registers out of their specified physreg.
567  for (auto &Val : RVLocs) {
568  Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(),
569  Val.getValVT(), InFlag).getValue(1);
570  InFlag = Chain.getValue(2);
571  InVals.push_back(Chain.getValue(0));
572  }
573 
574  return Chain;
575 }
576 
577 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
578  switch (CC) {
579  default:
580  break;
581  case ISD::SETULT:
582  case ISD::SETULE:
583  case ISD::SETLT:
584  case ISD::SETLE:
586  std::swap(LHS, RHS);
587  break;
588  }
589 }
590 
591 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
592  SDValue Chain = Op.getOperand(0);
593  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
594  SDValue LHS = Op.getOperand(2);
595  SDValue RHS = Op.getOperand(3);
596  SDValue Dest = Op.getOperand(4);
597  SDLoc DL(Op);
598 
599  if (!getHasJmpExt())
600  NegateCC(LHS, RHS, CC);
601 
602  return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
603  DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
604 }
605 
606 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
607  SDValue LHS = Op.getOperand(0);
608  SDValue RHS = Op.getOperand(1);
609  SDValue TrueV = Op.getOperand(2);
610  SDValue FalseV = Op.getOperand(3);
611  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
612  SDLoc DL(Op);
613 
614  if (!getHasJmpExt())
615  NegateCC(LHS, RHS, CC);
616 
617  SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
618  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
619  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
620 
621  return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops);
622 }
623 
624 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
625  switch ((BPFISD::NodeType)Opcode) {
627  break;
628  case BPFISD::RET_FLAG:
629  return "BPFISD::RET_FLAG";
630  case BPFISD::CALL:
631  return "BPFISD::CALL";
632  case BPFISD::SELECT_CC:
633  return "BPFISD::SELECT_CC";
634  case BPFISD::BR_CC:
635  return "BPFISD::BR_CC";
636  case BPFISD::Wrapper:
637  return "BPFISD::Wrapper";
638  case BPFISD::MEMCPY:
639  return "BPFISD::MEMCPY";
640  }
641  return nullptr;
642 }
643 
644 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
645  SelectionDAG &DAG) const {
646  auto N = cast<GlobalAddressSDNode>(Op);
647  assert(N->getOffset() == 0 && "Invalid offset for global address");
648 
649  SDLoc DL(Op);
650  const GlobalValue *GV = N->getGlobal();
651  SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64);
652 
653  return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA);
654 }
655 
656 unsigned
657 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
658  unsigned Reg, bool isSigned) const {
659  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
661  int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
662  MachineFunction *F = BB->getParent();
663  DebugLoc DL = MI.getDebugLoc();
664 
665  MachineRegisterInfo &RegInfo = F->getRegInfo();
666 
667  if (!isSigned) {
668  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
669  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
670  return PromotedReg0;
671  }
672  Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
673  Register PromotedReg1 = RegInfo.createVirtualRegister(RC);
674  Register PromotedReg2 = RegInfo.createVirtualRegister(RC);
675  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
676  BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1)
677  .addReg(PromotedReg0).addImm(32);
678  BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2)
679  .addReg(PromotedReg1).addImm(32);
680 
681  return PromotedReg2;
682 }
683 
685 BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
687  const {
688  MachineFunction *MF = MI.getParent()->getParent();
690  MachineInstrBuilder MIB(*MF, MI);
691  unsigned ScratchReg;
692 
693  // This function does custom insertion during lowering BPFISD::MEMCPY which
694  // only has two register operands from memcpy semantics, the copy source
695  // address and the copy destination address.
696  //
697  // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
698  // a third scratch register to serve as the destination register of load and
699  // source register of store.
700  //
701  // The scratch register here is with the Define | Dead | EarlyClobber flags.
702  // The EarlyClobber flag has the semantic property that the operand it is
703  // attached to is clobbered before the rest of the inputs are read. Hence it
704  // must be unique among the operands to the instruction. The Define flag is
705  // needed to coerce the machine verifier that an Undef value isn't a problem
706  // as we anyway is loading memory into it. The Dead flag is needed as the
707  // value in scratch isn't supposed to be used by any other instruction.
708  ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass);
709  MIB.addReg(ScratchReg,
711 
712  return BB;
713 }
714 
717  MachineBasicBlock *BB) const {
718  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
719  DebugLoc DL = MI.getDebugLoc();
720  unsigned Opc = MI.getOpcode();
721  bool isSelectRROp = (Opc == BPF::Select ||
722  Opc == BPF::Select_64_32 ||
723  Opc == BPF::Select_32 ||
724  Opc == BPF::Select_32_64);
725 
726  bool isMemcpyOp = Opc == BPF::MEMCPY;
727 
728 #ifndef NDEBUG
729  bool isSelectRIOp = (Opc == BPF::Select_Ri ||
730  Opc == BPF::Select_Ri_64_32 ||
731  Opc == BPF::Select_Ri_32 ||
732  Opc == BPF::Select_Ri_32_64);
733 
734 
735  assert((isSelectRROp || isSelectRIOp || isMemcpyOp) &&
736  "Unexpected instr type to insert");
737 #endif
738 
739  if (isMemcpyOp)
740  return EmitInstrWithCustomInserterMemcpy(MI, BB);
741 
742  bool is32BitCmp = (Opc == BPF::Select_32 ||
743  Opc == BPF::Select_32_64 ||
744  Opc == BPF::Select_Ri_32 ||
745  Opc == BPF::Select_Ri_32_64);
746 
747  // To "insert" a SELECT instruction, we actually have to insert the diamond
748  // control-flow pattern. The incoming instruction knows the destination vreg
749  // to set, the condition code register to branch on, the true/false values to
750  // select between, and a branch opcode to use.
751  const BasicBlock *LLVM_BB = BB->getBasicBlock();
752  MachineFunction::iterator I = ++BB->getIterator();
753 
754  // ThisMBB:
755  // ...
756  // TrueVal = ...
757  // jmp_XX r1, r2 goto Copy1MBB
758  // fallthrough --> Copy0MBB
759  MachineBasicBlock *ThisMBB = BB;
760  MachineFunction *F = BB->getParent();
761  MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
762  MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB);
763 
764  F->insert(I, Copy0MBB);
765  F->insert(I, Copy1MBB);
766  // Update machine-CFG edges by transferring all successors of the current
767  // block to the new block which will contain the Phi node for the select.
768  Copy1MBB->splice(Copy1MBB->begin(), BB,
769  std::next(MachineBasicBlock::iterator(MI)), BB->end());
771  // Next, add the true and fallthrough blocks as its successors.
772  BB->addSuccessor(Copy0MBB);
773  BB->addSuccessor(Copy1MBB);
774 
775  // Insert Branch if Flag
776  int CC = MI.getOperand(3).getImm();
777  int NewCC;
778  switch (CC) {
779 #define SET_NEWCC(X, Y) \
780  case ISD::X: \
781  if (is32BitCmp && HasJmp32) \
782  NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
783  else \
784  NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
785  break
786  SET_NEWCC(SETGT, JSGT);
787  SET_NEWCC(SETUGT, JUGT);
788  SET_NEWCC(SETGE, JSGE);
789  SET_NEWCC(SETUGE, JUGE);
790  SET_NEWCC(SETEQ, JEQ);
791  SET_NEWCC(SETNE, JNE);
792  SET_NEWCC(SETLT, JSLT);
793  SET_NEWCC(SETULT, JULT);
794  SET_NEWCC(SETLE, JSLE);
795  SET_NEWCC(SETULE, JULE);
796  default:
797  report_fatal_error("unimplemented select CondCode " + Twine(CC));
798  }
799 
800  Register LHS = MI.getOperand(1).getReg();
801  bool isSignedCmp = (CC == ISD::SETGT ||
802  CC == ISD::SETGE ||
803  CC == ISD::SETLT ||
804  CC == ISD::SETLE);
805 
806  // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
807  // to be promoted, however if the 32-bit comparison operands are destination
808  // registers then they are implicitly zero-extended already, there is no
809  // need of explicit zero-extend sequence for them.
810  //
811  // We simply do extension for all situations in this method, but we will
812  // try to remove those unnecessary in BPFMIPeephole pass.
813  if (is32BitCmp && !HasJmp32)
814  LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
815 
816  if (isSelectRROp) {
817  Register RHS = MI.getOperand(2).getReg();
818 
819  if (is32BitCmp && !HasJmp32)
820  RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
821 
822  BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
823  } else {
824  int64_t imm32 = MI.getOperand(2).getImm();
825  // sanity check before we build J*_ri instruction.
826  assert (isInt<32>(imm32));
827  BuildMI(BB, DL, TII.get(NewCC))
828  .addReg(LHS).addImm(imm32).addMBB(Copy1MBB);
829  }
830 
831  // Copy0MBB:
832  // %FalseValue = ...
833  // # fallthrough to Copy1MBB
834  BB = Copy0MBB;
835 
836  // Update machine-CFG edges
837  BB->addSuccessor(Copy1MBB);
838 
839  // Copy1MBB:
840  // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
841  // ...
842  BB = Copy1MBB;
843  BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg())
844  .addReg(MI.getOperand(5).getReg())
845  .addMBB(Copy0MBB)
846  .addReg(MI.getOperand(4).getReg())
847  .addMBB(ThisMBB);
848 
849  MI.eraseFromParent(); // The pseudo instruction is gone now.
850  return BB;
851 }
852 
854  EVT VT) const {
855  return getHasAlu32() ? MVT::i32 : MVT::i64;
856 }
857 
859  EVT VT) const {
860  return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
861 }
llvm::CallingConv::C
@ C
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
llvm::CCValAssign::getLocVT
MVT getLocVT() const
Definition: CallingConvLower.h:153
llvm::MachineRegisterInfo::addLiveIn
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
Definition: MachineRegisterInfo.h:944
i
i
Definition: README.txt:29
llvm::ISD::SETUGE
@ SETUGE
Definition: ISDOpcodes.h:1368
llvm::TargetLoweringBase::MaxStoresPerMemsetOptSize
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3112
llvm::CCValAssign::ZExt
@ ZExt
Definition: CallingConvLower.h:38
ValueTypes.h
llvm::BPFTargetLowering::getTargetNodeName
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
Definition: BPFISelLowering.cpp:624
llvm::SelectionDAG::getCALLSEQ_START
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:923
llvm::ISD::SETLE
@ SETLE
Definition: ISDOpcodes.h:1379
MI
IRTranslator LLVM IR MI
Definition: IRTranslator.cpp:103
llvm::MachineInstrBuilder::addImm
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Definition: MachineInstrBuilder.h:131
llvm
---------------------— PointerInfo ------------------------------------—
Definition: AllocatorList.h:23
BPFSubtarget.h
Reg
unsigned Reg
Definition: MachineSink.cpp:1566
llvm::SDLoc
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Definition: SelectionDAGNodes.h:1086
llvm::CCValAssign::Full
@ Full
Definition: CallingConvLower.h:36
llvm::DataLayout
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:112
llvm::ISD::SETGT
@ SETGT
Definition: ISDOpcodes.h:1376
llvm::ISD::SETNE
@ SETNE
Definition: ISDOpcodes.h:1380
llvm::MachineRegisterInfo::createVirtualRegister
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Definition: MachineRegisterInfo.cpp:158
llvm::TargetLowering::ConstraintType
ConstraintType
Definition: TargetLowering.h:4157
llvm::ISD::BR_JT
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:946
llvm::DiagnosticInfoUnsupported
Diagnostic information for unsupported feature in backend.
Definition: DiagnosticInfo.h:1004
llvm::BPFSubtarget::getHasJmp32
bool getHasJmp32() const
Definition: BPFSubtarget.h:72
llvm::CCState
CCState - This class holds information needed while lowering arguments and return values.
Definition: CallingConvLower.h:191
llvm::SelectionDAG::getCopyToReg
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:735
llvm::ISD::AssertSext
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
llvm::MachineRegisterInfo
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Definition: MachineRegisterInfo.h:52
llvm::SDValue::getNode
SDNode * getNode() const
get the SDNode which holds the desired result
Definition: SelectionDAGNodes.h:152
llvm::ISD::BSWAP
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:666
llvm::ISD::DYNAMIC_STACKALLOC
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:931
llvm::BPFISD::MEMCPY
@ MEMCPY
Definition: BPFISelLowering.h:31
llvm::ISD::CTTZ_ZERO_UNDEF
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:674
llvm::SelectionDAG::getValueType
SDValue getValueType(EVT)
Definition: SelectionDAG.cpp:1722
llvm::raw_string_ostream
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:625
llvm::BPFSubtarget::getRegisterInfo
const TargetRegisterInfo * getRegisterInfo() const override
Definition: BPFSubtarget.h:86
llvm::TargetLoweringBase::MaxStoresPerMemset
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
Definition: TargetLowering.h:3110
BPFISelLowering.h
llvm::BPFTargetLowering::getConstraintType
BPFTargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
Definition: BPFISelLowering.cpp:224
llvm::SmallVector
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1168
llvm::ISD::SETEQ
@ SETEQ
Definition: ISDOpcodes.h:1375
llvm::ISD::STACKRESTORE
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1012
llvm::SelectionDAG::getVTList
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
Definition: SelectionDAG.cpp:8551
llvm::BPFTargetLowering::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFISelLowering.h:62
llvm::BPFTargetLowering::getHasAlu32
bool getHasAlu32() const
Definition: BPFISelLowering.h:60
ErrorHandling.h
llvm::ISD::ANY_EXTEND
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:732
llvm::SDNode
Represents one node in the SelectionDAG.
Definition: SelectionDAGNodes.h:455
llvm::cl::Hidden
@ Hidden
Definition: CommandLine.h:143
llvm::MVT::Glue
@ Glue
Definition: MachineValueType.h:262
llvm::TargetRegisterInfo
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
Definition: TargetRegisterInfo.h:233
llvm::ISD::SETULE
@ SETULE
Definition: ISDOpcodes.h:1370
llvm::Type
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
llvm::ISD::SHL_PARTS
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:715
llvm::ISD::SETCC
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:702
llvm::ARMISD::MEMCPY
@ MEMCPY
Definition: ARMISelLowering.h:309
llvm::TargetLoweringBase::setMinFunctionAlignment
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
Definition: TargetLowering.h:2301
Results
Function Alias Analysis Results
Definition: AliasAnalysis.cpp:847
llvm::errs
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
Definition: raw_ostream.cpp:892
llvm::ISD::EXTLOAD
@ EXTLOAD
Definition: ISDOpcodes.h:1335
BPF.h
llvm::BPFISD::NodeType
NodeType
Definition: BPFISelLowering.h:24
TRI
unsigned const TargetRegisterInfo * TRI
Definition: MachineSink.cpp:1567
llvm::SelectionDAG::getContext
LLVMContext * getContext() const
Definition: SelectionDAG.h:447
llvm::ISD::ATOMIC_LOAD_OR
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1144
F
#define F(x, y, z)
Definition: MD5.cpp:56
llvm::ISD::BRCOND
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:956
MachineRegisterInfo.h
llvm::BPFTargetLowering::getRegForInlineAsmConstraint
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
Definition: BPFISelLowering.cpp:238
llvm::SelectionDAG::getRegister
SDValue getRegister(unsigned Reg, EVT VT)
Definition: SelectionDAG.cpp:1985
llvm::BasicBlock
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
llvm::MVT::SimpleValueType
SimpleValueType
Definition: MachineValueType.h:33
llvm::ISD::BRIND
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:942
llvm::ISD::ROTL
@ ROTL
Definition: ISDOpcodes.h:660
Arg
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Definition: AMDGPULibCalls.cpp:206
llvm::MVT::integer_valuetypes
static auto integer_valuetypes()
Definition: MachineValueType.h:1411
llvm::BPFISD::BR_CC
@ BR_CC
Definition: BPFISelLowering.h:29
llvm::BPFISD::FIRST_NUMBER
@ FIRST_NUMBER
Definition: BPFISelLowering.h:25
llvm::ISD::BR_CC
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:963
llvm::BPFTargetLowering::getSetCCResultType
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
Definition: BPFISelLowering.cpp:853
llvm::MVT::i1
@ i1
Definition: MachineValueType.h:43
llvm::TargetLowering::CallLoweringInfo::IsVarArg
bool IsVarArg
Definition: TargetLowering.h:3752
llvm::MachineFunction::getRegInfo
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
Definition: MachineFunction.h:636
llvm::CCValAssign::AExt
@ AExt
Definition: CallingConvLower.h:39
llvm::ISD::GlobalAddress
@ GlobalAddress
Definition: ISDOpcodes.h:78
llvm::ISD::SELECT_CC
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:694
llvm::ExternalSymbolSDNode
Definition: SelectionDAGNodes.h:2174
llvm::TargetInstrInfo
TargetInstrInfo - Interface to description of machine instruction set.
Definition: TargetInstrInfo.h:97
llvm::SDValue::getValueType
EVT getValueType() const
Return the ValueType of the referenced return value.
Definition: SelectionDAGNodes.h:1121
llvm::CCValAssign
CCValAssign - Represent assignment of one arg/retval to a location.
Definition: CallingConvLower.h:33
llvm::BPFISD::SELECT_CC
@ SELECT_CC
Definition: BPFISelLowering.h:28
llvm::ISD::CTLZ
@ CTLZ
Definition: ISDOpcodes.h:668
llvm::MachineInstrBuilder::addMBB
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Definition: MachineInstrBuilder.h:146
llvm::SelectionDAG
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:216
llvm::ISD::SELECT
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:679
llvm::ISD::ZERO_EXTEND
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:729
llvm::ISD::ArgFlagsTy::isByVal
bool isByVal() const
Definition: TargetCallingConv.h:85
llvm::ISD::SETGE
@ SETGE
Definition: ISDOpcodes.h:1377
E
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
llvm::CCValAssign::getLocReg
Register getLocReg() const
Definition: CallingConvLower.h:150
llvm::ISD::SIGN_EXTEND_INREG
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:747
llvm::EVT
Extended Value Type.
Definition: ValueTypes.h:35
llvm::SelectionDAG::getConstant
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
Definition: SelectionDAG.cpp:1373
llvm::TargetLowering
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
Definition: TargetLowering.h:3189
llvm::TargetRegisterClass
Definition: TargetRegisterInfo.h:46
llvm::ISD::TRUNCATE
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:735
llvm::MVT::SimpleTy
SimpleValueType SimpleTy
Definition: MachineValueType.h:321
llvm::BPFTargetLowering::BPFTargetLowering
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
Definition: BPFISelLowering.cpp:56
llvm::ISD::UDIVREM
@ UDIVREM
Definition: ISDOpcodes.h:256
TII
const HexagonInstrInfo * TII
Definition: HexagonCopyToCombine.cpp:129
llvm::TargetLoweringBase::addRegisterClass
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
Definition: TargetLowering.h:2181
llvm::ISD::ATOMIC_LOAD_AND
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1142
llvm::EVT::isInteger
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:145
llvm::MCID::Flag
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:146
llvm::BPFTargetLowering::EmitInstrWithCustomInserter
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
Definition: BPFISelLowering.cpp:716
llvm::report_fatal_error
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:140
llvm::raw_ostream::flush
void flush()
Definition: raw_ostream.h:186
llvm::TargetLoweringBase::MaxStoresPerMemcpy
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
Definition: TargetLowering.h:3125
llvm::RegState::Define
@ Define
Register definition.
Definition: MachineInstrBuilder.h:44
llvm::CCValAssign::getLocInfo
LocInfo getLocInfo() const
Definition: CallingConvLower.h:155
Align
uint64_t Align
Definition: ELFObjHandler.cpp:83
llvm::CCValAssign::isRegLoc
bool isRegLoc() const
Definition: CallingConvLower.h:145
llvm::CallingConv::ID
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition: CallingConv.h:24
llvm::MachineBasicBlock
Definition: MachineBasicBlock.h:95
llvm::ISD::SETUGT
@ SETUGT
Definition: ISDOpcodes.h:1367
llvm::SelectionDAG::getTargetGlobalAddress
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:683
llvm::Type::isIntegerTy
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:201
llvm::TargetLowering::CallLoweringInfo::Outs
SmallVector< ISD::OutputArg, 32 > Outs
Definition: TargetLowering.h:3775
G
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
llvm::TargetLoweringBase::setOperationAction
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
Definition: TargetLowering.h:2198
llvm::TargetLoweringBase::MaxStoresPerMemcpyOptSize
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3127
llvm::cl::opt< bool >
llvm::GlobalValue
Definition: GlobalValue.h:44
llvm::isInt< 32 >
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:373
llvm::BPFTargetLowering::LowerOperation
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
Definition: BPFISelLowering.cpp:281
llvm::EVT::getSizeInBits
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:341
llvm::Function::getReturnType
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:181
llvm::CCValAssign::SExt
@ SExt
Definition: CallingConvLower.h:37
llvm::MachineInstr
Representation of each machine instruction.
Definition: MachineInstr.h:64
llvm::MachineInstrBuilder
Definition: MachineInstrBuilder.h:69
llvm::ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1131
llvm::TargetLowering::CallLoweringInfo::Chain
SDValue Chain
Definition: TargetLowering.h:3748
llvm::ISD::AssertZext
@ AssertZext
Definition: ISDOpcodes.h:62
llvm::TargetLoweringBase::Promote
@ Promote
Definition: TargetLowering.h:198
llvm::LLVMContext
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:68
llvm::numbers::e
constexpr double e
Definition: MathExtras.h:57
llvm::SelectionDAG::getCopyFromReg
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:761
llvm::TargetLowering::CallLoweringInfo::CallConv
CallingConv::ID CallConv
Definition: TargetLowering.h:3769
llvm::TargetLoweringBase::setStackPointerRegisterToSaveRestore
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
Definition: TargetLowering.h:2147
fail
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
Definition: BPFISelLowering.cpp:38
llvm::ISD::CTLZ_ZERO_UNDEF
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:675
I
#define I(x, y, z)
Definition: MD5.cpp:59
llvm::SelectionDAG::getNode
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Definition: SelectionDAG.cpp:8316
llvm::TargetLoweringBase::setPrefFunctionAlignment
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
Definition: TargetLowering.h:2307
llvm::cl::init
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:443
llvm::ISD::ATOMIC_LOAD_ADD
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1140
llvm::TargetLowering::CallLoweringInfo::DL
SDLoc DL
Definition: TargetLowering.h:3773
llvm::Function::hasStructRetAttr
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition: Function.h:652
llvm::ISD::ZEXTLOAD
@ ZEXTLOAD
Definition: ISDOpcodes.h:1335
llvm::SDValue::getValue
SDValue getValue(unsigned R) const
Definition: SelectionDAGNodes.h:172
llvm::MVT::i8
@ i8
Definition: MachineValueType.h:44
llvm::TargetLowering::CallLoweringInfo
This structure contains all information that is necessary for lowering calls.
Definition: TargetLowering.h:3747
assert
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
llvm::BPFSubtarget::getSelectionDAGInfo
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
Definition: BPFSubtarget.h:83
llvm::TargetMachine
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:79
llvm::ISD::MULHS
@ MULHS
Definition: ISDOpcodes.h:615
llvm::MVT::Other
@ Other
Definition: MachineValueType.h:42
std::swap
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:840
llvm::ISD::SETULT
@ SETULT
Definition: ISDOpcodes.h:1369
llvm::SDNode::print
void print(raw_ostream &OS, const SelectionDAG *G=nullptr) const
Definition: SelectionDAGDumper.cpp:1048
BPFExpandMemcpyInOrder
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
llvm::ISD::CondCode
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1355
llvm::MVT
Machine Value Type.
Definition: MachineValueType.h:31
llvm::MachineInstrBuilder::addReg
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
Definition: MachineInstrBuilder.h:97
llvm::ISD::SRA_PARTS
@ SRA_PARTS
Definition: ISDOpcodes.h:716
NegateCC
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
Definition: BPFISelLowering.cpp:577
llvm::MachineFunction
Definition: MachineFunction.h:230
llvm::SelectionDAG::getCALLSEQ_END
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
Definition: SelectionDAG.h:935
llvm::TargetLowering::CallLoweringInfo::Ins
SmallVector< ISD::InputArg, 32 > Ins
Definition: TargetLowering.h:3777
llvm::TargetLowering::CallLoweringInfo::DAG
SelectionDAG & DAG
Definition: TargetLowering.h:3772
llvm::BPFSelectionDAGInfo::getCommonMaxStoresPerMemFunc
unsigned getCommonMaxStoresPerMemFunc() const
Definition: BPFSelectionDAGInfo.h:29
llvm::min
Expected< ExpressionValue > min(const ExpressionValue &Lhs, const ExpressionValue &Rhs)
Definition: FileCheck.cpp:357
llvm::RegState::EarlyClobber
@ EarlyClobber
Register definition happens before uses.
Definition: MachineInstrBuilder.h:54
llvm::MVT::i64
@ i64
Definition: MachineValueType.h:47
llvm::StringRef
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:58
llvm::MachineBasicBlock::splice
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
Definition: MachineBasicBlock.h:950
llvm_unreachable
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Definition: ErrorHandling.h:136
llvm::ISD::SREM
@ SREM
Definition: ISDOpcodes.h:244
llvm::ISD::UMUL_LOHI
@ UMUL_LOHI
Definition: ISDOpcodes.h:251
uint32_t
llvm::ISD::ArgFlagsTy
Definition: TargetCallingConv.h:27
llvm::TargetLoweringBase::MaxStoresPerMemmoveOptSize
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
Definition: TargetLowering.h:3162
DL
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Definition: AArch64SLSHardening.cpp:76
llvm::BPFISD::CALL
@ CALL
Definition: BPFISelLowering.h:27
llvm::ISD::SMUL_LOHI
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
llvm::TargetLowering::getRegForInlineAsmConstraint
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Definition: TargetLowering.cpp:4597
llvm::SDVTList
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
Definition: SelectionDAGNodes.h:79
llvm::ISD::ATOMIC_SWAP
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1139
llvm::ISD::SEXTLOAD
@ SEXTLOAD
Definition: ISDOpcodes.h:1335
MRI
unsigned const MachineRegisterInfo * MRI
Definition: AArch64AdvSIMDScalarPass.cpp:105
llvm::TargetLowering::C_RegisterClass
@ C_RegisterClass
Definition: TargetLowering.h:4159
llvm::Register
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Callee
amdgpu Simplify well known AMD library false FunctionCallee Callee
Definition: AMDGPULibCalls.cpp:206
SET_NEWCC
#define SET_NEWCC(X, Y)
llvm::EVT::getEVTString
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
Definition: ValueTypes.cpp:151
llvm::MCID::Select
@ Select
Definition: MCInstrDesc.h:162
llvm::TargetLowering::CallLoweringInfo::IsTailCall
bool IsTailCall
Definition: TargetLowering.h:3763
llvm::ISD::SETLT
@ SETLT
Definition: ISDOpcodes.h:1378
llvm::LLVMContext::diagnose
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Definition: LLVMContext.cpp:228
llvm::Twine
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:83
llvm::BPFTargetLowering::getScalarShiftAmountTy
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
EVT is not used in-tree, but is used by out-of-tree target.
Definition: BPFISelLowering.cpp:858
llvm::TargetLoweringBase::setLoadExtAction
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
Definition: TargetLowering.h:2206
llvm::GlobalAddressSDNode
Definition: SelectionDAGNodes.h:1718
llvm::MachineFunction::getFunction
Function & getFunction()
Return the LLVM function that this machine code represents.
Definition: MachineFunction.h:592
CallingConvLower.h
llvm::AMDGPU::SendMsg::Op
Op
Definition: SIDefines.h:321
llvm::ilist_iterator
Iterator for intrusive lists based on ilist_node.
Definition: ilist_iterator.h:57
MachineFrameInfo.h
llvm::ISD::getSetCCSwappedOperands
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
Definition: SelectionDAG.cpp:454
DiagnosticInfo.h
llvm::BPFSubtarget::getHasAlu32
bool getHasAlu32() const
Definition: BPFSubtarget.h:73
llvm::TargetLoweringBase::Custom
@ Custom
Definition: TargetLowering.h:201
llvm::SelectionDAG::getTargetExternalSymbol
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
Definition: SelectionDAG.cpp:1753
llvm::MVT::i32
@ i32
Definition: MachineValueType.h:46
llvm::SDValue
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
Definition: SelectionDAGNodes.h:138
llvm::TargetLoweringBase::ZeroOrOneBooleanContent
@ ZeroOrOneBooleanContent
Definition: TargetLowering.h:233
llvm::ISD::SRL_PARTS
@ SRL_PARTS
Definition: ISDOpcodes.h:717
llvm::RegState::Dead
@ Dead
Unused definition.
Definition: MachineInstrBuilder.h:50
llvm::CallingConv::Fast
@ Fast
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
llvm::BPFISD::Wrapper
@ Wrapper
Definition: BPFISelLowering.h:30
llvm::TargetLoweringBase::setBooleanContents
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
Definition: TargetLowering.h:2115
llvm::MachineBasicBlock::begin
iterator begin()
Definition: MachineBasicBlock.h:268
MachineInstrBuilder.h
llvm::ISD::UREM
@ UREM
Definition: ISDOpcodes.h:245
llvm::TargetLoweringBase::Expand
@ Expand
Definition: TargetLowering.h:199
llvm::BPFTargetLowering::isOffsetFoldingLegal
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
Definition: BPFISelLowering.cpp:187
llvm::BuildMI
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
Definition: MachineInstrBuilder.h:328
N
#define N
llvm::TargetLoweringBase::computeRegisterProperties
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
Definition: TargetLoweringBase.cpp:1287
llvm::ISD::CTTZ
@ CTTZ
Definition: ISDOpcodes.h:667
llvm::TargetLoweringBase::getRegClassFor
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
Definition: TargetLowering.h:852
llvm::MachineFunction::getDataLayout
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Definition: MachineFunction.cpp:260
llvm::MipsISD::Ins
@ Ins
Definition: MipsISelLowering.h:157
llvm::SmallVectorImpl
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:43
llvm::ISD::MULHU
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:614
llvm::TargetLowering::CallLoweringInfo::OutVals
SmallVector< SDValue, 32 > OutVals
Definition: TargetLowering.h:3776
llvm::MachineBasicBlock::transferSuccessorsAndUpdatePHIs
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
Definition: MachineBasicBlock.cpp:885
TM
const char LLVMTargetMachineRef TM
Definition: PassBuilderBindings.cpp:47
llvm::TargetLowering::CallLoweringInfo::Callee
SDValue Callee
Definition: TargetLowering.h:3770
llvm::BPFSubtarget::getHasJmpExt
bool getHasJmpExt() const
Definition: BPFSubtarget.h:71
llvm::MVT::i16
@ i16
Definition: MachineValueType.h:45
llvm::SelectionDAG::getMachineFunction
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:437
BB
Common register allocation spilling lr str ldr sxth r3 ldr mla r4 can lr mov lr str ldr sxth r3 mla r4 and then merge mul and lr str ldr sxth r3 mla r4 It also increase the likelihood the store may become dead bb27 Successors according to LLVM BB
Definition: README.txt:39
llvm::Type::isAggregateType
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:267
llvm::BPFISD::RET_FLAG
@ RET_FLAG
Definition: BPFISelLowering.h:26
llvm::ISD::SDIVREM
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
llvm::DebugLoc
A debug info location.
Definition: DebugLoc.h:33
llvm::cl::desc
Definition: CommandLine.h:414
llvm::ISD::ATOMIC_LOAD_XOR
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1145
llvm::ISD::SIGN_EXTEND
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:726
raw_ostream.h
llvm::StringRef::size
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:157
MachineFunction.h
DiagnosticPrinter.h
llvm::MachineInstrBundleIterator< MachineInstr >
llvm::ISD::STACKSAVE
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1008
llvm::TargetLoweringBase::MaxStoresPerMemmove
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
Definition: TargetLowering.h:3160
llvm::ISD::ROTR
@ ROTR
Definition: ISDOpcodes.h:661
Debug.h
llvm::TargetLowering::getConstraintType
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
Definition: TargetLowering.cpp:4457
llvm::TargetLoweringBase::getPointerTy
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Definition: TargetLowering.h:346
llvm::ISD::CTPOP
@ CTPOP
Definition: ISDOpcodes.h:669
llvm::Type::getPrimitiveSizeInBits
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:128
llvm::BPFSubtarget
Definition: BPFSubtarget.h:31
llvm::EVT::getSimpleVT
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:289
BPFTargetMachine.h
TargetLoweringObjectFileImpl.h