LLVM  8.0.0svn
BPFISelLowering.cpp
Go to the documentation of this file.
1 //===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2 //
3 // The LLVM Compiler Infrastructure
4 //
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
7 //
8 //===----------------------------------------------------------------------===//
9 //
10 // This file defines the interfaces that BPF uses to lower LLVM code into a
11 // selection DAG.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "BPFISelLowering.h"
16 #include "BPF.h"
17 #include "BPFSubtarget.h"
18 #include "BPFTargetMachine.h"
27 #include "llvm/IR/DiagnosticInfo.h"
29 #include "llvm/Support/Debug.h"
32 using namespace llvm;
33 
34 #define DEBUG_TYPE "bpf-lower"
35 
36 static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
37  cl::Hidden, cl::init(false),
38  cl::desc("Expand memcpy into load/store pairs in order"));
39 
40 static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) {
42  DAG.getContext()->diagnose(
44 }
45 
46 static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg,
47  SDValue Val) {
49  std::string Str;
50  raw_string_ostream OS(Str);
51  OS << Msg;
52  Val->print(OS);
53  OS.flush();
54  DAG.getContext()->diagnose(
56 }
57 
59  const BPFSubtarget &STI)
60  : TargetLowering(TM) {
61 
62  // Set up the register classes.
63  addRegisterClass(MVT::i64, &BPF::GPRRegClass);
64  if (STI.getHasAlu32())
65  addRegisterClass(MVT::i32, &BPF::GPR32RegClass);
66 
67  // Compute derived properties from the register classes
69 
71 
76 
78 
82 
83  for (auto VT : { MVT::i32, MVT::i64 }) {
84  if (VT == MVT::i32 && !STI.getHasAlu32())
85  continue;
86 
101 
105  }
106 
107  if (STI.getHasAlu32()) {
110  }
111 
116 
121 
122  // Extended load operations for i1 types must be promoted
123  for (MVT VT : MVT::integer_valuetypes()) {
127 
131  }
132 
134 
135  // Function alignments (log2)
138 
140  // LLVM generic code will try to expand memcpy into load/store pairs at this
141  // stage which is before quite a few IR optimization passes, therefore the
142  // loads and stores could potentially be moved apart from each other which
143  // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
144  // compilers.
145  //
146  // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
147  // of memcpy to later stage in IR optimization pipeline so those load/store
148  // pairs won't be touched and could be kept in order. Hence, we set
149  // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
150  // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
154  } else {
155  // inline memcpy() for kernel to see explicit copy
156  unsigned CommonMaxStores =
158 
159  MaxStoresPerMemset = MaxStoresPerMemsetOptSize = CommonMaxStores;
160  MaxStoresPerMemcpy = MaxStoresPerMemcpyOptSize = CommonMaxStores;
162  }
163 
164  // CPU/Feature control
165  HasAlu32 = STI.getHasAlu32();
166  HasJmpExt = STI.getHasJmpExt();
167 }
168 
170  return false;
171 }
172 
173 std::pair<unsigned, const TargetRegisterClass *>
175  StringRef Constraint,
176  MVT VT) const {
177  if (Constraint.size() == 1)
178  // GCC Constraint Letters
179  switch (Constraint[0]) {
180  case 'r': // GENERAL_REGS
181  return std::make_pair(0U, &BPF::GPRRegClass);
182  default:
183  break;
184  }
185 
186  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
187 }
188 
190  switch (Op.getOpcode()) {
191  case ISD::BR_CC:
192  return LowerBR_CC(Op, DAG);
193  case ISD::GlobalAddress:
194  return LowerGlobalAddress(Op, DAG);
195  case ISD::SELECT_CC:
196  return LowerSELECT_CC(Op, DAG);
197  default:
198  llvm_unreachable("unimplemented operand");
199  }
200 }
201 
202 // Calling Convention Implementation
203 #include "BPFGenCallingConv.inc"
204 
205 SDValue BPFTargetLowering::LowerFormalArguments(
206  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
207  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
208  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
209  switch (CallConv) {
210  default:
211  report_fatal_error("Unsupported calling convention");
212  case CallingConv::C:
213  case CallingConv::Fast:
214  break;
215  }
216 
218  MachineRegisterInfo &RegInfo = MF.getRegInfo();
219 
220  // Assign locations to all of the incoming arguments.
222  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
223  CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64);
224 
225  for (auto &VA : ArgLocs) {
226  if (VA.isRegLoc()) {
227  // Arguments passed in registers
228  EVT RegVT = VA.getLocVT();
229  MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
230  switch (SimpleTy) {
231  default: {
232  errs() << "LowerFormalArguments Unhandled argument type: "
233  << RegVT.getEVTString() << '\n';
234  llvm_unreachable(0);
235  }
236  case MVT::i32:
237  case MVT::i64:
238  unsigned VReg = RegInfo.createVirtualRegister(SimpleTy == MVT::i64 ?
239  &BPF::GPRRegClass :
240  &BPF::GPR32RegClass);
241  RegInfo.addLiveIn(VA.getLocReg(), VReg);
242  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
243 
244  // If this is an value that has been promoted to wider types, insert an
245  // assert[sz]ext to capture this, then truncate to the right size.
246  if (VA.getLocInfo() == CCValAssign::SExt)
247  ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue,
248  DAG.getValueType(VA.getValVT()));
249  else if (VA.getLocInfo() == CCValAssign::ZExt)
250  ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue,
251  DAG.getValueType(VA.getValVT()));
252 
253  if (VA.getLocInfo() != CCValAssign::Full)
254  ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue);
255 
256  InVals.push_back(ArgValue);
257 
258  break;
259  }
260  } else {
261  fail(DL, DAG, "defined with too many args");
262  InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT()));
263  }
264  }
265 
266  if (IsVarArg || MF.getFunction().hasStructRetAttr()) {
267  fail(DL, DAG, "functions with VarArgs or StructRet are not supported");
268  }
269 
270  return Chain;
271 }
272 
273 const unsigned BPFTargetLowering::MaxArgs = 5;
274 
275 SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
276  SmallVectorImpl<SDValue> &InVals) const {
277  SelectionDAG &DAG = CLI.DAG;
278  auto &Outs = CLI.Outs;
279  auto &OutVals = CLI.OutVals;
280  auto &Ins = CLI.Ins;
281  SDValue Chain = CLI.Chain;
282  SDValue Callee = CLI.Callee;
283  bool &IsTailCall = CLI.IsTailCall;
284  CallingConv::ID CallConv = CLI.CallConv;
285  bool IsVarArg = CLI.IsVarArg;
287 
288  // BPF target does not support tail call optimization.
289  IsTailCall = false;
290 
291  switch (CallConv) {
292  default:
293  report_fatal_error("Unsupported calling convention");
294  case CallingConv::Fast:
295  case CallingConv::C:
296  break;
297  }
298 
299  // Analyze operands of the call, assigning locations to each operand.
301  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
302 
303  CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64);
304 
305  unsigned NumBytes = CCInfo.getNextStackOffset();
306 
307  if (Outs.size() > MaxArgs)
308  fail(CLI.DL, DAG, "too many args to ", Callee);
309 
310  for (auto &Arg : Outs) {
311  ISD::ArgFlagsTy Flags = Arg.Flags;
312  if (!Flags.isByVal())
313  continue;
314 
315  fail(CLI.DL, DAG, "pass by value not supported ", Callee);
316  }
317 
318  auto PtrVT = getPointerTy(MF.getDataLayout());
319  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
320 
321  SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
322 
323  // Walk arg assignments
324  for (unsigned i = 0,
325  e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs);
326  i != e; ++i) {
327  CCValAssign &VA = ArgLocs[i];
328  SDValue Arg = OutVals[i];
329 
330  // Promote the value if needed.
331  switch (VA.getLocInfo()) {
332  default:
333  llvm_unreachable("Unknown loc info");
334  case CCValAssign::Full:
335  break;
336  case CCValAssign::SExt:
337  Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg);
338  break;
339  case CCValAssign::ZExt:
340  Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg);
341  break;
342  case CCValAssign::AExt:
343  Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg);
344  break;
345  }
346 
347  // Push arguments into RegsToPass vector
348  if (VA.isRegLoc())
349  RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
350  else
351  llvm_unreachable("call arg pass bug");
352  }
353 
354  SDValue InFlag;
355 
356  // Build a sequence of copy-to-reg nodes chained together with token chain and
357  // flag operands which copy the outgoing args into registers. The InFlag in
358  // necessary since all emitted instructions must be stuck together.
359  for (auto &Reg : RegsToPass) {
360  Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag);
361  InFlag = Chain.getValue(1);
362  }
363 
364  // If the callee is a GlobalAddress node (quite common, every direct call is)
365  // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
366  // Likewise ExternalSymbol -> TargetExternalSymbol.
367  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
368  Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
369  G->getOffset(), 0);
370  } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
371  Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
372  fail(CLI.DL, DAG, Twine("A call to built-in function '"
373  + StringRef(E->getSymbol())
374  + "' is not supported."));
375  }
376 
377  // Returns a chain & a flag for retval copy to use.
378  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
380  Ops.push_back(Chain);
381  Ops.push_back(Callee);
382 
383  // Add argument registers to the end of the list so that they are
384  // known live into the call.
385  for (auto &Reg : RegsToPass)
386  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
387 
388  if (InFlag.getNode())
389  Ops.push_back(InFlag);
390 
391  Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops);
392  InFlag = Chain.getValue(1);
393 
394  // Create the CALLSEQ_END node.
395  Chain = DAG.getCALLSEQ_END(
396  Chain, DAG.getConstant(NumBytes, CLI.DL, PtrVT, true),
397  DAG.getConstant(0, CLI.DL, PtrVT, true), InFlag, CLI.DL);
398  InFlag = Chain.getValue(1);
399 
400  // Handle result values, copying them out of physregs into vregs that we
401  // return.
402  return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG,
403  InVals);
404 }
405 
406 SDValue
407 BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
408  bool IsVarArg,
410  const SmallVectorImpl<SDValue> &OutVals,
411  const SDLoc &DL, SelectionDAG &DAG) const {
412  unsigned Opc = BPFISD::RET_FLAG;
413 
414  // CCValAssign - represent the assignment of the return value to a location
417 
418  // CCState - Info about the registers and stack slot.
419  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
420 
421  if (MF.getFunction().getReturnType()->isAggregateType()) {
422  fail(DL, DAG, "only integer returns supported");
423  return DAG.getNode(Opc, DL, MVT::Other, Chain);
424  }
425 
426  // Analize return values.
427  CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
428 
429  SDValue Flag;
430  SmallVector<SDValue, 4> RetOps(1, Chain);
431 
432  // Copy the result values into the output registers.
433  for (unsigned i = 0; i != RVLocs.size(); ++i) {
434  CCValAssign &VA = RVLocs[i];
435  assert(VA.isRegLoc() && "Can only return in registers!");
436 
437  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
438 
439  // Guarantee that all emitted copies are stuck together,
440  // avoiding something bad.
441  Flag = Chain.getValue(1);
442  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
443  }
444 
445  RetOps[0] = Chain; // Update chain.
446 
447  // Add the flag if we have it.
448  if (Flag.getNode())
449  RetOps.push_back(Flag);
450 
451  return DAG.getNode(Opc, DL, MVT::Other, RetOps);
452 }
453 
454 SDValue BPFTargetLowering::LowerCallResult(
455  SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
456  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
457  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
458 
460  // Assign locations to each value returned by this call.
462  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
463 
464  if (Ins.size() >= 2) {
465  fail(DL, DAG, "only small returns supported");
466  for (unsigned i = 0, e = Ins.size(); i != e; ++i)
467  InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT));
468  return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1);
469  }
470 
471  CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
472 
473  // Copy all of the result registers out of their specified physreg.
474  for (auto &Val : RVLocs) {
475  Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(),
476  Val.getValVT(), InFlag).getValue(1);
477  InFlag = Chain.getValue(2);
478  InVals.push_back(Chain.getValue(0));
479  }
480 
481  return Chain;
482 }
483 
484 static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
485  switch (CC) {
486  default:
487  break;
488  case ISD::SETULT:
489  case ISD::SETULE:
490  case ISD::SETLT:
491  case ISD::SETLE:
493  std::swap(LHS, RHS);
494  break;
495  }
496 }
497 
498 SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
499  SDValue Chain = Op.getOperand(0);
500  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
501  SDValue LHS = Op.getOperand(2);
502  SDValue RHS = Op.getOperand(3);
503  SDValue Dest = Op.getOperand(4);
504  SDLoc DL(Op);
505 
506  if (!getHasJmpExt())
507  NegateCC(LHS, RHS, CC);
508 
509  return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
510  DAG.getConstant(CC, DL, MVT::i64), Dest);
511 }
512 
513 SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
514  SDValue LHS = Op.getOperand(0);
515  SDValue RHS = Op.getOperand(1);
516  SDValue TrueV = Op.getOperand(2);
517  SDValue FalseV = Op.getOperand(3);
518  ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
519  SDLoc DL(Op);
520 
521  if (!getHasJmpExt())
522  NegateCC(LHS, RHS, CC);
523 
524  SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
525  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
526  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
527 
528  return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops);
529 }
530 
531 const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
532  switch ((BPFISD::NodeType)Opcode) {
534  break;
535  case BPFISD::RET_FLAG:
536  return "BPFISD::RET_FLAG";
537  case BPFISD::CALL:
538  return "BPFISD::CALL";
539  case BPFISD::SELECT_CC:
540  return "BPFISD::SELECT_CC";
541  case BPFISD::BR_CC:
542  return "BPFISD::BR_CC";
543  case BPFISD::Wrapper:
544  return "BPFISD::Wrapper";
545  case BPFISD::MEMCPY:
546  return "BPFISD::MEMCPY";
547  }
548  return nullptr;
549 }
550 
551 SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
552  SelectionDAG &DAG) const {
553  auto N = cast<GlobalAddressSDNode>(Op);
554  assert(N->getOffset() == 0 && "Invalid offset for global address");
555 
556  SDLoc DL(Op);
557  const GlobalValue *GV = N->getGlobal();
558  SDValue GA = DAG.getTargetGlobalAddress(GV, DL, MVT::i64);
559 
560  return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA);
561 }
562 
563 unsigned
564 BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
565  unsigned Reg, bool isSigned) const {
568  int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
569  MachineFunction *F = BB->getParent();
570  DebugLoc DL = MI.getDebugLoc();
571 
572  MachineRegisterInfo &RegInfo = F->getRegInfo();
573  unsigned PromotedReg0 = RegInfo.createVirtualRegister(RC);
574  unsigned PromotedReg1 = RegInfo.createVirtualRegister(RC);
575  unsigned PromotedReg2 = RegInfo.createVirtualRegister(RC);
576  BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
577  BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1)
578  .addReg(PromotedReg0).addImm(32);
579  BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2)
580  .addReg(PromotedReg1).addImm(32);
581 
582  return PromotedReg2;
583 }
584 
586 BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
587  MachineBasicBlock *BB)
588  const {
589  MachineFunction *MF = MI.getParent()->getParent();
591  MachineInstrBuilder MIB(*MF, MI);
592  unsigned ScratchReg;
593 
594  // This function does custom insertion during lowering BPFISD::MEMCPY which
595  // only has two register operands from memcpy semantics, the copy source
596  // address and the copy destination address.
597  //
598  // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
599  // a third scratch register to serve as the destination register of load and
600  // source register of store.
601  //
602  // The scratch register here is with the Define | Dead | EarlyClobber flags.
603  // The EarlyClobber flag has the semantic property that the operand it is
604  // attached to is clobbered before the rest of the inputs are read. Hence it
605  // must be unique among the operands to the instruction. The Define flag is
606  // needed to coerce the machine verifier that an Undef value isn't a problem
607  // as we anyway is loading memory into it. The Dead flag is needed as the
608  // value in scratch isn't supposed to be used by any other instruction.
609  ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass);
610  MIB.addReg(ScratchReg,
612 
613  return BB;
614 }
615 
618  MachineBasicBlock *BB) const {
620  DebugLoc DL = MI.getDebugLoc();
621  unsigned Opc = MI.getOpcode();
622  bool isSelectRROp = (Opc == BPF::Select ||
623  Opc == BPF::Select_64_32 ||
624  Opc == BPF::Select_32 ||
625  Opc == BPF::Select_32_64);
626 
627  bool isMemcpyOp = Opc == BPF::MEMCPY;
628 
629 #ifndef NDEBUG
630  bool isSelectRIOp = (Opc == BPF::Select_Ri ||
631  Opc == BPF::Select_Ri_64_32 ||
632  Opc == BPF::Select_Ri_32 ||
633  Opc == BPF::Select_Ri_32_64);
634 
635 
636  assert((isSelectRROp || isSelectRIOp || isMemcpyOp) &&
637  "Unexpected instr type to insert");
638 #endif
639 
640  if (isMemcpyOp)
641  return EmitInstrWithCustomInserterMemcpy(MI, BB);
642 
643  bool is32BitCmp = (Opc == BPF::Select_32 ||
644  Opc == BPF::Select_32_64 ||
645  Opc == BPF::Select_Ri_32 ||
646  Opc == BPF::Select_Ri_32_64);
647 
648  // To "insert" a SELECT instruction, we actually have to insert the diamond
649  // control-flow pattern. The incoming instruction knows the destination vreg
650  // to set, the condition code register to branch on, the true/false values to
651  // select between, and a branch opcode to use.
652  const BasicBlock *LLVM_BB = BB->getBasicBlock();
654 
655  // ThisMBB:
656  // ...
657  // TrueVal = ...
658  // jmp_XX r1, r2 goto Copy1MBB
659  // fallthrough --> Copy0MBB
660  MachineBasicBlock *ThisMBB = BB;
661  MachineFunction *F = BB->getParent();
662  MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
663  MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB);
664 
665  F->insert(I, Copy0MBB);
666  F->insert(I, Copy1MBB);
667  // Update machine-CFG edges by transferring all successors of the current
668  // block to the new block which will contain the Phi node for the select.
669  Copy1MBB->splice(Copy1MBB->begin(), BB,
670  std::next(MachineBasicBlock::iterator(MI)), BB->end());
671  Copy1MBB->transferSuccessorsAndUpdatePHIs(BB);
672  // Next, add the true and fallthrough blocks as its successors.
673  BB->addSuccessor(Copy0MBB);
674  BB->addSuccessor(Copy1MBB);
675 
676  // Insert Branch if Flag
677  int CC = MI.getOperand(3).getImm();
678  int NewCC;
679  switch (CC) {
680  case ISD::SETGT:
681  NewCC = isSelectRROp ? BPF::JSGT_rr : BPF::JSGT_ri;
682  break;
683  case ISD::SETUGT:
684  NewCC = isSelectRROp ? BPF::JUGT_rr : BPF::JUGT_ri;
685  break;
686  case ISD::SETGE:
687  NewCC = isSelectRROp ? BPF::JSGE_rr : BPF::JSGE_ri;
688  break;
689  case ISD::SETUGE:
690  NewCC = isSelectRROp ? BPF::JUGE_rr : BPF::JUGE_ri;
691  break;
692  case ISD::SETEQ:
693  NewCC = isSelectRROp ? BPF::JEQ_rr : BPF::JEQ_ri;
694  break;
695  case ISD::SETNE:
696  NewCC = isSelectRROp ? BPF::JNE_rr : BPF::JNE_ri;
697  break;
698  case ISD::SETLT:
699  NewCC = isSelectRROp ? BPF::JSLT_rr : BPF::JSLT_ri;
700  break;
701  case ISD::SETULT:
702  NewCC = isSelectRROp ? BPF::JULT_rr : BPF::JULT_ri;
703  break;
704  case ISD::SETLE:
705  NewCC = isSelectRROp ? BPF::JSLE_rr : BPF::JSLE_ri;
706  break;
707  case ISD::SETULE:
708  NewCC = isSelectRROp ? BPF::JULE_rr : BPF::JULE_ri;
709  break;
710  default:
711  report_fatal_error("unimplemented select CondCode " + Twine(CC));
712  }
713 
714  unsigned LHS = MI.getOperand(1).getReg();
715  bool isSignedCmp = (CC == ISD::SETGT ||
716  CC == ISD::SETGE ||
717  CC == ISD::SETLT ||
718  CC == ISD::SETLE);
719 
720  // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
721  // to be promoted, however if the 32-bit comparison operands are destination
722  // registers then they are implicitly zero-extended already, there is no
723  // need of explicit zero-extend sequence for them.
724  //
725  // We simply do extension for all situations in this method, but we will
726  // try to remove those unnecessary in BPFMIPeephole pass.
727  if (is32BitCmp)
728  LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
729 
730  if (isSelectRROp) {
731  unsigned RHS = MI.getOperand(2).getReg();
732 
733  if (is32BitCmp)
734  RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
735 
736  BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
737  } else {
738  int64_t imm32 = MI.getOperand(2).getImm();
739  // sanity check before we build J*_ri instruction.
740  assert (isInt<32>(imm32));
741  BuildMI(BB, DL, TII.get(NewCC))
742  .addReg(LHS).addImm(imm32).addMBB(Copy1MBB);
743  }
744 
745  // Copy0MBB:
746  // %FalseValue = ...
747  // # fallthrough to Copy1MBB
748  BB = Copy0MBB;
749 
750  // Update machine-CFG edges
751  BB->addSuccessor(Copy1MBB);
752 
753  // Copy1MBB:
754  // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
755  // ...
756  BB = Copy1MBB;
757  BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg())
758  .addReg(MI.getOperand(5).getReg())
759  .addMBB(Copy0MBB)
760  .addReg(MI.getOperand(4).getReg())
761  .addMBB(ThisMBB);
762 
763  MI.eraseFromParent(); // The pseudo instruction is gone now.
764  return BB;
765 }
766 
768  EVT VT) const {
769  return getHasAlu32() ? MVT::i32 : MVT::i64;
770 }
771 
773  EVT VT) const {
774  return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
775 }
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
EVT getValueType() const
Return the ValueType of the referenced return value.
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
raw_ostream & errs()
This returns a reference to a raw_ostream for standard error.
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
Diagnostic information for unsupported feature in backend.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it&#39;s not CSE&#39;d)...
Definition: SelectionDAG.h:851
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:619
Compute iterated dominance frontiers using a linear time algorithm.
Definition: AllocatorList.h:24
void addLiveIn(unsigned Reg, unsigned vreg=0)
addLiveIn - Add the specified register as a live-in.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:383
void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
unsigned getReg() const
getReg - Returns the register number.
LLVM_NODISCARD LLVM_ATTRIBUTE_ALWAYS_INLINE size_t size() const
size - Get the string size.
Definition: StringRef.h:138
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:668
unsigned Reg
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:253
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:43
virtual const TargetRegisterClass * getRegClassFor(MVT VT) const
Return the register class that should be used for the specified value type.
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:34
F(f)
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned char TargetFlags=0)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:415
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition: Function.h:579
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:210
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:436
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
EVT is not used in-tree, but is used by out-of-tree target.
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
const HexagonInstrInfo * TII
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:42
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
SimpleValueType SimpleTy
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:409
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LocInfo getLocInfo() const
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SmallVector< ISD::InputArg, 32 > Ins
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:664
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:395
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose...
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
SmallVector< ISD::OutputArg, 32 > Outs
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:928
virtual const TargetInstrInfo * getInstrInfo() const
amdgpu Simplify well known AMD library false Value * Callee
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:118
TargetInstrInfo - Interface to description of machine instruction set.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:410
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:169
unsigned const MachineRegisterInfo * MRI
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
Machine Value Type.
LLVM Basic Block Representation.
Definition: BasicBlock.h:58
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:69
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type...
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:839
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned getCommonMaxStoresPerMemFunc() const
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
std::string getEVTString() const
This function returns value type as a string, e.g. "i32".
Definition: ValueTypes.cpp:115
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:57
void setPrefFunctionAlignment(unsigned Align)
Set the target&#39;s preferred function alignment.
self_iterator getIterator()
Definition: ilist_node.h:82
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y)...
unsigned MaxStoresPerMemmove
Specify maximum bytes of store instructions per memmove call.
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:396
Extended Value Type.
Definition: ValueTypes.h:34
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This structure contains all information that is necessary for lowering calls.
size_t size() const
Definition: SmallVector.h:53
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
const TargetRegisterInfo * getRegisterInfo() const override
Definition: BPFSubtarget.h:82
Iterator for intrusive lists based on ilist_node.
CCState - This class holds information needed while lowering arguments and return values...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
constexpr bool isInt< 32 >(int64_t x)
Definition: MathExtras.h:309
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:222
This is a &#39;vector&#39; (really, a variable-sized array), optimized for the case when the array is small...
Definition: SmallVector.h:847
const DebugLoc & getDebugLoc() const
CCValAssign - Represent assignment of one arg/retval to a location.
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:613
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:258
const DataFlowGraph & G
Definition: RDFGraph.cpp:211
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:393
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:35
int64_t getImm() const
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
Definition: BPFSubtarget.h:79
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:676
const Function & getFunction() const
Return the LLVM function that this machine code represents.
static mvt_range integer_valuetypes()
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:941
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:400
void setMinFunctionAlignment(unsigned Align)
Set the target&#39;s minimum function alignment (in log2(bytes))
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:448
void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:451
amdgpu Simplify well known AMD library false Value Value * Arg
const MachineBasicBlock * getParent() const
Definition: MachineInstr.h:254
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:607
Representation of each machine instruction.
Definition: MachineInstr.h:64
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
SmallVector< SDValue, 32 > OutVals
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:702
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:206
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:466
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:45
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
unsigned MaxStoresPerMemmoveOptSize
Maximum number of store instructions that may be substituted for a call to memmove, used for functions with OptSize attribute.
unsigned MaxStoresPerMemcpyOptSize
Maximum number of store operations that may be substituted for a call to memcpy, used for functions w...
void setStackPointerRegisterToSaveRestore(unsigned R)
If set to a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save and restore.
bool getHasAlu32() const
Definition: BPFSubtarget.h:69
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
unsigned getOpcode() const
SDValue getValue(unsigned R) const
unsigned MaxStoresPerMemcpy
Specify maximum bytes of store instructions per memcpy call.
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
bool isRegLoc() const
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:483
void print(raw_ostream &OS, const SelectionDAG *G=nullptr) const
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the &#39;usesCustomInserter&#39; fla...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getValueType(EVT)
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:59
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:49
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:423
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
unsigned MaxStoresPerMemsetOptSize
Maximum number of stores operations that may be substituted for the call to memset, used for functions with OptSize attribute.
Conversion operators.
Definition: ISDOpcodes.h:445
const SDValue & getOperand(unsigned i) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
unsigned getLocReg() const
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:454
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:414
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
bool getHasJmpExt() const
Definition: BPFSubtarget.h:68
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
LLVMContext * getContext() const
Definition: SelectionDAG.h:404
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:619
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:364
BRIND - Indirect branch.
Definition: ISDOpcodes.h:603
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:592