LLVM 17.0.0git
BPFISelLowering.cpp
Go to the documentation of this file.
1//===-- BPFISelLowering.cpp - BPF DAG Lowering Implementation ------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that BPF uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "BPFISelLowering.h"
15#include "BPF.h"
16#include "BPFSubtarget.h"
17#include "BPFTargetMachine.h"
27#include "llvm/Support/Debug.h"
30using namespace llvm;
31
32#define DEBUG_TYPE "bpf-lower"
33
34static cl::opt<bool> BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order",
35 cl::Hidden, cl::init(false),
36 cl::desc("Expand memcpy into load/store pairs in order"));
37
38static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg) {
40 DAG.getContext()->diagnose(
41 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
42}
43
44static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg,
45 SDValue Val) {
47 std::string Str;
49 OS << Msg;
50 Val->print(OS);
51 OS.flush();
52 DAG.getContext()->diagnose(
53 DiagnosticInfoUnsupported(MF.getFunction(), Str, DL.getDebugLoc()));
54}
55
57 const BPFSubtarget &STI)
59
60 // Set up the register classes.
61 addRegisterClass(MVT::i64, &BPF::GPRRegClass);
62 if (STI.getHasAlu32())
63 addRegisterClass(MVT::i32, &BPF::GPR32RegClass);
64
65 // Compute derived properties from the register classes
67
69
74
76
80
81 // Set unsupported atomic operations as Custom so
82 // we can emit better error messages than fatal error
83 // from selectiondag.
84 for (auto VT : {MVT::i8, MVT::i16, MVT::i32}) {
85 if (VT == MVT::i32) {
86 if (STI.getHasAlu32())
87 continue;
88 } else {
90 }
91
97 }
98
99 for (auto VT : { MVT::i32, MVT::i64 }) {
100 if (VT == MVT::i32 && !STI.getHasAlu32())
101 continue;
102
116
120 }
121
122 if (STI.getHasAlu32()) {
125 STI.getHasJmp32() ? Custom : Promote);
126 }
127
132
137
138 // Extended load operations for i1 types must be promoted
139 for (MVT VT : MVT::integer_valuetypes()) {
143
147 }
148
150
151 // Function alignments
154
156 // LLVM generic code will try to expand memcpy into load/store pairs at this
157 // stage which is before quite a few IR optimization passes, therefore the
158 // loads and stores could potentially be moved apart from each other which
159 // will cause trouble to memcpy pattern matcher inside kernel eBPF JIT
160 // compilers.
161 //
162 // When -bpf-expand-memcpy-in-order specified, we want to defer the expand
163 // of memcpy to later stage in IR optimization pipeline so those load/store
164 // pairs won't be touched and could be kept in order. Hence, we set
165 // MaxStoresPerMem* to zero to disable the generic getMemcpyLoadsAndStores
166 // code path, and ask LLVM to use target expander EmitTargetCodeForMemcpy.
171 } else {
172 // inline memcpy() for kernel to see explicit copy
173 unsigned CommonMaxStores =
175
180 }
181
182 // CPU/Feature control
183 HasAlu32 = STI.getHasAlu32();
184 HasJmp32 = STI.getHasJmp32();
185 HasJmpExt = STI.getHasJmpExt();
186}
187
189 return false;
190}
191
192bool BPFTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
193 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
194 return false;
195 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
196 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
197 return NumBits1 > NumBits2;
198}
199
200bool BPFTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
201 if (!VT1.isInteger() || !VT2.isInteger())
202 return false;
203 unsigned NumBits1 = VT1.getSizeInBits();
204 unsigned NumBits2 = VT2.getSizeInBits();
205 return NumBits1 > NumBits2;
206}
207
208bool BPFTargetLowering::isZExtFree(Type *Ty1, Type *Ty2) const {
209 if (!getHasAlu32() || !Ty1->isIntegerTy() || !Ty2->isIntegerTy())
210 return false;
211 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
212 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
213 return NumBits1 == 32 && NumBits2 == 64;
214}
215
216bool BPFTargetLowering::isZExtFree(EVT VT1, EVT VT2) const {
217 if (!getHasAlu32() || !VT1.isInteger() || !VT2.isInteger())
218 return false;
219 unsigned NumBits1 = VT1.getSizeInBits();
220 unsigned NumBits2 = VT2.getSizeInBits();
221 return NumBits1 == 32 && NumBits2 == 64;
222}
223
226 if (Constraint.size() == 1) {
227 switch (Constraint[0]) {
228 default:
229 break;
230 case 'w':
231 return C_RegisterClass;
232 }
233 }
234
235 return TargetLowering::getConstraintType(Constraint);
236}
237
238std::pair<unsigned, const TargetRegisterClass *>
240 StringRef Constraint,
241 MVT VT) const {
242 if (Constraint.size() == 1)
243 // GCC Constraint Letters
244 switch (Constraint[0]) {
245 case 'r': // GENERAL_REGS
246 return std::make_pair(0U, &BPF::GPRRegClass);
247 case 'w':
248 if (HasAlu32)
249 return std::make_pair(0U, &BPF::GPR32RegClass);
250 break;
251 default:
252 break;
253 }
254
256}
257
258void BPFTargetLowering::ReplaceNodeResults(
260 const char *err_msg;
261 uint32_t Opcode = N->getOpcode();
262 switch (Opcode) {
263 default:
264 report_fatal_error("Unhandled custom legalization");
269 case ISD::ATOMIC_SWAP:
271 if (HasAlu32 || Opcode == ISD::ATOMIC_LOAD_ADD)
272 err_msg = "Unsupported atomic operations, please use 32/64 bit version";
273 else
274 err_msg = "Unsupported atomic operations, please use 64 bit version";
275 break;
276 }
277
278 SDLoc DL(N);
279 fail(DL, DAG, err_msg);
280}
281
283 switch (Op.getOpcode()) {
284 case ISD::BR_CC:
285 return LowerBR_CC(Op, DAG);
287 return LowerGlobalAddress(Op, DAG);
288 case ISD::SELECT_CC:
289 return LowerSELECT_CC(Op, DAG);
291 report_fatal_error("Unsupported dynamic stack allocation");
292 default:
293 llvm_unreachable("unimplemented operand");
294 }
295}
296
297// Calling Convention Implementation
298#include "BPFGenCallingConv.inc"
299
300SDValue BPFTargetLowering::LowerFormalArguments(
301 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
302 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
303 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
304 switch (CallConv) {
305 default:
306 report_fatal_error("Unsupported calling convention");
307 case CallingConv::C:
309 break;
310 }
311
314
315 // Assign locations to all of the incoming arguments.
317 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
318 CCInfo.AnalyzeFormalArguments(Ins, getHasAlu32() ? CC_BPF32 : CC_BPF64);
319
320 for (auto &VA : ArgLocs) {
321 if (VA.isRegLoc()) {
322 // Arguments passed in registers
323 EVT RegVT = VA.getLocVT();
324 MVT::SimpleValueType SimpleTy = RegVT.getSimpleVT().SimpleTy;
325 switch (SimpleTy) {
326 default: {
327 errs() << "LowerFormalArguments Unhandled argument type: "
328 << RegVT << '\n';
329 llvm_unreachable(nullptr);
330 }
331 case MVT::i32:
332 case MVT::i64:
333 Register VReg = RegInfo.createVirtualRegister(
334 SimpleTy == MVT::i64 ? &BPF::GPRRegClass : &BPF::GPR32RegClass);
335 RegInfo.addLiveIn(VA.getLocReg(), VReg);
336 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, VReg, RegVT);
337
338 // If this is an value that has been promoted to wider types, insert an
339 // assert[sz]ext to capture this, then truncate to the right size.
340 if (VA.getLocInfo() == CCValAssign::SExt)
341 ArgValue = DAG.getNode(ISD::AssertSext, DL, RegVT, ArgValue,
342 DAG.getValueType(VA.getValVT()));
343 else if (VA.getLocInfo() == CCValAssign::ZExt)
344 ArgValue = DAG.getNode(ISD::AssertZext, DL, RegVT, ArgValue,
345 DAG.getValueType(VA.getValVT()));
346
347 if (VA.getLocInfo() != CCValAssign::Full)
348 ArgValue = DAG.getNode(ISD::TRUNCATE, DL, VA.getValVT(), ArgValue);
349
350 InVals.push_back(ArgValue);
351
352 break;
353 }
354 } else {
355 fail(DL, DAG, "defined with too many args");
356 InVals.push_back(DAG.getConstant(0, DL, VA.getLocVT()));
357 }
358 }
359
360 if (IsVarArg || MF.getFunction().hasStructRetAttr()) {
361 fail(DL, DAG, "functions with VarArgs or StructRet are not supported");
362 }
363
364 return Chain;
365}
366
367const unsigned BPFTargetLowering::MaxArgs = 5;
368
369SDValue BPFTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
370 SmallVectorImpl<SDValue> &InVals) const {
371 SelectionDAG &DAG = CLI.DAG;
372 auto &Outs = CLI.Outs;
373 auto &OutVals = CLI.OutVals;
374 auto &Ins = CLI.Ins;
375 SDValue Chain = CLI.Chain;
376 SDValue Callee = CLI.Callee;
377 bool &IsTailCall = CLI.IsTailCall;
378 CallingConv::ID CallConv = CLI.CallConv;
379 bool IsVarArg = CLI.IsVarArg;
381
382 // BPF target does not support tail call optimization.
383 IsTailCall = false;
384
385 switch (CallConv) {
386 default:
387 report_fatal_error("Unsupported calling convention");
389 case CallingConv::C:
390 break;
391 }
392
393 // Analyze operands of the call, assigning locations to each operand.
395 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
396
397 CCInfo.AnalyzeCallOperands(Outs, getHasAlu32() ? CC_BPF32 : CC_BPF64);
398
399 unsigned NumBytes = CCInfo.getNextStackOffset();
400
401 if (Outs.size() > MaxArgs)
402 fail(CLI.DL, DAG, "too many args to ", Callee);
403
404 for (auto &Arg : Outs) {
405 ISD::ArgFlagsTy Flags = Arg.Flags;
406 if (!Flags.isByVal())
407 continue;
408
409 fail(CLI.DL, DAG, "pass by value not supported ", Callee);
410 }
411
412 auto PtrVT = getPointerTy(MF.getDataLayout());
413 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
414
415 SmallVector<std::pair<unsigned, SDValue>, MaxArgs> RegsToPass;
416
417 // Walk arg assignments
418 for (unsigned i = 0,
419 e = std::min(static_cast<unsigned>(ArgLocs.size()), MaxArgs);
420 i != e; ++i) {
421 CCValAssign &VA = ArgLocs[i];
422 SDValue Arg = OutVals[i];
423
424 // Promote the value if needed.
425 switch (VA.getLocInfo()) {
426 default:
427 llvm_unreachable("Unknown loc info");
429 break;
431 Arg = DAG.getNode(ISD::SIGN_EXTEND, CLI.DL, VA.getLocVT(), Arg);
432 break;
434 Arg = DAG.getNode(ISD::ZERO_EXTEND, CLI.DL, VA.getLocVT(), Arg);
435 break;
437 Arg = DAG.getNode(ISD::ANY_EXTEND, CLI.DL, VA.getLocVT(), Arg);
438 break;
439 }
440
441 // Push arguments into RegsToPass vector
442 if (VA.isRegLoc())
443 RegsToPass.push_back(std::make_pair(VA.getLocReg(), Arg));
444 else
445 llvm_unreachable("call arg pass bug");
446 }
447
448 SDValue InFlag;
449
450 // Build a sequence of copy-to-reg nodes chained together with token chain and
451 // flag operands which copy the outgoing args into registers. The InFlag in
452 // necessary since all emitted instructions must be stuck together.
453 for (auto &Reg : RegsToPass) {
454 Chain = DAG.getCopyToReg(Chain, CLI.DL, Reg.first, Reg.second, InFlag);
455 InFlag = Chain.getValue(1);
456 }
457
458 // If the callee is a GlobalAddress node (quite common, every direct call is)
459 // turn it into a TargetGlobalAddress node so that legalize doesn't hack it.
460 // Likewise ExternalSymbol -> TargetExternalSymbol.
461 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
462 Callee = DAG.getTargetGlobalAddress(G->getGlobal(), CLI.DL, PtrVT,
463 G->getOffset(), 0);
464 } else if (ExternalSymbolSDNode *E = dyn_cast<ExternalSymbolSDNode>(Callee)) {
465 Callee = DAG.getTargetExternalSymbol(E->getSymbol(), PtrVT, 0);
466 fail(CLI.DL, DAG, Twine("A call to built-in function '"
467 + StringRef(E->getSymbol())
468 + "' is not supported."));
469 }
470
471 // Returns a chain & a flag for retval copy to use.
472 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
474 Ops.push_back(Chain);
475 Ops.push_back(Callee);
476
477 // Add argument registers to the end of the list so that they are
478 // known live into the call.
479 for (auto &Reg : RegsToPass)
480 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
481
482 if (InFlag.getNode())
483 Ops.push_back(InFlag);
484
485 Chain = DAG.getNode(BPFISD::CALL, CLI.DL, NodeTys, Ops);
486 InFlag = Chain.getValue(1);
487
488 // Create the CALLSEQ_END node.
489 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, InFlag, CLI.DL);
490 InFlag = Chain.getValue(1);
491
492 // Handle result values, copying them out of physregs into vregs that we
493 // return.
494 return LowerCallResult(Chain, InFlag, CallConv, IsVarArg, Ins, CLI.DL, DAG,
495 InVals);
496}
497
499BPFTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
500 bool IsVarArg,
502 const SmallVectorImpl<SDValue> &OutVals,
503 const SDLoc &DL, SelectionDAG &DAG) const {
504 unsigned Opc = BPFISD::RET_FLAG;
505
506 // CCValAssign - represent the assignment of the return value to a location
509
510 // CCState - Info about the registers and stack slot.
511 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
512
514 fail(DL, DAG, "only integer returns supported");
515 return DAG.getNode(Opc, DL, MVT::Other, Chain);
516 }
517
518 // Analize return values.
519 CCInfo.AnalyzeReturn(Outs, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
520
522 SmallVector<SDValue, 4> RetOps(1, Chain);
523
524 // Copy the result values into the output registers.
525 for (unsigned i = 0; i != RVLocs.size(); ++i) {
526 CCValAssign &VA = RVLocs[i];
527 assert(VA.isRegLoc() && "Can only return in registers!");
528
529 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), OutVals[i], Flag);
530
531 // Guarantee that all emitted copies are stuck together,
532 // avoiding something bad.
533 Flag = Chain.getValue(1);
534 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
535 }
536
537 RetOps[0] = Chain; // Update chain.
538
539 // Add the flag if we have it.
540 if (Flag.getNode())
541 RetOps.push_back(Flag);
542
543 return DAG.getNode(Opc, DL, MVT::Other, RetOps);
544}
545
546SDValue BPFTargetLowering::LowerCallResult(
547 SDValue Chain, SDValue InFlag, CallingConv::ID CallConv, bool IsVarArg,
548 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
549 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
550
552 // Assign locations to each value returned by this call.
554 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
555
556 if (Ins.size() >= 2) {
557 fail(DL, DAG, "only small returns supported");
558 for (unsigned i = 0, e = Ins.size(); i != e; ++i)
559 InVals.push_back(DAG.getConstant(0, DL, Ins[i].VT));
560 return DAG.getCopyFromReg(Chain, DL, 1, Ins[0].VT, InFlag).getValue(1);
561 }
562
563 CCInfo.AnalyzeCallResult(Ins, getHasAlu32() ? RetCC_BPF32 : RetCC_BPF64);
564
565 // Copy all of the result registers out of their specified physreg.
566 for (auto &Val : RVLocs) {
567 Chain = DAG.getCopyFromReg(Chain, DL, Val.getLocReg(),
568 Val.getValVT(), InFlag).getValue(1);
569 InFlag = Chain.getValue(2);
570 InVals.push_back(Chain.getValue(0));
571 }
572
573 return Chain;
574}
575
576static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
577 switch (CC) {
578 default:
579 break;
580 case ISD::SETULT:
581 case ISD::SETULE:
582 case ISD::SETLT:
583 case ISD::SETLE:
585 std::swap(LHS, RHS);
586 break;
587 }
588}
589
590SDValue BPFTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
591 SDValue Chain = Op.getOperand(0);
592 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(1))->get();
593 SDValue LHS = Op.getOperand(2);
594 SDValue RHS = Op.getOperand(3);
595 SDValue Dest = Op.getOperand(4);
596 SDLoc DL(Op);
597
598 if (!getHasJmpExt())
599 NegateCC(LHS, RHS, CC);
600
601 return DAG.getNode(BPFISD::BR_CC, DL, Op.getValueType(), Chain, LHS, RHS,
602 DAG.getConstant(CC, DL, LHS.getValueType()), Dest);
603}
604
605SDValue BPFTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
606 SDValue LHS = Op.getOperand(0);
607 SDValue RHS = Op.getOperand(1);
608 SDValue TrueV = Op.getOperand(2);
609 SDValue FalseV = Op.getOperand(3);
610 ISD::CondCode CC = cast<CondCodeSDNode>(Op.getOperand(4))->get();
611 SDLoc DL(Op);
612
613 if (!getHasJmpExt())
614 NegateCC(LHS, RHS, CC);
615
616 SDValue TargetCC = DAG.getConstant(CC, DL, LHS.getValueType());
617 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
618 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
619
620 return DAG.getNode(BPFISD::SELECT_CC, DL, VTs, Ops);
621}
622
623const char *BPFTargetLowering::getTargetNodeName(unsigned Opcode) const {
624 switch ((BPFISD::NodeType)Opcode) {
626 break;
627 case BPFISD::RET_FLAG:
628 return "BPFISD::RET_FLAG";
629 case BPFISD::CALL:
630 return "BPFISD::CALL";
632 return "BPFISD::SELECT_CC";
633 case BPFISD::BR_CC:
634 return "BPFISD::BR_CC";
635 case BPFISD::Wrapper:
636 return "BPFISD::Wrapper";
637 case BPFISD::MEMCPY:
638 return "BPFISD::MEMCPY";
639 }
640 return nullptr;
641}
642
643SDValue BPFTargetLowering::LowerGlobalAddress(SDValue Op,
644 SelectionDAG &DAG) const {
645 auto N = cast<GlobalAddressSDNode>(Op);
646 assert(N->getOffset() == 0 && "Invalid offset for global address");
647
648 SDLoc DL(Op);
649 const GlobalValue *GV = N->getGlobal();
651
652 return DAG.getNode(BPFISD::Wrapper, DL, MVT::i64, GA);
653}
654
655unsigned
656BPFTargetLowering::EmitSubregExt(MachineInstr &MI, MachineBasicBlock *BB,
657 unsigned Reg, bool isSigned) const {
660 int RShiftOp = isSigned ? BPF::SRA_ri : BPF::SRL_ri;
661 MachineFunction *F = BB->getParent();
662 DebugLoc DL = MI.getDebugLoc();
663
664 MachineRegisterInfo &RegInfo = F->getRegInfo();
665
666 if (!isSigned) {
667 Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
668 BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
669 return PromotedReg0;
670 }
671 Register PromotedReg0 = RegInfo.createVirtualRegister(RC);
672 Register PromotedReg1 = RegInfo.createVirtualRegister(RC);
673 Register PromotedReg2 = RegInfo.createVirtualRegister(RC);
674 BuildMI(BB, DL, TII.get(BPF::MOV_32_64), PromotedReg0).addReg(Reg);
675 BuildMI(BB, DL, TII.get(BPF::SLL_ri), PromotedReg1)
676 .addReg(PromotedReg0).addImm(32);
677 BuildMI(BB, DL, TII.get(RShiftOp), PromotedReg2)
678 .addReg(PromotedReg1).addImm(32);
679
680 return PromotedReg2;
681}
682
684BPFTargetLowering::EmitInstrWithCustomInserterMemcpy(MachineInstr &MI,
686 const {
687 MachineFunction *MF = MI.getParent()->getParent();
689 MachineInstrBuilder MIB(*MF, MI);
690 unsigned ScratchReg;
691
692 // This function does custom insertion during lowering BPFISD::MEMCPY which
693 // only has two register operands from memcpy semantics, the copy source
694 // address and the copy destination address.
695 //
696 // Because we will expand BPFISD::MEMCPY into load/store pairs, we will need
697 // a third scratch register to serve as the destination register of load and
698 // source register of store.
699 //
700 // The scratch register here is with the Define | Dead | EarlyClobber flags.
701 // The EarlyClobber flag has the semantic property that the operand it is
702 // attached to is clobbered before the rest of the inputs are read. Hence it
703 // must be unique among the operands to the instruction. The Define flag is
704 // needed to coerce the machine verifier that an Undef value isn't a problem
705 // as we anyway is loading memory into it. The Dead flag is needed as the
706 // value in scratch isn't supposed to be used by any other instruction.
707 ScratchReg = MRI.createVirtualRegister(&BPF::GPRRegClass);
708 MIB.addReg(ScratchReg,
710
711 return BB;
712}
713
716 MachineBasicBlock *BB) const {
718 DebugLoc DL = MI.getDebugLoc();
719 unsigned Opc = MI.getOpcode();
720 bool isSelectRROp = (Opc == BPF::Select ||
721 Opc == BPF::Select_64_32 ||
722 Opc == BPF::Select_32 ||
723 Opc == BPF::Select_32_64);
724
725 bool isMemcpyOp = Opc == BPF::MEMCPY;
726
727#ifndef NDEBUG
728 bool isSelectRIOp = (Opc == BPF::Select_Ri ||
729 Opc == BPF::Select_Ri_64_32 ||
730 Opc == BPF::Select_Ri_32 ||
731 Opc == BPF::Select_Ri_32_64);
732
733
734 assert((isSelectRROp || isSelectRIOp || isMemcpyOp) &&
735 "Unexpected instr type to insert");
736#endif
737
738 if (isMemcpyOp)
739 return EmitInstrWithCustomInserterMemcpy(MI, BB);
740
741 bool is32BitCmp = (Opc == BPF::Select_32 ||
742 Opc == BPF::Select_32_64 ||
743 Opc == BPF::Select_Ri_32 ||
744 Opc == BPF::Select_Ri_32_64);
745
746 // To "insert" a SELECT instruction, we actually have to insert the diamond
747 // control-flow pattern. The incoming instruction knows the destination vreg
748 // to set, the condition code register to branch on, the true/false values to
749 // select between, and a branch opcode to use.
750 const BasicBlock *LLVM_BB = BB->getBasicBlock();
752
753 // ThisMBB:
754 // ...
755 // TrueVal = ...
756 // jmp_XX r1, r2 goto Copy1MBB
757 // fallthrough --> Copy0MBB
758 MachineBasicBlock *ThisMBB = BB;
759 MachineFunction *F = BB->getParent();
760 MachineBasicBlock *Copy0MBB = F->CreateMachineBasicBlock(LLVM_BB);
761 MachineBasicBlock *Copy1MBB = F->CreateMachineBasicBlock(LLVM_BB);
762
763 F->insert(I, Copy0MBB);
764 F->insert(I, Copy1MBB);
765 // Update machine-CFG edges by transferring all successors of the current
766 // block to the new block which will contain the Phi node for the select.
767 Copy1MBB->splice(Copy1MBB->begin(), BB,
768 std::next(MachineBasicBlock::iterator(MI)), BB->end());
770 // Next, add the true and fallthrough blocks as its successors.
771 BB->addSuccessor(Copy0MBB);
772 BB->addSuccessor(Copy1MBB);
773
774 // Insert Branch if Flag
775 int CC = MI.getOperand(3).getImm();
776 int NewCC;
777 switch (CC) {
778#define SET_NEWCC(X, Y) \
779 case ISD::X: \
780 if (is32BitCmp && HasJmp32) \
781 NewCC = isSelectRROp ? BPF::Y##_rr_32 : BPF::Y##_ri_32; \
782 else \
783 NewCC = isSelectRROp ? BPF::Y##_rr : BPF::Y##_ri; \
784 break
785 SET_NEWCC(SETGT, JSGT);
786 SET_NEWCC(SETUGT, JUGT);
787 SET_NEWCC(SETGE, JSGE);
788 SET_NEWCC(SETUGE, JUGE);
789 SET_NEWCC(SETEQ, JEQ);
790 SET_NEWCC(SETNE, JNE);
791 SET_NEWCC(SETLT, JSLT);
792 SET_NEWCC(SETULT, JULT);
793 SET_NEWCC(SETLE, JSLE);
794 SET_NEWCC(SETULE, JULE);
795 default:
796 report_fatal_error("unimplemented select CondCode " + Twine(CC));
797 }
798
799 Register LHS = MI.getOperand(1).getReg();
800 bool isSignedCmp = (CC == ISD::SETGT ||
801 CC == ISD::SETGE ||
802 CC == ISD::SETLT ||
803 CC == ISD::SETLE);
804
805 // eBPF at the moment only has 64-bit comparison. Any 32-bit comparison need
806 // to be promoted, however if the 32-bit comparison operands are destination
807 // registers then they are implicitly zero-extended already, there is no
808 // need of explicit zero-extend sequence for them.
809 //
810 // We simply do extension for all situations in this method, but we will
811 // try to remove those unnecessary in BPFMIPeephole pass.
812 if (is32BitCmp && !HasJmp32)
813 LHS = EmitSubregExt(MI, BB, LHS, isSignedCmp);
814
815 if (isSelectRROp) {
816 Register RHS = MI.getOperand(2).getReg();
817
818 if (is32BitCmp && !HasJmp32)
819 RHS = EmitSubregExt(MI, BB, RHS, isSignedCmp);
820
821 BuildMI(BB, DL, TII.get(NewCC)).addReg(LHS).addReg(RHS).addMBB(Copy1MBB);
822 } else {
823 int64_t imm32 = MI.getOperand(2).getImm();
824 // Check before we build J*_ri instruction.
825 assert (isInt<32>(imm32));
826 BuildMI(BB, DL, TII.get(NewCC))
827 .addReg(LHS).addImm(imm32).addMBB(Copy1MBB);
828 }
829
830 // Copy0MBB:
831 // %FalseValue = ...
832 // # fallthrough to Copy1MBB
833 BB = Copy0MBB;
834
835 // Update machine-CFG edges
836 BB->addSuccessor(Copy1MBB);
837
838 // Copy1MBB:
839 // %Result = phi [ %FalseValue, Copy0MBB ], [ %TrueValue, ThisMBB ]
840 // ...
841 BB = Copy1MBB;
842 BuildMI(*BB, BB->begin(), DL, TII.get(BPF::PHI), MI.getOperand(0).getReg())
843 .addReg(MI.getOperand(5).getReg())
844 .addMBB(Copy0MBB)
845 .addReg(MI.getOperand(4).getReg())
846 .addMBB(ThisMBB);
847
848 MI.eraseFromParent(); // The pseudo instruction is gone now.
849 return BB;
850}
851
853 EVT VT) const {
854 return getHasAlu32() ? MVT::i32 : MVT::i64;
855}
856
858 EVT VT) const {
859 return (getHasAlu32() && VT == MVT::i32) ? MVT::i32 : MVT::i64;
860}
861
862bool BPFTargetLowering::isLegalAddressingMode(const DataLayout &DL,
863 const AddrMode &AM, Type *Ty,
864 unsigned AS,
865 Instruction *I) const {
866 // No global is ever allowed as a base.
867 if (AM.BaseGV)
868 return false;
869
870 switch (AM.Scale) {
871 case 0: // "r+i" or just "i", depending on HasBaseReg.
872 break;
873 case 1:
874 if (!AM.HasBaseReg) // allow "r+i".
875 break;
876 return false; // disallow "r+r" or "r+r+i".
877 default:
878 return false;
879 }
880
881 return true;
882}
unsigned const MachineRegisterInfo * MRI
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
amdgpu Simplify well known AMD library false FunctionCallee Callee
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
Function Alias Analysis Results
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg)
static cl::opt< bool > BPFExpandMemcpyInOrder("bpf-expand-memcpy-in-order", cl::Hidden, cl::init(false), cl::desc("Expand memcpy into load/store pairs in order"))
static void NegateCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
#define SET_NEWCC(X, Y)
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static bool isSigned(unsigned int Opcode)
const HexagonInstrInfo * TII
IRTranslator LLVM IR MI
#define F(x, y, z)
Definition: MD5.cpp:55
#define I(x, y, z)
Definition: MD5.cpp:58
#define G(x, y, z)
Definition: MD5.cpp:56
unsigned const TargetRegisterInfo * TRI
const char LLVMTargetMachineRef TM
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
raw_pwrite_stream & OS
@ Flags
Definition: TextStubV5.cpp:93
Value * RHS
Value * LHS
unsigned getCommonMaxStoresPerMemFunc() const
bool getHasJmpExt() const
Definition: BPFSubtarget.h:71
const BPFSelectionDAGInfo * getSelectionDAGInfo() const override
Definition: BPFSubtarget.h:83
const TargetRegisterInfo * getRegisterInfo() const override
Definition: BPFSubtarget.h:86
bool getHasJmp32() const
Definition: BPFSubtarget.h:72
bool getHasAlu32() const
Definition: BPFSubtarget.h:73
BPFTargetLowering::ConstraintType getConstraintType(StringRef Constraint) const override
Given a constraint, return the type of constraint it is for this target.
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
BPFTargetLowering(const TargetMachine &TM, const BPFSubtarget &STI)
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the 'usesCustomInserter' fla...
MVT getScalarShiftAmountTy(const DataLayout &, EVT) const override
Return the type to use for a scalar shift opcode, given the shifted amount type.
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
LLVM Basic Block Representation.
Definition: BasicBlock.h:56
CCState - This class holds information needed while lowering arguments and return values.
CCValAssign - Represent assignment of one arg/retval to a location.
bool isRegLoc() const
Register getLocReg() const
LocInfo getLocInfo() const
A parsed version of the target data layout string in and methods for querying it.
Definition: DataLayout.h:110
A debug info location.
Definition: DebugLoc.h:33
Diagnostic information for unsupported feature in backend.
bool hasStructRetAttr() const
Determine if the function returns a structure through first or second pointer argument.
Definition: Function.h:630
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:179
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:67
void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
Machine Value Type.
SimpleValueType SimpleTy
static auto integer_valuetypes()
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
Definition: MachineInstr.h:68
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
void print(raw_ostream &OS, const SelectionDAG *G=nullptr) const
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:225
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:721
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
SDValue getRegister(unsigned Reg, EVT VT)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:773
SDValue getValueType(EVT)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:469
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:799
LLVMContext * getContext() const
Definition: SelectionDAG.h:485
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
size_t size() const
Definition: SmallVector.h:91
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: SmallVector.h:577
void push_back(const T &Elt)
Definition: SmallVector.h:416
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
Definition: SmallVector.h:1200
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:50
constexpr size_t size() const
size - Get the string size.
Definition: StringRef.h:137
TargetInstrInfo - Interface to description of machine instruction set.
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
unsigned MaxStoresPerMemcpyOptSize
Likewise for functions with the OptSize attribute.
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
unsigned MaxStoresPerMemsetOptSize
Likewise for functions with the OptSize attribute.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
unsigned MaxStoresPerMemmove
Specify maximum number of store instructions per memmove call.
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
unsigned MaxStoresPerMemmoveOptSize
Likewise for functions with the OptSize attribute.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
unsigned MaxStoresPerMemset
Specify maximum number of store instructions per memset call.
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
unsigned MaxStoresPerMemcpy
Specify maximum number of store instructions per memcpy call.
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:78
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition: Twine.h:81
The instances of the Type class are immutable: once they are created, they are never changed.
Definition: Type.h:45
bool isAggregateType() const
Return true if the type is an aggregate type.
Definition: Type.h:297
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:231
TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
self_iterator getIterator()
Definition: ilist_node.h:82
A raw_ostream that writes to an std::string.
Definition: raw_ostream.h:642
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:749
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
Definition: ISDOpcodes.h:1069
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:1065
@ CTLZ_ZERO_UNDEF
Definition: ISDOpcodes.h:722
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:250
@ BSWAP
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:713
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:779
@ GlobalAddress
Definition: ISDOpcodes.h:78
@ ATOMIC_CMP_SWAP_WITH_SUCCESS
Val, Success, OUTCHAIN = ATOMIC_CMP_SWAP_WITH_SUCCESS(INCHAIN, ptr, cmp, swap) N.b.
Definition: ISDOpcodes.h:1191
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:255
@ ATOMIC_LOAD_OR
Definition: ISDOpcodes.h:1204
@ ATOMIC_LOAD_XOR
Definition: ISDOpcodes.h:1205
@ SIGN_EXTEND
Conversion operators.
Definition: ISDOpcodes.h:773
@ CTTZ_ZERO_UNDEF
Bit counting operators with an undefined result for zero inputs.
Definition: ISDOpcodes.h:721
@ BR_CC
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:1020
@ BRIND
BRIND - Indirect branch.
Definition: ISDOpcodes.h:999
@ BR_JT
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:1003
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:726
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:650
@ ATOMIC_LOAD_AND
Definition: ISDOpcodes.h:1202
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition: ISDOpcodes.h:776
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:741
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
Definition: ISDOpcodes.h:988
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:794
@ ATOMIC_LOAD_ADD
Definition: ISDOpcodes.h:1200
@ ATOMIC_SWAP
Val, OUTCHAIN = ATOMIC_SWAP(INCHAIN, ptr, amt) Val, OUTCHAIN = ATOMIC_LOAD_[OpName](INCHAIN,...
Definition: ISDOpcodes.h:1199
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:782
@ BRCOND
BRCOND - Conditional branch.
Definition: ISDOpcodes.h:1013
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition: ISDOpcodes.h:762
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition: ISDOpcodes.h:61
@ AssertZext
Definition: ISDOpcodes.h:62
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
Definition: ISDOpcodes.h:1447
Flag
These should be considered private to the implementation of the MCInstrDesc class.
Definition: MCInstrDesc.h:148
@ Dead
Unused definition.
@ Define
Register definition.
@ EarlyClobber
Register definition happens before uses.
Reg
All possible values of the reg field in the ModR/M byte.
initializer< Ty > init(const Ty &Val)
Definition: CommandLine.h:445
This is an optimization pass for GlobalISel generic memory operations.
Definition: AddressRanges.h:18
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:145
raw_fd_ostream & errs()
This returns a reference to a raw_ostream for standard error.
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:860
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:39
Extended Value Type.
Definition: ValueTypes.h:34
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:351
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:299
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:144
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
SmallVector< SDValue, 32 > OutVals