LLVM  10.0.0svn
RISCVISelLowering.cpp
Go to the documentation of this file.
1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "RISCV.h"
17 #include "RISCVRegisterInfo.h"
18 #include "RISCVSubtarget.h"
19 #include "RISCVTargetMachine.h"
20 #include "Utils/RISCVMatInt.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
31 #include "llvm/IR/DiagnosticInfo.h"
33 #include "llvm/Support/Debug.h"
36 
37 using namespace llvm;
38 
39 #define DEBUG_TYPE "riscv-lower"
40 
41 STATISTIC(NumTailCalls, "Number of tail calls");
42 
44  const RISCVSubtarget &STI)
45  : TargetLowering(TM), Subtarget(STI) {
46 
47  if (Subtarget.isRV32E())
48  report_fatal_error("Codegen not yet implemented for RV32E");
49 
50  RISCVABI::ABI ABI = Subtarget.getTargetABI();
51  assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
52 
53  switch (ABI) {
54  default:
55  report_fatal_error("Don't know how to lower this ABI");
59  case RISCVABI::ABI_LP64:
62  break;
63  }
64 
65  MVT XLenVT = Subtarget.getXLenVT();
66 
67  // Set up the register classes.
68  addRegisterClass(XLenVT, &RISCV::GPRRegClass);
69 
70  if (Subtarget.hasStdExtF())
71  addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
72  if (Subtarget.hasStdExtD())
73  addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
74 
75  // Compute derived properties from the register classes.
77 
79 
80  for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
81  setLoadExtAction(N, XLenVT, MVT::i1, Promote);
82 
83  // TODO: add all necessary setOperationAction calls.
85 
90 
93 
98 
99  for (auto VT : {MVT::i1, MVT::i8, MVT::i16})
101 
102  if (Subtarget.is64Bit()) {
108  }
109 
110  if (!Subtarget.hasStdExtM()) {
118  }
119 
120  if (Subtarget.is64Bit() && Subtarget.hasStdExtM()) {
125  }
126 
131 
135 
142 
143  ISD::CondCode FPCCToExtend[] = {
147 
148  ISD::NodeType FPOpToExtend[] = {
150 
151  if (Subtarget.hasStdExtF()) {
154  for (auto CC : FPCCToExtend)
159  for (auto Op : FPOpToExtend)
161  }
162 
163  if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
165 
166  if (Subtarget.hasStdExtD()) {
169  for (auto CC : FPCCToExtend)
176  for (auto Op : FPOpToExtend)
178  }
179 
183 
185 
186  // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
187  // Unfortunately this can't be determined just from the ISA naming string.
189  Subtarget.is64Bit() ? Legal : Custom);
190 
191  if (Subtarget.hasStdExtA()) {
194  } else {
196  }
197 
199 
200  // Function alignments (log2).
201  unsigned FunctionAlignment = Subtarget.hasStdExtC() ? 1 : 2;
202  setMinFunctionAlignment(FunctionAlignment);
203  setPrefFunctionAlignment(FunctionAlignment);
204 
205  // Effectively disable jump table generation.
207 }
208 
210  EVT VT) const {
211  if (!VT.isVector())
212  return getPointerTy(DL);
214 }
215 
217  const CallInst &I,
218  MachineFunction &MF,
219  unsigned Intrinsic) const {
220  switch (Intrinsic) {
221  default:
222  return false;
223  case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
224  case Intrinsic::riscv_masked_atomicrmw_add_i32:
225  case Intrinsic::riscv_masked_atomicrmw_sub_i32:
226  case Intrinsic::riscv_masked_atomicrmw_nand_i32:
227  case Intrinsic::riscv_masked_atomicrmw_max_i32:
228  case Intrinsic::riscv_masked_atomicrmw_min_i32:
229  case Intrinsic::riscv_masked_atomicrmw_umax_i32:
230  case Intrinsic::riscv_masked_atomicrmw_umin_i32:
231  case Intrinsic::riscv_masked_cmpxchg_i32:
232  PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
234  Info.memVT = MVT::getVT(PtrTy->getElementType());
235  Info.ptrVal = I.getArgOperand(0);
236  Info.offset = 0;
237  Info.align = Align(4);
240  return true;
241  }
242 }
243 
245  const AddrMode &AM, Type *Ty,
246  unsigned AS,
247  Instruction *I) const {
248  // No global is ever allowed as a base.
249  if (AM.BaseGV)
250  return false;
251 
252  // Require a 12-bit signed offset.
253  if (!isInt<12>(AM.BaseOffs))
254  return false;
255 
256  switch (AM.Scale) {
257  case 0: // "r+i" or just "i", depending on HasBaseReg.
258  break;
259  case 1:
260  if (!AM.HasBaseReg) // allow "r+i".
261  break;
262  return false; // disallow "r+r" or "r+r+i".
263  default:
264  return false;
265  }
266 
267  return true;
268 }
269 
271  return isInt<12>(Imm);
272 }
273 
275  return isInt<12>(Imm);
276 }
277 
278 // On RV32, 64-bit integers are split into their high and low parts and held
279 // in two different registers, so the trunc is free since the low register can
280 // just be used.
281 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
282  if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
283  return false;
284  unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
285  unsigned DestBits = DstTy->getPrimitiveSizeInBits();
286  return (SrcBits == 64 && DestBits == 32);
287 }
288 
289 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
290  if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
291  !SrcVT.isInteger() || !DstVT.isInteger())
292  return false;
293  unsigned SrcBits = SrcVT.getSizeInBits();
294  unsigned DestBits = DstVT.getSizeInBits();
295  return (SrcBits == 64 && DestBits == 32);
296 }
297 
299  // Zexts are free if they can be combined with a load.
300  if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
301  EVT MemVT = LD->getMemoryVT();
302  if ((MemVT == MVT::i8 || MemVT == MVT::i16 ||
303  (Subtarget.is64Bit() && MemVT == MVT::i32)) &&
304  (LD->getExtensionType() == ISD::NON_EXTLOAD ||
305  LD->getExtensionType() == ISD::ZEXTLOAD))
306  return true;
307  }
308 
309  return TargetLowering::isZExtFree(Val, VT2);
310 }
311 
313  return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
314 }
315 
317  return (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
318  (VT == MVT::f64 && Subtarget.hasStdExtD());
319 }
320 
321 // Changes the condition code and swaps operands if necessary, so the SetCC
322 // operation matches one of the comparisons supported directly in the RISC-V
323 // ISA.
324 static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
325  switch (CC) {
326  default:
327  break;
328  case ISD::SETGT:
329  case ISD::SETLE:
330  case ISD::SETUGT:
331  case ISD::SETULE:
333  std::swap(LHS, RHS);
334  break;
335  }
336 }
337 
338 // Return the RISC-V branch opcode that matches the given DAG integer
339 // condition code. The CondCode must be one of those supported by the RISC-V
340 // ISA (see normaliseSetCC).
342  switch (CC) {
343  default:
344  llvm_unreachable("Unsupported CondCode");
345  case ISD::SETEQ:
346  return RISCV::BEQ;
347  case ISD::SETNE:
348  return RISCV::BNE;
349  case ISD::SETLT:
350  return RISCV::BLT;
351  case ISD::SETGE:
352  return RISCV::BGE;
353  case ISD::SETULT:
354  return RISCV::BLTU;
355  case ISD::SETUGE:
356  return RISCV::BGEU;
357  }
358 }
359 
361  SelectionDAG &DAG) const {
362  switch (Op.getOpcode()) {
363  default:
364  report_fatal_error("unimplemented operand");
365  case ISD::GlobalAddress:
366  return lowerGlobalAddress(Op, DAG);
367  case ISD::BlockAddress:
368  return lowerBlockAddress(Op, DAG);
369  case ISD::ConstantPool:
370  return lowerConstantPool(Op, DAG);
372  return lowerGlobalTLSAddress(Op, DAG);
373  case ISD::SELECT:
374  return lowerSELECT(Op, DAG);
375  case ISD::VASTART:
376  return lowerVASTART(Op, DAG);
377  case ISD::FRAMEADDR:
378  return lowerFRAMEADDR(Op, DAG);
379  case ISD::RETURNADDR:
380  return lowerRETURNADDR(Op, DAG);
381  case ISD::SHL_PARTS:
382  return lowerShiftLeftParts(Op, DAG);
383  case ISD::SRA_PARTS:
384  return lowerShiftRightParts(Op, DAG, true);
385  case ISD::SRL_PARTS:
386  return lowerShiftRightParts(Op, DAG, false);
387  case ISD::BITCAST: {
388  assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
389  "Unexpected custom legalisation");
390  SDLoc DL(Op);
391  SDValue Op0 = Op.getOperand(0);
392  if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32)
393  return SDValue();
394  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
395  SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
396  return FPConv;
397  }
398  }
399 }
400 
402  SelectionDAG &DAG, unsigned Flags) {
403  return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
404 }
405 
407  SelectionDAG &DAG, unsigned Flags) {
408  return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
409  Flags);
410 }
411 
413  SelectionDAG &DAG, unsigned Flags) {
414  return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
415  N->getOffset(), Flags);
416 }
417 
418 template <class NodeTy>
419 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
420  bool IsLocal) const {
421  SDLoc DL(N);
422  EVT Ty = getPointerTy(DAG.getDataLayout());
423 
424  if (isPositionIndependent()) {
425  SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
426  if (IsLocal)
427  // Use PC-relative addressing to access the symbol. This generates the
428  // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
429  // %pcrel_lo(auipc)).
430  return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
431 
432  // Use PC-relative addressing to access the GOT for this symbol, then load
433  // the address from the GOT. This generates the pattern (PseudoLA sym),
434  // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
435  return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
436  }
437 
438  switch (getTargetMachine().getCodeModel()) {
439  default:
440  report_fatal_error("Unsupported code model for lowering");
441  case CodeModel::Small: {
442  // Generate a sequence for accessing addresses within the first 2 GiB of
443  // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
444  SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
445  SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
446  SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
447  return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
448  }
449  case CodeModel::Medium: {
450  // Generate a sequence for accessing addresses within any 2GiB range within
451  // the address space. This generates the pattern (PseudoLLA sym), which
452  // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
453  SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
454  return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
455  }
456  }
457 }
458 
459 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
460  SelectionDAG &DAG) const {
461  SDLoc DL(Op);
462  EVT Ty = Op.getValueType();
463  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
464  int64_t Offset = N->getOffset();
465  MVT XLenVT = Subtarget.getXLenVT();
466 
467  const GlobalValue *GV = N->getGlobal();
468  bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
469  SDValue Addr = getAddr(N, DAG, IsLocal);
470 
471  // In order to maximise the opportunity for common subexpression elimination,
472  // emit a separate ADD node for the global address offset instead of folding
473  // it in the global address node. Later peephole optimisations may choose to
474  // fold it back in when profitable.
475  if (Offset != 0)
476  return DAG.getNode(ISD::ADD, DL, Ty, Addr,
477  DAG.getConstant(Offset, DL, XLenVT));
478  return Addr;
479 }
480 
481 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
482  SelectionDAG &DAG) const {
483  BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
484 
485  return getAddr(N, DAG);
486 }
487 
488 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
489  SelectionDAG &DAG) const {
490  ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
491 
492  return getAddr(N, DAG);
493 }
494 
495 SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
496  SelectionDAG &DAG,
497  bool UseGOT) const {
498  SDLoc DL(N);
499  EVT Ty = getPointerTy(DAG.getDataLayout());
500  const GlobalValue *GV = N->getGlobal();
501  MVT XLenVT = Subtarget.getXLenVT();
502 
503  if (UseGOT) {
504  // Use PC-relative addressing to access the GOT for this TLS symbol, then
505  // load the address from the GOT and add the thread pointer. This generates
506  // the pattern (PseudoLA_TLS_IE sym), which expands to
507  // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
508  SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
509  SDValue Load =
510  SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
511 
512  // Add the thread pointer.
513  SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
514  return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
515  }
516 
517  // Generate a sequence for accessing the address relative to the thread
518  // pointer, with the appropriate adjustment for the thread pointer offset.
519  // This generates the pattern
520  // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
521  SDValue AddrHi =
522  DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
523  SDValue AddrAdd =
524  DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
525  SDValue AddrLo =
526  DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
527 
528  SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
529  SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
530  SDValue MNAdd = SDValue(
531  DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
532  0);
533  return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
534 }
535 
536 SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
537  SelectionDAG &DAG) const {
538  SDLoc DL(N);
539  EVT Ty = getPointerTy(DAG.getDataLayout());
540  IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
541  const GlobalValue *GV = N->getGlobal();
542 
543  // Use a PC-relative addressing mode to access the global dynamic GOT address.
544  // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
545  // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
546  SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
547  SDValue Load =
548  SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
549 
550  // Prepare argument list to generate call.
551  ArgListTy Args;
552  ArgListEntry Entry;
553  Entry.Node = Load;
554  Entry.Ty = CallTy;
555  Args.push_back(Entry);
556 
557  // Setup call to __tls_get_addr.
559  CLI.setDebugLoc(DL)
560  .setChain(DAG.getEntryNode())
561  .setLibCallee(CallingConv::C, CallTy,
562  DAG.getExternalSymbol("__tls_get_addr", Ty),
563  std::move(Args));
564 
565  return LowerCallTo(CLI).first;
566 }
567 
568 SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
569  SelectionDAG &DAG) const {
570  SDLoc DL(Op);
571  EVT Ty = Op.getValueType();
572  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
573  int64_t Offset = N->getOffset();
574  MVT XLenVT = Subtarget.getXLenVT();
575 
576  // Non-PIC TLS lowering should always use the LocalExec model.
580 
581  SDValue Addr;
582  switch (Model) {
583  case TLSModel::LocalExec:
584  Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
585  break;
587  Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
588  break;
591  Addr = getDynamicTLSAddr(N, DAG);
592  break;
593  }
594 
595  // In order to maximise the opportunity for common subexpression elimination,
596  // emit a separate ADD node for the global address offset instead of folding
597  // it in the global address node. Later peephole optimisations may choose to
598  // fold it back in when profitable.
599  if (Offset != 0)
600  return DAG.getNode(ISD::ADD, DL, Ty, Addr,
601  DAG.getConstant(Offset, DL, XLenVT));
602  return Addr;
603 }
604 
605 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
606  SDValue CondV = Op.getOperand(0);
607  SDValue TrueV = Op.getOperand(1);
608  SDValue FalseV = Op.getOperand(2);
609  SDLoc DL(Op);
610  MVT XLenVT = Subtarget.getXLenVT();
611 
612  // If the result type is XLenVT and CondV is the output of a SETCC node
613  // which also operated on XLenVT inputs, then merge the SETCC node into the
614  // lowered RISCVISD::SELECT_CC to take advantage of the integer
615  // compare+branch instructions. i.e.:
616  // (select (setcc lhs, rhs, cc), truev, falsev)
617  // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
618  if (Op.getSimpleValueType() == XLenVT && CondV.getOpcode() == ISD::SETCC &&
619  CondV.getOperand(0).getSimpleValueType() == XLenVT) {
620  SDValue LHS = CondV.getOperand(0);
621  SDValue RHS = CondV.getOperand(1);
622  auto CC = cast<CondCodeSDNode>(CondV.getOperand(2));
623  ISD::CondCode CCVal = CC->get();
624 
625  normaliseSetCC(LHS, RHS, CCVal);
626 
627  SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
628  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
629  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
630  return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
631  }
632 
633  // Otherwise:
634  // (select condv, truev, falsev)
635  // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
636  SDValue Zero = DAG.getConstant(0, DL, XLenVT);
637  SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);
638 
639  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
640  SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
641 
642  return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
643 }
644 
645 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
648 
649  SDLoc DL(Op);
650  SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
652 
653  // vastart just stores the address of the VarArgsFrameIndex slot into the
654  // memory location argument.
655  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
656  return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
657  MachinePointerInfo(SV));
658 }
659 
660 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
661  SelectionDAG &DAG) const {
662  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
664  MachineFrameInfo &MFI = MF.getFrameInfo();
665  MFI.setFrameAddressIsTaken(true);
666  Register FrameReg = RI.getFrameRegister(MF);
667  int XLenInBytes = Subtarget.getXLen() / 8;
668 
669  EVT VT = Op.getValueType();
670  SDLoc DL(Op);
671  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
672  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
673  while (Depth--) {
674  int Offset = -(XLenInBytes * 2);
675  SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
676  DAG.getIntPtrConstant(Offset, DL));
677  FrameAddr =
678  DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
679  }
680  return FrameAddr;
681 }
682 
683 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
684  SelectionDAG &DAG) const {
685  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
687  MachineFrameInfo &MFI = MF.getFrameInfo();
688  MFI.setReturnAddressIsTaken(true);
689  MVT XLenVT = Subtarget.getXLenVT();
690  int XLenInBytes = Subtarget.getXLen() / 8;
691 
693  return SDValue();
694 
695  EVT VT = Op.getValueType();
696  SDLoc DL(Op);
697  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
698  if (Depth) {
699  int Off = -XLenInBytes;
700  SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
701  SDValue Offset = DAG.getConstant(Off, DL, VT);
702  return DAG.getLoad(VT, DL, DAG.getEntryNode(),
703  DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
705  }
706 
707  // Return the value of the return address register, marking it an implicit
708  // live-in.
709  Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
710  return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
711 }
712 
713 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
714  SelectionDAG &DAG) const {
715  SDLoc DL(Op);
716  SDValue Lo = Op.getOperand(0);
717  SDValue Hi = Op.getOperand(1);
718  SDValue Shamt = Op.getOperand(2);
719  EVT VT = Lo.getValueType();
720 
721  // if Shamt-XLEN < 0: // Shamt < XLEN
722  // Lo = Lo << Shamt
723  // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
724  // else:
725  // Lo = 0
726  // Hi = Lo << (Shamt-XLEN)
727 
728  SDValue Zero = DAG.getConstant(0, DL, VT);
729  SDValue One = DAG.getConstant(1, DL, VT);
730  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
731  SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
732  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
733  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
734 
735  SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
736  SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
737  SDValue ShiftRightLo =
738  DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
739  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
740  SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
741  SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
742 
743  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
744 
745  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
746  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
747 
748  SDValue Parts[2] = {Lo, Hi};
749  return DAG.getMergeValues(Parts, DL);
750 }
751 
752 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
753  bool IsSRA) const {
754  SDLoc DL(Op);
755  SDValue Lo = Op.getOperand(0);
756  SDValue Hi = Op.getOperand(1);
757  SDValue Shamt = Op.getOperand(2);
758  EVT VT = Lo.getValueType();
759 
760  // SRA expansion:
761  // if Shamt-XLEN < 0: // Shamt < XLEN
762  // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
763  // Hi = Hi >>s Shamt
764  // else:
765  // Lo = Hi >>s (Shamt-XLEN);
766  // Hi = Hi >>s (XLEN-1)
767  //
768  // SRL expansion:
769  // if Shamt-XLEN < 0: // Shamt < XLEN
770  // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
771  // Hi = Hi >>u Shamt
772  // else:
773  // Lo = Hi >>u (Shamt-XLEN);
774  // Hi = 0;
775 
776  unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
777 
778  SDValue Zero = DAG.getConstant(0, DL, VT);
779  SDValue One = DAG.getConstant(1, DL, VT);
780  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
781  SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
782  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
783  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
784 
785  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
786  SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
787  SDValue ShiftLeftHi =
788  DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
789  SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
790  SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
791  SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
792  SDValue HiFalse =
793  IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
794 
795  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
796 
797  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
798  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
799 
800  SDValue Parts[2] = {Lo, Hi};
801  return DAG.getMergeValues(Parts, DL);
802 }
803 
804 // Returns the opcode of the target-specific SDNode that implements the 32-bit
805 // form of the given Opcode.
806 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
807  switch (Opcode) {
808  default:
809  llvm_unreachable("Unexpected opcode");
810  case ISD::SHL:
811  return RISCVISD::SLLW;
812  case ISD::SRA:
813  return RISCVISD::SRAW;
814  case ISD::SRL:
815  return RISCVISD::SRLW;
816  case ISD::SDIV:
817  return RISCVISD::DIVW;
818  case ISD::UDIV:
819  return RISCVISD::DIVUW;
820  case ISD::UREM:
821  return RISCVISD::REMUW;
822  }
823 }
824 
825 // Converts the given 32-bit operation to a target-specific SelectionDAG node.
826 // Because i32 isn't a legal type for RV64, these operations would otherwise
827 // be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W
828 // later one because the fact the operation was originally of type i32 is
829 // lost.
831  SDLoc DL(N);
833  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
834  SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
835  SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
836  // ReplaceNodeResults requires we maintain the same type for the return value.
837  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
838 }
839 
840 // Converts the given 32-bit operation to a i64 operation with signed extension
841 // semantic to reduce the signed extension instructions.
843  SDLoc DL(N);
844  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
845  SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
846  SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
847  SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
848  DAG.getValueType(MVT::i32));
849  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
850 }
851 
854  SelectionDAG &DAG) const {
855  SDLoc DL(N);
856  switch (N->getOpcode()) {
857  default:
858  llvm_unreachable("Don't know how to custom type legalize this operation!");
859  case ISD::READCYCLECOUNTER: {
860  assert(!Subtarget.is64Bit() &&
861  "READCYCLECOUNTER only has custom type legalization on riscv32");
862 
864  SDValue RCW =
865  DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
866 
867  Results.push_back(RCW);
868  Results.push_back(RCW.getValue(1));
869  Results.push_back(RCW.getValue(2));
870  break;
871  }
872  case ISD::ADD:
873  case ISD::SUB:
874  case ISD::MUL:
875  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
876  "Unexpected custom legalisation");
877  if (N->getOperand(1).getOpcode() == ISD::Constant)
878  return;
879  Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
880  break;
881  case ISD::SHL:
882  case ISD::SRA:
883  case ISD::SRL:
884  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
885  "Unexpected custom legalisation");
886  if (N->getOperand(1).getOpcode() == ISD::Constant)
887  return;
888  Results.push_back(customLegalizeToWOp(N, DAG));
889  break;
890  case ISD::SDIV:
891  case ISD::UDIV:
892  case ISD::UREM:
893  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
894  Subtarget.hasStdExtM() && "Unexpected custom legalisation");
895  if (N->getOperand(0).getOpcode() == ISD::Constant ||
897  return;
898  Results.push_back(customLegalizeToWOp(N, DAG));
899  break;
900  case ISD::BITCAST: {
901  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
902  Subtarget.hasStdExtF() && "Unexpected custom legalisation");
903  SDLoc DL(N);
904  SDValue Op0 = N->getOperand(0);
905  if (Op0.getValueType() != MVT::f32)
906  return;
907  SDValue FPConv =
909  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
910  break;
911  }
912  }
913 }
914 
916  DAGCombinerInfo &DCI) const {
917  SelectionDAG &DAG = DCI.DAG;
918 
919  switch (N->getOpcode()) {
920  default:
921  break;
922  case RISCVISD::SplitF64: {
923  SDValue Op0 = N->getOperand(0);
924  // If the input to SplitF64 is just BuildPairF64 then the operation is
925  // redundant. Instead, use BuildPairF64's operands directly.
926  if (Op0->getOpcode() == RISCVISD::BuildPairF64)
927  return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
928 
929  SDLoc DL(N);
930 
931  // It's cheaper to materialise two 32-bit integers than to load a double
932  // from the constant pool and transfer it to integer registers through the
933  // stack.
934  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
935  APInt V = C->getValueAPF().bitcastToAPInt();
936  SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
937  SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
938  return DCI.CombineTo(N, Lo, Hi);
939  }
940 
941  // This is a target-specific version of a DAGCombine performed in
942  // DAGCombiner::visitBITCAST. It performs the equivalent of:
943  // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
944  // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
945  if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
946  !Op0.getNode()->hasOneUse())
947  break;
948  SDValue NewSplitF64 =
950  Op0.getOperand(0));
951  SDValue Lo = NewSplitF64.getValue(0);
952  SDValue Hi = NewSplitF64.getValue(1);
953  APInt SignBit = APInt::getSignMask(32);
954  if (Op0.getOpcode() == ISD::FNEG) {
955  SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
956  DAG.getConstant(SignBit, DL, MVT::i32));
957  return DCI.CombineTo(N, Lo, NewHi);
958  }
959  assert(Op0.getOpcode() == ISD::FABS);
960  SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
961  DAG.getConstant(~SignBit, DL, MVT::i32));
962  return DCI.CombineTo(N, Lo, NewHi);
963  }
964  case RISCVISD::SLLW:
965  case RISCVISD::SRAW:
966  case RISCVISD::SRLW: {
967  // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
968  SDValue LHS = N->getOperand(0);
969  SDValue RHS = N->getOperand(1);
970  APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
971  APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5);
972  if ((SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI)) ||
973  (SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)))
974  return SDValue();
975  break;
976  }
978  SDLoc DL(N);
979  SDValue Op0 = N->getOperand(0);
980  // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
981  // conversion is unnecessary and can be replaced with an ANY_EXTEND
982  // of the FMV_W_X_RV64 operand.
983  if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) {
984  SDValue AExtOp =
985  DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0.getOperand(0));
986  return DCI.CombineTo(N, AExtOp);
987  }
988 
989  // This is a target-specific version of a DAGCombine performed in
990  // DAGCombiner::visitBITCAST. It performs the equivalent of:
991  // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
992  // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
993  if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
994  !Op0.getNode()->hasOneUse())
995  break;
997  Op0.getOperand(0));
998  APInt SignBit = APInt::getSignMask(32).sext(64);
999  if (Op0.getOpcode() == ISD::FNEG) {
1000  return DCI.CombineTo(N,
1001  DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV,
1002  DAG.getConstant(SignBit, DL, MVT::i64)));
1003  }
1004  assert(Op0.getOpcode() == ISD::FABS);
1005  return DCI.CombineTo(N,
1006  DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
1007  DAG.getConstant(~SignBit, DL, MVT::i64)));
1008  }
1009  }
1010 
1011  return SDValue();
1012 }
1013 
1015  const SDNode *N, CombineLevel Level) const {
1016  // The following folds are only desirable if `(OP _, c1 << c2)` can be
1017  // materialised in fewer instructions than `(OP _, c1)`:
1018  //
1019  // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
1020  // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
1021  SDValue N0 = N->getOperand(0);
1022  EVT Ty = N0.getValueType();
1023  if (Ty.isScalarInteger() &&
1024  (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
1025  auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
1026  auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
1027  if (C1 && C2) {
1028  APInt C1Int = C1->getAPIntValue();
1029  APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
1030 
1031  // We can materialise `c1 << c2` into an add immediate, so it's "free",
1032  // and the combine should happen, to potentially allow further combines
1033  // later.
1034  if (ShiftedC1Int.getMinSignedBits() <= 64 &&
1035  isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
1036  return true;
1037 
1038  // We can materialise `c1` in an add immediate, so it's "free", and the
1039  // combine should be prevented.
1040  if (C1Int.getMinSignedBits() <= 64 &&
1042  return false;
1043 
1044  // Neither constant will fit into an immediate, so find materialisation
1045  // costs.
1046  int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
1047  Subtarget.is64Bit());
1048  int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
1049  ShiftedC1Int, Ty.getSizeInBits(), Subtarget.is64Bit());
1050 
1051  // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
1052  // combine should be prevented.
1053  if (C1Cost < ShiftedC1Cost)
1054  return false;
1055  }
1056  }
1057  return true;
1058 }
1059 
1061  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1062  unsigned Depth) const {
1063  switch (Op.getOpcode()) {
1064  default:
1065  break;
1066  case RISCVISD::SLLW:
1067  case RISCVISD::SRAW:
1068  case RISCVISD::SRLW:
1069  case RISCVISD::DIVW:
1070  case RISCVISD::DIVUW:
1071  case RISCVISD::REMUW:
1072  // TODO: As the result is sign-extended, this is conservatively correct. A
1073  // more precise answer could be calculated for SRAW depending on known
1074  // bits in the shift amount.
1075  return 33;
1076  }
1077 
1078  return 1;
1079 }
1080 
1082  MachineBasicBlock *BB) {
1083  assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction");
1084 
1085  // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
1086  // Should the count have wrapped while it was being read, we need to try
1087  // again.
1088  // ...
1089  // read:
1090  // rdcycleh x3 # load high word of cycle
1091  // rdcycle x2 # load low word of cycle
1092  // rdcycleh x4 # load high word of cycle
1093  // bne x3, x4, read # check if high word reads match, otherwise try again
1094  // ...
1095 
1096  MachineFunction &MF = *BB->getParent();
1097  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1099 
1100  MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1101  MF.insert(It, LoopMBB);
1102 
1103  MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1104  MF.insert(It, DoneMBB);
1105 
1106  // Transfer the remainder of BB and its successor edges to DoneMBB.
1107  DoneMBB->splice(DoneMBB->begin(), BB,
1108  std::next(MachineBasicBlock::iterator(MI)), BB->end());
1109  DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
1110 
1111  BB->addSuccessor(LoopMBB);
1112 
1113  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1114  Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1115  Register LoReg = MI.getOperand(0).getReg();
1116  Register HiReg = MI.getOperand(1).getReg();
1117  DebugLoc DL = MI.getDebugLoc();
1118 
1119  const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1120  BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
1121  .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1122  .addReg(RISCV::X0);
1123  BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
1124  .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
1125  .addReg(RISCV::X0);
1126  BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
1127  .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1128  .addReg(RISCV::X0);
1129 
1130  BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
1131  .addReg(HiReg)
1132  .addReg(ReadAgainReg)
1133  .addMBB(LoopMBB);
1134 
1135  LoopMBB->addSuccessor(LoopMBB);
1136  LoopMBB->addSuccessor(DoneMBB);
1137 
1138  MI.eraseFromParent();
1139 
1140  return DoneMBB;
1141 }
1142 
1144  MachineBasicBlock *BB) {
1145  assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
1146 
1147  MachineFunction &MF = *BB->getParent();
1148  DebugLoc DL = MI.getDebugLoc();
1149  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1150  const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1151  Register LoReg = MI.getOperand(0).getReg();
1152  Register HiReg = MI.getOperand(1).getReg();
1153  Register SrcReg = MI.getOperand(2).getReg();
1154  const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
1155  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1156 
1157  TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
1158  RI);
1159  MachineMemOperand *MMO =
1160  MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1162  BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
1163  .addFrameIndex(FI)
1164  .addImm(0)
1165  .addMemOperand(MMO);
1166  BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
1167  .addFrameIndex(FI)
1168  .addImm(4)
1169  .addMemOperand(MMO);
1170  MI.eraseFromParent(); // The pseudo instruction is gone now.
1171  return BB;
1172 }
1173 
1175  MachineBasicBlock *BB) {
1176  assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
1177  "Unexpected instruction");
1178 
1179  MachineFunction &MF = *BB->getParent();
1180  DebugLoc DL = MI.getDebugLoc();
1181  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1182  const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1183  Register DstReg = MI.getOperand(0).getReg();
1184  Register LoReg = MI.getOperand(1).getReg();
1185  Register HiReg = MI.getOperand(2).getReg();
1186  const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
1187  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1188 
1189  MachineMemOperand *MMO =
1190  MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1192  BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1193  .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
1194  .addFrameIndex(FI)
1195  .addImm(0)
1196  .addMemOperand(MMO);
1197  BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1198  .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
1199  .addFrameIndex(FI)
1200  .addImm(4)
1201  .addMemOperand(MMO);
1202  TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
1203  MI.eraseFromParent(); // The pseudo instruction is gone now.
1204  return BB;
1205 }
1206 
1208  switch (MI.getOpcode()) {
1209  default:
1210  return false;
1211  case RISCV::Select_GPR_Using_CC_GPR:
1212  case RISCV::Select_FPR32_Using_CC_GPR:
1213  case RISCV::Select_FPR64_Using_CC_GPR:
1214  return true;
1215  }
1216 }
1217 
1219  MachineBasicBlock *BB) {
1220  // To "insert" Select_* instructions, we actually have to insert the triangle
1221  // control-flow pattern. The incoming instructions know the destination vreg
1222  // to set, the condition code register to branch on, the true/false values to
1223  // select between, and the condcode to use to select the appropriate branch.
1224  //
1225  // We produce the following control flow:
1226  // HeadMBB
1227  // | \
1228  // | IfFalseMBB
1229  // | /
1230  // TailMBB
1231  //
1232  // When we find a sequence of selects we attempt to optimize their emission
1233  // by sharing the control flow. Currently we only handle cases where we have
1234  // multiple selects with the exact same condition (same LHS, RHS and CC).
1235  // The selects may be interleaved with other instructions if the other
1236  // instructions meet some requirements we deem safe:
1237  // - They are debug instructions. Otherwise,
1238  // - They do not have side-effects, do not access memory and their inputs do
1239  // not depend on the results of the select pseudo-instructions.
1240  // The TrueV/FalseV operands of the selects cannot depend on the result of
1241  // previous selects in the sequence.
1242  // These conditions could be further relaxed. See the X86 target for a
1243  // related approach and more information.
1244  Register LHS = MI.getOperand(1).getReg();
1245  Register RHS = MI.getOperand(2).getReg();
1246  auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm());
1247 
1248  SmallVector<MachineInstr *, 4> SelectDebugValues;
1249  SmallSet<Register, 4> SelectDests;
1250  SelectDests.insert(MI.getOperand(0).getReg());
1251 
1252  MachineInstr *LastSelectPseudo = &MI;
1253 
1254  for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
1255  SequenceMBBI != E; ++SequenceMBBI) {
1256  if (SequenceMBBI->isDebugInstr())
1257  continue;
1258  else if (isSelectPseudo(*SequenceMBBI)) {
1259  if (SequenceMBBI->getOperand(1).getReg() != LHS ||
1260  SequenceMBBI->getOperand(2).getReg() != RHS ||
1261  SequenceMBBI->getOperand(3).getImm() != CC ||
1262  SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
1263  SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
1264  break;
1265  LastSelectPseudo = &*SequenceMBBI;
1266  SequenceMBBI->collectDebugValues(SelectDebugValues);
1267  SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
1268  } else {
1269  if (SequenceMBBI->hasUnmodeledSideEffects() ||
1270  SequenceMBBI->mayLoadOrStore())
1271  break;
1272  if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
1273  return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
1274  }))
1275  break;
1276  }
1277  }
1278 
1279  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1280  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1281  DebugLoc DL = MI.getDebugLoc();
1283 
1284  MachineBasicBlock *HeadMBB = BB;
1285  MachineFunction *F = BB->getParent();
1286  MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
1287  MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
1288 
1289  F->insert(I, IfFalseMBB);
1290  F->insert(I, TailMBB);
1291 
1292  // Transfer debug instructions associated with the selects to TailMBB.
1293  for (MachineInstr *DebugInstr : SelectDebugValues) {
1294  TailMBB->push_back(DebugInstr->removeFromParent());
1295  }
1296 
1297  // Move all instructions after the sequence to TailMBB.
1298  TailMBB->splice(TailMBB->end(), HeadMBB,
1299  std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
1300  // Update machine-CFG edges by transferring all successors of the current
1301  // block to the new block which will contain the Phi nodes for the selects.
1302  TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
1303  // Set the successors for HeadMBB.
1304  HeadMBB->addSuccessor(IfFalseMBB);
1305  HeadMBB->addSuccessor(TailMBB);
1306 
1307  // Insert appropriate branch.
1308  unsigned Opcode = getBranchOpcodeForIntCondCode(CC);
1309 
1310  BuildMI(HeadMBB, DL, TII.get(Opcode))
1311  .addReg(LHS)
1312  .addReg(RHS)
1313  .addMBB(TailMBB);
1314 
1315  // IfFalseMBB just falls through to TailMBB.
1316  IfFalseMBB->addSuccessor(TailMBB);
1317 
1318  // Create PHIs for all of the select pseudo-instructions.
1319  auto SelectMBBI = MI.getIterator();
1320  auto SelectEnd = std::next(LastSelectPseudo->getIterator());
1321  auto InsertionPoint = TailMBB->begin();
1322  while (SelectMBBI != SelectEnd) {
1323  auto Next = std::next(SelectMBBI);
1324  if (isSelectPseudo(*SelectMBBI)) {
1325  // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
1326  BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
1327  TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
1328  .addReg(SelectMBBI->getOperand(4).getReg())
1329  .addMBB(HeadMBB)
1330  .addReg(SelectMBBI->getOperand(5).getReg())
1331  .addMBB(IfFalseMBB);
1332  SelectMBBI->eraseFromParent();
1333  }
1334  SelectMBBI = Next;
1335  }
1336 
1338  return TailMBB;
1339 }
1340 
1343  MachineBasicBlock *BB) const {
1344  switch (MI.getOpcode()) {
1345  default:
1346  llvm_unreachable("Unexpected instr type to insert");
1347  case RISCV::ReadCycleWide:
1348  assert(!Subtarget.is64Bit() &&
1349  "ReadCycleWrite is only to be used on riscv32");
1350  return emitReadCycleWidePseudo(MI, BB);
1351  case RISCV::Select_GPR_Using_CC_GPR:
1352  case RISCV::Select_FPR32_Using_CC_GPR:
1353  case RISCV::Select_FPR64_Using_CC_GPR:
1354  return emitSelectPseudo(MI, BB);
1355  case RISCV::BuildPairF64Pseudo:
1356  return emitBuildPairF64Pseudo(MI, BB);
1357  case RISCV::SplitF64Pseudo:
1358  return emitSplitF64Pseudo(MI, BB);
1359  }
1360 }
1361 
1362 // Calling Convention Implementation.
1363 // The expectations for frontend ABI lowering vary from target to target.
1364 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
1365 // details, but this is a longer term goal. For now, we simply try to keep the
1366 // role of the frontend as simple and well-defined as possible. The rules can
1367 // be summarised as:
1368 // * Never split up large scalar arguments. We handle them here.
1369 // * If a hardfloat calling convention is being used, and the struct may be
1370 // passed in a pair of registers (fp+fp, int+fp), and both registers are
1371 // available, then pass as two separate arguments. If either the GPRs or FPRs
1372 // are exhausted, then pass according to the rule below.
1373 // * If a struct could never be passed in registers or directly in a stack
1374 // slot (as it is larger than 2*XLEN and the floating point rules don't
1375 // apply), then pass it using a pointer with the byval attribute.
1376 // * If a struct is less than 2*XLEN, then coerce to either a two-element
1377 // word-sized array or a 2*XLEN scalar (depending on alignment).
1378 // * The frontend can determine whether a struct is returned by reference or
1379 // not based on its size and fields. If it will be returned by reference, the
1380 // frontend must modify the prototype so a pointer with the sret annotation is
1381 // passed as the first argument. This is not necessary for large scalar
1382 // returns.
1383 // * Struct return values and varargs should be coerced to structs containing
1384 // register-size fields in the same situations they would be for fixed
1385 // arguments.
1386 
1387 static const MCPhysReg ArgGPRs[] = {
1388  RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
1389  RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
1390 };
1391 static const MCPhysReg ArgFPR32s[] = {
1392  RISCV::F10_32, RISCV::F11_32, RISCV::F12_32, RISCV::F13_32,
1393  RISCV::F14_32, RISCV::F15_32, RISCV::F16_32, RISCV::F17_32
1394 };
1395 static const MCPhysReg ArgFPR64s[] = {
1396  RISCV::F10_64, RISCV::F11_64, RISCV::F12_64, RISCV::F13_64,
1397  RISCV::F14_64, RISCV::F15_64, RISCV::F16_64, RISCV::F17_64
1398 };
1399 
1400 // Pass a 2*XLEN argument that has been split into two XLEN values through
1401 // registers or the stack as necessary.
1402 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
1403  ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
1404  MVT ValVT2, MVT LocVT2,
1405  ISD::ArgFlagsTy ArgFlags2) {
1406  unsigned XLenInBytes = XLen / 8;
1407  if (Register Reg = State.AllocateReg(ArgGPRs)) {
1408  // At least one half can be passed via register.
1409  State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
1410  VA1.getLocVT(), CCValAssign::Full));
1411  } else {
1412  // Both halves must be passed on the stack, with proper alignment.
1413  unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
1414  State.addLoc(
1415  CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
1416  State.AllocateStack(XLenInBytes, StackAlign),
1417  VA1.getLocVT(), CCValAssign::Full));
1419  ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1421  return false;
1422  }
1423 
1424  if (Register Reg = State.AllocateReg(ArgGPRs)) {
1425  // The second half can also be passed via register.
1426  State.addLoc(
1427  CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
1428  } else {
1429  // The second half is passed via the stack, without additional alignment.
1431  ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1433  }
1434 
1435  return false;
1436 }
1437 
1438 // Implements the RISC-V calling convention. Returns true upon failure.
1439 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
1440  MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
1441  ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
1442  bool IsRet, Type *OrigTy) {
1443  unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
1444  assert(XLen == 32 || XLen == 64);
1445  MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
1446 
1447  // Any return value split in to more than two values can't be returned
1448  // directly.
1449  if (IsRet && ValNo > 1)
1450  return true;
1451 
1452  // UseGPRForF32 if targeting one of the soft-float ABIs, if passing a
1453  // variadic argument, or if no F32 argument registers are available.
1454  bool UseGPRForF32 = true;
1455  // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
1456  // variadic argument, or if no F64 argument registers are available.
1457  bool UseGPRForF64 = true;
1458 
1459  switch (ABI) {
1460  default:
1461  llvm_unreachable("Unexpected ABI");
1462  case RISCVABI::ABI_ILP32:
1463  case RISCVABI::ABI_LP64:
1464  break;
1465  case RISCVABI::ABI_ILP32F:
1466  case RISCVABI::ABI_LP64F:
1467  UseGPRForF32 = !IsFixed;
1468  break;
1469  case RISCVABI::ABI_ILP32D:
1470  case RISCVABI::ABI_LP64D:
1471  UseGPRForF32 = !IsFixed;
1472  UseGPRForF64 = !IsFixed;
1473  break;
1474  }
1475 
1477  UseGPRForF32 = true;
1479  UseGPRForF64 = true;
1480 
1481  // From this point on, rely on UseGPRForF32, UseGPRForF64 and similar local
1482  // variables rather than directly checking against the target ABI.
1483 
1484  if (UseGPRForF32 && ValVT == MVT::f32) {
1485  LocVT = XLenVT;
1486  LocInfo = CCValAssign::BCvt;
1487  } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
1488  LocVT = MVT::i64;
1489  LocInfo = CCValAssign::BCvt;
1490  }
1491 
1492  // If this is a variadic argument, the RISC-V calling convention requires
1493  // that it is assigned an 'even' or 'aligned' register if it has 8-byte
1494  // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
1495  // be used regardless of whether the original argument was split during
1496  // legalisation or not. The argument will not be passed by registers if the
1497  // original type is larger than 2*XLEN, so the register alignment rule does
1498  // not apply.
1499  unsigned TwoXLenInBytes = (2 * XLen) / 8;
1500  if (!IsFixed && ArgFlags.getOrigAlign() == TwoXLenInBytes &&
1501  DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
1502  unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
1503  // Skip 'odd' register if necessary.
1504  if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
1505  State.AllocateReg(ArgGPRs);
1506  }
1507 
1508  SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
1509  SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
1510  State.getPendingArgFlags();
1511 
1512  assert(PendingLocs.size() == PendingArgFlags.size() &&
1513  "PendingLocs and PendingArgFlags out of sync");
1514 
1515  // Handle passing f64 on RV32D with a soft float ABI or when floating point
1516  // registers are exhausted.
1517  if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
1518  assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
1519  "Can't lower f64 if it is split");
1520  // Depending on available argument GPRS, f64 may be passed in a pair of
1521  // GPRs, split between a GPR and the stack, or passed completely on the
1522  // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
1523  // cases.
1524  Register Reg = State.AllocateReg(ArgGPRs);
1525  LocVT = MVT::i32;
1526  if (!Reg) {
1527  unsigned StackOffset = State.AllocateStack(8, 8);
1528  State.addLoc(
1529  CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1530  return false;
1531  }
1532  if (!State.AllocateReg(ArgGPRs))
1533  State.AllocateStack(4, 4);
1534  State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1535  return false;
1536  }
1537 
1538  // Split arguments might be passed indirectly, so keep track of the pending
1539  // values.
1540  if (ArgFlags.isSplit() || !PendingLocs.empty()) {
1541  LocVT = XLenVT;
1542  LocInfo = CCValAssign::Indirect;
1543  PendingLocs.push_back(
1544  CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
1545  PendingArgFlags.push_back(ArgFlags);
1546  if (!ArgFlags.isSplitEnd()) {
1547  return false;
1548  }
1549  }
1550 
1551  // If the split argument only had two elements, it should be passed directly
1552  // in registers or on the stack.
1553  if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) {
1554  assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
1555  // Apply the normal calling convention rules to the first half of the
1556  // split argument.
1557  CCValAssign VA = PendingLocs[0];
1558  ISD::ArgFlagsTy AF = PendingArgFlags[0];
1559  PendingLocs.clear();
1560  PendingArgFlags.clear();
1561  return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
1562  ArgFlags);
1563  }
1564 
1565  // Allocate to a register if possible, or else a stack slot.
1566  Register Reg;
1567  if (ValVT == MVT::f32 && !UseGPRForF32)
1568  Reg = State.AllocateReg(ArgFPR32s, ArgFPR64s);
1569  else if (ValVT == MVT::f64 && !UseGPRForF64)
1570  Reg = State.AllocateReg(ArgFPR64s, ArgFPR32s);
1571  else
1572  Reg = State.AllocateReg(ArgGPRs);
1573  unsigned StackOffset = Reg ? 0 : State.AllocateStack(XLen / 8, XLen / 8);
1574 
1575  // If we reach this point and PendingLocs is non-empty, we must be at the
1576  // end of a split argument that must be passed indirectly.
1577  if (!PendingLocs.empty()) {
1578  assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
1579  assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
1580 
1581  for (auto &It : PendingLocs) {
1582  if (Reg)
1583  It.convertToReg(Reg);
1584  else
1585  It.convertToMem(StackOffset);
1586  State.addLoc(It);
1587  }
1588  PendingLocs.clear();
1589  PendingArgFlags.clear();
1590  return false;
1591  }
1592 
1593  assert((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) &&
1594  "Expected an XLenVT at this stage");
1595 
1596  if (Reg) {
1597  State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1598  return false;
1599  }
1600 
1601  // When an f32 or f64 is passed on the stack, no bit-conversion is needed.
1602  if (ValVT == MVT::f32 || ValVT == MVT::f64) {
1603  LocVT = ValVT;
1604  LocInfo = CCValAssign::Full;
1605  }
1606  State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1607  return false;
1608 }
1609 
1610 void RISCVTargetLowering::analyzeInputArgs(
1611  MachineFunction &MF, CCState &CCInfo,
1612  const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
1613  unsigned NumArgs = Ins.size();
1614  FunctionType *FType = MF.getFunction().getFunctionType();
1615 
1616  for (unsigned i = 0; i != NumArgs; ++i) {
1617  MVT ArgVT = Ins[i].VT;
1618  ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
1619 
1620  Type *ArgTy = nullptr;
1621  if (IsRet)
1622  ArgTy = FType->getReturnType();
1623  else if (Ins[i].isOrigArg())
1624  ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
1625 
1626  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1627  if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1628  ArgFlags, CCInfo, /*IsRet=*/true, IsRet, ArgTy)) {
1629  LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
1630  << EVT(ArgVT).getEVTString() << '\n');
1631  llvm_unreachable(nullptr);
1632  }
1633  }
1634 }
1635 
1636 void RISCVTargetLowering::analyzeOutputArgs(
1637  MachineFunction &MF, CCState &CCInfo,
1638  const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
1639  CallLoweringInfo *CLI) const {
1640  unsigned NumArgs = Outs.size();
1641 
1642  for (unsigned i = 0; i != NumArgs; i++) {
1643  MVT ArgVT = Outs[i].VT;
1644  ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
1645  Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
1646 
1647  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1648  if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1649  ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) {
1650  LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
1651  << EVT(ArgVT).getEVTString() << "\n");
1652  llvm_unreachable(nullptr);
1653  }
1654  }
1655 }
1656 
1657 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
1658 // values.
1660  const CCValAssign &VA, const SDLoc &DL) {
1661  switch (VA.getLocInfo()) {
1662  default:
1663  llvm_unreachable("Unexpected CCValAssign::LocInfo");
1664  case CCValAssign::Full:
1665  break;
1666  case CCValAssign::BCvt:
1667  if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1668  Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
1669  break;
1670  }
1671  Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
1672  break;
1673  }
1674  return Val;
1675 }
1676 
1677 // The caller is responsible for loading the full value if the argument is
1678 // passed with CCValAssign::Indirect.
1680  const CCValAssign &VA, const SDLoc &DL) {
1681  MachineFunction &MF = DAG.getMachineFunction();
1682  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1683  EVT LocVT = VA.getLocVT();
1684  SDValue Val;
1685  const TargetRegisterClass *RC;
1686 
1687  switch (LocVT.getSimpleVT().SimpleTy) {
1688  default:
1689  llvm_unreachable("Unexpected register type");
1690  case MVT::i32:
1691  case MVT::i64:
1692  RC = &RISCV::GPRRegClass;
1693  break;
1694  case MVT::f32:
1695  RC = &RISCV::FPR32RegClass;
1696  break;
1697  case MVT::f64:
1698  RC = &RISCV::FPR64RegClass;
1699  break;
1700  }
1701 
1702  Register VReg = RegInfo.createVirtualRegister(RC);
1703  RegInfo.addLiveIn(VA.getLocReg(), VReg);
1704  Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1705 
1706  if (VA.getLocInfo() == CCValAssign::Indirect)
1707  return Val;
1708 
1709  return convertLocVTToValVT(DAG, Val, VA, DL);
1710 }
1711 
1713  const CCValAssign &VA, const SDLoc &DL) {
1714  EVT LocVT = VA.getLocVT();
1715 
1716  switch (VA.getLocInfo()) {
1717  default:
1718  llvm_unreachable("Unexpected CCValAssign::LocInfo");
1719  case CCValAssign::Full:
1720  break;
1721  case CCValAssign::BCvt:
1722  if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1723  Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
1724  break;
1725  }
1726  Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
1727  break;
1728  }
1729  return Val;
1730 }
1731 
1732 // The caller is responsible for loading the full value if the argument is
1733 // passed with CCValAssign::Indirect.
1735  const CCValAssign &VA, const SDLoc &DL) {
1736  MachineFunction &MF = DAG.getMachineFunction();
1737  MachineFrameInfo &MFI = MF.getFrameInfo();
1738  EVT LocVT = VA.getLocVT();
1739  EVT ValVT = VA.getValVT();
1741  int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
1742  VA.getLocMemOffset(), /*Immutable=*/true);
1743  SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1744  SDValue Val;
1745 
1747  switch (VA.getLocInfo()) {
1748  default:
1749  llvm_unreachable("Unexpected CCValAssign::LocInfo");
1750  case CCValAssign::Full:
1751  case CCValAssign::Indirect:
1752  case CCValAssign::BCvt:
1753  ExtType = ISD::NON_EXTLOAD;
1754  break;
1755  }
1756  Val = DAG.getExtLoad(
1757  ExtType, DL, LocVT, Chain, FIN,
1759  return Val;
1760 }
1761 
1763  const CCValAssign &VA, const SDLoc &DL) {
1764  assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
1765  "Unexpected VA");
1766  MachineFunction &MF = DAG.getMachineFunction();
1767  MachineFrameInfo &MFI = MF.getFrameInfo();
1768  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1769 
1770  if (VA.isMemLoc()) {
1771  // f64 is passed on the stack.
1772  int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true);
1773  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1774  return DAG.getLoad(MVT::f64, DL, Chain, FIN,
1776  }
1777 
1778  assert(VA.isRegLoc() && "Expected register VA assignment");
1779 
1780  Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1781  RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
1782  SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
1783  SDValue Hi;
1784  if (VA.getLocReg() == RISCV::X17) {
1785  // Second half of f64 is passed on the stack.
1786  int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true);
1787  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1788  Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
1790  } else {
1791  // Second half of f64 is passed in another GPR.
1792  Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1793  RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
1794  Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
1795  }
1796  return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1797 }
1798 
1799 // Transform physical registers into virtual registers.
1800 SDValue RISCVTargetLowering::LowerFormalArguments(
1801  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1802  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1803  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1804 
1805  switch (CallConv) {
1806  default:
1807  report_fatal_error("Unsupported calling convention");
1808  case CallingConv::C:
1809  case CallingConv::Fast:
1810  break;
1811  }
1812 
1813  MachineFunction &MF = DAG.getMachineFunction();
1814 
1815  const Function &Func = MF.getFunction();
1816  if (Func.hasFnAttribute("interrupt")) {
1817  if (!Func.arg_empty())
1819  "Functions with the interrupt attribute cannot have arguments!");
1820 
1821  StringRef Kind =
1822  MF.getFunction().getFnAttribute("interrupt").getValueAsString();
1823 
1824  if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
1826  "Function interrupt attribute argument not supported!");
1827  }
1828 
1829  EVT PtrVT = getPointerTy(DAG.getDataLayout());
1830  MVT XLenVT = Subtarget.getXLenVT();
1831  unsigned XLenInBytes = Subtarget.getXLen() / 8;
1832  // Used with vargs to acumulate store chains.
1833  std::vector<SDValue> OutChains;
1834 
1835  // Assign locations to all of the incoming arguments.
1837  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1838  analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);
1839 
1840  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1841  CCValAssign &VA = ArgLocs[i];
1842  SDValue ArgValue;
1843  // Passing f64 on RV32D with a soft float ABI must be handled as a special
1844  // case.
1845  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
1846  ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
1847  else if (VA.isRegLoc())
1848  ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL);
1849  else
1850  ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
1851 
1852  if (VA.getLocInfo() == CCValAssign::Indirect) {
1853  // If the original argument was split and passed by reference (e.g. i128
1854  // on RV32), we need to load all parts of it here (using the same
1855  // address).
1856  InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1857  MachinePointerInfo()));
1858  unsigned ArgIndex = Ins[i].OrigArgIndex;
1859  assert(Ins[i].PartOffset == 0);
1860  while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
1861  CCValAssign &PartVA = ArgLocs[i + 1];
1862  unsigned PartOffset = Ins[i + 1].PartOffset;
1863  SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1864  DAG.getIntPtrConstant(PartOffset, DL));
1865  InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1866  MachinePointerInfo()));
1867  ++i;
1868  }
1869  continue;
1870  }
1871  InVals.push_back(ArgValue);
1872  }
1873 
1874  if (IsVarArg) {
1876  unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
1877  const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1878  MachineFrameInfo &MFI = MF.getFrameInfo();
1879  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1881 
1882  // Offset of the first variable argument from stack pointer, and size of
1883  // the vararg save area. For now, the varargs save area is either zero or
1884  // large enough to hold a0-a7.
1885  int VaArgOffset, VarArgsSaveSize;
1886 
1887  // If all registers are allocated, then all varargs must be passed on the
1888  // stack and we don't need to save any argregs.
1889  if (ArgRegs.size() == Idx) {
1890  VaArgOffset = CCInfo.getNextStackOffset();
1891  VarArgsSaveSize = 0;
1892  } else {
1893  VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
1894  VaArgOffset = -VarArgsSaveSize;
1895  }
1896 
1897  // Record the frame index of the first variable argument
1898  // which is a value necessary to VASTART.
1899  int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1900  RVFI->setVarArgsFrameIndex(FI);
1901 
1902  // If saving an odd number of registers then create an extra stack slot to
1903  // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
1904  // offsets to even-numbered registered remain 2*XLEN-aligned.
1905  if (Idx % 2) {
1906  FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes,
1907  true);
1908  VarArgsSaveSize += XLenInBytes;
1909  }
1910 
1911  // Copy the integer registers that may have been used for passing varargs
1912  // to the vararg save area.
1913  for (unsigned I = Idx; I < ArgRegs.size();
1914  ++I, VaArgOffset += XLenInBytes) {
1915  const Register Reg = RegInfo.createVirtualRegister(RC);
1916  RegInfo.addLiveIn(ArgRegs[I], Reg);
1917  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
1918  FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1919  SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1920  SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
1922  cast<StoreSDNode>(Store.getNode())
1923  ->getMemOperand()
1924  ->setValue((Value *)nullptr);
1925  OutChains.push_back(Store);
1926  }
1927  RVFI->setVarArgsSaveSize(VarArgsSaveSize);
1928  }
1929 
1930  // All stores are grouped in one node to allow the matching between
1931  // the size of Ins and InVals. This only happens for vararg functions.
1932  if (!OutChains.empty()) {
1933  OutChains.push_back(Chain);
1934  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
1935  }
1936 
1937  return Chain;
1938 }
1939 
1940 /// isEligibleForTailCallOptimization - Check whether the call is eligible
1941 /// for tail call optimization.
1942 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
1943 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
1944  CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
1945  const SmallVector<CCValAssign, 16> &ArgLocs) const {
1946 
1947  auto &Callee = CLI.Callee;
1948  auto CalleeCC = CLI.CallConv;
1949  auto IsVarArg = CLI.IsVarArg;
1950  auto &Outs = CLI.Outs;
1951  auto &Caller = MF.getFunction();
1952  auto CallerCC = Caller.getCallingConv();
1953 
1954  // Do not tail call opt functions with "disable-tail-calls" attribute.
1955  if (Caller.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
1956  return false;
1957 
1958  // Exception-handling functions need a special set of instructions to
1959  // indicate a return to the hardware. Tail-calling another function would
1960  // probably break this.
1961  // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
1962  // should be expanded as new function attributes are introduced.
1963  if (Caller.hasFnAttribute("interrupt"))
1964  return false;
1965 
1966  // Do not tail call opt functions with varargs.
1967  if (IsVarArg)
1968  return false;
1969 
1970  // Do not tail call opt if the stack is used to pass parameters.
1971  if (CCInfo.getNextStackOffset() != 0)
1972  return false;
1973 
1974  // Do not tail call opt if any parameters need to be passed indirectly.
1975  // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
1976  // passed indirectly. So the address of the value will be passed in a
1977  // register, or if not available, then the address is put on the stack. In
1978  // order to pass indirectly, space on the stack often needs to be allocated
1979  // in order to store the value. In this case the CCInfo.getNextStackOffset()
1980  // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
1981  // are passed CCValAssign::Indirect.
1982  for (auto &VA : ArgLocs)
1983  if (VA.getLocInfo() == CCValAssign::Indirect)
1984  return false;
1985 
1986  // Do not tail call opt if either caller or callee uses struct return
1987  // semantics.
1988  auto IsCallerStructRet = Caller.hasStructRetAttr();
1989  auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
1990  if (IsCallerStructRet || IsCalleeStructRet)
1991  return false;
1992 
1993  // Externally-defined functions with weak linkage should not be
1994  // tail-called. The behaviour of branch instructions in this situation (as
1995  // used for tail calls) is implementation-defined, so we cannot rely on the
1996  // linker replacing the tail call with a return.
1997  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1998  const GlobalValue *GV = G->getGlobal();
1999  if (GV->hasExternalWeakLinkage())
2000  return false;
2001  }
2002 
2003  // The callee has to preserve all registers the caller needs to preserve.
2004  const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
2005  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2006  if (CalleeCC != CallerCC) {
2007  const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2008  if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
2009  return false;
2010  }
2011 
2012  // Byval parameters hand the function a pointer directly into the stack area
2013  // we want to reuse during a tail call. Working around this *is* possible
2014  // but less efficient and uglier in LowerCall.
2015  for (auto &Arg : Outs)
2016  if (Arg.Flags.isByVal())
2017  return false;
2018 
2019  return true;
2020 }
2021 
2022 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
2023 // and output parameter nodes.
2024 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
2025  SmallVectorImpl<SDValue> &InVals) const {
2026  SelectionDAG &DAG = CLI.DAG;
2027  SDLoc &DL = CLI.DL;
2028  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2029  SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2030  SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2031  SDValue Chain = CLI.Chain;
2032  SDValue Callee = CLI.Callee;
2033  bool &IsTailCall = CLI.IsTailCall;
2034  CallingConv::ID CallConv = CLI.CallConv;
2035  bool IsVarArg = CLI.IsVarArg;
2036  EVT PtrVT = getPointerTy(DAG.getDataLayout());
2037  MVT XLenVT = Subtarget.getXLenVT();
2038 
2039  MachineFunction &MF = DAG.getMachineFunction();
2040 
2041  // Analyze the operands of the call, assigning locations to each operand.
2043  CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2044  analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI);
2045 
2046  // Check if it's really possible to do a tail call.
2047  if (IsTailCall)
2048  IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
2049 
2050  if (IsTailCall)
2051  ++NumTailCalls;
2052  else if (CLI.CS && CLI.CS.isMustTailCall())
2053  report_fatal_error("failed to perform tail call elimination on a call "
2054  "site marked musttail");
2055 
2056  // Get a count of how many bytes are to be pushed on the stack.
2057  unsigned NumBytes = ArgCCInfo.getNextStackOffset();
2058 
2059  // Create local copies for byval args
2060  SmallVector<SDValue, 8> ByValArgs;
2061  for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2062  ISD::ArgFlagsTy Flags = Outs[i].Flags;
2063  if (!Flags.isByVal())
2064  continue;
2065 
2066  SDValue Arg = OutVals[i];
2067  unsigned Size = Flags.getByValSize();
2068  unsigned Align = Flags.getByValAlign();
2069 
2070  int FI = MF.getFrameInfo().CreateStackObject(Size, Align, /*isSS=*/false);
2071  SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2072  SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
2073 
2074  Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Align,
2075  /*IsVolatile=*/false,
2076  /*AlwaysInline=*/false,
2077  IsTailCall, MachinePointerInfo(),
2078  MachinePointerInfo());
2079  ByValArgs.push_back(FIPtr);
2080  }
2081 
2082  if (!IsTailCall)
2083  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
2084 
2085  // Copy argument values to their designated locations.
2087  SmallVector<SDValue, 8> MemOpChains;
2088  SDValue StackPtr;
2089  for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
2090  CCValAssign &VA = ArgLocs[i];
2091  SDValue ArgValue = OutVals[i];
2092  ISD::ArgFlagsTy Flags = Outs[i].Flags;
2093 
2094  // Handle passing f64 on RV32D with a soft float ABI as a special case.
2095  bool IsF64OnRV32DSoftABI =
2096  VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
2097  if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
2098  SDValue SplitF64 = DAG.getNode(
2099  RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
2100  SDValue Lo = SplitF64.getValue(0);
2101  SDValue Hi = SplitF64.getValue(1);
2102 
2103  Register RegLo = VA.getLocReg();
2104  RegsToPass.push_back(std::make_pair(RegLo, Lo));
2105 
2106  if (RegLo == RISCV::X17) {
2107  // Second half of f64 is passed on the stack.
2108  // Work out the address of the stack slot.
2109  if (!StackPtr.getNode())
2110  StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2111  // Emit the store.
2112  MemOpChains.push_back(
2113  DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
2114  } else {
2115  // Second half of f64 is passed in another GPR.
2116  Register RegHigh = RegLo + 1;
2117  RegsToPass.push_back(std::make_pair(RegHigh, Hi));
2118  }
2119  continue;
2120  }
2121 
2122  // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
2123  // as any other MemLoc.
2124 
2125  // Promote the value if needed.
2126  // For now, only handle fully promoted and indirect arguments.
2127  if (VA.getLocInfo() == CCValAssign::Indirect) {
2128  // Store the argument in a stack slot and pass its address.
2129  SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
2130  int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
2131  MemOpChains.push_back(
2132  DAG.getStore(Chain, DL, ArgValue, SpillSlot,
2134  // If the original argument was split (e.g. i128), we need
2135  // to store all parts of it here (and pass just one address).
2136  unsigned ArgIndex = Outs[i].OrigArgIndex;
2137  assert(Outs[i].PartOffset == 0);
2138  while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
2139  SDValue PartValue = OutVals[i + 1];
2140  unsigned PartOffset = Outs[i + 1].PartOffset;
2141  SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
2142  DAG.getIntPtrConstant(PartOffset, DL));
2143  MemOpChains.push_back(
2144  DAG.getStore(Chain, DL, PartValue, Address,
2146  ++i;
2147  }
2148  ArgValue = SpillSlot;
2149  } else {
2150  ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL);
2151  }
2152 
2153  // Use local copy if it is a byval arg.
2154  if (Flags.isByVal())
2155  ArgValue = ByValArgs[j++];
2156 
2157  if (VA.isRegLoc()) {
2158  // Queue up the argument copies and emit them at the end.
2159  RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
2160  } else {
2161  assert(VA.isMemLoc() && "Argument not register or memory");
2162  assert(!IsTailCall && "Tail call not allowed if stack is used "
2163  "for passing parameters");
2164 
2165  // Work out the address of the stack slot.
2166  if (!StackPtr.getNode())
2167  StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2168  SDValue Address =
2169  DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
2170  DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
2171 
2172  // Emit the store.
2173  MemOpChains.push_back(
2174  DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
2175  }
2176  }
2177 
2178  // Join the stores, which are independent of one another.
2179  if (!MemOpChains.empty())
2180  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2181 
2182  SDValue Glue;
2183 
2184  // Build a sequence of copy-to-reg nodes, chained and glued together.
2185  for (auto &Reg : RegsToPass) {
2186  Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
2187  Glue = Chain.getValue(1);
2188  }
2189 
2190  // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
2191  // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
2192  // split it and then direct call can be matched by PseudoCALL.
2193  if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
2194  const GlobalValue *GV = S->getGlobal();
2195 
2196  unsigned OpFlags = RISCVII::MO_CALL;
2197  if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
2198  OpFlags = RISCVII::MO_PLT;
2199 
2200  Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
2201  } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2202  unsigned OpFlags = RISCVII::MO_CALL;
2203 
2204  if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
2205  nullptr))
2206  OpFlags = RISCVII::MO_PLT;
2207 
2208  Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
2209  }
2210 
2211  // The first call operand is the chain and the second is the target address.
2213  Ops.push_back(Chain);
2214  Ops.push_back(Callee);
2215 
2216  // Add argument registers to the end of the list so that they are
2217  // known live into the call.
2218  for (auto &Reg : RegsToPass)
2219  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
2220 
2221  if (!IsTailCall) {
2222  // Add a register mask operand representing the call-preserved registers.
2223  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2224  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2225  assert(Mask && "Missing call preserved mask for calling convention");
2226  Ops.push_back(DAG.getRegisterMask(Mask));
2227  }
2228 
2229  // Glue the call to the argument copies, if any.
2230  if (Glue.getNode())
2231  Ops.push_back(Glue);
2232 
2233  // Emit the call.
2234  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2235 
2236  if (IsTailCall) {
2238  return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
2239  }
2240 
2241  Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
2242  Glue = Chain.getValue(1);
2243 
2244  // Mark the end of the call, which is glued to the call itself.
2245  Chain = DAG.getCALLSEQ_END(Chain,
2246  DAG.getConstant(NumBytes, DL, PtrVT, true),
2247  DAG.getConstant(0, DL, PtrVT, true),
2248  Glue, DL);
2249  Glue = Chain.getValue(1);
2250 
2251  // Assign locations to each value returned by this call.
2253  CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
2254  analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);
2255 
2256  // Copy all of the result registers out of their specified physreg.
2257  for (auto &VA : RVLocs) {
2258  // Copy the value out
2259  SDValue RetValue =
2260  DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
2261  // Glue the RetValue to the end of the call sequence
2262  Chain = RetValue.getValue(1);
2263  Glue = RetValue.getValue(2);
2264 
2265  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2266  assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
2267  SDValue RetValue2 =
2268  DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
2269  Chain = RetValue2.getValue(1);
2270  Glue = RetValue2.getValue(2);
2271  RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
2272  RetValue2);
2273  }
2274 
2275  RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
2276 
2277  InVals.push_back(RetValue);
2278  }
2279 
2280  return Chain;
2281 }
2282 
2283 bool RISCVTargetLowering::CanLowerReturn(
2284  CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2285  const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2287  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2288  for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2289  MVT VT = Outs[i].VT;
2290  ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
2291  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
2292  if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
2293  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr))
2294  return false;
2295  }
2296  return true;
2297 }
2298 
2299 SDValue
2300 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2301  bool IsVarArg,
2302  const SmallVectorImpl<ISD::OutputArg> &Outs,
2303  const SmallVectorImpl<SDValue> &OutVals,
2304  const SDLoc &DL, SelectionDAG &DAG) const {
2305  // Stores the assignment of the return value to a location.
2307 
2308  // Info about the registers and stack slot.
2309  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2310  *DAG.getContext());
2311 
2312  analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
2313  nullptr);
2314 
2315  SDValue Glue;
2316  SmallVector<SDValue, 4> RetOps(1, Chain);
2317 
2318  // Copy the result values into the output registers.
2319  for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
2320  SDValue Val = OutVals[i];
2321  CCValAssign &VA = RVLocs[i];
2322  assert(VA.isRegLoc() && "Can only return in registers!");
2323 
2324  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2325  // Handle returning f64 on RV32D with a soft float ABI.
2326  assert(VA.isRegLoc() && "Expected return via registers");
2328  DAG.getVTList(MVT::i32, MVT::i32), Val);
2329  SDValue Lo = SplitF64.getValue(0);
2330  SDValue Hi = SplitF64.getValue(1);
2331  Register RegLo = VA.getLocReg();
2332  Register RegHi = RegLo + 1;
2333  Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
2334  Glue = Chain.getValue(1);
2335  RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
2336  Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
2337  Glue = Chain.getValue(1);
2338  RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
2339  } else {
2340  // Handle a 'normal' return.
2341  Val = convertValVTToLocVT(DAG, Val, VA, DL);
2342  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
2343 
2344  // Guarantee that all emitted copies are stuck together.
2345  Glue = Chain.getValue(1);
2346  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2347  }
2348  }
2349 
2350  RetOps[0] = Chain; // Update chain.
2351 
2352  // Add the glue node if we have it.
2353  if (Glue.getNode()) {
2354  RetOps.push_back(Glue);
2355  }
2356 
2357  // Interrupt service routines use different return instructions.
2358  const Function &Func = DAG.getMachineFunction().getFunction();
2359  if (Func.hasFnAttribute("interrupt")) {
2360  if (!Func.getReturnType()->isVoidTy())
2362  "Functions with the interrupt attribute must have void return type!");
2363 
2364  MachineFunction &MF = DAG.getMachineFunction();
2365  StringRef Kind =
2366  MF.getFunction().getFnAttribute("interrupt").getValueAsString();
2367 
2368  unsigned RetOpc;
2369  if (Kind == "user")
2370  RetOpc = RISCVISD::URET_FLAG;
2371  else if (Kind == "supervisor")
2372  RetOpc = RISCVISD::SRET_FLAG;
2373  else
2374  RetOpc = RISCVISD::MRET_FLAG;
2375 
2376  return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
2377  }
2378 
2379  return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps);
2380 }
2381 
2382 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
2383  switch ((RISCVISD::NodeType)Opcode) {
2385  break;
2386  case RISCVISD::RET_FLAG:
2387  return "RISCVISD::RET_FLAG";
2388  case RISCVISD::URET_FLAG:
2389  return "RISCVISD::URET_FLAG";
2390  case RISCVISD::SRET_FLAG:
2391  return "RISCVISD::SRET_FLAG";
2392  case RISCVISD::MRET_FLAG:
2393  return "RISCVISD::MRET_FLAG";
2394  case RISCVISD::CALL:
2395  return "RISCVISD::CALL";
2396  case RISCVISD::SELECT_CC:
2397  return "RISCVISD::SELECT_CC";
2399  return "RISCVISD::BuildPairF64";
2400  case RISCVISD::SplitF64:
2401  return "RISCVISD::SplitF64";
2402  case RISCVISD::TAIL:
2403  return "RISCVISD::TAIL";
2404  case RISCVISD::SLLW:
2405  return "RISCVISD::SLLW";
2406  case RISCVISD::SRAW:
2407  return "RISCVISD::SRAW";
2408  case RISCVISD::SRLW:
2409  return "RISCVISD::SRLW";
2410  case RISCVISD::DIVW:
2411  return "RISCVISD::DIVW";
2412  case RISCVISD::DIVUW:
2413  return "RISCVISD::DIVUW";
2414  case RISCVISD::REMUW:
2415  return "RISCVISD::REMUW";
2417  return "RISCVISD::FMV_W_X_RV64";
2419  return "RISCVISD::FMV_X_ANYEXTW_RV64";
2421  return "RISCVISD::READ_CYCLE_WIDE";
2422  }
2423  return nullptr;
2424 }
2425 
2426 /// getConstraintType - Given a constraint letter, return the type of
2427 /// constraint it is for this target.
2430  if (Constraint.size() == 1) {
2431  switch (Constraint[0]) {
2432  default:
2433  break;
2434  case 'f':
2435  return C_RegisterClass;
2436  case 'I':
2437  case 'J':
2438  case 'K':
2439  return C_Immediate;
2440  case 'A':
2441  return C_Memory;
2442  }
2443  }
2444  return TargetLowering::getConstraintType(Constraint);
2445 }
2446 
2447 std::pair<unsigned, const TargetRegisterClass *>
2449  StringRef Constraint,
2450  MVT VT) const {
2451  // First, see if this is a constraint that directly corresponds to a
2452  // RISCV register class.
2453  if (Constraint.size() == 1) {
2454  switch (Constraint[0]) {
2455  case 'r':
2456  return std::make_pair(0U, &RISCV::GPRRegClass);
2457  case 'f':
2458  if (Subtarget.hasStdExtF() && VT == MVT::f32)
2459  return std::make_pair(0U, &RISCV::FPR32RegClass);
2460  if (Subtarget.hasStdExtD() && VT == MVT::f64)
2461  return std::make_pair(0U, &RISCV::FPR64RegClass);
2462  break;
2463  default:
2464  break;
2465  }
2466  }
2467 
2468  // Clang will correctly decode the usage of register name aliases into their
2469  // official names. However, other frontends like `rustc` do not. This allows
2470  // users of these frontends to use the ABI names for registers in LLVM-style
2471  // register constraints.
2472  Register XRegFromAlias = StringSwitch<Register>(Constraint.lower())
2473  .Case("{zero}", RISCV::X0)
2474  .Case("{ra}", RISCV::X1)
2475  .Case("{sp}", RISCV::X2)
2476  .Case("{gp}", RISCV::X3)
2477  .Case("{tp}", RISCV::X4)
2478  .Case("{t0}", RISCV::X5)
2479  .Case("{t1}", RISCV::X6)
2480  .Case("{t2}", RISCV::X7)
2481  .Cases("{s0}", "{fp}", RISCV::X8)
2482  .Case("{s1}", RISCV::X9)
2483  .Case("{a0}", RISCV::X10)
2484  .Case("{a1}", RISCV::X11)
2485  .Case("{a2}", RISCV::X12)
2486  .Case("{a3}", RISCV::X13)
2487  .Case("{a4}", RISCV::X14)
2488  .Case("{a5}", RISCV::X15)
2489  .Case("{a6}", RISCV::X16)
2490  .Case("{a7}", RISCV::X17)
2491  .Case("{s2}", RISCV::X18)
2492  .Case("{s3}", RISCV::X19)
2493  .Case("{s4}", RISCV::X20)
2494  .Case("{s5}", RISCV::X21)
2495  .Case("{s6}", RISCV::X22)
2496  .Case("{s7}", RISCV::X23)
2497  .Case("{s8}", RISCV::X24)
2498  .Case("{s9}", RISCV::X25)
2499  .Case("{s10}", RISCV::X26)
2500  .Case("{s11}", RISCV::X27)
2501  .Case("{t3}", RISCV::X28)
2502  .Case("{t4}", RISCV::X29)
2503  .Case("{t5}", RISCV::X30)
2504  .Case("{t6}", RISCV::X31)
2505  .Default(RISCV::NoRegister);
2506  if (XRegFromAlias != RISCV::NoRegister)
2507  return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
2508 
2509  // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
2510  // TableGen record rather than the AsmName to choose registers for InlineAsm
2511  // constraints, plus we want to match those names to the widest floating point
2512  // register type available, manually select floating point registers here.
2513  //
2514  // The second case is the ABI name of the register, so that frontends can also
2515  // use the ABI names in register constraint lists.
2516  if (Subtarget.hasStdExtF() || Subtarget.hasStdExtD()) {
2517  std::pair<Register, Register> FReg =
2519  .Cases("{f0}", "{ft0}", {RISCV::F0_32, RISCV::F0_64})
2520  .Cases("{f1}", "{ft1}", {RISCV::F1_32, RISCV::F1_64})
2521  .Cases("{f2}", "{ft2}", {RISCV::F2_32, RISCV::F2_64})
2522  .Cases("{f3}", "{ft3}", {RISCV::F3_32, RISCV::F3_64})
2523  .Cases("{f4}", "{ft4}", {RISCV::F4_32, RISCV::F4_64})
2524  .Cases("{f5}", "{ft5}", {RISCV::F5_32, RISCV::F5_64})
2525  .Cases("{f6}", "{ft6}", {RISCV::F6_32, RISCV::F6_64})
2526  .Cases("{f7}", "{ft7}", {RISCV::F7_32, RISCV::F7_64})
2527  .Cases("{f8}", "{fs0}", {RISCV::F8_32, RISCV::F8_64})
2528  .Cases("{f9}", "{fs1}", {RISCV::F9_32, RISCV::F9_64})
2529  .Cases("{f10}", "{fa0}", {RISCV::F10_32, RISCV::F10_64})
2530  .Cases("{f11}", "{fa1}", {RISCV::F11_32, RISCV::F11_64})
2531  .Cases("{f12}", "{fa2}", {RISCV::F12_32, RISCV::F12_64})
2532  .Cases("{f13}", "{fa3}", {RISCV::F13_32, RISCV::F13_64})
2533  .Cases("{f14}", "{fa4}", {RISCV::F14_32, RISCV::F14_64})
2534  .Cases("{f15}", "{fa5}", {RISCV::F15_32, RISCV::F15_64})
2535  .Cases("{f16}", "{fa6}", {RISCV::F16_32, RISCV::F16_64})
2536  .Cases("{f17}", "{fa7}", {RISCV::F17_32, RISCV::F17_64})
2537  .Cases("{f18}", "{fs2}", {RISCV::F18_32, RISCV::F18_64})
2538  .Cases("{f19}", "{fs3}", {RISCV::F19_32, RISCV::F19_64})
2539  .Cases("{f20}", "{fs4}", {RISCV::F20_32, RISCV::F20_64})
2540  .Cases("{f21}", "{fs5}", {RISCV::F21_32, RISCV::F21_64})
2541  .Cases("{f22}", "{fs6}", {RISCV::F22_32, RISCV::F22_64})
2542  .Cases("{f23}", "{fs7}", {RISCV::F23_32, RISCV::F23_64})
2543  .Cases("{f24}", "{fs8}", {RISCV::F24_32, RISCV::F24_64})
2544  .Cases("{f25}", "{fs9}", {RISCV::F25_32, RISCV::F25_64})
2545  .Cases("{f26}", "{fs10}", {RISCV::F26_32, RISCV::F26_64})
2546  .Cases("{f27}", "{fs11}", {RISCV::F27_32, RISCV::F27_64})
2547  .Cases("{f28}", "{ft8}", {RISCV::F28_32, RISCV::F28_64})
2548  .Cases("{f29}", "{ft9}", {RISCV::F29_32, RISCV::F29_64})
2549  .Cases("{f30}", "{ft10}", {RISCV::F30_32, RISCV::F30_64})
2550  .Cases("{f31}", "{ft11}", {RISCV::F31_32, RISCV::F31_64})
2551  .Default({RISCV::NoRegister, RISCV::NoRegister});
2552  if (FReg.first != RISCV::NoRegister)
2553  return Subtarget.hasStdExtD()
2554  ? std::make_pair(FReg.second, &RISCV::FPR64RegClass)
2555  : std::make_pair(FReg.first, &RISCV::FPR32RegClass);
2556  }
2557 
2558  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
2559 }
2560 
2561 unsigned
2563  // Currently only support length 1 constraints.
2564  if (ConstraintCode.size() == 1) {
2565  switch (ConstraintCode[0]) {
2566  case 'A':
2567  return InlineAsm::Constraint_A;
2568  default:
2569  break;
2570  }
2571  }
2572 
2573  return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
2574 }
2575 
2577  SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2578  SelectionDAG &DAG) const {
2579  // Currently only support length 1 constraints.
2580  if (Constraint.length() == 1) {
2581  switch (Constraint[0]) {
2582  case 'I':
2583  // Validate & create a 12-bit signed immediate operand.
2584  if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2585  uint64_t CVal = C->getSExtValue();
2586  if (isInt<12>(CVal))
2587  Ops.push_back(
2588  DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2589  }
2590  return;
2591  case 'J':
2592  // Validate & create an integer zero operand.
2593  if (auto *C = dyn_cast<ConstantSDNode>(Op))
2594  if (C->getZExtValue() == 0)
2595  Ops.push_back(
2596  DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
2597  return;
2598  case 'K':
2599  // Validate & create a 5-bit unsigned immediate operand.
2600  if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2601  uint64_t CVal = C->getZExtValue();
2602  if (isUInt<5>(CVal))
2603  Ops.push_back(
2604  DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2605  }
2606  return;
2607  default:
2608  break;
2609  }
2610  }
2611  TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
2612 }
2613 
2615  Instruction *Inst,
2616  AtomicOrdering Ord) const {
2617  if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
2618  return Builder.CreateFence(Ord);
2619  if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
2620  return Builder.CreateFence(AtomicOrdering::Release);
2621  return nullptr;
2622 }
2623 
2625  Instruction *Inst,
2626  AtomicOrdering Ord) const {
2627  if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
2628  return Builder.CreateFence(AtomicOrdering::Acquire);
2629  return nullptr;
2630 }
2631 
2633 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
2634  // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
2635  // point operations can't be used in an lr/sc sequence without breaking the
2636  // forward-progress guarantee.
2637  if (AI->isFloatingPointOperation())
2639 
2640  unsigned Size = AI->getType()->getPrimitiveSizeInBits();
2641  if (Size == 8 || Size == 16)
2644 }
2645 
2646 static Intrinsic::ID
2648  if (XLen == 32) {
2649  switch (BinOp) {
2650  default:
2651  llvm_unreachable("Unexpected AtomicRMW BinOp");
2652  case AtomicRMWInst::Xchg:
2653  return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
2654  case AtomicRMWInst::Add:
2655  return Intrinsic::riscv_masked_atomicrmw_add_i32;
2656  case AtomicRMWInst::Sub:
2657  return Intrinsic::riscv_masked_atomicrmw_sub_i32;
2658  case AtomicRMWInst::Nand:
2659  return Intrinsic::riscv_masked_atomicrmw_nand_i32;
2660  case AtomicRMWInst::Max:
2661  return Intrinsic::riscv_masked_atomicrmw_max_i32;
2662  case AtomicRMWInst::Min:
2663  return Intrinsic::riscv_masked_atomicrmw_min_i32;
2664  case AtomicRMWInst::UMax:
2665  return Intrinsic::riscv_masked_atomicrmw_umax_i32;
2666  case AtomicRMWInst::UMin:
2667  return Intrinsic::riscv_masked_atomicrmw_umin_i32;
2668  }
2669  }
2670 
2671  if (XLen == 64) {
2672  switch (BinOp) {
2673  default:
2674  llvm_unreachable("Unexpected AtomicRMW BinOp");
2675  case AtomicRMWInst::Xchg:
2676  return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
2677  case AtomicRMWInst::Add:
2678  return Intrinsic::riscv_masked_atomicrmw_add_i64;
2679  case AtomicRMWInst::Sub:
2680  return Intrinsic::riscv_masked_atomicrmw_sub_i64;
2681  case AtomicRMWInst::Nand:
2682  return Intrinsic::riscv_masked_atomicrmw_nand_i64;
2683  case AtomicRMWInst::Max:
2684  return Intrinsic::riscv_masked_atomicrmw_max_i64;
2685  case AtomicRMWInst::Min:
2686  return Intrinsic::riscv_masked_atomicrmw_min_i64;
2687  case AtomicRMWInst::UMax:
2688  return Intrinsic::riscv_masked_atomicrmw_umax_i64;
2689  case AtomicRMWInst::UMin:
2690  return Intrinsic::riscv_masked_atomicrmw_umin_i64;
2691  }
2692  }
2693 
2694  llvm_unreachable("Unexpected XLen\n");
2695 }
2696 
2697 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
2698  IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
2699  Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
2700  unsigned XLen = Subtarget.getXLen();
2701  Value *Ordering =
2702  Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
2703  Type *Tys[] = {AlignedAddr->getType()};
2704  Function *LrwOpScwLoop = Intrinsic::getDeclaration(
2705  AI->getModule(),
2707 
2708  if (XLen == 64) {
2709  Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
2710  Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2711  ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
2712  }
2713 
2714  Value *Result;
2715 
2716  // Must pass the shift amount needed to sign extend the loaded value prior
2717  // to performing a signed comparison for min/max. ShiftAmt is the number of
2718  // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
2719  // is the number of bits to left+right shift the value in order to
2720  // sign-extend.
2721  if (AI->getOperation() == AtomicRMWInst::Min ||
2722  AI->getOperation() == AtomicRMWInst::Max) {
2723  const DataLayout &DL = AI->getModule()->getDataLayout();
2724  unsigned ValWidth =
2726  Value *SextShamt =
2727  Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
2728  Result = Builder.CreateCall(LrwOpScwLoop,
2729  {AlignedAddr, Incr, Mask, SextShamt, Ordering});
2730  } else {
2731  Result =
2732  Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
2733  }
2734 
2735  if (XLen == 64)
2736  Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2737  return Result;
2738 }
2739 
2741 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
2742  AtomicCmpXchgInst *CI) const {
2743  unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
2744  if (Size == 8 || Size == 16)
2747 }
2748 
2749 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
2750  IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
2751  Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
2752  unsigned XLen = Subtarget.getXLen();
2753  Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
2754  Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
2755  if (XLen == 64) {
2756  CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
2757  NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
2758  Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2759  CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
2760  }
2761  Type *Tys[] = {AlignedAddr->getType()};
2762  Function *MaskedCmpXchg =
2763  Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
2764  Value *Result = Builder.CreateCall(
2765  MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
2766  if (XLen == 64)
2767  Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2768  return Result;
2769 }
2770 
2772  const Constant *PersonalityFn) const {
2773  return RISCV::X10;
2774 }
2775 
2777  const Constant *PersonalityFn) const {
2778  return RISCV::X11;
2779 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
void setFrameAddressIsTaken(bool T)
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:595
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set, or Regs.size() if they are all allocated.
static MVT getIntegerVT(unsigned BitWidth)
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:111
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
uint64_t getTypeStoreSizeInBits(Type *Ty) const
Returns the maximum number of bits that may be overwritten by storing the specified type; always a mu...
Definition: DataLayout.h:453
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:622
EVT getValueType() const
Return the ValueType of the referenced return value.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
RISCVABI::ABI getTargetABI() const
void collectDebugValues(SmallVectorImpl< MachineInstr *> &DbgValues)
Scan instructions following MI and collect any matching DBG_VALUEs.
const GlobalValue * getGlobal() const
ConstraintType getConstraintType(StringRef Constraint) const override
getConstraintType - Given a constraint letter, return the type of constraint it is for this target...
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:836
*p = old <signed v ? old : v
Definition: Instructions.h:723
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
StackOffset is a wrapper around scalable and non-scalable offsets and is used in several functions su...
LLVMContext & Context
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:959
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it&#39;s not CSE&#39;d)...
Definition: SelectionDAG.h:858
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:678
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
MachineFunctionProperties & reset(Property P)
Register getLocReg() const
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
Register getFrameRegister(const MachineFunction &MF) const override
Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
An instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:530
void addLiveIn(unsigned Reg, unsigned vreg=0)
addLiveIn - Add the specified register as a live-in.
const MachineFunctionProperties & getProperties() const
Get the function properties.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:385
unsigned addLiveIn(unsigned PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition: ValueTypes.h:145
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:730
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
const RISCVRegisterInfo * getRegisterInfo() const override
*p = old <unsigned v ? old : v
Definition: Instructions.h:727
Function Alias Analysis Results
*p = old >unsigned v ? old : v
Definition: Instructions.h:725
unsigned getValNo() const
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC)
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:323
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:813
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:389
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1882
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
MachineBasicBlock * emitReadCycleWidePseudo(MachineInstr &MI, MachineBasicBlock *BB)
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:140
F(f)
CallLoweringInfo & setDebugLoc(const SDLoc &dl)
static const MCPhysReg ArgFPR32s[]
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:446
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:693
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
*p = old >signed v ? old : v
Definition: Instructions.h:721
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
StringSwitch & Case(StringLiteral S, T Value)
Definition: StringSwitch.h:67
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:459
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
SDValue getExternalSymbol(const char *Sym, EVT VT)
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:158
bool isMemLoc() const
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:383
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:480
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
bool hasOneUse() const
Return true if there is exactly one use of this node.
A description of a memory reference used in the backend.
static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the &#39;usesCustomInserter&#39; fla...
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
const HexagonInstrInfo * TII
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:388
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
unsigned getXLen() const
Shift and rotation operations.
Definition: ISDOpcodes.h:434
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
BinOp getOperation() const
Definition: Instructions.h:752
CallLoweringInfo & setChain(SDValue InChain)
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
void addLoc(const CCValAssign &V)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:196
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:779
LLVM_NODISCARD R Default(T Value)
Definition: StringSwitch.h:181
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB)
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn&#39;t supported on the target and indicate what to d...
SimpleValueType SimpleTy
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:411
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:473
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:413
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
const BlockAddress * getBlockAddress() const
LocInfo getLocInfo() const
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const override
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
AtomicOrdering
Atomic ordering for LLVM&#39;s memory model.
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:726
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
unsigned getNextStackOffset() const
getNextStackOffset - Return the next stack offset such that all stack slots satisfy their alignment r...
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1581
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:245
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:410
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose...
bool isRV32E() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
SDValue getRegisterMask(const uint32_t *RegMask)
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition: Instructions.h:705
bool arg_empty() const
Definition: Function.h:729
SmallVectorImpl< CCValAssign > & getPendingLocs()
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it&#39;s free to truncate a value of type FromTy to type ToTy.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
bool hasStdExtA() const
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB)
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
static const MCPhysReg ArgFPR64s[]
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1135
TLSModel::Model getTLSModel(const GlobalValue *GV) const
Returns the TLS model which should be used for the given global variable.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:1012
virtual const TargetInstrInfo * getInstrInfo() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:592
READCYCLECOUNTER - This corresponds to the readcyclecounter intrinsic.
Definition: ISDOpcodes.h:778
virtual unsigned getInlineAsmMemConstraint(StringRef ConstraintCode) const
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1043
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Definition: IRBuilder.h:354
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
Class to represent pointers.
Definition: DerivedTypes.h:544
unsigned getByValSize() const
unsigned getKillRegState(bool B)
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension. ...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition: MCRegister.h:19
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
TargetInstrInfo - Interface to description of machine instruction set.
static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
A switch()-like statement whose cases are string literals.
Definition: StringSwitch.h:42
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function...
MachineInstrBundleIterator< MachineInstr > iterator
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
StringSwitch & Cases(StringLiteral S0, StringLiteral S1, T Value)
Definition: StringSwitch.h:88
void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
bool shouldAssumeDSOLocal(const Module &M, const GlobalValue *GV) const
Machine Value Type.
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
unsigned getOrigAlign() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
bool hasStdExtF() const
This is an important base class in LLVM.
Definition: Constant.h:41
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE...
Definition: ISDOpcodes.h:759
bool isFloatingPointOperation() const
Definition: Instructions.h:825
Instruction * emitTrailingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const override
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:987
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:846
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
bool isAcquireOrStronger(AtomicOrdering ao)
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
CombineLevel
Definition: DAGCombine.h:15
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1184
bool hasStdExtM() const
unsigned getExceptionSelectorRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
unsigned getLargestLegalIntTypeSizeInBits() const
Returns the size of largest legal integer type size, or 0 if none are set.
Definition: DataLayout.cpp:794
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
void setPrefFunctionAlignment(unsigned Align)
Set the target&#39;s preferred function alignment.
self_iterator getIterator()
Definition: ilist_node.h:81
Class to represent integer types.
Definition: DerivedTypes.h:40
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y)...
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:442
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:750
const MachineInstrBuilder & addFrameIndex(int Idx) const
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it&#39;s implicit...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool hasBitPreservingFPLogic(EVT VT) const override
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
std::vector< ArgListEntry > ArgListTy
Extended Value Type.
Definition: ValueTypes.h:33
static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
This structure contains all information that is necessary for lowering calls.
bool isPositionIndependent() const
size_t size() const
Definition: SmallVector.h:52
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:791
const TargetMachine & getTargetMachine() const
This class contains a discriminated union of information about pointers in memory operands...
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1874
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition: Alignment.h:40
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:970
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:686
The memory access writes data.
bool isReleaseOrStronger(AtomicOrdering ao)
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount though its operand...
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:554
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:49
Value * getValOperand()
Definition: Instructions.h:817
static const MCPhysReg ArgGPRs[]
bool hasStdExtD() const
SDValue getTargetConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:654
Iterator for intrusive lists based on ilist_node.
CCState - This class holds information needed while lowering arguments and return values...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
Definition: SelectionDAG.h:638
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
MachineOperand class - Representation of each machine instruction operand.
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG)
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
CCValAssign - Represent assignment of one arg/retval to a location.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:1035
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:95
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Instruction * emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:437
Type * getReturnType() const
Definition: DerivedTypes.h:124
static bool isSelectPseudo(MachineInstr &MI)
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
static IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition: Type.cpp:179
const Constant * getConstVal() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
int64_t getImm() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:691
const Function & getFunction() const
Return the LLVM function that this machine code represents.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:163
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
Class for arbitrary precision integers.
Definition: APInt.h:69
unsigned getByValAlign() const
CodeModel::Model getCodeModel() const
Returns the code model.
amdgpu Simplify well known AMD library false FunctionCallee Callee
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:444
void setMinFunctionAlignment(unsigned Align)
Set the target&#39;s minimum function alignment (in log2(bytes))
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
const SysReg * lookupSysRegByName(StringRef)
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:495
virtual void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Store the specified register of the given register class to the specified stack frame index...
bool is64Bit() const
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
int getIntMatCost(const APInt &Val, unsigned Size, bool IsRV64)
Definition: RISCVMatInt.cpp:78
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:470
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:666
Representation of each machine instruction.
Definition: MachineInstr.h:64
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the source.
Definition: ISDOpcodes.h:755
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:150
unsigned AllocateStack(unsigned Size, unsigned Alignment)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:411
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Definition: IRBuilder.h:1656
unsigned getExceptionPointerRegister(const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:717
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getLocMemOffset() const
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:223
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:510
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
PointerUnion< const Value *, const PseudoSourceValue * > ptrVal
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2)
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
void setStackPointerRegisterToSaveRestore(unsigned R)
If set to a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save and restore.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
static CCValAssign getMem(unsigned ValNo, MVT ValVT, unsigned Offset, MVT LocVT, LocInfo HTP)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2223
unsigned getOpcode() const
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:636
SDValue getValue(unsigned R) const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
bool isRegLoc() const
LLVM_NODISCARD std::string lower() const
Definition: StringRef.cpp:107
bool hasStdExtC() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
unsigned getMinSignedBits() const
Get the minimum bit size for this signed APInt.
Definition: APInt.h:1558
void insert(iterator MBBI, MachineBasicBlock *MBB)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
void setReturnAddressIsTaken(bool s)
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:114
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:73
SDValue getRegister(unsigned Reg, EVT VT)
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
SDValue getValueType(EVT)
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:333
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:467
Register getReg() const
getReg - Returns the register number.
const SDValue & getOperand(unsigned i) const
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:498
#define LLVM_DEBUG(X)
Definition: Debug.h:122
unsigned AllocateReg(unsigned Reg)
AllocateReg - Attempt to allocate one register.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:416
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:610
Function Alias Analysis false
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Load the specified register of the given register class from the specified stack frame index...
LLVMContext * getContext() const
Definition: SelectionDAG.h:420
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
Type * getElementType() const
Definition: DerivedTypes.h:563
Wrapper class representing virtual and physical registers.
Definition: Register.h:19
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:404
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:651