LLVM  9.0.0svn
RISCVISelLowering.cpp
Go to the documentation of this file.
1 //===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation --------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the interfaces that RISCV uses to lower LLVM code into a
10 // selection DAG.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "RISCVISelLowering.h"
15 #include "RISCV.h"
17 #include "RISCVRegisterInfo.h"
18 #include "RISCVSubtarget.h"
19 #include "RISCVTargetMachine.h"
20 #include "Utils/RISCVMatInt.h"
21 #include "llvm/ADT/SmallSet.h"
22 #include "llvm/ADT/Statistic.h"
31 #include "llvm/IR/DiagnosticInfo.h"
33 #include "llvm/Support/Debug.h"
36 
37 using namespace llvm;
38 
39 #define DEBUG_TYPE "riscv-lower"
40 
41 STATISTIC(NumTailCalls, "Number of tail calls");
42 
44  const RISCVSubtarget &STI)
45  : TargetLowering(TM), Subtarget(STI) {
46 
47  if (Subtarget.isRV32E())
48  report_fatal_error("Codegen not yet implemented for RV32E");
49 
50  RISCVABI::ABI ABI = Subtarget.getTargetABI();
51  assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
52 
53  switch (ABI) {
54  default:
55  report_fatal_error("Don't know how to lower this ABI");
59  case RISCVABI::ABI_LP64:
62  break;
63  }
64 
65  MVT XLenVT = Subtarget.getXLenVT();
66 
67  // Set up the register classes.
68  addRegisterClass(XLenVT, &RISCV::GPRRegClass);
69 
70  if (Subtarget.hasStdExtF())
71  addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
72  if (Subtarget.hasStdExtD())
73  addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
74 
75  // Compute derived properties from the register classes.
77 
79 
80  for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
81  setLoadExtAction(N, XLenVT, MVT::i1, Promote);
82 
83  // TODO: add all necessary setOperationAction calls.
85 
90 
93 
98 
99  for (auto VT : {MVT::i1, MVT::i8, MVT::i16})
101 
102  if (Subtarget.is64Bit()) {
106  }
107 
108  if (!Subtarget.hasStdExtM()) {
116  }
117 
118  if (Subtarget.is64Bit() && Subtarget.hasStdExtM()) {
122  }
123 
128 
132 
139 
140  ISD::CondCode FPCCToExtend[] = {
144 
145  ISD::NodeType FPOpToExtend[] = {
147 
148  if (Subtarget.hasStdExtF()) {
151  for (auto CC : FPCCToExtend)
156  for (auto Op : FPOpToExtend)
158  }
159 
160  if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
162 
163  if (Subtarget.hasStdExtD()) {
166  for (auto CC : FPCCToExtend)
173  for (auto Op : FPOpToExtend)
175  }
176 
180 
181  if (Subtarget.hasStdExtA()) {
184  } else {
186  }
187 
189 
190  // Function alignments (log2).
191  unsigned FunctionAlignment = Subtarget.hasStdExtC() ? 1 : 2;
192  setMinFunctionAlignment(FunctionAlignment);
193  setPrefFunctionAlignment(FunctionAlignment);
194 
195  // Effectively disable jump table generation.
197 }
198 
200  EVT VT) const {
201  if (!VT.isVector())
202  return getPointerTy(DL);
204 }
205 
207  const CallInst &I,
208  MachineFunction &MF,
209  unsigned Intrinsic) const {
210  switch (Intrinsic) {
211  default:
212  return false;
213  case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
214  case Intrinsic::riscv_masked_atomicrmw_add_i32:
215  case Intrinsic::riscv_masked_atomicrmw_sub_i32:
216  case Intrinsic::riscv_masked_atomicrmw_nand_i32:
217  case Intrinsic::riscv_masked_atomicrmw_max_i32:
218  case Intrinsic::riscv_masked_atomicrmw_min_i32:
219  case Intrinsic::riscv_masked_atomicrmw_umax_i32:
220  case Intrinsic::riscv_masked_atomicrmw_umin_i32:
221  case Intrinsic::riscv_masked_cmpxchg_i32:
222  PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
224  Info.memVT = MVT::getVT(PtrTy->getElementType());
225  Info.ptrVal = I.getArgOperand(0);
226  Info.offset = 0;
227  Info.align = 4;
230  return true;
231  }
232 }
233 
235  const AddrMode &AM, Type *Ty,
236  unsigned AS,
237  Instruction *I) const {
238  // No global is ever allowed as a base.
239  if (AM.BaseGV)
240  return false;
241 
242  // Require a 12-bit signed offset.
243  if (!isInt<12>(AM.BaseOffs))
244  return false;
245 
246  switch (AM.Scale) {
247  case 0: // "r+i" or just "i", depending on HasBaseReg.
248  break;
249  case 1:
250  if (!AM.HasBaseReg) // allow "r+i".
251  break;
252  return false; // disallow "r+r" or "r+r+i".
253  default:
254  return false;
255  }
256 
257  return true;
258 }
259 
261  return isInt<12>(Imm);
262 }
263 
265  return isInt<12>(Imm);
266 }
267 
268 // On RV32, 64-bit integers are split into their high and low parts and held
269 // in two different registers, so the trunc is free since the low register can
270 // just be used.
271 bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
272  if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
273  return false;
274  unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
275  unsigned DestBits = DstTy->getPrimitiveSizeInBits();
276  return (SrcBits == 64 && DestBits == 32);
277 }
278 
279 bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
280  if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
281  !SrcVT.isInteger() || !DstVT.isInteger())
282  return false;
283  unsigned SrcBits = SrcVT.getSizeInBits();
284  unsigned DestBits = DstVT.getSizeInBits();
285  return (SrcBits == 64 && DestBits == 32);
286 }
287 
289  // Zexts are free if they can be combined with a load.
290  if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
291  EVT MemVT = LD->getMemoryVT();
292  if ((MemVT == MVT::i8 || MemVT == MVT::i16 ||
293  (Subtarget.is64Bit() && MemVT == MVT::i32)) &&
294  (LD->getExtensionType() == ISD::NON_EXTLOAD ||
295  LD->getExtensionType() == ISD::ZEXTLOAD))
296  return true;
297  }
298 
299  return TargetLowering::isZExtFree(Val, VT2);
300 }
301 
303  return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
304 }
305 
307  return (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
308  (VT == MVT::f64 && Subtarget.hasStdExtD());
309 }
310 
311 // Changes the condition code and swaps operands if necessary, so the SetCC
312 // operation matches one of the comparisons supported directly in the RISC-V
313 // ISA.
314 static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
315  switch (CC) {
316  default:
317  break;
318  case ISD::SETGT:
319  case ISD::SETLE:
320  case ISD::SETUGT:
321  case ISD::SETULE:
323  std::swap(LHS, RHS);
324  break;
325  }
326 }
327 
328 // Return the RISC-V branch opcode that matches the given DAG integer
329 // condition code. The CondCode must be one of those supported by the RISC-V
330 // ISA (see normaliseSetCC).
332  switch (CC) {
333  default:
334  llvm_unreachable("Unsupported CondCode");
335  case ISD::SETEQ:
336  return RISCV::BEQ;
337  case ISD::SETNE:
338  return RISCV::BNE;
339  case ISD::SETLT:
340  return RISCV::BLT;
341  case ISD::SETGE:
342  return RISCV::BGE;
343  case ISD::SETULT:
344  return RISCV::BLTU;
345  case ISD::SETUGE:
346  return RISCV::BGEU;
347  }
348 }
349 
351  SelectionDAG &DAG) const {
352  switch (Op.getOpcode()) {
353  default:
354  report_fatal_error("unimplemented operand");
355  case ISD::GlobalAddress:
356  return lowerGlobalAddress(Op, DAG);
357  case ISD::BlockAddress:
358  return lowerBlockAddress(Op, DAG);
359  case ISD::ConstantPool:
360  return lowerConstantPool(Op, DAG);
361  case ISD::SELECT:
362  return lowerSELECT(Op, DAG);
363  case ISD::VASTART:
364  return lowerVASTART(Op, DAG);
365  case ISD::FRAMEADDR:
366  return lowerFRAMEADDR(Op, DAG);
367  case ISD::RETURNADDR:
368  return lowerRETURNADDR(Op, DAG);
369  case ISD::SHL_PARTS:
370  return lowerShiftLeftParts(Op, DAG);
371  case ISD::SRA_PARTS:
372  return lowerShiftRightParts(Op, DAG, true);
373  case ISD::SRL_PARTS:
374  return lowerShiftRightParts(Op, DAG, false);
375  case ISD::BITCAST: {
376  assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
377  "Unexpected custom legalisation");
378  SDLoc DL(Op);
379  SDValue Op0 = Op.getOperand(0);
380  if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32)
381  return SDValue();
382  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
383  SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
384  return FPConv;
385  }
386  }
387 }
388 
390  SelectionDAG &DAG, unsigned Flags) {
391  return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
392 }
393 
395  SelectionDAG &DAG, unsigned Flags) {
396  return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
397  Flags);
398 }
399 
401  SelectionDAG &DAG, unsigned Flags) {
402  return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
403  N->getOffset(), Flags);
404 }
405 
406 template <class NodeTy>
407 SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
408  bool IsLocal) const {
409  SDLoc DL(N);
410  EVT Ty = getPointerTy(DAG.getDataLayout());
411 
412  if (isPositionIndependent()) {
413  SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
414  if (IsLocal)
415  // Use PC-relative addressing to access the symbol. This generates the
416  // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
417  // %pcrel_lo(auipc)).
418  return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
419 
420  // Use PC-relative addressing to access the GOT for this symbol, then load
421  // the address from the GOT. This generates the pattern (PseudoLA sym),
422  // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
423  return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
424  }
425 
426  switch (getTargetMachine().getCodeModel()) {
427  default:
428  report_fatal_error("Unsupported code model for lowering");
429  case CodeModel::Small: {
430  // Generate a sequence for accessing addresses within the first 2 GiB of
431  // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
432  SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
433  SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
434  SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
435  return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
436  }
437  case CodeModel::Medium: {
438  // Generate a sequence for accessing addresses within any 2GiB range within
439  // the address space. This generates the pattern (PseudoLLA sym), which
440  // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
441  SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
442  return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
443  }
444  }
445 }
446 
447 SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
448  SelectionDAG &DAG) const {
449  SDLoc DL(Op);
450  EVT Ty = Op.getValueType();
451  GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
452  int64_t Offset = N->getOffset();
453  MVT XLenVT = Subtarget.getXLenVT();
454 
455  const GlobalValue *GV = N->getGlobal();
456  bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
457  SDValue Addr = getAddr(N, DAG, IsLocal);
458 
459  // In order to maximise the opportunity for common subexpression elimination,
460  // emit a separate ADD node for the global address offset instead of folding
461  // it in the global address node. Later peephole optimisations may choose to
462  // fold it back in when profitable.
463  if (Offset != 0)
464  return DAG.getNode(ISD::ADD, DL, Ty, Addr,
465  DAG.getConstant(Offset, DL, XLenVT));
466  return Addr;
467 }
468 
469 SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
470  SelectionDAG &DAG) const {
471  BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
472 
473  return getAddr(N, DAG);
474 }
475 
476 SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
477  SelectionDAG &DAG) const {
478  ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
479 
480  return getAddr(N, DAG);
481 }
482 
483 SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
484  SDValue CondV = Op.getOperand(0);
485  SDValue TrueV = Op.getOperand(1);
486  SDValue FalseV = Op.getOperand(2);
487  SDLoc DL(Op);
488  MVT XLenVT = Subtarget.getXLenVT();
489 
490  // If the result type is XLenVT and CondV is the output of a SETCC node
491  // which also operated on XLenVT inputs, then merge the SETCC node into the
492  // lowered RISCVISD::SELECT_CC to take advantage of the integer
493  // compare+branch instructions. i.e.:
494  // (select (setcc lhs, rhs, cc), truev, falsev)
495  // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
496  if (Op.getSimpleValueType() == XLenVT && CondV.getOpcode() == ISD::SETCC &&
497  CondV.getOperand(0).getSimpleValueType() == XLenVT) {
498  SDValue LHS = CondV.getOperand(0);
499  SDValue RHS = CondV.getOperand(1);
500  auto CC = cast<CondCodeSDNode>(CondV.getOperand(2));
501  ISD::CondCode CCVal = CC->get();
502 
503  normaliseSetCC(LHS, RHS, CCVal);
504 
505  SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
506  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
507  SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
508  return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
509  }
510 
511  // Otherwise:
512  // (select condv, truev, falsev)
513  // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
514  SDValue Zero = DAG.getConstant(0, DL, XLenVT);
515  SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);
516 
517  SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
518  SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
519 
520  return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
521 }
522 
523 SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
526 
527  SDLoc DL(Op);
528  SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
530 
531  // vastart just stores the address of the VarArgsFrameIndex slot into the
532  // memory location argument.
533  const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
534  return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
535  MachinePointerInfo(SV));
536 }
537 
538 SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
539  SelectionDAG &DAG) const {
540  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
542  MachineFrameInfo &MFI = MF.getFrameInfo();
543  MFI.setFrameAddressIsTaken(true);
544  unsigned FrameReg = RI.getFrameRegister(MF);
545  int XLenInBytes = Subtarget.getXLen() / 8;
546 
547  EVT VT = Op.getValueType();
548  SDLoc DL(Op);
549  SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
550  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
551  while (Depth--) {
552  int Offset = -(XLenInBytes * 2);
553  SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
554  DAG.getIntPtrConstant(Offset, DL));
555  FrameAddr =
556  DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
557  }
558  return FrameAddr;
559 }
560 
561 SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
562  SelectionDAG &DAG) const {
563  const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
565  MachineFrameInfo &MFI = MF.getFrameInfo();
566  MFI.setReturnAddressIsTaken(true);
567  MVT XLenVT = Subtarget.getXLenVT();
568  int XLenInBytes = Subtarget.getXLen() / 8;
569 
571  return SDValue();
572 
573  EVT VT = Op.getValueType();
574  SDLoc DL(Op);
575  unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
576  if (Depth) {
577  int Off = -XLenInBytes;
578  SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
579  SDValue Offset = DAG.getConstant(Off, DL, VT);
580  return DAG.getLoad(VT, DL, DAG.getEntryNode(),
581  DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
583  }
584 
585  // Return the value of the return address register, marking it an implicit
586  // live-in.
587  unsigned Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
588  return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
589 }
590 
591 SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
592  SelectionDAG &DAG) const {
593  SDLoc DL(Op);
594  SDValue Lo = Op.getOperand(0);
595  SDValue Hi = Op.getOperand(1);
596  SDValue Shamt = Op.getOperand(2);
597  EVT VT = Lo.getValueType();
598 
599  // if Shamt-XLEN < 0: // Shamt < XLEN
600  // Lo = Lo << Shamt
601  // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
602  // else:
603  // Lo = 0
604  // Hi = Lo << (Shamt-XLEN)
605 
606  SDValue Zero = DAG.getConstant(0, DL, VT);
607  SDValue One = DAG.getConstant(1, DL, VT);
608  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
609  SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
610  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
611  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
612 
613  SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
614  SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
615  SDValue ShiftRightLo =
616  DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
617  SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
618  SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
619  SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
620 
621  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
622 
623  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
624  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
625 
626  SDValue Parts[2] = {Lo, Hi};
627  return DAG.getMergeValues(Parts, DL);
628 }
629 
630 SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
631  bool IsSRA) const {
632  SDLoc DL(Op);
633  SDValue Lo = Op.getOperand(0);
634  SDValue Hi = Op.getOperand(1);
635  SDValue Shamt = Op.getOperand(2);
636  EVT VT = Lo.getValueType();
637 
638  // SRA expansion:
639  // if Shamt-XLEN < 0: // Shamt < XLEN
640  // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
641  // Hi = Hi >>s Shamt
642  // else:
643  // Lo = Hi >>s (Shamt-XLEN);
644  // Hi = Hi >>s (XLEN-1)
645  //
646  // SRL expansion:
647  // if Shamt-XLEN < 0: // Shamt < XLEN
648  // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
649  // Hi = Hi >>u Shamt
650  // else:
651  // Lo = Hi >>u (Shamt-XLEN);
652  // Hi = 0;
653 
654  unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
655 
656  SDValue Zero = DAG.getConstant(0, DL, VT);
657  SDValue One = DAG.getConstant(1, DL, VT);
658  SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
659  SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
660  SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
661  SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
662 
663  SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
664  SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
665  SDValue ShiftLeftHi =
666  DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
667  SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
668  SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
669  SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
670  SDValue HiFalse =
671  IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
672 
673  SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
674 
675  Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
676  Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
677 
678  SDValue Parts[2] = {Lo, Hi};
679  return DAG.getMergeValues(Parts, DL);
680 }
681 
682 // Returns the opcode of the target-specific SDNode that implements the 32-bit
683 // form of the given Opcode.
684 static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
685  switch (Opcode) {
686  default:
687  llvm_unreachable("Unexpected opcode");
688  case ISD::SHL:
689  return RISCVISD::SLLW;
690  case ISD::SRA:
691  return RISCVISD::SRAW;
692  case ISD::SRL:
693  return RISCVISD::SRLW;
694  case ISD::SDIV:
695  return RISCVISD::DIVW;
696  case ISD::UDIV:
697  return RISCVISD::DIVUW;
698  case ISD::UREM:
699  return RISCVISD::REMUW;
700  }
701 }
702 
703 // Converts the given 32-bit operation to a target-specific SelectionDAG node.
704 // Because i32 isn't a legal type for RV64, these operations would otherwise
705 // be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W
706 // later one because the fact the operation was originally of type i32 is
707 // lost.
709  SDLoc DL(N);
711  SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
712  SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
713  SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
714  // ReplaceNodeResults requires we maintain the same type for the return value.
715  return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
716 }
717 
720  SelectionDAG &DAG) const {
721  SDLoc DL(N);
722  switch (N->getOpcode()) {
723  default:
724  llvm_unreachable("Don't know how to custom type legalize this operation!");
725  case ISD::SHL:
726  case ISD::SRA:
727  case ISD::SRL:
728  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
729  "Unexpected custom legalisation");
730  if (N->getOperand(1).getOpcode() == ISD::Constant)
731  return;
732  Results.push_back(customLegalizeToWOp(N, DAG));
733  break;
734  case ISD::SDIV:
735  case ISD::UDIV:
736  case ISD::UREM:
737  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
738  Subtarget.hasStdExtM() && "Unexpected custom legalisation");
739  if (N->getOperand(0).getOpcode() == ISD::Constant ||
741  return;
742  Results.push_back(customLegalizeToWOp(N, DAG));
743  break;
744  case ISD::BITCAST: {
745  assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
746  Subtarget.hasStdExtF() && "Unexpected custom legalisation");
747  SDLoc DL(N);
748  SDValue Op0 = N->getOperand(0);
749  if (Op0.getValueType() != MVT::f32)
750  return;
751  SDValue FPConv =
753  Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
754  break;
755  }
756  }
757 }
758 
760  DAGCombinerInfo &DCI) const {
761  SelectionDAG &DAG = DCI.DAG;
762 
763  switch (N->getOpcode()) {
764  default:
765  break;
766  case RISCVISD::SplitF64: {
767  SDValue Op0 = N->getOperand(0);
768  // If the input to SplitF64 is just BuildPairF64 then the operation is
769  // redundant. Instead, use BuildPairF64's operands directly.
770  if (Op0->getOpcode() == RISCVISD::BuildPairF64)
771  return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
772 
773  SDLoc DL(N);
774 
775  // It's cheaper to materialise two 32-bit integers than to load a double
776  // from the constant pool and transfer it to integer registers through the
777  // stack.
778  if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
779  APInt V = C->getValueAPF().bitcastToAPInt();
780  SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
781  SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
782  return DCI.CombineTo(N, Lo, Hi);
783  }
784 
785  // This is a target-specific version of a DAGCombine performed in
786  // DAGCombiner::visitBITCAST. It performs the equivalent of:
787  // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
788  // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
789  if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
790  !Op0.getNode()->hasOneUse())
791  break;
792  SDValue NewSplitF64 =
794  Op0.getOperand(0));
795  SDValue Lo = NewSplitF64.getValue(0);
796  SDValue Hi = NewSplitF64.getValue(1);
797  APInt SignBit = APInt::getSignMask(32);
798  if (Op0.getOpcode() == ISD::FNEG) {
799  SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
800  DAG.getConstant(SignBit, DL, MVT::i32));
801  return DCI.CombineTo(N, Lo, NewHi);
802  }
803  assert(Op0.getOpcode() == ISD::FABS);
804  SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
805  DAG.getConstant(~SignBit, DL, MVT::i32));
806  return DCI.CombineTo(N, Lo, NewHi);
807  }
808  case RISCVISD::SLLW:
809  case RISCVISD::SRAW:
810  case RISCVISD::SRLW: {
811  // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
812  SDValue LHS = N->getOperand(0);
813  SDValue RHS = N->getOperand(1);
814  APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
815  APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5);
816  if ((SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI)) ||
817  (SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)))
818  return SDValue();
819  break;
820  }
822  SDLoc DL(N);
823  SDValue Op0 = N->getOperand(0);
824  // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
825  // conversion is unnecessary and can be replaced with an ANY_EXTEND
826  // of the FMV_W_X_RV64 operand.
827  if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) {
828  SDValue AExtOp =
829  DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0.getOperand(0));
830  return DCI.CombineTo(N, AExtOp);
831  }
832 
833  // This is a target-specific version of a DAGCombine performed in
834  // DAGCombiner::visitBITCAST. It performs the equivalent of:
835  // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
836  // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
837  if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
838  !Op0.getNode()->hasOneUse())
839  break;
841  Op0.getOperand(0));
842  APInt SignBit = APInt::getSignMask(32).sext(64);
843  if (Op0.getOpcode() == ISD::FNEG) {
844  return DCI.CombineTo(N,
845  DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV,
846  DAG.getConstant(SignBit, DL, MVT::i64)));
847  }
848  assert(Op0.getOpcode() == ISD::FABS);
849  return DCI.CombineTo(N,
850  DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
851  DAG.getConstant(~SignBit, DL, MVT::i64)));
852  }
853  }
854 
855  return SDValue();
856 }
857 
859  const SDNode *N, CombineLevel Level) const {
860  // The following folds are only desirable if `(OP _, c1 << c2)` can be
861  // materialised in fewer instructions than `(OP _, c1)`:
862  //
863  // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
864  // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
865  SDValue N0 = N->getOperand(0);
866  MVT Ty = N0.getSimpleValueType();
867  if (Ty.isScalarInteger() &&
868  (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
869  auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
870  auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
871  if (C1 && C2) {
872  APInt C1Int = C1->getAPIntValue();
873  APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
874 
875  // We can materialise `c1 << c2` into an add immediate, so it's "free",
876  // and the combine should happen, to potentially allow further combines
877  // later.
878  if (isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
879  return true;
880 
881  // We can materialise `c1` in an add immediate, so it's "free", and the
882  // combine should be prevented.
883  if (isLegalAddImmediate(C1Int.getSExtValue()))
884  return false;
885 
886  // Neither constant will fit into an immediate, so find materialisation
887  // costs.
888  int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
889  Subtarget.is64Bit());
890  int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
891  ShiftedC1Int, Ty.getSizeInBits(), Subtarget.is64Bit());
892 
893  // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
894  // combine should be prevented.
895  if (C1Cost < ShiftedC1Cost)
896  return false;
897  }
898  }
899  return true;
900 }
901 
903  SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
904  unsigned Depth) const {
905  switch (Op.getOpcode()) {
906  default:
907  break;
908  case RISCVISD::SLLW:
909  case RISCVISD::SRAW:
910  case RISCVISD::SRLW:
911  case RISCVISD::DIVW:
912  case RISCVISD::DIVUW:
913  case RISCVISD::REMUW:
914  // TODO: As the result is sign-extended, this is conservatively correct. A
915  // more precise answer could be calculated for SRAW depending on known
916  // bits in the shift amount.
917  return 33;
918  }
919 
920  return 1;
921 }
922 
924  MachineBasicBlock *BB) {
925  assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
926 
927  MachineFunction &MF = *BB->getParent();
928  DebugLoc DL = MI.getDebugLoc();
929  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
930  const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
931  unsigned LoReg = MI.getOperand(0).getReg();
932  unsigned HiReg = MI.getOperand(1).getReg();
933  unsigned SrcReg = MI.getOperand(2).getReg();
934  const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
935  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
936 
937  TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
938  RI);
939  MachineMemOperand *MMO =
940  MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
942  BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
943  .addFrameIndex(FI)
944  .addImm(0)
945  .addMemOperand(MMO);
946  BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
947  .addFrameIndex(FI)
948  .addImm(4)
949  .addMemOperand(MMO);
950  MI.eraseFromParent(); // The pseudo instruction is gone now.
951  return BB;
952 }
953 
955  MachineBasicBlock *BB) {
956  assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
957  "Unexpected instruction");
958 
959  MachineFunction &MF = *BB->getParent();
960  DebugLoc DL = MI.getDebugLoc();
961  const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
962  const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
963  unsigned DstReg = MI.getOperand(0).getReg();
964  unsigned LoReg = MI.getOperand(1).getReg();
965  unsigned HiReg = MI.getOperand(2).getReg();
966  const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
967  int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
968 
969  MachineMemOperand *MMO =
970  MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
972  BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
973  .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
974  .addFrameIndex(FI)
975  .addImm(0)
976  .addMemOperand(MMO);
977  BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
978  .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
979  .addFrameIndex(FI)
980  .addImm(4)
981  .addMemOperand(MMO);
982  TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
983  MI.eraseFromParent(); // The pseudo instruction is gone now.
984  return BB;
985 }
986 
988  switch (MI.getOpcode()) {
989  default:
990  return false;
991  case RISCV::Select_GPR_Using_CC_GPR:
992  case RISCV::Select_FPR32_Using_CC_GPR:
993  case RISCV::Select_FPR64_Using_CC_GPR:
994  return true;
995  }
996 }
997 
999  MachineBasicBlock *BB) {
1000  // To "insert" Select_* instructions, we actually have to insert the triangle
1001  // control-flow pattern. The incoming instructions know the destination vreg
1002  // to set, the condition code register to branch on, the true/false values to
1003  // select between, and the condcode to use to select the appropriate branch.
1004  //
1005  // We produce the following control flow:
1006  // HeadMBB
1007  // | \
1008  // | IfFalseMBB
1009  // | /
1010  // TailMBB
1011  //
1012  // When we find a sequence of selects we attempt to optimize their emission
1013  // by sharing the control flow. Currently we only handle cases where we have
1014  // multiple selects with the exact same condition (same LHS, RHS and CC).
1015  // The selects may be interleaved with other instructions if the other
1016  // instructions meet some requirements we deem safe:
1017  // - They are debug instructions. Otherwise,
1018  // - They do not have side-effects, do not access memory and their inputs do
1019  // not depend on the results of the select pseudo-instructions.
1020  // The TrueV/FalseV operands of the selects cannot depend on the result of
1021  // previous selects in the sequence.
1022  // These conditions could be further relaxed. See the X86 target for a
1023  // related approach and more information.
1024  unsigned LHS = MI.getOperand(1).getReg();
1025  unsigned RHS = MI.getOperand(2).getReg();
1026  auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm());
1027 
1028  SmallVector<MachineInstr *, 4> SelectDebugValues;
1029  SmallSet<unsigned, 4> SelectDests;
1030  SelectDests.insert(MI.getOperand(0).getReg());
1031 
1032  MachineInstr *LastSelectPseudo = &MI;
1033 
1034  for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
1035  SequenceMBBI != E; ++SequenceMBBI) {
1036  if (SequenceMBBI->isDebugInstr())
1037  continue;
1038  else if (isSelectPseudo(*SequenceMBBI)) {
1039  if (SequenceMBBI->getOperand(1).getReg() != LHS ||
1040  SequenceMBBI->getOperand(2).getReg() != RHS ||
1041  SequenceMBBI->getOperand(3).getImm() != CC ||
1042  SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
1043  SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
1044  break;
1045  LastSelectPseudo = &*SequenceMBBI;
1046  SequenceMBBI->collectDebugValues(SelectDebugValues);
1047  SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
1048  } else {
1049  if (SequenceMBBI->hasUnmodeledSideEffects() ||
1050  SequenceMBBI->mayLoadOrStore())
1051  break;
1052  if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
1053  return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
1054  }))
1055  break;
1056  }
1057  }
1058 
1059  const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1060  const BasicBlock *LLVM_BB = BB->getBasicBlock();
1061  DebugLoc DL = MI.getDebugLoc();
1063 
1064  MachineBasicBlock *HeadMBB = BB;
1065  MachineFunction *F = BB->getParent();
1066  MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
1067  MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
1068 
1069  F->insert(I, IfFalseMBB);
1070  F->insert(I, TailMBB);
1071 
1072  // Transfer debug instructions associated with the selects to TailMBB.
1073  for (MachineInstr *DebugInstr : SelectDebugValues) {
1074  TailMBB->push_back(DebugInstr->removeFromParent());
1075  }
1076 
1077  // Move all instructions after the sequence to TailMBB.
1078  TailMBB->splice(TailMBB->end(), HeadMBB,
1079  std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
1080  // Update machine-CFG edges by transferring all successors of the current
1081  // block to the new block which will contain the Phi nodes for the selects.
1082  TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
1083  // Set the successors for HeadMBB.
1084  HeadMBB->addSuccessor(IfFalseMBB);
1085  HeadMBB->addSuccessor(TailMBB);
1086 
1087  // Insert appropriate branch.
1088  unsigned Opcode = getBranchOpcodeForIntCondCode(CC);
1089 
1090  BuildMI(HeadMBB, DL, TII.get(Opcode))
1091  .addReg(LHS)
1092  .addReg(RHS)
1093  .addMBB(TailMBB);
1094 
1095  // IfFalseMBB just falls through to TailMBB.
1096  IfFalseMBB->addSuccessor(TailMBB);
1097 
1098  // Create PHIs for all of the select pseudo-instructions.
1099  auto SelectMBBI = MI.getIterator();
1100  auto SelectEnd = std::next(LastSelectPseudo->getIterator());
1101  auto InsertionPoint = TailMBB->begin();
1102  while (SelectMBBI != SelectEnd) {
1103  auto Next = std::next(SelectMBBI);
1104  if (isSelectPseudo(*SelectMBBI)) {
1105  // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
1106  BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
1107  TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
1108  .addReg(SelectMBBI->getOperand(4).getReg())
1109  .addMBB(HeadMBB)
1110  .addReg(SelectMBBI->getOperand(5).getReg())
1111  .addMBB(IfFalseMBB);
1112  SelectMBBI->eraseFromParent();
1113  }
1114  SelectMBBI = Next;
1115  }
1116 
1117  return TailMBB;
1118 }
1119 
1122  MachineBasicBlock *BB) const {
1123  switch (MI.getOpcode()) {
1124  default:
1125  llvm_unreachable("Unexpected instr type to insert");
1126  case RISCV::Select_GPR_Using_CC_GPR:
1127  case RISCV::Select_FPR32_Using_CC_GPR:
1128  case RISCV::Select_FPR64_Using_CC_GPR:
1129  return emitSelectPseudo(MI, BB);
1130  case RISCV::BuildPairF64Pseudo:
1131  return emitBuildPairF64Pseudo(MI, BB);
1132  case RISCV::SplitF64Pseudo:
1133  return emitSplitF64Pseudo(MI, BB);
1134  }
1135 }
1136 
1137 // Calling Convention Implementation.
1138 // The expectations for frontend ABI lowering vary from target to target.
1139 // Ideally, an LLVM frontend would be able to avoid worrying about many ABI
1140 // details, but this is a longer term goal. For now, we simply try to keep the
1141 // role of the frontend as simple and well-defined as possible. The rules can
1142 // be summarised as:
1143 // * Never split up large scalar arguments. We handle them here.
1144 // * If a hardfloat calling convention is being used, and the struct may be
1145 // passed in a pair of registers (fp+fp, int+fp), and both registers are
1146 // available, then pass as two separate arguments. If either the GPRs or FPRs
1147 // are exhausted, then pass according to the rule below.
1148 // * If a struct could never be passed in registers or directly in a stack
1149 // slot (as it is larger than 2*XLEN and the floating point rules don't
1150 // apply), then pass it using a pointer with the byval attribute.
1151 // * If a struct is less than 2*XLEN, then coerce to either a two-element
1152 // word-sized array or a 2*XLEN scalar (depending on alignment).
1153 // * The frontend can determine whether a struct is returned by reference or
1154 // not based on its size and fields. If it will be returned by reference, the
1155 // frontend must modify the prototype so a pointer with the sret annotation is
1156 // passed as the first argument. This is not necessary for large scalar
1157 // returns.
1158 // * Struct return values and varargs should be coerced to structs containing
1159 // register-size fields in the same situations they would be for fixed
1160 // arguments.
1161 
1162 static const MCPhysReg ArgGPRs[] = {
1163  RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
1164  RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
1165 };
1166 static const MCPhysReg ArgFPR32s[] = {
1167  RISCV::F10_32, RISCV::F11_32, RISCV::F12_32, RISCV::F13_32,
1168  RISCV::F14_32, RISCV::F15_32, RISCV::F16_32, RISCV::F17_32
1169 };
1170 static const MCPhysReg ArgFPR64s[] = {
1171  RISCV::F10_64, RISCV::F11_64, RISCV::F12_64, RISCV::F13_64,
1172  RISCV::F14_64, RISCV::F15_64, RISCV::F16_64, RISCV::F17_64
1173 };
1174 
1175 // Pass a 2*XLEN argument that has been split into two XLEN values through
1176 // registers or the stack as necessary.
1177 static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
1178  ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
1179  MVT ValVT2, MVT LocVT2,
1180  ISD::ArgFlagsTy ArgFlags2) {
1181  unsigned XLenInBytes = XLen / 8;
1182  if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
1183  // At least one half can be passed via register.
1184  State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
1185  VA1.getLocVT(), CCValAssign::Full));
1186  } else {
1187  // Both halves must be passed on the stack, with proper alignment.
1188  unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
1189  State.addLoc(
1190  CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
1191  State.AllocateStack(XLenInBytes, StackAlign),
1192  VA1.getLocVT(), CCValAssign::Full));
1194  ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1196  return false;
1197  }
1198 
1199  if (unsigned Reg = State.AllocateReg(ArgGPRs)) {
1200  // The second half can also be passed via register.
1201  State.addLoc(
1202  CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
1203  } else {
1204  // The second half is passed via the stack, without additional alignment.
1206  ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1208  }
1209 
1210  return false;
1211 }
1212 
1213 // Implements the RISC-V calling convention. Returns true upon failure.
1214 static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
1215  MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
1216  ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
1217  bool IsRet, Type *OrigTy) {
1218  unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
1219  assert(XLen == 32 || XLen == 64);
1220  MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
1221 
1222  // Any return value split in to more than two values can't be returned
1223  // directly.
1224  if (IsRet && ValNo > 1)
1225  return true;
1226 
1227  // UseGPRForF32 if targeting one of the soft-float ABIs, if passing a
1228  // variadic argument, or if no F32 argument registers are available.
1229  bool UseGPRForF32 = true;
1230  // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
1231  // variadic argument, or if no F64 argument registers are available.
1232  bool UseGPRForF64 = true;
1233 
1234  switch (ABI) {
1235  default:
1236  llvm_unreachable("Unexpected ABI");
1237  case RISCVABI::ABI_ILP32:
1238  case RISCVABI::ABI_LP64:
1239  break;
1240  case RISCVABI::ABI_ILP32F:
1241  case RISCVABI::ABI_LP64F:
1242  UseGPRForF32 = !IsFixed;
1243  break;
1244  case RISCVABI::ABI_ILP32D:
1245  case RISCVABI::ABI_LP64D:
1246  UseGPRForF32 = !IsFixed;
1247  UseGPRForF64 = !IsFixed;
1248  break;
1249  }
1250 
1252  UseGPRForF32 = true;
1254  UseGPRForF64 = true;
1255 
1256  // From this point on, rely on UseGPRForF32, UseGPRForF64 and similar local
1257  // variables rather than directly checking against the target ABI.
1258 
1259  if (UseGPRForF32 && ValVT == MVT::f32) {
1260  LocVT = XLenVT;
1261  LocInfo = CCValAssign::BCvt;
1262  } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
1263  LocVT = MVT::i64;
1264  LocInfo = CCValAssign::BCvt;
1265  }
1266 
1267  // If this is a variadic argument, the RISC-V calling convention requires
1268  // that it is assigned an 'even' or 'aligned' register if it has 8-byte
1269  // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
1270  // be used regardless of whether the original argument was split during
1271  // legalisation or not. The argument will not be passed by registers if the
1272  // original type is larger than 2*XLEN, so the register alignment rule does
1273  // not apply.
1274  unsigned TwoXLenInBytes = (2 * XLen) / 8;
1275  if (!IsFixed && ArgFlags.getOrigAlign() == TwoXLenInBytes &&
1276  DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
1277  unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
1278  // Skip 'odd' register if necessary.
1279  if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
1280  State.AllocateReg(ArgGPRs);
1281  }
1282 
1283  SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
1284  SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
1285  State.getPendingArgFlags();
1286 
1287  assert(PendingLocs.size() == PendingArgFlags.size() &&
1288  "PendingLocs and PendingArgFlags out of sync");
1289 
1290  // Handle passing f64 on RV32D with a soft float ABI or when floating point
1291  // registers are exhausted.
1292  if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
1293  assert(!ArgFlags.isSplit() && PendingLocs.empty() &&
1294  "Can't lower f64 if it is split");
1295  // Depending on available argument GPRS, f64 may be passed in a pair of
1296  // GPRs, split between a GPR and the stack, or passed completely on the
1297  // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
1298  // cases.
1299  unsigned Reg = State.AllocateReg(ArgGPRs);
1300  LocVT = MVT::i32;
1301  if (!Reg) {
1302  unsigned StackOffset = State.AllocateStack(8, 8);
1303  State.addLoc(
1304  CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1305  return false;
1306  }
1307  if (!State.AllocateReg(ArgGPRs))
1308  State.AllocateStack(4, 4);
1309  State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1310  return false;
1311  }
1312 
1313  // Split arguments might be passed indirectly, so keep track of the pending
1314  // values.
1315  if (ArgFlags.isSplit() || !PendingLocs.empty()) {
1316  LocVT = XLenVT;
1317  LocInfo = CCValAssign::Indirect;
1318  PendingLocs.push_back(
1319  CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
1320  PendingArgFlags.push_back(ArgFlags);
1321  if (!ArgFlags.isSplitEnd()) {
1322  return false;
1323  }
1324  }
1325 
1326  // If the split argument only had two elements, it should be passed directly
1327  // in registers or on the stack.
1328  if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) {
1329  assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
1330  // Apply the normal calling convention rules to the first half of the
1331  // split argument.
1332  CCValAssign VA = PendingLocs[0];
1333  ISD::ArgFlagsTy AF = PendingArgFlags[0];
1334  PendingLocs.clear();
1335  PendingArgFlags.clear();
1336  return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
1337  ArgFlags);
1338  }
1339 
1340  // Allocate to a register if possible, or else a stack slot.
1341  unsigned Reg;
1342  if (ValVT == MVT::f32 && !UseGPRForF32)
1343  Reg = State.AllocateReg(ArgFPR32s, ArgFPR64s);
1344  else if (ValVT == MVT::f64 && !UseGPRForF64)
1345  Reg = State.AllocateReg(ArgFPR64s, ArgFPR32s);
1346  else
1347  Reg = State.AllocateReg(ArgGPRs);
1348  unsigned StackOffset = Reg ? 0 : State.AllocateStack(XLen / 8, XLen / 8);
1349 
1350  // If we reach this point and PendingLocs is non-empty, we must be at the
1351  // end of a split argument that must be passed indirectly.
1352  if (!PendingLocs.empty()) {
1353  assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
1354  assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
1355 
1356  for (auto &It : PendingLocs) {
1357  if (Reg)
1358  It.convertToReg(Reg);
1359  else
1360  It.convertToMem(StackOffset);
1361  State.addLoc(It);
1362  }
1363  PendingLocs.clear();
1364  PendingArgFlags.clear();
1365  return false;
1366  }
1367 
1368  assert((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) &&
1369  "Expected an XLenVT at this stage");
1370 
1371  if (Reg) {
1372  State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1373  return false;
1374  }
1375 
1376  // When an f32 or f64 is passed on the stack, no bit-conversion is needed.
1377  if (ValVT == MVT::f32 || ValVT == MVT::f64) {
1378  LocVT = ValVT;
1379  LocInfo = CCValAssign::Full;
1380  }
1381  State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1382  return false;
1383 }
1384 
1385 void RISCVTargetLowering::analyzeInputArgs(
1386  MachineFunction &MF, CCState &CCInfo,
1387  const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
1388  unsigned NumArgs = Ins.size();
1389  FunctionType *FType = MF.getFunction().getFunctionType();
1390 
1391  for (unsigned i = 0; i != NumArgs; ++i) {
1392  MVT ArgVT = Ins[i].VT;
1393  ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
1394 
1395  Type *ArgTy = nullptr;
1396  if (IsRet)
1397  ArgTy = FType->getReturnType();
1398  else if (Ins[i].isOrigArg())
1399  ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
1400 
1401  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1402  if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1403  ArgFlags, CCInfo, /*IsRet=*/true, IsRet, ArgTy)) {
1404  LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "
1405  << EVT(ArgVT).getEVTString() << '\n');
1406  llvm_unreachable(nullptr);
1407  }
1408  }
1409 }
1410 
1411 void RISCVTargetLowering::analyzeOutputArgs(
1412  MachineFunction &MF, CCState &CCInfo,
1413  const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
1414  CallLoweringInfo *CLI) const {
1415  unsigned NumArgs = Outs.size();
1416 
1417  for (unsigned i = 0; i != NumArgs; i++) {
1418  MVT ArgVT = Outs[i].VT;
1419  ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
1420  Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
1421 
1422  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1423  if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1424  ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) {
1425  LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "
1426  << EVT(ArgVT).getEVTString() << "\n");
1427  llvm_unreachable(nullptr);
1428  }
1429  }
1430 }
1431 
1432 // Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
1433 // values.
1435  const CCValAssign &VA, const SDLoc &DL) {
1436  switch (VA.getLocInfo()) {
1437  default:
1438  llvm_unreachable("Unexpected CCValAssign::LocInfo");
1439  case CCValAssign::Full:
1440  break;
1441  case CCValAssign::BCvt:
1442  if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1443  Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
1444  break;
1445  }
1446  Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
1447  break;
1448  }
1449  return Val;
1450 }
1451 
1452 // The caller is responsible for loading the full value if the argument is
1453 // passed with CCValAssign::Indirect.
1455  const CCValAssign &VA, const SDLoc &DL) {
1456  MachineFunction &MF = DAG.getMachineFunction();
1457  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1458  EVT LocVT = VA.getLocVT();
1459  SDValue Val;
1460  const TargetRegisterClass *RC;
1461 
1462  switch (LocVT.getSimpleVT().SimpleTy) {
1463  default:
1464  llvm_unreachable("Unexpected register type");
1465  case MVT::i32:
1466  case MVT::i64:
1467  RC = &RISCV::GPRRegClass;
1468  break;
1469  case MVT::f32:
1470  RC = &RISCV::FPR32RegClass;
1471  break;
1472  case MVT::f64:
1473  RC = &RISCV::FPR64RegClass;
1474  break;
1475  }
1476 
1477  unsigned VReg = RegInfo.createVirtualRegister(RC);
1478  RegInfo.addLiveIn(VA.getLocReg(), VReg);
1479  Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1480 
1481  if (VA.getLocInfo() == CCValAssign::Indirect)
1482  return Val;
1483 
1484  return convertLocVTToValVT(DAG, Val, VA, DL);
1485 }
1486 
1488  const CCValAssign &VA, const SDLoc &DL) {
1489  EVT LocVT = VA.getLocVT();
1490 
1491  switch (VA.getLocInfo()) {
1492  default:
1493  llvm_unreachable("Unexpected CCValAssign::LocInfo");
1494  case CCValAssign::Full:
1495  break;
1496  case CCValAssign::BCvt:
1497  if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1498  Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
1499  break;
1500  }
1501  Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
1502  break;
1503  }
1504  return Val;
1505 }
1506 
1507 // The caller is responsible for loading the full value if the argument is
1508 // passed with CCValAssign::Indirect.
1510  const CCValAssign &VA, const SDLoc &DL) {
1511  MachineFunction &MF = DAG.getMachineFunction();
1512  MachineFrameInfo &MFI = MF.getFrameInfo();
1513  EVT LocVT = VA.getLocVT();
1514  EVT ValVT = VA.getValVT();
1516  int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
1517  VA.getLocMemOffset(), /*Immutable=*/true);
1518  SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1519  SDValue Val;
1520 
1522  switch (VA.getLocInfo()) {
1523  default:
1524  llvm_unreachable("Unexpected CCValAssign::LocInfo");
1525  case CCValAssign::Full:
1526  case CCValAssign::Indirect:
1527  case CCValAssign::BCvt:
1528  ExtType = ISD::NON_EXTLOAD;
1529  break;
1530  }
1531  Val = DAG.getExtLoad(
1532  ExtType, DL, LocVT, Chain, FIN,
1534  return Val;
1535 }
1536 
1538  const CCValAssign &VA, const SDLoc &DL) {
1539  assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
1540  "Unexpected VA");
1541  MachineFunction &MF = DAG.getMachineFunction();
1542  MachineFrameInfo &MFI = MF.getFrameInfo();
1543  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1544 
1545  if (VA.isMemLoc()) {
1546  // f64 is passed on the stack.
1547  int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true);
1548  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1549  return DAG.getLoad(MVT::f64, DL, Chain, FIN,
1551  }
1552 
1553  assert(VA.isRegLoc() && "Expected register VA assignment");
1554 
1555  unsigned LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1556  RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
1557  SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
1558  SDValue Hi;
1559  if (VA.getLocReg() == RISCV::X17) {
1560  // Second half of f64 is passed on the stack.
1561  int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true);
1562  SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1563  Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
1565  } else {
1566  // Second half of f64 is passed in another GPR.
1567  unsigned HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1568  RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
1569  Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
1570  }
1571  return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1572 }
1573 
1574 // Transform physical registers into virtual registers.
1575 SDValue RISCVTargetLowering::LowerFormalArguments(
1576  SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1577  const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1578  SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1579 
1580  switch (CallConv) {
1581  default:
1582  report_fatal_error("Unsupported calling convention");
1583  case CallingConv::C:
1584  case CallingConv::Fast:
1585  break;
1586  }
1587 
1588  MachineFunction &MF = DAG.getMachineFunction();
1589 
1590  const Function &Func = MF.getFunction();
1591  if (Func.hasFnAttribute("interrupt")) {
1592  if (!Func.arg_empty())
1594  "Functions with the interrupt attribute cannot have arguments!");
1595 
1596  StringRef Kind =
1597  MF.getFunction().getFnAttribute("interrupt").getValueAsString();
1598 
1599  if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
1601  "Function interrupt attribute argument not supported!");
1602  }
1603 
1604  EVT PtrVT = getPointerTy(DAG.getDataLayout());
1605  MVT XLenVT = Subtarget.getXLenVT();
1606  unsigned XLenInBytes = Subtarget.getXLen() / 8;
1607  // Used with vargs to acumulate store chains.
1608  std::vector<SDValue> OutChains;
1609 
1610  // Assign locations to all of the incoming arguments.
1612  CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1613  analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);
1614 
1615  for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1616  CCValAssign &VA = ArgLocs[i];
1617  SDValue ArgValue;
1618  // Passing f64 on RV32D with a soft float ABI must be handled as a special
1619  // case.
1620  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
1621  ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
1622  else if (VA.isRegLoc())
1623  ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL);
1624  else
1625  ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
1626 
1627  if (VA.getLocInfo() == CCValAssign::Indirect) {
1628  // If the original argument was split and passed by reference (e.g. i128
1629  // on RV32), we need to load all parts of it here (using the same
1630  // address).
1631  InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1632  MachinePointerInfo()));
1633  unsigned ArgIndex = Ins[i].OrigArgIndex;
1634  assert(Ins[i].PartOffset == 0);
1635  while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
1636  CCValAssign &PartVA = ArgLocs[i + 1];
1637  unsigned PartOffset = Ins[i + 1].PartOffset;
1638  SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1639  DAG.getIntPtrConstant(PartOffset, DL));
1640  InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1641  MachinePointerInfo()));
1642  ++i;
1643  }
1644  continue;
1645  }
1646  InVals.push_back(ArgValue);
1647  }
1648 
1649  if (IsVarArg) {
1651  unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
1652  const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1653  MachineFrameInfo &MFI = MF.getFrameInfo();
1654  MachineRegisterInfo &RegInfo = MF.getRegInfo();
1656 
1657  // Offset of the first variable argument from stack pointer, and size of
1658  // the vararg save area. For now, the varargs save area is either zero or
1659  // large enough to hold a0-a7.
1660  int VaArgOffset, VarArgsSaveSize;
1661 
1662  // If all registers are allocated, then all varargs must be passed on the
1663  // stack and we don't need to save any argregs.
1664  if (ArgRegs.size() == Idx) {
1665  VaArgOffset = CCInfo.getNextStackOffset();
1666  VarArgsSaveSize = 0;
1667  } else {
1668  VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
1669  VaArgOffset = -VarArgsSaveSize;
1670  }
1671 
1672  // Record the frame index of the first variable argument
1673  // which is a value necessary to VASTART.
1674  int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1675  RVFI->setVarArgsFrameIndex(FI);
1676 
1677  // If saving an odd number of registers then create an extra stack slot to
1678  // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
1679  // offsets to even-numbered registered remain 2*XLEN-aligned.
1680  if (Idx % 2) {
1681  FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes,
1682  true);
1683  VarArgsSaveSize += XLenInBytes;
1684  }
1685 
1686  // Copy the integer registers that may have been used for passing varargs
1687  // to the vararg save area.
1688  for (unsigned I = Idx; I < ArgRegs.size();
1689  ++I, VaArgOffset += XLenInBytes) {
1690  const unsigned Reg = RegInfo.createVirtualRegister(RC);
1691  RegInfo.addLiveIn(ArgRegs[I], Reg);
1692  SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
1693  FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1694  SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1695  SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
1697  cast<StoreSDNode>(Store.getNode())
1698  ->getMemOperand()
1699  ->setValue((Value *)nullptr);
1700  OutChains.push_back(Store);
1701  }
1702  RVFI->setVarArgsSaveSize(VarArgsSaveSize);
1703  }
1704 
1705  // All stores are grouped in one node to allow the matching between
1706  // the size of Ins and InVals. This only happens for vararg functions.
1707  if (!OutChains.empty()) {
1708  OutChains.push_back(Chain);
1709  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
1710  }
1711 
1712  return Chain;
1713 }
1714 
1715 /// isEligibleForTailCallOptimization - Check whether the call is eligible
1716 /// for tail call optimization.
1717 /// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
1718 bool RISCVTargetLowering::isEligibleForTailCallOptimization(
1719  CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
1720  const SmallVector<CCValAssign, 16> &ArgLocs) const {
1721 
1722  auto &Callee = CLI.Callee;
1723  auto CalleeCC = CLI.CallConv;
1724  auto IsVarArg = CLI.IsVarArg;
1725  auto &Outs = CLI.Outs;
1726  auto &Caller = MF.getFunction();
1727  auto CallerCC = Caller.getCallingConv();
1728 
1729  // Do not tail call opt functions with "disable-tail-calls" attribute.
1730  if (Caller.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
1731  return false;
1732 
1733  // Exception-handling functions need a special set of instructions to
1734  // indicate a return to the hardware. Tail-calling another function would
1735  // probably break this.
1736  // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
1737  // should be expanded as new function attributes are introduced.
1738  if (Caller.hasFnAttribute("interrupt"))
1739  return false;
1740 
1741  // Do not tail call opt functions with varargs.
1742  if (IsVarArg)
1743  return false;
1744 
1745  // Do not tail call opt if the stack is used to pass parameters.
1746  if (CCInfo.getNextStackOffset() != 0)
1747  return false;
1748 
1749  // Do not tail call opt if any parameters need to be passed indirectly.
1750  // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
1751  // passed indirectly. So the address of the value will be passed in a
1752  // register, or if not available, then the address is put on the stack. In
1753  // order to pass indirectly, space on the stack often needs to be allocated
1754  // in order to store the value. In this case the CCInfo.getNextStackOffset()
1755  // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
1756  // are passed CCValAssign::Indirect.
1757  for (auto &VA : ArgLocs)
1758  if (VA.getLocInfo() == CCValAssign::Indirect)
1759  return false;
1760 
1761  // Do not tail call opt if either caller or callee uses struct return
1762  // semantics.
1763  auto IsCallerStructRet = Caller.hasStructRetAttr();
1764  auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
1765  if (IsCallerStructRet || IsCalleeStructRet)
1766  return false;
1767 
1768  // Externally-defined functions with weak linkage should not be
1769  // tail-called. The behaviour of branch instructions in this situation (as
1770  // used for tail calls) is implementation-defined, so we cannot rely on the
1771  // linker replacing the tail call with a return.
1772  if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
1773  const GlobalValue *GV = G->getGlobal();
1774  if (GV->hasExternalWeakLinkage())
1775  return false;
1776  }
1777 
1778  // The callee has to preserve all registers the caller needs to preserve.
1779  const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
1780  const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
1781  if (CalleeCC != CallerCC) {
1782  const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
1783  if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
1784  return false;
1785  }
1786 
1787  // Byval parameters hand the function a pointer directly into the stack area
1788  // we want to reuse during a tail call. Working around this *is* possible
1789  // but less efficient and uglier in LowerCall.
1790  for (auto &Arg : Outs)
1791  if (Arg.Flags.isByVal())
1792  return false;
1793 
1794  return true;
1795 }
1796 
1797 // Lower a call to a callseq_start + CALL + callseq_end chain, and add input
1798 // and output parameter nodes.
1799 SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
1800  SmallVectorImpl<SDValue> &InVals) const {
1801  SelectionDAG &DAG = CLI.DAG;
1802  SDLoc &DL = CLI.DL;
1803  SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1804  SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1805  SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1806  SDValue Chain = CLI.Chain;
1807  SDValue Callee = CLI.Callee;
1808  bool &IsTailCall = CLI.IsTailCall;
1809  CallingConv::ID CallConv = CLI.CallConv;
1810  bool IsVarArg = CLI.IsVarArg;
1811  EVT PtrVT = getPointerTy(DAG.getDataLayout());
1812  MVT XLenVT = Subtarget.getXLenVT();
1813 
1814  MachineFunction &MF = DAG.getMachineFunction();
1815 
1816  // Analyze the operands of the call, assigning locations to each operand.
1818  CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1819  analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI);
1820 
1821  // Check if it's really possible to do a tail call.
1822  if (IsTailCall)
1823  IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
1824 
1825  if (IsTailCall)
1826  ++NumTailCalls;
1827  else if (CLI.CS && CLI.CS.isMustTailCall())
1828  report_fatal_error("failed to perform tail call elimination on a call "
1829  "site marked musttail");
1830 
1831  // Get a count of how many bytes are to be pushed on the stack.
1832  unsigned NumBytes = ArgCCInfo.getNextStackOffset();
1833 
1834  // Create local copies for byval args
1835  SmallVector<SDValue, 8> ByValArgs;
1836  for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
1837  ISD::ArgFlagsTy Flags = Outs[i].Flags;
1838  if (!Flags.isByVal())
1839  continue;
1840 
1841  SDValue Arg = OutVals[i];
1842  unsigned Size = Flags.getByValSize();
1843  unsigned Align = Flags.getByValAlign();
1844 
1845  int FI = MF.getFrameInfo().CreateStackObject(Size, Align, /*isSS=*/false);
1846  SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1847  SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
1848 
1849  Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Align,
1850  /*IsVolatile=*/false,
1851  /*AlwaysInline=*/false,
1852  IsTailCall, MachinePointerInfo(),
1853  MachinePointerInfo());
1854  ByValArgs.push_back(FIPtr);
1855  }
1856 
1857  if (!IsTailCall)
1858  Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
1859 
1860  // Copy argument values to their designated locations.
1862  SmallVector<SDValue, 8> MemOpChains;
1863  SDValue StackPtr;
1864  for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
1865  CCValAssign &VA = ArgLocs[i];
1866  SDValue ArgValue = OutVals[i];
1867  ISD::ArgFlagsTy Flags = Outs[i].Flags;
1868 
1869  // Handle passing f64 on RV32D with a soft float ABI as a special case.
1870  bool IsF64OnRV32DSoftABI =
1871  VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
1872  if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
1873  SDValue SplitF64 = DAG.getNode(
1874  RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
1875  SDValue Lo = SplitF64.getValue(0);
1876  SDValue Hi = SplitF64.getValue(1);
1877 
1878  unsigned RegLo = VA.getLocReg();
1879  RegsToPass.push_back(std::make_pair(RegLo, Lo));
1880 
1881  if (RegLo == RISCV::X17) {
1882  // Second half of f64 is passed on the stack.
1883  // Work out the address of the stack slot.
1884  if (!StackPtr.getNode())
1885  StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
1886  // Emit the store.
1887  MemOpChains.push_back(
1888  DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
1889  } else {
1890  // Second half of f64 is passed in another GPR.
1891  unsigned RegHigh = RegLo + 1;
1892  RegsToPass.push_back(std::make_pair(RegHigh, Hi));
1893  }
1894  continue;
1895  }
1896 
1897  // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
1898  // as any other MemLoc.
1899 
1900  // Promote the value if needed.
1901  // For now, only handle fully promoted and indirect arguments.
1902  if (VA.getLocInfo() == CCValAssign::Indirect) {
1903  // Store the argument in a stack slot and pass its address.
1904  SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
1905  int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
1906  MemOpChains.push_back(
1907  DAG.getStore(Chain, DL, ArgValue, SpillSlot,
1909  // If the original argument was split (e.g. i128), we need
1910  // to store all parts of it here (and pass just one address).
1911  unsigned ArgIndex = Outs[i].OrigArgIndex;
1912  assert(Outs[i].PartOffset == 0);
1913  while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
1914  SDValue PartValue = OutVals[i + 1];
1915  unsigned PartOffset = Outs[i + 1].PartOffset;
1916  SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
1917  DAG.getIntPtrConstant(PartOffset, DL));
1918  MemOpChains.push_back(
1919  DAG.getStore(Chain, DL, PartValue, Address,
1921  ++i;
1922  }
1923  ArgValue = SpillSlot;
1924  } else {
1925  ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL);
1926  }
1927 
1928  // Use local copy if it is a byval arg.
1929  if (Flags.isByVal())
1930  ArgValue = ByValArgs[j++];
1931 
1932  if (VA.isRegLoc()) {
1933  // Queue up the argument copies and emit them at the end.
1934  RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
1935  } else {
1936  assert(VA.isMemLoc() && "Argument not register or memory");
1937  assert(!IsTailCall && "Tail call not allowed if stack is used "
1938  "for passing parameters");
1939 
1940  // Work out the address of the stack slot.
1941  if (!StackPtr.getNode())
1942  StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
1943  SDValue Address =
1944  DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
1945  DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
1946 
1947  // Emit the store.
1948  MemOpChains.push_back(
1949  DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
1950  }
1951  }
1952 
1953  // Join the stores, which are independent of one another.
1954  if (!MemOpChains.empty())
1955  Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
1956 
1957  SDValue Glue;
1958 
1959  // Build a sequence of copy-to-reg nodes, chained and glued together.
1960  for (auto &Reg : RegsToPass) {
1961  Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
1962  Glue = Chain.getValue(1);
1963  }
1964 
1965  // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
1966  // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
1967  // split it and then direct call can be matched by PseudoCALL.
1968  if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
1969  const GlobalValue *GV = S->getGlobal();
1970 
1971  unsigned OpFlags = RISCVII::MO_CALL;
1972  if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
1973  OpFlags = RISCVII::MO_PLT;
1974 
1975  Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
1976  } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
1977  unsigned OpFlags = RISCVII::MO_CALL;
1978 
1979  if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
1980  nullptr))
1981  OpFlags = RISCVII::MO_PLT;
1982 
1983  Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
1984  }
1985 
1986  // The first call operand is the chain and the second is the target address.
1988  Ops.push_back(Chain);
1989  Ops.push_back(Callee);
1990 
1991  // Add argument registers to the end of the list so that they are
1992  // known live into the call.
1993  for (auto &Reg : RegsToPass)
1994  Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
1995 
1996  if (!IsTailCall) {
1997  // Add a register mask operand representing the call-preserved registers.
1998  const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
1999  const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2000  assert(Mask && "Missing call preserved mask for calling convention");
2001  Ops.push_back(DAG.getRegisterMask(Mask));
2002  }
2003 
2004  // Glue the call to the argument copies, if any.
2005  if (Glue.getNode())
2006  Ops.push_back(Glue);
2007 
2008  // Emit the call.
2009  SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2010 
2011  if (IsTailCall) {
2013  return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
2014  }
2015 
2016  Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
2017  Glue = Chain.getValue(1);
2018 
2019  // Mark the end of the call, which is glued to the call itself.
2020  Chain = DAG.getCALLSEQ_END(Chain,
2021  DAG.getConstant(NumBytes, DL, PtrVT, true),
2022  DAG.getConstant(0, DL, PtrVT, true),
2023  Glue, DL);
2024  Glue = Chain.getValue(1);
2025 
2026  // Assign locations to each value returned by this call.
2028  CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
2029  analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);
2030 
2031  // Copy all of the result registers out of their specified physreg.
2032  for (auto &VA : RVLocs) {
2033  // Copy the value out
2034  SDValue RetValue =
2035  DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
2036  // Glue the RetValue to the end of the call sequence
2037  Chain = RetValue.getValue(1);
2038  Glue = RetValue.getValue(2);
2039 
2040  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2041  assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment");
2042  SDValue RetValue2 =
2043  DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
2044  Chain = RetValue2.getValue(1);
2045  Glue = RetValue2.getValue(2);
2046  RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
2047  RetValue2);
2048  }
2049 
2050  RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
2051 
2052  InVals.push_back(RetValue);
2053  }
2054 
2055  return Chain;
2056 }
2057 
2058 bool RISCVTargetLowering::CanLowerReturn(
2059  CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2060  const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2062  CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2063  for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2064  MVT VT = Outs[i].VT;
2065  ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
2066  RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
2067  if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
2068  ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr))
2069  return false;
2070  }
2071  return true;
2072 }
2073 
2074 SDValue
2075 RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2076  bool IsVarArg,
2077  const SmallVectorImpl<ISD::OutputArg> &Outs,
2078  const SmallVectorImpl<SDValue> &OutVals,
2079  const SDLoc &DL, SelectionDAG &DAG) const {
2080  // Stores the assignment of the return value to a location.
2082 
2083  // Info about the registers and stack slot.
2084  CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2085  *DAG.getContext());
2086 
2087  analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
2088  nullptr);
2089 
2090  SDValue Glue;
2091  SmallVector<SDValue, 4> RetOps(1, Chain);
2092 
2093  // Copy the result values into the output registers.
2094  for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
2095  SDValue Val = OutVals[i];
2096  CCValAssign &VA = RVLocs[i];
2097  assert(VA.isRegLoc() && "Can only return in registers!");
2098 
2099  if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2100  // Handle returning f64 on RV32D with a soft float ABI.
2101  assert(VA.isRegLoc() && "Expected return via registers");
2103  DAG.getVTList(MVT::i32, MVT::i32), Val);
2104  SDValue Lo = SplitF64.getValue(0);
2105  SDValue Hi = SplitF64.getValue(1);
2106  unsigned RegLo = VA.getLocReg();
2107  unsigned RegHi = RegLo + 1;
2108  Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
2109  Glue = Chain.getValue(1);
2110  RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
2111  Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
2112  Glue = Chain.getValue(1);
2113  RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
2114  } else {
2115  // Handle a 'normal' return.
2116  Val = convertValVTToLocVT(DAG, Val, VA, DL);
2117  Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
2118 
2119  // Guarantee that all emitted copies are stuck together.
2120  Glue = Chain.getValue(1);
2121  RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2122  }
2123  }
2124 
2125  RetOps[0] = Chain; // Update chain.
2126 
2127  // Add the glue node if we have it.
2128  if (Glue.getNode()) {
2129  RetOps.push_back(Glue);
2130  }
2131 
2132  // Interrupt service routines use different return instructions.
2133  const Function &Func = DAG.getMachineFunction().getFunction();
2134  if (Func.hasFnAttribute("interrupt")) {
2135  if (!Func.getReturnType()->isVoidTy())
2137  "Functions with the interrupt attribute must have void return type!");
2138 
2139  MachineFunction &MF = DAG.getMachineFunction();
2140  StringRef Kind =
2141  MF.getFunction().getFnAttribute("interrupt").getValueAsString();
2142 
2143  unsigned RetOpc;
2144  if (Kind == "user")
2145  RetOpc = RISCVISD::URET_FLAG;
2146  else if (Kind == "supervisor")
2147  RetOpc = RISCVISD::SRET_FLAG;
2148  else
2149  RetOpc = RISCVISD::MRET_FLAG;
2150 
2151  return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
2152  }
2153 
2154  return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps);
2155 }
2156 
2157 const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
2158  switch ((RISCVISD::NodeType)Opcode) {
2160  break;
2161  case RISCVISD::RET_FLAG:
2162  return "RISCVISD::RET_FLAG";
2163  case RISCVISD::URET_FLAG:
2164  return "RISCVISD::URET_FLAG";
2165  case RISCVISD::SRET_FLAG:
2166  return "RISCVISD::SRET_FLAG";
2167  case RISCVISD::MRET_FLAG:
2168  return "RISCVISD::MRET_FLAG";
2169  case RISCVISD::CALL:
2170  return "RISCVISD::CALL";
2171  case RISCVISD::SELECT_CC:
2172  return "RISCVISD::SELECT_CC";
2174  return "RISCVISD::BuildPairF64";
2175  case RISCVISD::SplitF64:
2176  return "RISCVISD::SplitF64";
2177  case RISCVISD::TAIL:
2178  return "RISCVISD::TAIL";
2179  case RISCVISD::SLLW:
2180  return "RISCVISD::SLLW";
2181  case RISCVISD::SRAW:
2182  return "RISCVISD::SRAW";
2183  case RISCVISD::SRLW:
2184  return "RISCVISD::SRLW";
2185  case RISCVISD::DIVW:
2186  return "RISCVISD::DIVW";
2187  case RISCVISD::DIVUW:
2188  return "RISCVISD::DIVUW";
2189  case RISCVISD::REMUW:
2190  return "RISCVISD::REMUW";
2192  return "RISCVISD::FMV_W_X_RV64";
2194  return "RISCVISD::FMV_X_ANYEXTW_RV64";
2195  }
2196  return nullptr;
2197 }
2198 
2199 std::pair<unsigned, const TargetRegisterClass *>
2201  StringRef Constraint,
2202  MVT VT) const {
2203  // First, see if this is a constraint that directly corresponds to a
2204  // RISCV register class.
2205  if (Constraint.size() == 1) {
2206  switch (Constraint[0]) {
2207  case 'r':
2208  return std::make_pair(0U, &RISCV::GPRRegClass);
2209  default:
2210  break;
2211  }
2212  }
2213 
2214  return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
2215 }
2216 
2218  SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2219  SelectionDAG &DAG) const {
2220  // Currently only support length 1 constraints.
2221  if (Constraint.length() == 1) {
2222  switch (Constraint[0]) {
2223  case 'I':
2224  // Validate & create a 12-bit signed immediate operand.
2225  if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2226  uint64_t CVal = C->getSExtValue();
2227  if (isInt<12>(CVal))
2228  Ops.push_back(
2229  DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2230  }
2231  return;
2232  case 'J':
2233  // Validate & create an integer zero operand.
2234  if (auto *C = dyn_cast<ConstantSDNode>(Op))
2235  if (C->getZExtValue() == 0)
2236  Ops.push_back(
2237  DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
2238  return;
2239  case 'K':
2240  // Validate & create a 5-bit unsigned immediate operand.
2241  if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2242  uint64_t CVal = C->getZExtValue();
2243  if (isUInt<5>(CVal))
2244  Ops.push_back(
2245  DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2246  }
2247  return;
2248  default:
2249  break;
2250  }
2251  }
2252  TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
2253 }
2254 
2256  Instruction *Inst,
2257  AtomicOrdering Ord) const {
2258  if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
2259  return Builder.CreateFence(Ord);
2260  if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
2261  return Builder.CreateFence(AtomicOrdering::Release);
2262  return nullptr;
2263 }
2264 
2266  Instruction *Inst,
2267  AtomicOrdering Ord) const {
2268  if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
2269  return Builder.CreateFence(AtomicOrdering::Acquire);
2270  return nullptr;
2271 }
2272 
2274 RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
2275  // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
2276  // point operations can't be used in an lr/sc sequence without breaking the
2277  // forward-progress guarantee.
2278  if (AI->isFloatingPointOperation())
2280 
2281  unsigned Size = AI->getType()->getPrimitiveSizeInBits();
2282  if (Size == 8 || Size == 16)
2285 }
2286 
2287 static Intrinsic::ID
2289  if (XLen == 32) {
2290  switch (BinOp) {
2291  default:
2292  llvm_unreachable("Unexpected AtomicRMW BinOp");
2293  case AtomicRMWInst::Xchg:
2294  return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
2295  case AtomicRMWInst::Add:
2296  return Intrinsic::riscv_masked_atomicrmw_add_i32;
2297  case AtomicRMWInst::Sub:
2298  return Intrinsic::riscv_masked_atomicrmw_sub_i32;
2299  case AtomicRMWInst::Nand:
2300  return Intrinsic::riscv_masked_atomicrmw_nand_i32;
2301  case AtomicRMWInst::Max:
2302  return Intrinsic::riscv_masked_atomicrmw_max_i32;
2303  case AtomicRMWInst::Min:
2304  return Intrinsic::riscv_masked_atomicrmw_min_i32;
2305  case AtomicRMWInst::UMax:
2306  return Intrinsic::riscv_masked_atomicrmw_umax_i32;
2307  case AtomicRMWInst::UMin:
2308  return Intrinsic::riscv_masked_atomicrmw_umin_i32;
2309  }
2310  }
2311 
2312  if (XLen == 64) {
2313  switch (BinOp) {
2314  default:
2315  llvm_unreachable("Unexpected AtomicRMW BinOp");
2316  case AtomicRMWInst::Xchg:
2317  return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
2318  case AtomicRMWInst::Add:
2319  return Intrinsic::riscv_masked_atomicrmw_add_i64;
2320  case AtomicRMWInst::Sub:
2321  return Intrinsic::riscv_masked_atomicrmw_sub_i64;
2322  case AtomicRMWInst::Nand:
2323  return Intrinsic::riscv_masked_atomicrmw_nand_i64;
2324  case AtomicRMWInst::Max:
2325  return Intrinsic::riscv_masked_atomicrmw_max_i64;
2326  case AtomicRMWInst::Min:
2327  return Intrinsic::riscv_masked_atomicrmw_min_i64;
2328  case AtomicRMWInst::UMax:
2329  return Intrinsic::riscv_masked_atomicrmw_umax_i64;
2330  case AtomicRMWInst::UMin:
2331  return Intrinsic::riscv_masked_atomicrmw_umin_i64;
2332  }
2333  }
2334 
2335  llvm_unreachable("Unexpected XLen\n");
2336 }
2337 
2338 Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
2339  IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
2340  Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
2341  unsigned XLen = Subtarget.getXLen();
2342  Value *Ordering =
2343  Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
2344  Type *Tys[] = {AlignedAddr->getType()};
2345  Function *LrwOpScwLoop = Intrinsic::getDeclaration(
2346  AI->getModule(),
2348 
2349  if (XLen == 64) {
2350  Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
2351  Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2352  ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
2353  }
2354 
2355  Value *Result;
2356 
2357  // Must pass the shift amount needed to sign extend the loaded value prior
2358  // to performing a signed comparison for min/max. ShiftAmt is the number of
2359  // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
2360  // is the number of bits to left+right shift the value in order to
2361  // sign-extend.
2362  if (AI->getOperation() == AtomicRMWInst::Min ||
2363  AI->getOperation() == AtomicRMWInst::Max) {
2364  const DataLayout &DL = AI->getModule()->getDataLayout();
2365  unsigned ValWidth =
2367  Value *SextShamt =
2368  Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
2369  Result = Builder.CreateCall(LrwOpScwLoop,
2370  {AlignedAddr, Incr, Mask, SextShamt, Ordering});
2371  } else {
2372  Result =
2373  Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
2374  }
2375 
2376  if (XLen == 64)
2377  Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2378  return Result;
2379 }
2380 
2382 RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
2383  AtomicCmpXchgInst *CI) const {
2384  unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
2385  if (Size == 8 || Size == 16)
2388 }
2389 
2390 Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
2391  IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
2392  Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
2393  unsigned XLen = Subtarget.getXLen();
2394  Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
2395  Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
2396  if (XLen == 64) {
2397  CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
2398  NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
2399  Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2400  CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
2401  }
2402  Type *Tys[] = {AlignedAddr->getType()};
2403  Function *MaskedCmpXchg =
2404  Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
2405  Value *Result = Builder.CreateCall(
2406  MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
2407  if (XLen == 64)
2408  Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2409  return Result;
2410 }
SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
void setFrameAddressIsTaken(bool T)
uint64_t CallInst * C
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
Definition: ISDOpcodes.h:595
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set, or Regs.size() if they are all allocated.
static MVT getIntegerVT(unsigned BitWidth)
A parsed version of the target data layout string in and methods for querying it. ...
Definition: DataLayout.h:110
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
uint64_t getTypeStoreSizeInBits(Type *Ty) const
Returns the maximum number of bits that may be overwritten by storing the specified type; always a mu...
Definition: DataLayout.h:452
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
FMINNUM/FMAXNUM - Perform floating-point minimum or maximum on two values.
Definition: ISDOpcodes.h:622
EVT getValueType() const
Return the ValueType of the referenced return value.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg If BaseGV is null...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
RISCVABI::ABI getTargetABI() const
void collectDebugValues(SmallVectorImpl< MachineInstr *> &DbgValues)
Scan instructions following MI and collect any matching DBG_VALUEs.
const GlobalValue * getGlobal() const
APInt sext(unsigned width) const
Sign extend to a new width.
Definition: APInt.cpp:833
*p = old <signed v ? old : v
Definition: Instructions.h:721
GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2)
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
LLVMContext & Context
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond)
Helper function to make it easier to build SetCC&#39;s if you just have an ISD::CondCode instead of an SD...
Definition: SelectionDAG.h:953
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it&#39;s not CSE&#39;d)...
Definition: SelectionDAG.h:852
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
BR_CC - Conditional branch.
Definition: ISDOpcodes.h:678
LLVM_ATTRIBUTE_NORETURN void report_fatal_error(Error Err, bool gen_crash_diag=true)
Report a serious error, calling any installed error handler.
Definition: Error.cpp:139
This class represents lattice values for constants.
Definition: AllocatorList.h:23
Type * getParamType(unsigned i) const
Parameter type accessors.
Definition: DerivedTypes.h:135
an instruction that atomically checks whether a specified value is in a memory location, and, if it is, stores a new value there.
Definition: Instructions.h:528
void addLiveIn(unsigned Reg, unsigned vreg=0)
addLiveIn - Add the specified register as a live-in.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
virtual void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
const DebugLoc & getDebugLoc() const
Returns the debug location id of this MachineInstr.
Definition: MachineInstr.h:384
unsigned addLiveIn(unsigned PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
unsigned getReg() const
getReg - Returns the register number.
void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain...
Definition: ISDOpcodes.h:730
This class represents a function call, abstracting a target machine&#39;s calling convention.
unsigned Reg
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Get a value with low bits set.
Definition: APInt.h:647
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition: ValueTypes.h:252
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
const RISCVRegisterInfo * getRegisterInfo() const override
*p = old <unsigned v ? old : v
Definition: Instructions.h:725
Function Alias Analysis Results
*p = old >unsigned v ? old : v
Definition: Instructions.h:723
unsigned getValNo() const
std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const override
Given a physical register constraint (e.g.
static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC)
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition: Function.h:323
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
APInt trunc(unsigned width) const
Truncate to new width.
Definition: APInt.cpp:810
unsigned getPointerSizeInBits(unsigned AS=0) const
Layout pointer size, in bits FIXME: The defaults need to be removed once all of the backends/clients ...
Definition: DataLayout.h:388
Value * CreateSExt(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1740
STATISTIC(NumFunctions, "Total number of functions")
unsigned const TargetRegisterInfo * TRI
A debug info location.
Definition: DebugLoc.h:33
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition: ValueTypes.h:140
F(f)
static const MCPhysReg ArgFPR32s[]
bool hasExternalWeakLinkage() const
Definition: GlobalValue.h:446
an instruction that atomically reads a memory location, combines it with another value, and then stores the result back.
Definition: Instructions.h:691
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned char TargetFlags=0)
SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
*p = old >signed v ? old : v
Definition: Instructions.h:719
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
unsigned getValueSizeInBits() const
Returns the size of the value in bits.
int CreateStackObject(uint64_t Size, unsigned Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it...
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition: ISDOpcodes.h:459
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition: ISDOpcodes.h:38
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition: ISDOpcodes.h:158
bool isMemLoc() const
IntegerType * getInt32Ty()
Fetch the type representing a 32-bit integer.
Definition: IRBuilder.h:346
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition: ISDOpcodes.h:209
Value * getArgOperand(unsigned i) const
Definition: InstrTypes.h:1241
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations...
Definition: ISDOpcodes.h:480
SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
bool hasOneUse() const
Return true if there is exactly one use of this node.
A description of a memory reference used in the backend.
static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed, bool IsRet, Type *OrigTy)
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
MachineBasicBlock * EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *BB) const override
This method should be implemented by targets that mark instructions with the &#39;usesCustomInserter&#39; fla...
const DataLayout & getDataLayout() const
Get the data layout for the module&#39;s target platform.
Definition: Module.cpp:369
const HexagonInstrInfo * TII
IntegerType * getInt64Ty()
Fetch the type representing a 64-bit integer.
Definition: IRBuilder.h:351
ArrayRef< T > makeArrayRef(const T &OneElt)
Construct an ArrayRef from a single element.
Definition: ArrayRef.h:450
unsigned getXLen() const
Shift and rotation operations.
Definition: ISDOpcodes.h:434
MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s), MachineInstr opcode, and operands.
BinOp getOperation() const
Definition: Instructions.h:750
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
Definition: APFloat.h:41
void eraseFromParent()
Unlink &#39;this&#39; from the containing basic block and delete it.
void addLoc(const CCValAssign &V)
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition: Type.h:196
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition: IRBuilder.h:742
static MachineBasicBlock * emitSplitF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB)
void setCondCodeAction(ISD::CondCode CC, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn&#39;t supported on the target and indicate what to d...
SimpleValueType SimpleTy
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted...
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
Definition: MachineInstr.h:410
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
Definition: SelectionDAG.h:463
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
const DataLayout & getDataLayout() const
Definition: SelectionDAG.h:403
SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG...
Definition: ISDOpcodes.h:72
const BlockAddress * getBlockAddress() const
LocInfo getLocInfo() const
static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
This represents a list of ValueType&#39;s that has been intern&#39;d by a SelectionDAG.
SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
AtomicOrdering
Atomic ordering for LLVM&#39;s memory model.
STACKSAVE - STACKSAVE has one operand, an input chain.
Definition: ISDOpcodes.h:726
SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
unsigned getSizeInBits() const
Class to represent function types.
Definition: DerivedTypes.h:103
unsigned getSizeInBits() const
Return the size of the specified value type in bits.
Definition: ValueTypes.h:291
unsigned getNextStackOffset() const
getNextStackOffset - Return the next stack offset such that all stack slots satisfy their alignment r...
int64_t getSExtValue() const
Get sign extended value.
Definition: APInt.h:1574
Type * getType() const
All values are typed, get the type of this value.
Definition: Value.h:244
MachineFunction & getMachineFunction() const
Definition: SelectionDAG.h:400
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose...
bool isRV32E() const
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
SDValue getRegisterMask(const uint32_t *RegMask)
BinOp
This enumeration lists the possible modifications atomicrmw can make.
Definition: Instructions.h:703
bool arg_empty() const
Definition: Function.h:715
SmallVectorImpl< CCValAssign > & getPendingLocs()
bool isTruncateFree(Type *SrcTy, Type *DstTy) const override
Return true if it&#39;s free to truncate a value of type FromTy to type ToTy.
Simple integer binary arithmetic operators.
Definition: ISDOpcodes.h:200
bool hasStdExtA() const
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB)
SmallVectorImpl< ISD::ArgFlagsTy > & getPendingArgFlags()
LLVM_NODISCARD size_t size() const
size - Get the string size.
Definition: StringRef.h:130
static const MCPhysReg ArgFPR64s[]
Value * CreateSub(Value *LHS, Value *RHS, const Twine &Name="", bool HasNUW=false, bool HasNSW=false)
Definition: IRBuilder.h:1066
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out...
Definition: ISDOpcodes.h:995
virtual const TargetInstrInfo * getInstrInfo() const
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
Definition: SelectionDAG.h:582
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
Function * getDeclaration(Module *M, ID id, ArrayRef< Type *> Tys=None)
Create or insert an LLVM Function declaration for an intrinsic, and return it.
Definition: Function.cpp:1044
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *bb=nullptr)
CreateMachineBasicBlock - Allocate a new MachineBasicBlock.
Analysis containing CSE Info
Definition: CSEInfo.cpp:20
ConstantInt * getIntN(unsigned N, uint64_t C)
Get a constant N-bit value, zero extended or truncated from a 64-bit value.
Definition: IRBuilder.h:317
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const override
Class to represent pointers.
Definition: DerivedTypes.h:544
unsigned getByValSize() const
unsigned getKillRegState(bool B)
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension. ...
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
static CCValAssign getReg(unsigned ValNo, MVT ValVT, unsigned RegNo, MVT LocVT, LocInfo HTP)
TargetInstrInfo - Interface to description of machine instruction set.
static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC)
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
bool isVoidTy() const
Return true if this is &#39;void&#39;.
Definition: Type.h:140
The memory access is volatile.
MachineInstrBuilder BuildMI(MachineFunction &MF, const DebugLoc &DL, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
virtual const uint32_t * getCallPreservedMask(const MachineFunction &MF, CallingConv::ID) const
Return a mask of call-preserved registers for the given calling convention on the current function...
MachineInstrBundleIterator< MachineInstr > iterator
Type * getReturnType() const
Returns the type of the ret val.
Definition: Function.h:168
static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode)
void LowerAsmOperandForConstraint(SDValue Op, std::string &Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const override
Lower the specified operand into the Ops vector.
bool shouldAssumeDSOLocal(const Module &M, const GlobalValue *GV) const
Machine Value Type.
LLVM Basic Block Representation.
Definition: BasicBlock.h:57
unsigned getOrigAlign() const
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
The instances of the Type class are immutable: once they are created, they are never changed...
Definition: Type.h:45
This is an important class for using LLVM in a threaded context.
Definition: LLVMContext.h:64
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type...
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
size_t size() const
size - Get the array size.
Definition: ArrayRef.h:148
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
bool hasStdExtF() const
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE...
Definition: ISDOpcodes.h:759
bool isFloatingPointOperation() const
Definition: Instructions.h:823
Instruction * emitTrailingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const override
const SDValue & getOperand(unsigned Num) const
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
Definition: ISDOpcodes.h:970
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition: SmallSet.h:134
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
Definition: SelectionDAG.h:840
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
bool isAcquireOrStronger(AtomicOrdering ao)
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
CombineLevel
Definition: DAGCombine.h:15
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB)
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly...
Definition: STLExtras.h:1199
bool hasStdExtM() const
unsigned getLargestLegalIntTypeSizeInBits() const
Returns the size of largest legal integer type size, or 0 if none are set.
Definition: DataLayout.cpp:791
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
amdgpu Simplify well known AMD library false FunctionCallee Value * Arg
void setPrefFunctionAlignment(unsigned Align)
Set the target&#39;s preferred function alignment.
self_iterator getIterator()
Definition: ilist_node.h:81
CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y)...
static MVT getVT(Type *Ty, bool HandleUnknown=false)
Return the value type corresponding to the specified type.
Definition: ValueTypes.cpp:288
std::pair< NoneType, bool > insert(const T &V)
insert - Insert an element into the set if it isn&#39;t already there.
Definition: SmallSet.h:180
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
Definition: ISDOpcodes.h:750
const MachineInstrBuilder & addFrameIndex(int Idx) const
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it&#39;s implicit...
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
bool hasBitPreservingFPLogic(EVT VT) const override
Return true if it is safe to transform an integer-domain bitwise operation into the equivalent floati...
Extended Value Type.
Definition: ValueTypes.h:33
static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
bool isPositionIndependent() const
size_t size() const
Definition: SmallVector.h:52
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
Definition: Instructions.h:789
const TargetMachine & getTargetMachine() const
This class contains a discriminated union of information about pointers in memory operands...
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp)
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
Value * CreateTrunc(Value *V, Type *DestTy, const Twine &Name="")
Definition: IRBuilder.h:1732
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition: APInt.h:970
SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, unsigned Alignment=0, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr)
Loads are not normal binary operators: their result type is not determined by their operands...
SDValue CreateStackTemporary(EVT VT, unsigned minAlign=1)
Create a stack temporary, suitable for holding the specified value type.
The memory access writes data.
unsigned getFrameRegister(const MachineFunction &MF) const override
bool isReleaseOrStronger(AtomicOrdering ao)
bool isDesirableToCommuteWithShift(const SDNode *N, CombineLevel Level) const override
Return true if it is profitable to move this shift by a constant amount though its operand...
SDValue getTargetConstantPool(const Constant *C, EVT VT, unsigned Align=0, int Offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:645
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition: APInt.h:554
TokenFactor - This node takes multiple tokens as input and produces a single token result...
Definition: ISDOpcodes.h:49
Value * getValOperand()
Definition: Instructions.h:815
static const MCPhysReg ArgGPRs[]
bool hasStdExtD() const
Iterator for intrusive lists based on ilist_node.
CCState - This class holds information needed while lowering arguments and return values...
void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
Fast - This calling convention attempts to make calls as fast as possible (e.g.
Definition: CallingConv.h:42
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
Definition: SelectionDAG.h:221
MachineOperand class - Representation of each machine instruction operand.
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG)
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
CCValAssign - Represent assignment of one arg/retval to a location.
SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, unsigned Align, bool isVol, bool AlwaysInline, bool isTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo)
constexpr size_t array_lengthof(T(&)[N])
Find the length of an array.
Definition: STLExtras.h:1050
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition: ValueTypes.h:95
const DataFlowGraph & G
Definition: RDFGraph.cpp:202
const MachineInstrBuilder & addMemOperand(MachineMemOperand *MMO) const
Instruction * emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst, AtomicOrdering Ord) const override
Inserts in the IR a target-specific intrinsic specifying a fence.
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
Byte Swap and Counting operators.
Definition: ISDOpcodes.h:437
Type * getReturnType() const
Definition: DerivedTypes.h:124
static bool isSelectPseudo(MachineInstr &MI)
int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
const Constant * getConstVal() const
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
int64_t getImm() const
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N)
Definition: SelectionDAG.h:685
const Function & getFunction() const
Return the LLVM function that this machine code represents.
raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition: Debug.cpp:132
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition: BitVector.h:940
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition: Function.h:163
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
Definition: Instruction.cpp:55
Class for arbitrary precision integers.
Definition: APInt.h:69
unsigned getByValAlign() const
CodeModel::Model getCodeModel() const
Returns the code model.
amdgpu Simplify well known AMD library false FunctionCallee Callee
Select(COND, TRUEVAL, FALSEVAL).
Definition: ISDOpcodes.h:444
void setMinFunctionAlignment(unsigned Align)
Set the target&#39;s minimum function alignment (in log2(bytes))
RISCVTargetLowering(const TargetMachine &TM, const RISCVSubtarget &STI)
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition: ISDOpcodes.h:495
virtual void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Store the specified register of the given register class to the specified stack frame index...
bool is64Bit() const
MachineRegisterInfo - Keep track of information for virtual and physical registers, including vreg register classes, use/def chains for registers, etc.
The memory access reads data.
int getIntMatCost(const APInt &Val, unsigned Size, bool IsRV64)
Definition: RISCVMatInt.cpp:78
static MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
uint64_t getTypeAllocSize(Type *Ty) const
Returns the offset in bytes between successive objects of the specified type, including alignment pad...
Definition: DataLayout.h:469
BR_JT - Jumptable branch.
Definition: ISDOpcodes.h:666
Representation of each machine instruction.
Definition: MachineInstr.h:63
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer, a SRCVALUE for the destination, and a SRCVALUE for the source.
Definition: ISDOpcodes.h:755
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:679
bool isVector() const
Return true if this is a vector value type.
Definition: ValueTypes.h:150
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
Bitwise operators - logical and, logical or, logical xor.
Definition: ISDOpcodes.h:411
FenceInst * CreateFence(AtomicOrdering Ordering, SyncScope::ID SSID=SyncScope::System, const Twine &Name="")
Definition: IRBuilder.h:1514
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB &#39;Other&#39; at the position From, and insert it into this MBB right before &#39;...
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT)
Definition: SelectionDAG.h:711
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
unsigned getLocMemOffset() const
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition: ISDOpcodes.h:205
LLVM_NODISCARD bool empty() const
Definition: SmallVector.h:55
StringRef getValueAsString() const
Return the attribute&#39;s value as a string.
Definition: Attributes.cpp:223
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition: ISDOpcodes.h:510
const MCInstrDesc & get(unsigned Opcode) const
Return the machine instruction descriptor that corresponds to the specified instruction opcode...
Definition: MCInstrInfo.h:44
PointerUnion< const Value *, const PseudoSourceValue * > ptrVal
static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2)
void push_back(MachineInstr *MI)
#define I(x, y, z)
Definition: MD5.cpp:58
#define N
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
void setStackPointerRegisterToSaveRestore(unsigned R)
If set to a physical register, this specifies the register that llvm.savestack/llvm.restorestack should save and restore.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
LLVM_NODISCARD std::enable_if<!is_simple_type< Y >::value, typename cast_retty< X, const Y >::ret_type >::type dyn_cast(const Y &Val)
Definition: Casting.h:332
uint32_t Size
Definition: Profile.cpp:46
static CCValAssign getMem(unsigned ValNo, MVT ValVT, unsigned Offset, MVT LocVT, LocInfo HTP)
CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value *> Args=None, const Twine &Name="", MDNode *FPMathTag=nullptr)
Definition: IRBuilder.h:2051
const MachineInstrBuilder & addReg(unsigned RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
unsigned getOpcode() const
FSINCOS - Compute both fsin and fcos as a single operation.
Definition: ISDOpcodes.h:636
SDValue getValue(unsigned R) const
SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
bool isRegLoc() const
bool hasStdExtC() const
assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())
void insert(iterator MBBI, MachineBasicBlock *MBB)
SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
void setReturnAddressIsTaken(bool s)
RISCVMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private RISCV-...
C - The default llvm calling convention, compatible with C.
Definition: CallingConv.h:34
unsigned getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition: Type.cpp:114
Module * getParent()
Get the module that this global value is contained inside of...
Definition: GlobalValue.h:575
LLVM Value Representation.
Definition: Value.h:72
SDValue getRegister(unsigned Reg, EVT VT)
const char * getTargetNodeName(unsigned Opcode) const override
This method returns the name of a target specific DAG node.
std::underlying_type< E >::type Mask()
Get a bitmask with 1s in all places up to the high-order bit of E&#39;s largest value.
Definition: BitmaskEnum.h:80
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition: Function.h:333
Primary interface to the complete machine description for the target machine.
Definition: TargetMachine.h:65
IRTranslator LLVM IR MI
StringRef - Represent a constant reference to a string, i.e.
Definition: StringRef.h:48
SetCC operator - This evaluates to a true value iff the condition is true.
Definition: ISDOpcodes.h:467
const SDValue & getOperand(unsigned i) const
bool verifyReturnAddressArgumentIsConstant(SDValue Op, SelectionDAG &DAG) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned char TargetFlags=0) const
unsigned getLocReg() const
TRUNCATE - Completely drop the high bits.
Definition: ISDOpcodes.h:498
#define LLVM_DEBUG(X)
Definition: Debug.h:122
unsigned AllocateReg(unsigned Reg)
AllocateReg - Attempt to allocate one register.
const MachineOperand & getOperand(unsigned i) const
Definition: MachineInstr.h:415
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation...
Perform various unary floating-point operations inspired by libm.
Definition: ISDOpcodes.h:610
unsigned AllocateStack(unsigned Size, unsigned Align)
AllocateStack - Allocate a chunk of stack space with the specified size and alignment.
Function Alias Analysis false
static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
virtual void loadRegFromStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, unsigned DestReg, int FrameIndex, const TargetRegisterClass *RC, const TargetRegisterInfo *TRI) const
Load the specified register of the given register class from the specified stack frame index...
LLVMContext * getContext() const
Definition: SelectionDAG.h:410
Type * getElementType() const
Definition: DerivedTypes.h:563
unsigned createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned char TargetFlags=0)
Definition: SelectionDAG.h:628
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition: ISDOpcodes.h:404
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition: SmallSet.h:164
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary...
Definition: ISDOpcodes.h:651