Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1163, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name RISCVISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/Target/RISCV -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/include -I /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/build-llvm/lib/Target/RISCV -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-12-07-102640-14763-1 -x c++ /build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp

/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp

1//===-- RISCVISelLowering.cpp - RISCV DAG Lowering Implementation --------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISCV uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "RISCV.h"
16#include "RISCVMachineFunctionInfo.h"
17#include "RISCVRegisterInfo.h"
18#include "RISCVSubtarget.h"
19#include "RISCVTargetMachine.h"
20#include "Utils/RISCVMatInt.h"
21#include "llvm/ADT/SmallSet.h"
22#include "llvm/ADT/Statistic.h"
23#include "llvm/CodeGen/CallingConvLower.h"
24#include "llvm/CodeGen/MachineFrameInfo.h"
25#include "llvm/CodeGen/MachineFunction.h"
26#include "llvm/CodeGen/MachineInstrBuilder.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/SelectionDAGISel.h"
29#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
30#include "llvm/CodeGen/ValueTypes.h"
31#include "llvm/IR/DiagnosticInfo.h"
32#include "llvm/IR/DiagnosticPrinter.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/raw_ostream.h"
36
37using namespace llvm;
38
39#define DEBUG_TYPE"riscv-lower" "riscv-lower"
40
41STATISTIC(NumTailCalls, "Number of tail calls")static llvm::Statistic NumTailCalls = {"riscv-lower", "NumTailCalls"
, "Number of tail calls"}
;
42
43RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
44 const RISCVSubtarget &STI)
45 : TargetLowering(TM), Subtarget(STI) {
46
47 if (Subtarget.isRV32E())
48 report_fatal_error("Codegen not yet implemented for RV32E");
49
50 RISCVABI::ABI ABI = Subtarget.getTargetABI();
51 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI")((ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI"
) ? static_cast<void> (0) : __assert_fail ("ABI != RISCVABI::ABI_Unknown && \"Improperly initialised target ABI\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 51, __PRETTY_FUNCTION__))
;
52
53 switch (ABI) {
54 default:
55 report_fatal_error("Don't know how to lower this ABI");
56 case RISCVABI::ABI_ILP32:
57 case RISCVABI::ABI_ILP32F:
58 case RISCVABI::ABI_ILP32D:
59 case RISCVABI::ABI_LP64:
60 case RISCVABI::ABI_LP64F:
61 case RISCVABI::ABI_LP64D:
62 break;
63 }
64
65 MVT XLenVT = Subtarget.getXLenVT();
66
67 // Set up the register classes.
68 addRegisterClass(XLenVT, &RISCV::GPRRegClass);
69
70 if (Subtarget.hasStdExtF())
71 addRegisterClass(MVT::f32, &RISCV::FPR32RegClass);
72 if (Subtarget.hasStdExtD())
73 addRegisterClass(MVT::f64, &RISCV::FPR64RegClass);
74
75 // Compute derived properties from the register classes.
76 computeRegisterProperties(STI.getRegisterInfo());
77
78 setStackPointerRegisterToSaveRestore(RISCV::X2);
79
80 for (auto N : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD})
81 setLoadExtAction(N, XLenVT, MVT::i1, Promote);
82
83 // TODO: add all necessary setOperationAction calls.
84 setOperationAction(ISD::DYNAMIC_STACKALLOC, XLenVT, Expand);
85
86 setOperationAction(ISD::BR_JT, MVT::Other, Expand);
87 setOperationAction(ISD::BR_CC, XLenVT, Expand);
88 setOperationAction(ISD::SELECT, XLenVT, Custom);
89 setOperationAction(ISD::SELECT_CC, XLenVT, Expand);
90
91 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
92 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
93
94 setOperationAction(ISD::VASTART, MVT::Other, Custom);
95 setOperationAction(ISD::VAARG, MVT::Other, Expand);
96 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
97 setOperationAction(ISD::VAEND, MVT::Other, Expand);
98
99 for (auto VT : {MVT::i1, MVT::i8, MVT::i16})
100 setOperationAction(ISD::SIGN_EXTEND_INREG, VT, Expand);
101
102 if (Subtarget.is64Bit()) {
103 setOperationAction(ISD::ADD, MVT::i32, Custom);
104 setOperationAction(ISD::SUB, MVT::i32, Custom);
105 setOperationAction(ISD::SHL, MVT::i32, Custom);
106 setOperationAction(ISD::SRA, MVT::i32, Custom);
107 setOperationAction(ISD::SRL, MVT::i32, Custom);
108 }
109
110 if (!Subtarget.hasStdExtM()) {
111 setOperationAction(ISD::MUL, XLenVT, Expand);
112 setOperationAction(ISD::MULHS, XLenVT, Expand);
113 setOperationAction(ISD::MULHU, XLenVT, Expand);
114 setOperationAction(ISD::SDIV, XLenVT, Expand);
115 setOperationAction(ISD::UDIV, XLenVT, Expand);
116 setOperationAction(ISD::SREM, XLenVT, Expand);
117 setOperationAction(ISD::UREM, XLenVT, Expand);
118 }
119
120 if (Subtarget.is64Bit() && Subtarget.hasStdExtM()) {
121 setOperationAction(ISD::MUL, MVT::i32, Custom);
122 setOperationAction(ISD::SDIV, MVT::i32, Custom);
123 setOperationAction(ISD::UDIV, MVT::i32, Custom);
124 setOperationAction(ISD::UREM, MVT::i32, Custom);
125 }
126
127 setOperationAction(ISD::SDIVREM, XLenVT, Expand);
128 setOperationAction(ISD::UDIVREM, XLenVT, Expand);
129 setOperationAction(ISD::SMUL_LOHI, XLenVT, Expand);
130 setOperationAction(ISD::UMUL_LOHI, XLenVT, Expand);
131
132 setOperationAction(ISD::SHL_PARTS, XLenVT, Custom);
133 setOperationAction(ISD::SRL_PARTS, XLenVT, Custom);
134 setOperationAction(ISD::SRA_PARTS, XLenVT, Custom);
135
136 setOperationAction(ISD::ROTL, XLenVT, Expand);
137 setOperationAction(ISD::ROTR, XLenVT, Expand);
138 setOperationAction(ISD::BSWAP, XLenVT, Expand);
139 setOperationAction(ISD::CTTZ, XLenVT, Expand);
140 setOperationAction(ISD::CTLZ, XLenVT, Expand);
141 setOperationAction(ISD::CTPOP, XLenVT, Expand);
142
143 ISD::CondCode FPCCToExtend[] = {
144 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
145 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
146 ISD::SETGE, ISD::SETNE};
147
148 ISD::NodeType FPOpToExtend[] = {
149 ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FP16_TO_FP,
150 ISD::FP_TO_FP16};
151
152 if (Subtarget.hasStdExtF()) {
153 setOperationAction(ISD::FMINNUM, MVT::f32, Legal);
154 setOperationAction(ISD::FMAXNUM, MVT::f32, Legal);
155 for (auto CC : FPCCToExtend)
156 setCondCodeAction(CC, MVT::f32, Expand);
157 setOperationAction(ISD::SELECT_CC, MVT::f32, Expand);
158 setOperationAction(ISD::SELECT, MVT::f32, Custom);
159 setOperationAction(ISD::BR_CC, MVT::f32, Expand);
160 for (auto Op : FPOpToExtend)
161 setOperationAction(Op, MVT::f32, Expand);
162 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
163 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
164 }
165
166 if (Subtarget.hasStdExtF() && Subtarget.is64Bit())
167 setOperationAction(ISD::BITCAST, MVT::i32, Custom);
168
169 if (Subtarget.hasStdExtD()) {
170 setOperationAction(ISD::FMINNUM, MVT::f64, Legal);
171 setOperationAction(ISD::FMAXNUM, MVT::f64, Legal);
172 for (auto CC : FPCCToExtend)
173 setCondCodeAction(CC, MVT::f64, Expand);
174 setOperationAction(ISD::SELECT_CC, MVT::f64, Expand);
175 setOperationAction(ISD::SELECT, MVT::f64, Custom);
176 setOperationAction(ISD::BR_CC, MVT::f64, Expand);
177 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
178 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
179 for (auto Op : FPOpToExtend)
180 setOperationAction(Op, MVT::f64, Expand);
181 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
182 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
183 }
184
185 setOperationAction(ISD::GlobalAddress, XLenVT, Custom);
186 setOperationAction(ISD::BlockAddress, XLenVT, Custom);
187 setOperationAction(ISD::ConstantPool, XLenVT, Custom);
188
189 setOperationAction(ISD::GlobalTLSAddress, XLenVT, Custom);
190
191 // TODO: On M-mode only targets, the cycle[h] CSR may not be present.
192 // Unfortunately this can't be determined just from the ISA naming string.
193 setOperationAction(ISD::READCYCLECOUNTER, MVT::i64,
194 Subtarget.is64Bit() ? Legal : Custom);
195
196 setOperationAction(ISD::TRAP, MVT::Other, Legal);
197 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
198
199 if (Subtarget.hasStdExtA()) {
200 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
201 setMinCmpXchgSizeInBits(32);
202 } else {
203 setMaxAtomicSizeInBitsSupported(0);
204 }
205
206 setBooleanContents(ZeroOrOneBooleanContent);
207
208 // Function alignments.
209 const Align FunctionAlignment(Subtarget.hasStdExtC() ? 2 : 4);
210 setMinFunctionAlignment(FunctionAlignment);
211 setPrefFunctionAlignment(FunctionAlignment);
212
213 // Effectively disable jump table generation.
214 setMinimumJumpTableEntries(INT_MAX2147483647);
215}
216
217EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &,
218 EVT VT) const {
219 if (!VT.isVector())
220 return getPointerTy(DL);
221 return VT.changeVectorElementTypeToInteger();
222}
223
224bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
225 const CallInst &I,
226 MachineFunction &MF,
227 unsigned Intrinsic) const {
228 switch (Intrinsic) {
229 default:
230 return false;
231 case Intrinsic::riscv_masked_atomicrmw_xchg_i32:
232 case Intrinsic::riscv_masked_atomicrmw_add_i32:
233 case Intrinsic::riscv_masked_atomicrmw_sub_i32:
234 case Intrinsic::riscv_masked_atomicrmw_nand_i32:
235 case Intrinsic::riscv_masked_atomicrmw_max_i32:
236 case Intrinsic::riscv_masked_atomicrmw_min_i32:
237 case Intrinsic::riscv_masked_atomicrmw_umax_i32:
238 case Intrinsic::riscv_masked_atomicrmw_umin_i32:
239 case Intrinsic::riscv_masked_cmpxchg_i32:
240 PointerType *PtrTy = cast<PointerType>(I.getArgOperand(0)->getType());
241 Info.opc = ISD::INTRINSIC_W_CHAIN;
242 Info.memVT = MVT::getVT(PtrTy->getElementType());
243 Info.ptrVal = I.getArgOperand(0);
244 Info.offset = 0;
245 Info.align = Align(4);
246 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
247 MachineMemOperand::MOVolatile;
248 return true;
249 }
250}
251
252bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
253 const AddrMode &AM, Type *Ty,
254 unsigned AS,
255 Instruction *I) const {
256 // No global is ever allowed as a base.
257 if (AM.BaseGV)
258 return false;
259
260 // Require a 12-bit signed offset.
261 if (!isInt<12>(AM.BaseOffs))
262 return false;
263
264 switch (AM.Scale) {
265 case 0: // "r+i" or just "i", depending on HasBaseReg.
266 break;
267 case 1:
268 if (!AM.HasBaseReg) // allow "r+i".
269 break;
270 return false; // disallow "r+r" or "r+r+i".
271 default:
272 return false;
273 }
274
275 return true;
276}
277
278bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
279 return isInt<12>(Imm);
280}
281
282bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
283 return isInt<12>(Imm);
284}
285
286// On RV32, 64-bit integers are split into their high and low parts and held
287// in two different registers, so the trunc is free since the low register can
288// just be used.
289bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
290 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
291 return false;
292 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
293 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
294 return (SrcBits == 64 && DestBits == 32);
295}
296
297bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
298 if (Subtarget.is64Bit() || SrcVT.isVector() || DstVT.isVector() ||
299 !SrcVT.isInteger() || !DstVT.isInteger())
300 return false;
301 unsigned SrcBits = SrcVT.getSizeInBits();
302 unsigned DestBits = DstVT.getSizeInBits();
303 return (SrcBits == 64 && DestBits == 32);
304}
305
306bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
307 // Zexts are free if they can be combined with a load.
308 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
1
Assuming 'LD' is null
2
Taking false branch
309 EVT MemVT = LD->getMemoryVT();
310 if ((MemVT == MVT::i8 || MemVT == MVT::i16 ||
311 (Subtarget.is64Bit() && MemVT == MVT::i32)) &&
312 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
313 LD->getExtensionType() == ISD::ZEXTLOAD))
314 return true;
315 }
316
317 return TargetLowering::isZExtFree(Val, VT2);
3
Value assigned to 'Val.Node'
4
Calling 'TargetLoweringBase::isZExtFree'
318}
319
320bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
321 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
322}
323
324bool RISCVTargetLowering::hasBitPreservingFPLogic(EVT VT) const {
325 return (VT == MVT::f32 && Subtarget.hasStdExtF()) ||
326 (VT == MVT::f64 && Subtarget.hasStdExtD());
327}
328
329// Changes the condition code and swaps operands if necessary, so the SetCC
330// operation matches one of the comparisons supported directly in the RISC-V
331// ISA.
332static void normaliseSetCC(SDValue &LHS, SDValue &RHS, ISD::CondCode &CC) {
333 switch (CC) {
334 default:
335 break;
336 case ISD::SETGT:
337 case ISD::SETLE:
338 case ISD::SETUGT:
339 case ISD::SETULE:
340 CC = ISD::getSetCCSwappedOperands(CC);
341 std::swap(LHS, RHS);
342 break;
343 }
344}
345
346// Return the RISC-V branch opcode that matches the given DAG integer
347// condition code. The CondCode must be one of those supported by the RISC-V
348// ISA (see normaliseSetCC).
349static unsigned getBranchOpcodeForIntCondCode(ISD::CondCode CC) {
350 switch (CC) {
351 default:
352 llvm_unreachable("Unsupported CondCode")::llvm::llvm_unreachable_internal("Unsupported CondCode", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 352)
;
353 case ISD::SETEQ:
354 return RISCV::BEQ;
355 case ISD::SETNE:
356 return RISCV::BNE;
357 case ISD::SETLT:
358 return RISCV::BLT;
359 case ISD::SETGE:
360 return RISCV::BGE;
361 case ISD::SETULT:
362 return RISCV::BLTU;
363 case ISD::SETUGE:
364 return RISCV::BGEU;
365 }
366}
367
368SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
369 SelectionDAG &DAG) const {
370 switch (Op.getOpcode()) {
371 default:
372 report_fatal_error("unimplemented operand");
373 case ISD::GlobalAddress:
374 return lowerGlobalAddress(Op, DAG);
375 case ISD::BlockAddress:
376 return lowerBlockAddress(Op, DAG);
377 case ISD::ConstantPool:
378 return lowerConstantPool(Op, DAG);
379 case ISD::GlobalTLSAddress:
380 return lowerGlobalTLSAddress(Op, DAG);
381 case ISD::SELECT:
382 return lowerSELECT(Op, DAG);
383 case ISD::VASTART:
384 return lowerVASTART(Op, DAG);
385 case ISD::FRAMEADDR:
386 return lowerFRAMEADDR(Op, DAG);
387 case ISD::RETURNADDR:
388 return lowerRETURNADDR(Op, DAG);
389 case ISD::SHL_PARTS:
390 return lowerShiftLeftParts(Op, DAG);
391 case ISD::SRA_PARTS:
392 return lowerShiftRightParts(Op, DAG, true);
393 case ISD::SRL_PARTS:
394 return lowerShiftRightParts(Op, DAG, false);
395 case ISD::BITCAST: {
396 assert(Subtarget.is64Bit() && Subtarget.hasStdExtF() &&((Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
"Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("Subtarget.is64Bit() && Subtarget.hasStdExtF() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 397, __PRETTY_FUNCTION__))
397 "Unexpected custom legalisation")((Subtarget.is64Bit() && Subtarget.hasStdExtF() &&
"Unexpected custom legalisation") ? static_cast<void> (
0) : __assert_fail ("Subtarget.is64Bit() && Subtarget.hasStdExtF() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 397, __PRETTY_FUNCTION__))
;
398 SDLoc DL(Op);
399 SDValue Op0 = Op.getOperand(0);
400 if (Op.getValueType() != MVT::f32 || Op0.getValueType() != MVT::i32)
401 return SDValue();
402 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
403 SDValue FPConv = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, NewOp0);
404 return FPConv;
405 }
406 }
407}
408
409static SDValue getTargetNode(GlobalAddressSDNode *N, SDLoc DL, EVT Ty,
410 SelectionDAG &DAG, unsigned Flags) {
411 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
412}
413
414static SDValue getTargetNode(BlockAddressSDNode *N, SDLoc DL, EVT Ty,
415 SelectionDAG &DAG, unsigned Flags) {
416 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
417 Flags);
418}
419
420static SDValue getTargetNode(ConstantPoolSDNode *N, SDLoc DL, EVT Ty,
421 SelectionDAG &DAG, unsigned Flags) {
422 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlignment(),
423 N->getOffset(), Flags);
424}
425
426template <class NodeTy>
427SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
428 bool IsLocal) const {
429 SDLoc DL(N);
430 EVT Ty = getPointerTy(DAG.getDataLayout());
431
432 if (isPositionIndependent()) {
433 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
434 if (IsLocal)
435 // Use PC-relative addressing to access the symbol. This generates the
436 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
437 // %pcrel_lo(auipc)).
438 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
439
440 // Use PC-relative addressing to access the GOT for this symbol, then load
441 // the address from the GOT. This generates the pattern (PseudoLA sym),
442 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
443 return SDValue(DAG.getMachineNode(RISCV::PseudoLA, DL, Ty, Addr), 0);
444 }
445
446 switch (getTargetMachine().getCodeModel()) {
447 default:
448 report_fatal_error("Unsupported code model for lowering");
449 case CodeModel::Small: {
450 // Generate a sequence for accessing addresses within the first 2 GiB of
451 // address space. This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
452 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
453 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
454 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
455 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNHi, AddrLo), 0);
456 }
457 case CodeModel::Medium: {
458 // Generate a sequence for accessing addresses within any 2GiB range within
459 // the address space. This generates the pattern (PseudoLLA sym), which
460 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
461 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
462 return SDValue(DAG.getMachineNode(RISCV::PseudoLLA, DL, Ty, Addr), 0);
463 }
464 }
465}
466
467SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
468 SelectionDAG &DAG) const {
469 SDLoc DL(Op);
470 EVT Ty = Op.getValueType();
471 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
472 int64_t Offset = N->getOffset();
473 MVT XLenVT = Subtarget.getXLenVT();
474
475 const GlobalValue *GV = N->getGlobal();
476 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV);
477 SDValue Addr = getAddr(N, DAG, IsLocal);
478
479 // In order to maximise the opportunity for common subexpression elimination,
480 // emit a separate ADD node for the global address offset instead of folding
481 // it in the global address node. Later peephole optimisations may choose to
482 // fold it back in when profitable.
483 if (Offset != 0)
484 return DAG.getNode(ISD::ADD, DL, Ty, Addr,
485 DAG.getConstant(Offset, DL, XLenVT));
486 return Addr;
487}
488
489SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
490 SelectionDAG &DAG) const {
491 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Op);
492
493 return getAddr(N, DAG);
494}
495
496SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
497 SelectionDAG &DAG) const {
498 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Op);
499
500 return getAddr(N, DAG);
501}
502
503SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
504 SelectionDAG &DAG,
505 bool UseGOT) const {
506 SDLoc DL(N);
507 EVT Ty = getPointerTy(DAG.getDataLayout());
508 const GlobalValue *GV = N->getGlobal();
509 MVT XLenVT = Subtarget.getXLenVT();
510
511 if (UseGOT) {
512 // Use PC-relative addressing to access the GOT for this TLS symbol, then
513 // load the address from the GOT and add the thread pointer. This generates
514 // the pattern (PseudoLA_TLS_IE sym), which expands to
515 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
516 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
517 SDValue Load =
518 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_IE, DL, Ty, Addr), 0);
519
520 // Add the thread pointer.
521 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
522 return DAG.getNode(ISD::ADD, DL, Ty, Load, TPReg);
523 }
524
525 // Generate a sequence for accessing the address relative to the thread
526 // pointer, with the appropriate adjustment for the thread pointer offset.
527 // This generates the pattern
528 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
529 SDValue AddrHi =
530 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_HI);
531 SDValue AddrAdd =
532 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_ADD);
533 SDValue AddrLo =
534 DAG.getTargetGlobalAddress(GV, DL, Ty, 0, RISCVII::MO_TPREL_LO);
535
536 SDValue MNHi = SDValue(DAG.getMachineNode(RISCV::LUI, DL, Ty, AddrHi), 0);
537 SDValue TPReg = DAG.getRegister(RISCV::X4, XLenVT);
538 SDValue MNAdd = SDValue(
539 DAG.getMachineNode(RISCV::PseudoAddTPRel, DL, Ty, MNHi, TPReg, AddrAdd),
540 0);
541 return SDValue(DAG.getMachineNode(RISCV::ADDI, DL, Ty, MNAdd, AddrLo), 0);
542}
543
544SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
545 SelectionDAG &DAG) const {
546 SDLoc DL(N);
547 EVT Ty = getPointerTy(DAG.getDataLayout());
548 IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
549 const GlobalValue *GV = N->getGlobal();
550
551 // Use a PC-relative addressing mode to access the global dynamic GOT address.
552 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
553 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
554 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
555 SDValue Load =
556 SDValue(DAG.getMachineNode(RISCV::PseudoLA_TLS_GD, DL, Ty, Addr), 0);
557
558 // Prepare argument list to generate call.
559 ArgListTy Args;
560 ArgListEntry Entry;
561 Entry.Node = Load;
562 Entry.Ty = CallTy;
563 Args.push_back(Entry);
564
565 // Setup call to __tls_get_addr.
566 TargetLowering::CallLoweringInfo CLI(DAG);
567 CLI.setDebugLoc(DL)
568 .setChain(DAG.getEntryNode())
569 .setLibCallee(CallingConv::C, CallTy,
570 DAG.getExternalSymbol("__tls_get_addr", Ty),
571 std::move(Args));
572
573 return LowerCallTo(CLI).first;
574}
575
576SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
577 SelectionDAG &DAG) const {
578 SDLoc DL(Op);
579 EVT Ty = Op.getValueType();
580 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
581 int64_t Offset = N->getOffset();
582 MVT XLenVT = Subtarget.getXLenVT();
583
584 // Non-PIC TLS lowering should always use the LocalExec model.
585 TLSModel::Model Model = isPositionIndependent()
586 ? getTargetMachine().getTLSModel(N->getGlobal())
587 : TLSModel::LocalExec;
588
589 SDValue Addr;
590 switch (Model) {
591 case TLSModel::LocalExec:
592 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
593 break;
594 case TLSModel::InitialExec:
595 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
596 break;
597 case TLSModel::LocalDynamic:
598 case TLSModel::GeneralDynamic:
599 Addr = getDynamicTLSAddr(N, DAG);
600 break;
601 }
602
603 // In order to maximise the opportunity for common subexpression elimination,
604 // emit a separate ADD node for the global address offset instead of folding
605 // it in the global address node. Later peephole optimisations may choose to
606 // fold it back in when profitable.
607 if (Offset != 0)
608 return DAG.getNode(ISD::ADD, DL, Ty, Addr,
609 DAG.getConstant(Offset, DL, XLenVT));
610 return Addr;
611}
612
613SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
614 SDValue CondV = Op.getOperand(0);
615 SDValue TrueV = Op.getOperand(1);
616 SDValue FalseV = Op.getOperand(2);
617 SDLoc DL(Op);
618 MVT XLenVT = Subtarget.getXLenVT();
619
620 // If the result type is XLenVT and CondV is the output of a SETCC node
621 // which also operated on XLenVT inputs, then merge the SETCC node into the
622 // lowered RISCVISD::SELECT_CC to take advantage of the integer
623 // compare+branch instructions. i.e.:
624 // (select (setcc lhs, rhs, cc), truev, falsev)
625 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
626 if (Op.getSimpleValueType() == XLenVT && CondV.getOpcode() == ISD::SETCC &&
627 CondV.getOperand(0).getSimpleValueType() == XLenVT) {
628 SDValue LHS = CondV.getOperand(0);
629 SDValue RHS = CondV.getOperand(1);
630 auto CC = cast<CondCodeSDNode>(CondV.getOperand(2));
631 ISD::CondCode CCVal = CC->get();
632
633 normaliseSetCC(LHS, RHS, CCVal);
634
635 SDValue TargetCC = DAG.getConstant(CCVal, DL, XLenVT);
636 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
637 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
638 return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
639 }
640
641 // Otherwise:
642 // (select condv, truev, falsev)
643 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
644 SDValue Zero = DAG.getConstant(0, DL, XLenVT);
645 SDValue SetNE = DAG.getConstant(ISD::SETNE, DL, XLenVT);
646
647 SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::Glue);
648 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
649
650 return DAG.getNode(RISCVISD::SELECT_CC, DL, VTs, Ops);
651}
652
653SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
654 MachineFunction &MF = DAG.getMachineFunction();
655 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
656
657 SDLoc DL(Op);
658 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
659 getPointerTy(MF.getDataLayout()));
660
661 // vastart just stores the address of the VarArgsFrameIndex slot into the
662 // memory location argument.
663 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
664 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
665 MachinePointerInfo(SV));
666}
667
668SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
669 SelectionDAG &DAG) const {
670 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
671 MachineFunction &MF = DAG.getMachineFunction();
672 MachineFrameInfo &MFI = MF.getFrameInfo();
673 MFI.setFrameAddressIsTaken(true);
674 Register FrameReg = RI.getFrameRegister(MF);
675 int XLenInBytes = Subtarget.getXLen() / 8;
676
677 EVT VT = Op.getValueType();
678 SDLoc DL(Op);
679 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
680 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
681 while (Depth--) {
682 int Offset = -(XLenInBytes * 2);
683 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
684 DAG.getIntPtrConstant(Offset, DL));
685 FrameAddr =
686 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
687 }
688 return FrameAddr;
689}
690
691SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
692 SelectionDAG &DAG) const {
693 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
694 MachineFunction &MF = DAG.getMachineFunction();
695 MachineFrameInfo &MFI = MF.getFrameInfo();
696 MFI.setReturnAddressIsTaken(true);
697 MVT XLenVT = Subtarget.getXLenVT();
698 int XLenInBytes = Subtarget.getXLen() / 8;
699
700 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
701 return SDValue();
702
703 EVT VT = Op.getValueType();
704 SDLoc DL(Op);
705 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
706 if (Depth) {
707 int Off = -XLenInBytes;
708 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
709 SDValue Offset = DAG.getConstant(Off, DL, VT);
710 return DAG.getLoad(VT, DL, DAG.getEntryNode(),
711 DAG.getNode(ISD::ADD, DL, VT, FrameAddr, Offset),
712 MachinePointerInfo());
713 }
714
715 // Return the value of the return address register, marking it an implicit
716 // live-in.
717 Register Reg = MF.addLiveIn(RI.getRARegister(), getRegClassFor(XLenVT));
718 return DAG.getCopyFromReg(DAG.getEntryNode(), DL, Reg, XLenVT);
719}
720
721SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
722 SelectionDAG &DAG) const {
723 SDLoc DL(Op);
724 SDValue Lo = Op.getOperand(0);
725 SDValue Hi = Op.getOperand(1);
726 SDValue Shamt = Op.getOperand(2);
727 EVT VT = Lo.getValueType();
728
729 // if Shamt-XLEN < 0: // Shamt < XLEN
730 // Lo = Lo << Shamt
731 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
732 // else:
733 // Lo = 0
734 // Hi = Lo << (Shamt-XLEN)
735
736 SDValue Zero = DAG.getConstant(0, DL, VT);
737 SDValue One = DAG.getConstant(1, DL, VT);
738 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
739 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
740 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
741 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
742
743 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
744 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
745 SDValue ShiftRightLo =
746 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, XLenMinus1Shamt);
747 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
748 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
749 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusXLen);
750
751 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
752
753 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
754 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
755
756 SDValue Parts[2] = {Lo, Hi};
757 return DAG.getMergeValues(Parts, DL);
758}
759
760SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
761 bool IsSRA) const {
762 SDLoc DL(Op);
763 SDValue Lo = Op.getOperand(0);
764 SDValue Hi = Op.getOperand(1);
765 SDValue Shamt = Op.getOperand(2);
766 EVT VT = Lo.getValueType();
767
768 // SRA expansion:
769 // if Shamt-XLEN < 0: // Shamt < XLEN
770 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
771 // Hi = Hi >>s Shamt
772 // else:
773 // Lo = Hi >>s (Shamt-XLEN);
774 // Hi = Hi >>s (XLEN-1)
775 //
776 // SRL expansion:
777 // if Shamt-XLEN < 0: // Shamt < XLEN
778 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - Shamt))
779 // Hi = Hi >>u Shamt
780 // else:
781 // Lo = Hi >>u (Shamt-XLEN);
782 // Hi = 0;
783
784 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
785
786 SDValue Zero = DAG.getConstant(0, DL, VT);
787 SDValue One = DAG.getConstant(1, DL, VT);
788 SDValue MinusXLen = DAG.getConstant(-(int)Subtarget.getXLen(), DL, VT);
789 SDValue XLenMinus1 = DAG.getConstant(Subtarget.getXLen() - 1, DL, VT);
790 SDValue ShamtMinusXLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusXLen);
791 SDValue XLenMinus1Shamt = DAG.getNode(ISD::SUB, DL, VT, XLenMinus1, Shamt);
792
793 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
794 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
795 SDValue ShiftLeftHi =
796 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, XLenMinus1Shamt);
797 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
798 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
799 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusXLen);
800 SDValue HiFalse =
801 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, XLenMinus1) : Zero;
802
803 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusXLen, Zero, ISD::SETLT);
804
805 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
806 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
807
808 SDValue Parts[2] = {Lo, Hi};
809 return DAG.getMergeValues(Parts, DL);
810}
811
812// Returns the opcode of the target-specific SDNode that implements the 32-bit
813// form of the given Opcode.
814static RISCVISD::NodeType getRISCVWOpcode(unsigned Opcode) {
815 switch (Opcode) {
816 default:
817 llvm_unreachable("Unexpected opcode")::llvm::llvm_unreachable_internal("Unexpected opcode", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 817)
;
818 case ISD::SHL:
819 return RISCVISD::SLLW;
820 case ISD::SRA:
821 return RISCVISD::SRAW;
822 case ISD::SRL:
823 return RISCVISD::SRLW;
824 case ISD::SDIV:
825 return RISCVISD::DIVW;
826 case ISD::UDIV:
827 return RISCVISD::DIVUW;
828 case ISD::UREM:
829 return RISCVISD::REMUW;
830 }
831}
832
833// Converts the given 32-bit operation to a target-specific SelectionDAG node.
834// Because i32 isn't a legal type for RV64, these operations would otherwise
835// be promoted to i64, making it difficult to select the SLLW/DIVUW/.../*W
836// later one because the fact the operation was originally of type i32 is
837// lost.
838static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG) {
839 SDLoc DL(N);
840 RISCVISD::NodeType WOpcode = getRISCVWOpcode(N->getOpcode());
841 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
842 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
843 SDValue NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
844 // ReplaceNodeResults requires we maintain the same type for the return value.
845 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
846}
847
848// Converts the given 32-bit operation to a i64 operation with signed extension
849// semantic to reduce the signed extension instructions.
850static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
851 SDLoc DL(N);
852 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
853 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
854 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
855 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
856 DAG.getValueType(MVT::i32));
857 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
858}
859
860void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
861 SmallVectorImpl<SDValue> &Results,
862 SelectionDAG &DAG) const {
863 SDLoc DL(N);
864 switch (N->getOpcode()) {
865 default:
866 llvm_unreachable("Don't know how to custom type legalize this operation!")::llvm::llvm_unreachable_internal("Don't know how to custom type legalize this operation!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 866)
;
867 case ISD::READCYCLECOUNTER: {
868 assert(!Subtarget.is64Bit() &&((!Subtarget.is64Bit() && "READCYCLECOUNTER only has custom type legalization on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"READCYCLECOUNTER only has custom type legalization on riscv32\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 869, __PRETTY_FUNCTION__))
869 "READCYCLECOUNTER only has custom type legalization on riscv32")((!Subtarget.is64Bit() && "READCYCLECOUNTER only has custom type legalization on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"READCYCLECOUNTER only has custom type legalization on riscv32\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 869, __PRETTY_FUNCTION__))
;
870
871 SDVTList VTs = DAG.getVTList(MVT::i32, MVT::i32, MVT::Other);
872 SDValue RCW =
873 DAG.getNode(RISCVISD::READ_CYCLE_WIDE, DL, VTs, N->getOperand(0));
874
875 Results.push_back(RCW);
876 Results.push_back(RCW.getValue(1));
877 Results.push_back(RCW.getValue(2));
878 break;
879 }
880 case ISD::ADD:
881 case ISD::SUB:
882 case ISD::MUL:
883 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 884, __PRETTY_FUNCTION__))
884 "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 884, __PRETTY_FUNCTION__))
;
885 if (N->getOperand(1).getOpcode() == ISD::Constant)
886 return;
887 Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
888 break;
889 case ISD::SHL:
890 case ISD::SRA:
891 case ISD::SRL:
892 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 893, __PRETTY_FUNCTION__))
893 "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && "Unexpected custom legalisation") ? static_cast
<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 893, __PRETTY_FUNCTION__))
;
894 if (N->getOperand(1).getOpcode() == ISD::Constant)
895 return;
896 Results.push_back(customLegalizeToWOp(N, DAG));
897 break;
898 case ISD::SDIV:
899 case ISD::UDIV:
900 case ISD::UREM:
901 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtM() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtM() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 902, __PRETTY_FUNCTION__))
902 Subtarget.hasStdExtM() && "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtM() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtM() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 902, __PRETTY_FUNCTION__))
;
903 if (N->getOperand(0).getOpcode() == ISD::Constant ||
904 N->getOperand(1).getOpcode() == ISD::Constant)
905 return;
906 Results.push_back(customLegalizeToWOp(N, DAG));
907 break;
908 case ISD::BITCAST: {
909 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtF() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtF() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 910, __PRETTY_FUNCTION__))
910 Subtarget.hasStdExtF() && "Unexpected custom legalisation")((N->getValueType(0) == MVT::i32 && Subtarget.is64Bit
() && Subtarget.hasStdExtF() && "Unexpected custom legalisation"
) ? static_cast<void> (0) : __assert_fail ("N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() && Subtarget.hasStdExtF() && \"Unexpected custom legalisation\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 910, __PRETTY_FUNCTION__))
;
911 SDLoc DL(N);
912 SDValue Op0 = N->getOperand(0);
913 if (Op0.getValueType() != MVT::f32)
914 return;
915 SDValue FPConv =
916 DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Op0);
917 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, FPConv));
918 break;
919 }
920 }
921}
922
923SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
924 DAGCombinerInfo &DCI) const {
925 SelectionDAG &DAG = DCI.DAG;
926
927 switch (N->getOpcode()) {
928 default:
929 break;
930 case RISCVISD::SplitF64: {
931 SDValue Op0 = N->getOperand(0);
932 // If the input to SplitF64 is just BuildPairF64 then the operation is
933 // redundant. Instead, use BuildPairF64's operands directly.
934 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
935 return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
936
937 SDLoc DL(N);
938
939 // It's cheaper to materialise two 32-bit integers than to load a double
940 // from the constant pool and transfer it to integer registers through the
941 // stack.
942 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Op0)) {
943 APInt V = C->getValueAPF().bitcastToAPInt();
944 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
945 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
946 return DCI.CombineTo(N, Lo, Hi);
947 }
948
949 // This is a target-specific version of a DAGCombine performed in
950 // DAGCombiner::visitBITCAST. It performs the equivalent of:
951 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
952 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
953 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
954 !Op0.getNode()->hasOneUse())
955 break;
956 SDValue NewSplitF64 =
957 DAG.getNode(RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32),
958 Op0.getOperand(0));
959 SDValue Lo = NewSplitF64.getValue(0);
960 SDValue Hi = NewSplitF64.getValue(1);
961 APInt SignBit = APInt::getSignMask(32);
962 if (Op0.getOpcode() == ISD::FNEG) {
963 SDValue NewHi = DAG.getNode(ISD::XOR, DL, MVT::i32, Hi,
964 DAG.getConstant(SignBit, DL, MVT::i32));
965 return DCI.CombineTo(N, Lo, NewHi);
966 }
967 assert(Op0.getOpcode() == ISD::FABS)((Op0.getOpcode() == ISD::FABS) ? static_cast<void> (0)
: __assert_fail ("Op0.getOpcode() == ISD::FABS", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 967, __PRETTY_FUNCTION__))
;
968 SDValue NewHi = DAG.getNode(ISD::AND, DL, MVT::i32, Hi,
969 DAG.getConstant(~SignBit, DL, MVT::i32));
970 return DCI.CombineTo(N, Lo, NewHi);
971 }
972 case RISCVISD::SLLW:
973 case RISCVISD::SRAW:
974 case RISCVISD::SRLW: {
975 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
976 SDValue LHS = N->getOperand(0);
977 SDValue RHS = N->getOperand(1);
978 APInt LHSMask = APInt::getLowBitsSet(LHS.getValueSizeInBits(), 32);
979 APInt RHSMask = APInt::getLowBitsSet(RHS.getValueSizeInBits(), 5);
980 if ((SimplifyDemandedBits(N->getOperand(0), LHSMask, DCI)) ||
981 (SimplifyDemandedBits(N->getOperand(1), RHSMask, DCI)))
982 return SDValue();
983 break;
984 }
985 case RISCVISD::FMV_X_ANYEXTW_RV64: {
986 SDLoc DL(N);
987 SDValue Op0 = N->getOperand(0);
988 // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
989 // conversion is unnecessary and can be replaced with an ANY_EXTEND
990 // of the FMV_W_X_RV64 operand.
991 if (Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) {
992 SDValue AExtOp =
993 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0.getOperand(0));
994 return DCI.CombineTo(N, AExtOp);
995 }
996
997 // This is a target-specific version of a DAGCombine performed in
998 // DAGCombiner::visitBITCAST. It performs the equivalent of:
999 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
1000 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
1001 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
1002 !Op0.getNode()->hasOneUse())
1003 break;
1004 SDValue NewFMV = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64,
1005 Op0.getOperand(0));
1006 APInt SignBit = APInt::getSignMask(32).sext(64);
1007 if (Op0.getOpcode() == ISD::FNEG) {
1008 return DCI.CombineTo(N,
1009 DAG.getNode(ISD::XOR, DL, MVT::i64, NewFMV,
1010 DAG.getConstant(SignBit, DL, MVT::i64)));
1011 }
1012 assert(Op0.getOpcode() == ISD::FABS)((Op0.getOpcode() == ISD::FABS) ? static_cast<void> (0)
: __assert_fail ("Op0.getOpcode() == ISD::FABS", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1012, __PRETTY_FUNCTION__))
;
1013 return DCI.CombineTo(N,
1014 DAG.getNode(ISD::AND, DL, MVT::i64, NewFMV,
1015 DAG.getConstant(~SignBit, DL, MVT::i64)));
1016 }
1017 }
1018
1019 return SDValue();
1020}
1021
1022bool RISCVTargetLowering::isDesirableToCommuteWithShift(
1023 const SDNode *N, CombineLevel Level) const {
1024 // The following folds are only desirable if `(OP _, c1 << c2)` can be
1025 // materialised in fewer instructions than `(OP _, c1)`:
1026 //
1027 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
1028 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
1029 SDValue N0 = N->getOperand(0);
1030 EVT Ty = N0.getValueType();
1031 if (Ty.isScalarInteger() &&
1032 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
1033 auto *C1 = dyn_cast<ConstantSDNode>(N0->getOperand(1));
1034 auto *C2 = dyn_cast<ConstantSDNode>(N->getOperand(1));
1035 if (C1 && C2) {
1036 APInt C1Int = C1->getAPIntValue();
1037 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
1038
1039 // We can materialise `c1 << c2` into an add immediate, so it's "free",
1040 // and the combine should happen, to potentially allow further combines
1041 // later.
1042 if (ShiftedC1Int.getMinSignedBits() <= 64 &&
1043 isLegalAddImmediate(ShiftedC1Int.getSExtValue()))
1044 return true;
1045
1046 // We can materialise `c1` in an add immediate, so it's "free", and the
1047 // combine should be prevented.
1048 if (C1Int.getMinSignedBits() <= 64 &&
1049 isLegalAddImmediate(C1Int.getSExtValue()))
1050 return false;
1051
1052 // Neither constant will fit into an immediate, so find materialisation
1053 // costs.
1054 int C1Cost = RISCVMatInt::getIntMatCost(C1Int, Ty.getSizeInBits(),
1055 Subtarget.is64Bit());
1056 int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
1057 ShiftedC1Int, Ty.getSizeInBits(), Subtarget.is64Bit());
1058
1059 // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
1060 // combine should be prevented.
1061 if (C1Cost < ShiftedC1Cost)
1062 return false;
1063 }
1064 }
1065 return true;
1066}
1067
1068unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
1069 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
1070 unsigned Depth) const {
1071 switch (Op.getOpcode()) {
1072 default:
1073 break;
1074 case RISCVISD::SLLW:
1075 case RISCVISD::SRAW:
1076 case RISCVISD::SRLW:
1077 case RISCVISD::DIVW:
1078 case RISCVISD::DIVUW:
1079 case RISCVISD::REMUW:
1080 // TODO: As the result is sign-extended, this is conservatively correct. A
1081 // more precise answer could be calculated for SRAW depending on known
1082 // bits in the shift amount.
1083 return 33;
1084 }
1085
1086 return 1;
1087}
1088
1089static MachineBasicBlock *emitReadCycleWidePseudo(MachineInstr &MI,
1090 MachineBasicBlock *BB) {
1091 assert(MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction")((MI.getOpcode() == RISCV::ReadCycleWide && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::ReadCycleWide && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1091, __PRETTY_FUNCTION__))
;
1092
1093 // To read the 64-bit cycle CSR on a 32-bit target, we read the two halves.
1094 // Should the count have wrapped while it was being read, we need to try
1095 // again.
1096 // ...
1097 // read:
1098 // rdcycleh x3 # load high word of cycle
1099 // rdcycle x2 # load low word of cycle
1100 // rdcycleh x4 # load high word of cycle
1101 // bne x3, x4, read # check if high word reads match, otherwise try again
1102 // ...
1103
1104 MachineFunction &MF = *BB->getParent();
1105 const BasicBlock *LLVM_BB = BB->getBasicBlock();
1106 MachineFunction::iterator It = ++BB->getIterator();
1107
1108 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1109 MF.insert(It, LoopMBB);
1110
1111 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(LLVM_BB);
1112 MF.insert(It, DoneMBB);
1113
1114 // Transfer the remainder of BB and its successor edges to DoneMBB.
1115 DoneMBB->splice(DoneMBB->begin(), BB,
1116 std::next(MachineBasicBlock::iterator(MI)), BB->end());
1117 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
1118
1119 BB->addSuccessor(LoopMBB);
1120
1121 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1122 Register ReadAgainReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1123 Register LoReg = MI.getOperand(0).getReg();
1124 Register HiReg = MI.getOperand(1).getReg();
1125 DebugLoc DL = MI.getDebugLoc();
1126
1127 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
1128 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), HiReg)
1129 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1130 .addReg(RISCV::X0);
1131 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), LoReg)
1132 .addImm(RISCVSysReg::lookupSysRegByName("CYCLE")->Encoding)
1133 .addReg(RISCV::X0);
1134 BuildMI(LoopMBB, DL, TII->get(RISCV::CSRRS), ReadAgainReg)
1135 .addImm(RISCVSysReg::lookupSysRegByName("CYCLEH")->Encoding)
1136 .addReg(RISCV::X0);
1137
1138 BuildMI(LoopMBB, DL, TII->get(RISCV::BNE))
1139 .addReg(HiReg)
1140 .addReg(ReadAgainReg)
1141 .addMBB(LoopMBB);
1142
1143 LoopMBB->addSuccessor(LoopMBB);
1144 LoopMBB->addSuccessor(DoneMBB);
1145
1146 MI.eraseFromParent();
1147
1148 return DoneMBB;
1149}
1150
1151static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
1152 MachineBasicBlock *BB) {
1153 assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction")((MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::SplitF64Pseudo && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1153, __PRETTY_FUNCTION__))
;
1154
1155 MachineFunction &MF = *BB->getParent();
1156 DebugLoc DL = MI.getDebugLoc();
1157 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1158 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1159 Register LoReg = MI.getOperand(0).getReg();
1160 Register HiReg = MI.getOperand(1).getReg();
1161 Register SrcReg = MI.getOperand(2).getReg();
1162 const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
1163 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1164
1165 TII.storeRegToStackSlot(*BB, MI, SrcReg, MI.getOperand(2).isKill(), FI, SrcRC,
1166 RI);
1167 MachineMemOperand *MMO =
1168 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1169 MachineMemOperand::MOLoad, 8, 8);
1170 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), LoReg)
1171 .addFrameIndex(FI)
1172 .addImm(0)
1173 .addMemOperand(MMO);
1174 BuildMI(*BB, MI, DL, TII.get(RISCV::LW), HiReg)
1175 .addFrameIndex(FI)
1176 .addImm(4)
1177 .addMemOperand(MMO);
1178 MI.eraseFromParent(); // The pseudo instruction is gone now.
1179 return BB;
1180}
1181
1182static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
1183 MachineBasicBlock *BB) {
1184 assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&((MI.getOpcode() == RISCV::BuildPairF64Pseudo && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::BuildPairF64Pseudo && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1185, __PRETTY_FUNCTION__))
1185 "Unexpected instruction")((MI.getOpcode() == RISCV::BuildPairF64Pseudo && "Unexpected instruction"
) ? static_cast<void> (0) : __assert_fail ("MI.getOpcode() == RISCV::BuildPairF64Pseudo && \"Unexpected instruction\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1185, __PRETTY_FUNCTION__))
;
1186
1187 MachineFunction &MF = *BB->getParent();
1188 DebugLoc DL = MI.getDebugLoc();
1189 const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo();
1190 const TargetRegisterInfo *RI = MF.getSubtarget().getRegisterInfo();
1191 Register DstReg = MI.getOperand(0).getReg();
1192 Register LoReg = MI.getOperand(1).getReg();
1193 Register HiReg = MI.getOperand(2).getReg();
1194 const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
1195 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex();
1196
1197 MachineMemOperand *MMO =
1198 MF.getMachineMemOperand(MachinePointerInfo::getFixedStack(MF, FI),
1199 MachineMemOperand::MOStore, 8, 8);
1200 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1201 .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()))
1202 .addFrameIndex(FI)
1203 .addImm(0)
1204 .addMemOperand(MMO);
1205 BuildMI(*BB, MI, DL, TII.get(RISCV::SW))
1206 .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()))
1207 .addFrameIndex(FI)
1208 .addImm(4)
1209 .addMemOperand(MMO);
1210 TII.loadRegFromStackSlot(*BB, MI, DstReg, FI, DstRC, RI);
1211 MI.eraseFromParent(); // The pseudo instruction is gone now.
1212 return BB;
1213}
1214
1215static bool isSelectPseudo(MachineInstr &MI) {
1216 switch (MI.getOpcode()) {
1217 default:
1218 return false;
1219 case RISCV::Select_GPR_Using_CC_GPR:
1220 case RISCV::Select_FPR32_Using_CC_GPR:
1221 case RISCV::Select_FPR64_Using_CC_GPR:
1222 return true;
1223 }
1224}
1225
1226static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
1227 MachineBasicBlock *BB) {
1228 // To "insert" Select_* instructions, we actually have to insert the triangle
1229 // control-flow pattern. The incoming instructions know the destination vreg
1230 // to set, the condition code register to branch on, the true/false values to
1231 // select between, and the condcode to use to select the appropriate branch.
1232 //
1233 // We produce the following control flow:
1234 // HeadMBB
1235 // | \
1236 // | IfFalseMBB
1237 // | /
1238 // TailMBB
1239 //
1240 // When we find a sequence of selects we attempt to optimize their emission
1241 // by sharing the control flow. Currently we only handle cases where we have
1242 // multiple selects with the exact same condition (same LHS, RHS and CC).
1243 // The selects may be interleaved with other instructions if the other
1244 // instructions meet some requirements we deem safe:
1245 // - They are debug instructions. Otherwise,
1246 // - They do not have side-effects, do not access memory and their inputs do
1247 // not depend on the results of the select pseudo-instructions.
1248 // The TrueV/FalseV operands of the selects cannot depend on the result of
1249 // previous selects in the sequence.
1250 // These conditions could be further relaxed. See the X86 target for a
1251 // related approach and more information.
1252 Register LHS = MI.getOperand(1).getReg();
1253 Register RHS = MI.getOperand(2).getReg();
1254 auto CC = static_cast<ISD::CondCode>(MI.getOperand(3).getImm());
1255
1256 SmallVector<MachineInstr *, 4> SelectDebugValues;
1257 SmallSet<Register, 4> SelectDests;
1258 SelectDests.insert(MI.getOperand(0).getReg());
1259
1260 MachineInstr *LastSelectPseudo = &MI;
1261
1262 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
1263 SequenceMBBI != E; ++SequenceMBBI) {
1264 if (SequenceMBBI->isDebugInstr())
1265 continue;
1266 else if (isSelectPseudo(*SequenceMBBI)) {
1267 if (SequenceMBBI->getOperand(1).getReg() != LHS ||
1268 SequenceMBBI->getOperand(2).getReg() != RHS ||
1269 SequenceMBBI->getOperand(3).getImm() != CC ||
1270 SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
1271 SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
1272 break;
1273 LastSelectPseudo = &*SequenceMBBI;
1274 SequenceMBBI->collectDebugValues(SelectDebugValues);
1275 SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
1276 } else {
1277 if (SequenceMBBI->hasUnmodeledSideEffects() ||
1278 SequenceMBBI->mayLoadOrStore())
1279 break;
1280 if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
1281 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
1282 }))
1283 break;
1284 }
1285 }
1286
1287 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
1288 const BasicBlock *LLVM_BB = BB->getBasicBlock();
1289 DebugLoc DL = MI.getDebugLoc();
1290 MachineFunction::iterator I = ++BB->getIterator();
1291
1292 MachineBasicBlock *HeadMBB = BB;
1293 MachineFunction *F = BB->getParent();
1294 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
1295 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
1296
1297 F->insert(I, IfFalseMBB);
1298 F->insert(I, TailMBB);
1299
1300 // Transfer debug instructions associated with the selects to TailMBB.
1301 for (MachineInstr *DebugInstr : SelectDebugValues) {
1302 TailMBB->push_back(DebugInstr->removeFromParent());
1303 }
1304
1305 // Move all instructions after the sequence to TailMBB.
1306 TailMBB->splice(TailMBB->end(), HeadMBB,
1307 std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
1308 // Update machine-CFG edges by transferring all successors of the current
1309 // block to the new block which will contain the Phi nodes for the selects.
1310 TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
1311 // Set the successors for HeadMBB.
1312 HeadMBB->addSuccessor(IfFalseMBB);
1313 HeadMBB->addSuccessor(TailMBB);
1314
1315 // Insert appropriate branch.
1316 unsigned Opcode = getBranchOpcodeForIntCondCode(CC);
1317
1318 BuildMI(HeadMBB, DL, TII.get(Opcode))
1319 .addReg(LHS)
1320 .addReg(RHS)
1321 .addMBB(TailMBB);
1322
1323 // IfFalseMBB just falls through to TailMBB.
1324 IfFalseMBB->addSuccessor(TailMBB);
1325
1326 // Create PHIs for all of the select pseudo-instructions.
1327 auto SelectMBBI = MI.getIterator();
1328 auto SelectEnd = std::next(LastSelectPseudo->getIterator());
1329 auto InsertionPoint = TailMBB->begin();
1330 while (SelectMBBI != SelectEnd) {
1331 auto Next = std::next(SelectMBBI);
1332 if (isSelectPseudo(*SelectMBBI)) {
1333 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
1334 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
1335 TII.get(RISCV::PHI), SelectMBBI->getOperand(0).getReg())
1336 .addReg(SelectMBBI->getOperand(4).getReg())
1337 .addMBB(HeadMBB)
1338 .addReg(SelectMBBI->getOperand(5).getReg())
1339 .addMBB(IfFalseMBB);
1340 SelectMBBI->eraseFromParent();
1341 }
1342 SelectMBBI = Next;
1343 }
1344
1345 F->getProperties().reset(MachineFunctionProperties::Property::NoPHIs);
1346 return TailMBB;
1347}
1348
1349MachineBasicBlock *
1350RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
1351 MachineBasicBlock *BB) const {
1352 switch (MI.getOpcode()) {
1353 default:
1354 llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1354)
;
1355 case RISCV::ReadCycleWide:
1356 assert(!Subtarget.is64Bit() &&((!Subtarget.is64Bit() && "ReadCycleWrite is only to be used on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"ReadCycleWrite is only to be used on riscv32\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1357, __PRETTY_FUNCTION__))
1357 "ReadCycleWrite is only to be used on riscv32")((!Subtarget.is64Bit() && "ReadCycleWrite is only to be used on riscv32"
) ? static_cast<void> (0) : __assert_fail ("!Subtarget.is64Bit() && \"ReadCycleWrite is only to be used on riscv32\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1357, __PRETTY_FUNCTION__))
;
1358 return emitReadCycleWidePseudo(MI, BB);
1359 case RISCV::Select_GPR_Using_CC_GPR:
1360 case RISCV::Select_FPR32_Using_CC_GPR:
1361 case RISCV::Select_FPR64_Using_CC_GPR:
1362 return emitSelectPseudo(MI, BB);
1363 case RISCV::BuildPairF64Pseudo:
1364 return emitBuildPairF64Pseudo(MI, BB);
1365 case RISCV::SplitF64Pseudo:
1366 return emitSplitF64Pseudo(MI, BB);
1367 }
1368}
1369
1370// Calling Convention Implementation.
1371// The expectations for frontend ABI lowering vary from target to target.
1372// Ideally, an LLVM frontend would be able to avoid worrying about many ABI
1373// details, but this is a longer term goal. For now, we simply try to keep the
1374// role of the frontend as simple and well-defined as possible. The rules can
1375// be summarised as:
1376// * Never split up large scalar arguments. We handle them here.
1377// * If a hardfloat calling convention is being used, and the struct may be
1378// passed in a pair of registers (fp+fp, int+fp), and both registers are
1379// available, then pass as two separate arguments. If either the GPRs or FPRs
1380// are exhausted, then pass according to the rule below.
1381// * If a struct could never be passed in registers or directly in a stack
1382// slot (as it is larger than 2*XLEN and the floating point rules don't
1383// apply), then pass it using a pointer with the byval attribute.
1384// * If a struct is less than 2*XLEN, then coerce to either a two-element
1385// word-sized array or a 2*XLEN scalar (depending on alignment).
1386// * The frontend can determine whether a struct is returned by reference or
1387// not based on its size and fields. If it will be returned by reference, the
1388// frontend must modify the prototype so a pointer with the sret annotation is
1389// passed as the first argument. This is not necessary for large scalar
1390// returns.
1391// * Struct return values and varargs should be coerced to structs containing
1392// register-size fields in the same situations they would be for fixed
1393// arguments.
1394
1395static const MCPhysReg ArgGPRs[] = {
1396 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13,
1397 RISCV::X14, RISCV::X15, RISCV::X16, RISCV::X17
1398};
1399static const MCPhysReg ArgFPR32s[] = {
1400 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F,
1401 RISCV::F14_F, RISCV::F15_F, RISCV::F16_F, RISCV::F17_F
1402};
1403static const MCPhysReg ArgFPR64s[] = {
1404 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D,
1405 RISCV::F14_D, RISCV::F15_D, RISCV::F16_D, RISCV::F17_D
1406};
1407
1408// Pass a 2*XLEN argument that has been split into two XLEN values through
1409// registers or the stack as necessary.
1410static bool CC_RISCVAssign2XLen(unsigned XLen, CCState &State, CCValAssign VA1,
1411 ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2,
1412 MVT ValVT2, MVT LocVT2,
1413 ISD::ArgFlagsTy ArgFlags2) {
1414 unsigned XLenInBytes = XLen / 8;
1415 if (Register Reg = State.AllocateReg(ArgGPRs)) {
1416 // At least one half can be passed via register.
1417 State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
1418 VA1.getLocVT(), CCValAssign::Full));
1419 } else {
1420 // Both halves must be passed on the stack, with proper alignment.
1421 unsigned StackAlign = std::max(XLenInBytes, ArgFlags1.getOrigAlign());
1422 State.addLoc(
1423 CCValAssign::getMem(VA1.getValNo(), VA1.getValVT(),
1424 State.AllocateStack(XLenInBytes, StackAlign),
1425 VA1.getLocVT(), CCValAssign::Full));
1426 State.addLoc(CCValAssign::getMem(
1427 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1428 CCValAssign::Full));
1429 return false;
1430 }
1431
1432 if (Register Reg = State.AllocateReg(ArgGPRs)) {
1433 // The second half can also be passed via register.
1434 State.addLoc(
1435 CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
1436 } else {
1437 // The second half is passed via the stack, without additional alignment.
1438 State.addLoc(CCValAssign::getMem(
1439 ValNo2, ValVT2, State.AllocateStack(XLenInBytes, XLenInBytes), LocVT2,
1440 CCValAssign::Full));
1441 }
1442
1443 return false;
1444}
1445
1446// Implements the RISC-V calling convention. Returns true upon failure.
1447static bool CC_RISCV(const DataLayout &DL, RISCVABI::ABI ABI, unsigned ValNo,
1448 MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo,
1449 ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsFixed,
1450 bool IsRet, Type *OrigTy) {
1451 unsigned XLen = DL.getLargestLegalIntTypeSizeInBits();
1452 assert(XLen == 32 || XLen == 64)((XLen == 32 || XLen == 64) ? static_cast<void> (0) : __assert_fail
("XLen == 32 || XLen == 64", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1452, __PRETTY_FUNCTION__))
;
1453 MVT XLenVT = XLen == 32 ? MVT::i32 : MVT::i64;
1454
1455 // Any return value split in to more than two values can't be returned
1456 // directly.
1457 if (IsRet && ValNo > 1)
1458 return true;
1459
1460 // UseGPRForF32 if targeting one of the soft-float ABIs, if passing a
1461 // variadic argument, or if no F32 argument registers are available.
1462 bool UseGPRForF32 = true;
1463 // UseGPRForF64 if targeting soft-float ABIs or an FLEN=32 ABI, if passing a
1464 // variadic argument, or if no F64 argument registers are available.
1465 bool UseGPRForF64 = true;
1466
1467 switch (ABI) {
1468 default:
1469 llvm_unreachable("Unexpected ABI")::llvm::llvm_unreachable_internal("Unexpected ABI", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1469)
;
1470 case RISCVABI::ABI_ILP32:
1471 case RISCVABI::ABI_LP64:
1472 break;
1473 case RISCVABI::ABI_ILP32F:
1474 case RISCVABI::ABI_LP64F:
1475 UseGPRForF32 = !IsFixed;
1476 break;
1477 case RISCVABI::ABI_ILP32D:
1478 case RISCVABI::ABI_LP64D:
1479 UseGPRForF32 = !IsFixed;
1480 UseGPRForF64 = !IsFixed;
1481 break;
1482 }
1483
1484 if (State.getFirstUnallocated(ArgFPR32s) == array_lengthof(ArgFPR32s))
1485 UseGPRForF32 = true;
1486 if (State.getFirstUnallocated(ArgFPR64s) == array_lengthof(ArgFPR64s))
1487 UseGPRForF64 = true;
1488
1489 // From this point on, rely on UseGPRForF32, UseGPRForF64 and similar local
1490 // variables rather than directly checking against the target ABI.
1491
1492 if (UseGPRForF32 && ValVT == MVT::f32) {
1493 LocVT = XLenVT;
1494 LocInfo = CCValAssign::BCvt;
1495 } else if (UseGPRForF64 && XLen == 64 && ValVT == MVT::f64) {
1496 LocVT = MVT::i64;
1497 LocInfo = CCValAssign::BCvt;
1498 }
1499
1500 // If this is a variadic argument, the RISC-V calling convention requires
1501 // that it is assigned an 'even' or 'aligned' register if it has 8-byte
1502 // alignment (RV32) or 16-byte alignment (RV64). An aligned register should
1503 // be used regardless of whether the original argument was split during
1504 // legalisation or not. The argument will not be passed by registers if the
1505 // original type is larger than 2*XLEN, so the register alignment rule does
1506 // not apply.
1507 unsigned TwoXLenInBytes = (2 * XLen) / 8;
1508 if (!IsFixed && ArgFlags.getOrigAlign() == TwoXLenInBytes &&
1509 DL.getTypeAllocSize(OrigTy) == TwoXLenInBytes) {
1510 unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
1511 // Skip 'odd' register if necessary.
1512 if (RegIdx != array_lengthof(ArgGPRs) && RegIdx % 2 == 1)
1513 State.AllocateReg(ArgGPRs);
1514 }
1515
1516 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
1517 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
1518 State.getPendingArgFlags();
1519
1520 assert(PendingLocs.size() == PendingArgFlags.size() &&((PendingLocs.size() == PendingArgFlags.size() && "PendingLocs and PendingArgFlags out of sync"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() == PendingArgFlags.size() && \"PendingLocs and PendingArgFlags out of sync\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1521, __PRETTY_FUNCTION__))
1521 "PendingLocs and PendingArgFlags out of sync")((PendingLocs.size() == PendingArgFlags.size() && "PendingLocs and PendingArgFlags out of sync"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() == PendingArgFlags.size() && \"PendingLocs and PendingArgFlags out of sync\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1521, __PRETTY_FUNCTION__))
;
1522
1523 // Handle passing f64 on RV32D with a soft float ABI or when floating point
1524 // registers are exhausted.
1525 if (UseGPRForF64 && XLen == 32 && ValVT == MVT::f64) {
1526 assert(!ArgFlags.isSplit() && PendingLocs.empty() &&((!ArgFlags.isSplit() && PendingLocs.empty() &&
"Can't lower f64 if it is split") ? static_cast<void> (
0) : __assert_fail ("!ArgFlags.isSplit() && PendingLocs.empty() && \"Can't lower f64 if it is split\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1527, __PRETTY_FUNCTION__))
1527 "Can't lower f64 if it is split")((!ArgFlags.isSplit() && PendingLocs.empty() &&
"Can't lower f64 if it is split") ? static_cast<void> (
0) : __assert_fail ("!ArgFlags.isSplit() && PendingLocs.empty() && \"Can't lower f64 if it is split\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1527, __PRETTY_FUNCTION__))
;
1528 // Depending on available argument GPRS, f64 may be passed in a pair of
1529 // GPRs, split between a GPR and the stack, or passed completely on the
1530 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
1531 // cases.
1532 Register Reg = State.AllocateReg(ArgGPRs);
1533 LocVT = MVT::i32;
1534 if (!Reg) {
1535 unsigned StackOffset = State.AllocateStack(8, 8);
1536 State.addLoc(
1537 CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1538 return false;
1539 }
1540 if (!State.AllocateReg(ArgGPRs))
1541 State.AllocateStack(4, 4);
1542 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1543 return false;
1544 }
1545
1546 // Split arguments might be passed indirectly, so keep track of the pending
1547 // values.
1548 if (ArgFlags.isSplit() || !PendingLocs.empty()) {
1549 LocVT = XLenVT;
1550 LocInfo = CCValAssign::Indirect;
1551 PendingLocs.push_back(
1552 CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
1553 PendingArgFlags.push_back(ArgFlags);
1554 if (!ArgFlags.isSplitEnd()) {
1555 return false;
1556 }
1557 }
1558
1559 // If the split argument only had two elements, it should be passed directly
1560 // in registers or on the stack.
1561 if (ArgFlags.isSplitEnd() && PendingLocs.size() <= 2) {
1562 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()")((PendingLocs.size() == 2 && "Unexpected PendingLocs.size()"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() == 2 && \"Unexpected PendingLocs.size()\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1562, __PRETTY_FUNCTION__))
;
1563 // Apply the normal calling convention rules to the first half of the
1564 // split argument.
1565 CCValAssign VA = PendingLocs[0];
1566 ISD::ArgFlagsTy AF = PendingArgFlags[0];
1567 PendingLocs.clear();
1568 PendingArgFlags.clear();
1569 return CC_RISCVAssign2XLen(XLen, State, VA, AF, ValNo, ValVT, LocVT,
1570 ArgFlags);
1571 }
1572
1573 // Allocate to a register if possible, or else a stack slot.
1574 Register Reg;
1575 if (ValVT == MVT::f32 && !UseGPRForF32)
1576 Reg = State.AllocateReg(ArgFPR32s, ArgFPR64s);
1577 else if (ValVT == MVT::f64 && !UseGPRForF64)
1578 Reg = State.AllocateReg(ArgFPR64s, ArgFPR32s);
1579 else
1580 Reg = State.AllocateReg(ArgGPRs);
1581 unsigned StackOffset = Reg ? 0 : State.AllocateStack(XLen / 8, XLen / 8);
1582
1583 // If we reach this point and PendingLocs is non-empty, we must be at the
1584 // end of a split argument that must be passed indirectly.
1585 if (!PendingLocs.empty()) {
1586 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()")((ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()"
) ? static_cast<void> (0) : __assert_fail ("ArgFlags.isSplitEnd() && \"Expected ArgFlags.isSplitEnd()\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1586, __PRETTY_FUNCTION__))
;
1587 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()")((PendingLocs.size() > 2 && "Unexpected PendingLocs.size()"
) ? static_cast<void> (0) : __assert_fail ("PendingLocs.size() > 2 && \"Unexpected PendingLocs.size()\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1587, __PRETTY_FUNCTION__))
;
1588
1589 for (auto &It : PendingLocs) {
1590 if (Reg)
1591 It.convertToReg(Reg);
1592 else
1593 It.convertToMem(StackOffset);
1594 State.addLoc(It);
1595 }
1596 PendingLocs.clear();
1597 PendingArgFlags.clear();
1598 return false;
1599 }
1600
1601 assert((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) &&(((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) &&
"Expected an XLenVT at this stage") ? static_cast<void>
(0) : __assert_fail ("(!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) && \"Expected an XLenVT at this stage\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1602, __PRETTY_FUNCTION__))
1602 "Expected an XLenVT at this stage")(((!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) &&
"Expected an XLenVT at this stage") ? static_cast<void>
(0) : __assert_fail ("(!UseGPRForF32 || !UseGPRForF64 || LocVT == XLenVT) && \"Expected an XLenVT at this stage\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1602, __PRETTY_FUNCTION__))
;
1603
1604 if (Reg) {
1605 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1606 return false;
1607 }
1608
1609 // When an f32 or f64 is passed on the stack, no bit-conversion is needed.
1610 if (ValVT == MVT::f32 || ValVT == MVT::f64) {
1611 LocVT = ValVT;
1612 LocInfo = CCValAssign::Full;
1613 }
1614 State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
1615 return false;
1616}
1617
1618void RISCVTargetLowering::analyzeInputArgs(
1619 MachineFunction &MF, CCState &CCInfo,
1620 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet) const {
1621 unsigned NumArgs = Ins.size();
1622 FunctionType *FType = MF.getFunction().getFunctionType();
1623
1624 for (unsigned i = 0; i != NumArgs; ++i) {
1625 MVT ArgVT = Ins[i].VT;
1626 ISD::ArgFlagsTy ArgFlags = Ins[i].Flags;
1627
1628 Type *ArgTy = nullptr;
1629 if (IsRet)
1630 ArgTy = FType->getReturnType();
1631 else if (Ins[i].isOrigArg())
1632 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
1633
1634 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1635 if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1636 ArgFlags, CCInfo, /*IsRet=*/true, IsRet, ArgTy)) {
1637 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "InputArg #" << i <<
" has unhandled type " << EVT(ArgVT).getEVTString() <<
'\n'; } } while (false)
1638 << EVT(ArgVT).getEVTString() << '\n')do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "InputArg #" << i <<
" has unhandled type " << EVT(ArgVT).getEVTString() <<
'\n'; } } while (false)
;
1639 llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1639)
;
1640 }
1641 }
1642}
1643
1644void RISCVTargetLowering::analyzeOutputArgs(
1645 MachineFunction &MF, CCState &CCInfo,
1646 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
1647 CallLoweringInfo *CLI) const {
1648 unsigned NumArgs = Outs.size();
1649
1650 for (unsigned i = 0; i != NumArgs; i++) {
1651 MVT ArgVT = Outs[i].VT;
1652 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
1653 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
1654
1655 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
1656 if (CC_RISCV(MF.getDataLayout(), ABI, i, ArgVT, ArgVT, CCValAssign::Full,
1657 ArgFlags, CCInfo, Outs[i].IsFixed, IsRet, OrigTy)) {
1658 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type "do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "OutputArg #" << i <<
" has unhandled type " << EVT(ArgVT).getEVTString() <<
"\n"; } } while (false)
1659 << EVT(ArgVT).getEVTString() << "\n")do { if (::llvm::DebugFlag && ::llvm::isCurrentDebugType
("riscv-lower")) { dbgs() << "OutputArg #" << i <<
" has unhandled type " << EVT(ArgVT).getEVTString() <<
"\n"; } } while (false)
;
1660 llvm_unreachable(nullptr)::llvm::llvm_unreachable_internal(nullptr, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1660)
;
1661 }
1662 }
1663}
1664
1665// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
1666// values.
1667static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
1668 const CCValAssign &VA, const SDLoc &DL) {
1669 switch (VA.getLocInfo()) {
1670 default:
1671 llvm_unreachable("Unexpected CCValAssign::LocInfo")::llvm::llvm_unreachable_internal("Unexpected CCValAssign::LocInfo"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1671)
;
1672 case CCValAssign::Full:
1673 break;
1674 case CCValAssign::BCvt:
1675 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1676 Val = DAG.getNode(RISCVISD::FMV_W_X_RV64, DL, MVT::f32, Val);
1677 break;
1678 }
1679 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
1680 break;
1681 }
1682 return Val;
1683}
1684
1685// The caller is responsible for loading the full value if the argument is
1686// passed with CCValAssign::Indirect.
1687static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
1688 const CCValAssign &VA, const SDLoc &DL) {
1689 MachineFunction &MF = DAG.getMachineFunction();
1690 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1691 EVT LocVT = VA.getLocVT();
1692 SDValue Val;
1693 const TargetRegisterClass *RC;
1694
1695 switch (LocVT.getSimpleVT().SimpleTy) {
1696 default:
1697 llvm_unreachable("Unexpected register type")::llvm::llvm_unreachable_internal("Unexpected register type",
"/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1697)
;
1698 case MVT::i32:
1699 case MVT::i64:
1700 RC = &RISCV::GPRRegClass;
1701 break;
1702 case MVT::f32:
1703 RC = &RISCV::FPR32RegClass;
1704 break;
1705 case MVT::f64:
1706 RC = &RISCV::FPR64RegClass;
1707 break;
1708 }
1709
1710 Register VReg = RegInfo.createVirtualRegister(RC);
1711 RegInfo.addLiveIn(VA.getLocReg(), VReg);
1712 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
1713
1714 if (VA.getLocInfo() == CCValAssign::Indirect)
1715 return Val;
1716
1717 return convertLocVTToValVT(DAG, Val, VA, DL);
1718}
1719
1720static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
1721 const CCValAssign &VA, const SDLoc &DL) {
1722 EVT LocVT = VA.getLocVT();
1723
1724 switch (VA.getLocInfo()) {
1725 default:
1726 llvm_unreachable("Unexpected CCValAssign::LocInfo")::llvm::llvm_unreachable_internal("Unexpected CCValAssign::LocInfo"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1726)
;
1727 case CCValAssign::Full:
1728 break;
1729 case CCValAssign::BCvt:
1730 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32) {
1731 Val = DAG.getNode(RISCVISD::FMV_X_ANYEXTW_RV64, DL, MVT::i64, Val);
1732 break;
1733 }
1734 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
1735 break;
1736 }
1737 return Val;
1738}
1739
1740// The caller is responsible for loading the full value if the argument is
1741// passed with CCValAssign::Indirect.
1742static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
1743 const CCValAssign &VA, const SDLoc &DL) {
1744 MachineFunction &MF = DAG.getMachineFunction();
1745 MachineFrameInfo &MFI = MF.getFrameInfo();
1746 EVT LocVT = VA.getLocVT();
1747 EVT ValVT = VA.getValVT();
1748 EVT PtrVT = MVT::getIntegerVT(DAG.getDataLayout().getPointerSizeInBits(0));
1749 int FI = MFI.CreateFixedObject(ValVT.getSizeInBits() / 8,
1750 VA.getLocMemOffset(), /*Immutable=*/true);
1751 SDValue FIN = DAG.getFrameIndex(FI, PtrVT);
1752 SDValue Val;
1753
1754 ISD::LoadExtType ExtType;
1755 switch (VA.getLocInfo()) {
1756 default:
1757 llvm_unreachable("Unexpected CCValAssign::LocInfo")::llvm::llvm_unreachable_internal("Unexpected CCValAssign::LocInfo"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1757)
;
1758 case CCValAssign::Full:
1759 case CCValAssign::Indirect:
1760 case CCValAssign::BCvt:
1761 ExtType = ISD::NON_EXTLOAD;
1762 break;
1763 }
1764 Val = DAG.getExtLoad(
1765 ExtType, DL, LocVT, Chain, FIN,
1766 MachinePointerInfo::getFixedStack(DAG.getMachineFunction(), FI), ValVT);
1767 return Val;
1768}
1769
1770static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
1771 const CCValAssign &VA, const SDLoc &DL) {
1772 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&((VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::
f64 && "Unexpected VA") ? static_cast<void> (0)
: __assert_fail ("VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 && \"Unexpected VA\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1773, __PRETTY_FUNCTION__))
1773 "Unexpected VA")((VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::
f64 && "Unexpected VA") ? static_cast<void> (0)
: __assert_fail ("VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 && \"Unexpected VA\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1773, __PRETTY_FUNCTION__))
;
1774 MachineFunction &MF = DAG.getMachineFunction();
1775 MachineFrameInfo &MFI = MF.getFrameInfo();
1776 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1777
1778 if (VA.isMemLoc()) {
1779 // f64 is passed on the stack.
1780 int FI = MFI.CreateFixedObject(8, VA.getLocMemOffset(), /*Immutable=*/true);
1781 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1782 return DAG.getLoad(MVT::f64, DL, Chain, FIN,
1783 MachinePointerInfo::getFixedStack(MF, FI));
1784 }
1785
1786 assert(VA.isRegLoc() && "Expected register VA assignment")((VA.isRegLoc() && "Expected register VA assignment")
? static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Expected register VA assignment\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1786, __PRETTY_FUNCTION__))
;
1787
1788 Register LoVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1789 RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
1790 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
1791 SDValue Hi;
1792 if (VA.getLocReg() == RISCV::X17) {
1793 // Second half of f64 is passed on the stack.
1794 int FI = MFI.CreateFixedObject(4, 0, /*Immutable=*/true);
1795 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
1796 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
1797 MachinePointerInfo::getFixedStack(MF, FI));
1798 } else {
1799 // Second half of f64 is passed in another GPR.
1800 Register HiVReg = RegInfo.createVirtualRegister(&RISCV::GPRRegClass);
1801 RegInfo.addLiveIn(VA.getLocReg() + 1, HiVReg);
1802 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
1803 }
1804 return DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, Lo, Hi);
1805}
1806
1807// FastCC has less than 1% performance improvement for some particular
1808// benchmark. But theoretically, it may has benenfit for some cases.
1809static bool CC_RISCV_FastCC(unsigned ValNo, MVT ValVT, MVT LocVT,
1810 CCValAssign::LocInfo LocInfo,
1811 ISD::ArgFlagsTy ArgFlags, CCState &State) {
1812
1813 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
1814 // X5 and X6 might be used for save-restore libcall.
1815 static const MCPhysReg GPRList[] = {
1816 RISCV::X10, RISCV::X11, RISCV::X12, RISCV::X13, RISCV::X14,
1817 RISCV::X15, RISCV::X16, RISCV::X17, RISCV::X7, RISCV::X28,
1818 RISCV::X29, RISCV::X30, RISCV::X31};
1819 if (unsigned Reg = State.AllocateReg(GPRList)) {
1820 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1821 return false;
1822 }
1823 }
1824
1825 if (LocVT == MVT::f32) {
1826 static const MCPhysReg FPR32List[] = {
1827 RISCV::F10_F, RISCV::F11_F, RISCV::F12_F, RISCV::F13_F, RISCV::F14_F,
1828 RISCV::F15_F, RISCV::F16_F, RISCV::F17_F, RISCV::F0_F, RISCV::F1_F,
1829 RISCV::F2_F, RISCV::F3_F, RISCV::F4_F, RISCV::F5_F, RISCV::F6_F,
1830 RISCV::F7_F, RISCV::F28_F, RISCV::F29_F, RISCV::F30_F, RISCV::F31_F};
1831 if (unsigned Reg = State.AllocateReg(FPR32List)) {
1832 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1833 return false;
1834 }
1835 }
1836
1837 if (LocVT == MVT::f64) {
1838 static const MCPhysReg FPR64List[] = {
1839 RISCV::F10_D, RISCV::F11_D, RISCV::F12_D, RISCV::F13_D, RISCV::F14_D,
1840 RISCV::F15_D, RISCV::F16_D, RISCV::F17_D, RISCV::F0_D, RISCV::F1_D,
1841 RISCV::F2_D, RISCV::F3_D, RISCV::F4_D, RISCV::F5_D, RISCV::F6_D,
1842 RISCV::F7_D, RISCV::F28_D, RISCV::F29_D, RISCV::F30_D, RISCV::F31_D};
1843 if (unsigned Reg = State.AllocateReg(FPR64List)) {
1844 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
1845 return false;
1846 }
1847 }
1848
1849 if (LocVT == MVT::i32 || LocVT == MVT::f32) {
1850 unsigned Offset4 = State.AllocateStack(4, 4);
1851 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset4, LocVT, LocInfo));
1852 return false;
1853 }
1854
1855 if (LocVT == MVT::i64 || LocVT == MVT::f64) {
1856 unsigned Offset5 = State.AllocateStack(8, 8);
1857 State.addLoc(CCValAssign::getMem(ValNo, ValVT, Offset5, LocVT, LocInfo));
1858 return false;
1859 }
1860
1861 return true; // CC didn't match.
1862}
1863
1864// Transform physical registers into virtual registers.
1865SDValue RISCVTargetLowering::LowerFormalArguments(
1866 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1867 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1868 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1869
1870 switch (CallConv) {
1871 default:
1872 report_fatal_error("Unsupported calling convention");
1873 case CallingConv::C:
1874 case CallingConv::Fast:
1875 break;
1876 }
1877
1878 MachineFunction &MF = DAG.getMachineFunction();
1879
1880 const Function &Func = MF.getFunction();
1881 if (Func.hasFnAttribute("interrupt")) {
1882 if (!Func.arg_empty())
1883 report_fatal_error(
1884 "Functions with the interrupt attribute cannot have arguments!");
1885
1886 StringRef Kind =
1887 MF.getFunction().getFnAttribute("interrupt").getValueAsString();
1888
1889 if (!(Kind == "user" || Kind == "supervisor" || Kind == "machine"))
1890 report_fatal_error(
1891 "Function interrupt attribute argument not supported!");
1892 }
1893
1894 EVT PtrVT = getPointerTy(DAG.getDataLayout());
1895 MVT XLenVT = Subtarget.getXLenVT();
1896 unsigned XLenInBytes = Subtarget.getXLen() / 8;
1897 // Used with vargs to acumulate store chains.
1898 std::vector<SDValue> OutChains;
1899
1900 // Assign locations to all of the incoming arguments.
1901 SmallVector<CCValAssign, 16> ArgLocs;
1902 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1903
1904 if (CallConv == CallingConv::Fast)
1905 CCInfo.AnalyzeFormalArguments(Ins, CC_RISCV_FastCC);
1906 else
1907 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false);
1908
1909 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
1910 CCValAssign &VA = ArgLocs[i];
1911 SDValue ArgValue;
1912 // Passing f64 on RV32D with a soft float ABI must be handled as a special
1913 // case.
1914 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64)
1915 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, DL);
1916 else if (VA.isRegLoc())
1917 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL);
1918 else
1919 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
1920
1921 if (VA.getLocInfo() == CCValAssign::Indirect) {
1922 // If the original argument was split and passed by reference (e.g. i128
1923 // on RV32), we need to load all parts of it here (using the same
1924 // address).
1925 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
1926 MachinePointerInfo()));
1927 unsigned ArgIndex = Ins[i].OrigArgIndex;
1928 assert(Ins[i].PartOffset == 0)((Ins[i].PartOffset == 0) ? static_cast<void> (0) : __assert_fail
("Ins[i].PartOffset == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 1928, __PRETTY_FUNCTION__))
;
1929 while (i + 1 != e && Ins[i + 1].OrigArgIndex == ArgIndex) {
1930 CCValAssign &PartVA = ArgLocs[i + 1];
1931 unsigned PartOffset = Ins[i + 1].PartOffset;
1932 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue,
1933 DAG.getIntPtrConstant(PartOffset, DL));
1934 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
1935 MachinePointerInfo()));
1936 ++i;
1937 }
1938 continue;
1939 }
1940 InVals.push_back(ArgValue);
1941 }
1942
1943 if (IsVarArg) {
1944 ArrayRef<MCPhysReg> ArgRegs = makeArrayRef(ArgGPRs);
1945 unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
1946 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
1947 MachineFrameInfo &MFI = MF.getFrameInfo();
1948 MachineRegisterInfo &RegInfo = MF.getRegInfo();
1949 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
1950
1951 // Offset of the first variable argument from stack pointer, and size of
1952 // the vararg save area. For now, the varargs save area is either zero or
1953 // large enough to hold a0-a7.
1954 int VaArgOffset, VarArgsSaveSize;
1955
1956 // If all registers are allocated, then all varargs must be passed on the
1957 // stack and we don't need to save any argregs.
1958 if (ArgRegs.size() == Idx) {
1959 VaArgOffset = CCInfo.getNextStackOffset();
1960 VarArgsSaveSize = 0;
1961 } else {
1962 VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
1963 VaArgOffset = -VarArgsSaveSize;
1964 }
1965
1966 // Record the frame index of the first variable argument
1967 // which is a value necessary to VASTART.
1968 int FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1969 RVFI->setVarArgsFrameIndex(FI);
1970
1971 // If saving an odd number of registers then create an extra stack slot to
1972 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
1973 // offsets to even-numbered registered remain 2*XLEN-aligned.
1974 if (Idx % 2) {
1975 MFI.CreateFixedObject(XLenInBytes, VaArgOffset - (int)XLenInBytes, true);
1976 VarArgsSaveSize += XLenInBytes;
1977 }
1978
1979 // Copy the integer registers that may have been used for passing varargs
1980 // to the vararg save area.
1981 for (unsigned I = Idx; I < ArgRegs.size();
1982 ++I, VaArgOffset += XLenInBytes) {
1983 const Register Reg = RegInfo.createVirtualRegister(RC);
1984 RegInfo.addLiveIn(ArgRegs[I], Reg);
1985 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, XLenVT);
1986 FI = MFI.CreateFixedObject(XLenInBytes, VaArgOffset, true);
1987 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
1988 SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
1989 MachinePointerInfo::getFixedStack(MF, FI));
1990 cast<StoreSDNode>(Store.getNode())
1991 ->getMemOperand()
1992 ->setValue((Value *)nullptr);
1993 OutChains.push_back(Store);
1994 }
1995 RVFI->setVarArgsSaveSize(VarArgsSaveSize);
1996 }
1997
1998 // All stores are grouped in one node to allow the matching between
1999 // the size of Ins and InVals. This only happens for vararg functions.
2000 if (!OutChains.empty()) {
2001 OutChains.push_back(Chain);
2002 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
2003 }
2004
2005 return Chain;
2006}
2007
2008/// isEligibleForTailCallOptimization - Check whether the call is eligible
2009/// for tail call optimization.
2010/// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
2011bool RISCVTargetLowering::isEligibleForTailCallOptimization(
2012 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
2013 const SmallVector<CCValAssign, 16> &ArgLocs) const {
2014
2015 auto &Callee = CLI.Callee;
2016 auto CalleeCC = CLI.CallConv;
2017 auto &Outs = CLI.Outs;
2018 auto &Caller = MF.getFunction();
2019 auto CallerCC = Caller.getCallingConv();
2020
2021 // Do not tail call opt functions with "disable-tail-calls" attribute.
2022 if (Caller.getFnAttribute("disable-tail-calls").getValueAsString() == "true")
2023 return false;
2024
2025 // Exception-handling functions need a special set of instructions to
2026 // indicate a return to the hardware. Tail-calling another function would
2027 // probably break this.
2028 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
2029 // should be expanded as new function attributes are introduced.
2030 if (Caller.hasFnAttribute("interrupt"))
2031 return false;
2032
2033 // Do not tail call opt if the stack is used to pass parameters.
2034 if (CCInfo.getNextStackOffset() != 0)
2035 return false;
2036
2037 // Do not tail call opt if any parameters need to be passed indirectly.
2038 // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
2039 // passed indirectly. So the address of the value will be passed in a
2040 // register, or if not available, then the address is put on the stack. In
2041 // order to pass indirectly, space on the stack often needs to be allocated
2042 // in order to store the value. In this case the CCInfo.getNextStackOffset()
2043 // != 0 check is not enough and we need to check if any CCValAssign ArgsLocs
2044 // are passed CCValAssign::Indirect.
2045 for (auto &VA : ArgLocs)
2046 if (VA.getLocInfo() == CCValAssign::Indirect)
2047 return false;
2048
2049 // Do not tail call opt if either caller or callee uses struct return
2050 // semantics.
2051 auto IsCallerStructRet = Caller.hasStructRetAttr();
2052 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
2053 if (IsCallerStructRet || IsCalleeStructRet)
2054 return false;
2055
2056 // Externally-defined functions with weak linkage should not be
2057 // tail-called. The behaviour of branch instructions in this situation (as
2058 // used for tail calls) is implementation-defined, so we cannot rely on the
2059 // linker replacing the tail call with a return.
2060 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Callee)) {
2061 const GlobalValue *GV = G->getGlobal();
2062 if (GV->hasExternalWeakLinkage())
2063 return false;
2064 }
2065
2066 // The callee has to preserve all registers the caller needs to preserve.
2067 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
2068 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
2069 if (CalleeCC != CallerCC) {
2070 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
2071 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
2072 return false;
2073 }
2074
2075 // Byval parameters hand the function a pointer directly into the stack area
2076 // we want to reuse during a tail call. Working around this *is* possible
2077 // but less efficient and uglier in LowerCall.
2078 for (auto &Arg : Outs)
2079 if (Arg.Flags.isByVal())
2080 return false;
2081
2082 return true;
2083}
2084
2085// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
2086// and output parameter nodes.
2087SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
2088 SmallVectorImpl<SDValue> &InVals) const {
2089 SelectionDAG &DAG = CLI.DAG;
2090 SDLoc &DL = CLI.DL;
2091 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
2092 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
2093 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
2094 SDValue Chain = CLI.Chain;
2095 SDValue Callee = CLI.Callee;
2096 bool &IsTailCall = CLI.IsTailCall;
2097 CallingConv::ID CallConv = CLI.CallConv;
2098 bool IsVarArg = CLI.IsVarArg;
2099 EVT PtrVT = getPointerTy(DAG.getDataLayout());
2100 MVT XLenVT = Subtarget.getXLenVT();
2101
2102 MachineFunction &MF = DAG.getMachineFunction();
2103
2104 // Analyze the operands of the call, assigning locations to each operand.
2105 SmallVector<CCValAssign, 16> ArgLocs;
2106 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
2107
2108 if (CallConv == CallingConv::Fast)
2109 ArgCCInfo.AnalyzeCallOperands(Outs, CC_RISCV_FastCC);
2110 else
2111 analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI);
2112
2113 // Check if it's really possible to do a tail call.
2114 if (IsTailCall)
2115 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
2116
2117 if (IsTailCall)
2118 ++NumTailCalls;
2119 else if (CLI.CS && CLI.CS.isMustTailCall())
2120 report_fatal_error("failed to perform tail call elimination on a call "
2121 "site marked musttail");
2122
2123 // Get a count of how many bytes are to be pushed on the stack.
2124 unsigned NumBytes = ArgCCInfo.getNextStackOffset();
2125
2126 // Create local copies for byval args
2127 SmallVector<SDValue, 8> ByValArgs;
2128 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2129 ISD::ArgFlagsTy Flags = Outs[i].Flags;
2130 if (!Flags.isByVal())
2131 continue;
2132
2133 SDValue Arg = OutVals[i];
2134 unsigned Size = Flags.getByValSize();
2135 unsigned Align = Flags.getByValAlign();
2136
2137 int FI = MF.getFrameInfo().CreateStackObject(Size, Align, /*isSS=*/false);
2138 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
2139 SDValue SizeNode = DAG.getConstant(Size, DL, XLenVT);
2140
2141 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Align,
2142 /*IsVolatile=*/false,
2143 /*AlwaysInline=*/false,
2144 IsTailCall, MachinePointerInfo(),
2145 MachinePointerInfo());
2146 ByValArgs.push_back(FIPtr);
2147 }
2148
2149 if (!IsTailCall)
2150 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
2151
2152 // Copy argument values to their designated locations.
2153 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
2154 SmallVector<SDValue, 8> MemOpChains;
2155 SDValue StackPtr;
2156 for (unsigned i = 0, j = 0, e = ArgLocs.size(); i != e; ++i) {
2157 CCValAssign &VA = ArgLocs[i];
2158 SDValue ArgValue = OutVals[i];
2159 ISD::ArgFlagsTy Flags = Outs[i].Flags;
2160
2161 // Handle passing f64 on RV32D with a soft float ABI as a special case.
2162 bool IsF64OnRV32DSoftABI =
2163 VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64;
2164 if (IsF64OnRV32DSoftABI && VA.isRegLoc()) {
2165 SDValue SplitF64 = DAG.getNode(
2166 RISCVISD::SplitF64, DL, DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
2167 SDValue Lo = SplitF64.getValue(0);
2168 SDValue Hi = SplitF64.getValue(1);
2169
2170 Register RegLo = VA.getLocReg();
2171 RegsToPass.push_back(std::make_pair(RegLo, Lo));
2172
2173 if (RegLo == RISCV::X17) {
2174 // Second half of f64 is passed on the stack.
2175 // Work out the address of the stack slot.
2176 if (!StackPtr.getNode())
2177 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2178 // Emit the store.
2179 MemOpChains.push_back(
2180 DAG.getStore(Chain, DL, Hi, StackPtr, MachinePointerInfo()));
2181 } else {
2182 // Second half of f64 is passed in another GPR.
2183 assert(RegLo < RISCV::X31 && "Invalid register pair")((RegLo < RISCV::X31 && "Invalid register pair") ?
static_cast<void> (0) : __assert_fail ("RegLo < RISCV::X31 && \"Invalid register pair\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2183, __PRETTY_FUNCTION__))
;
2184 Register RegHigh = RegLo + 1;
2185 RegsToPass.push_back(std::make_pair(RegHigh, Hi));
2186 }
2187 continue;
2188 }
2189
2190 // IsF64OnRV32DSoftABI && VA.isMemLoc() is handled below in the same way
2191 // as any other MemLoc.
2192
2193 // Promote the value if needed.
2194 // For now, only handle fully promoted and indirect arguments.
2195 if (VA.getLocInfo() == CCValAssign::Indirect) {
2196 // Store the argument in a stack slot and pass its address.
2197 SDValue SpillSlot = DAG.CreateStackTemporary(Outs[i].ArgVT);
2198 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
2199 MemOpChains.push_back(
2200 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
2201 MachinePointerInfo::getFixedStack(MF, FI)));
2202 // If the original argument was split (e.g. i128), we need
2203 // to store all parts of it here (and pass just one address).
2204 unsigned ArgIndex = Outs[i].OrigArgIndex;
2205 assert(Outs[i].PartOffset == 0)((Outs[i].PartOffset == 0) ? static_cast<void> (0) : __assert_fail
("Outs[i].PartOffset == 0", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2205, __PRETTY_FUNCTION__))
;
2206 while (i + 1 != e && Outs[i + 1].OrigArgIndex == ArgIndex) {
2207 SDValue PartValue = OutVals[i + 1];
2208 unsigned PartOffset = Outs[i + 1].PartOffset;
2209 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot,
2210 DAG.getIntPtrConstant(PartOffset, DL));
2211 MemOpChains.push_back(
2212 DAG.getStore(Chain, DL, PartValue, Address,
2213 MachinePointerInfo::getFixedStack(MF, FI)));
2214 ++i;
2215 }
2216 ArgValue = SpillSlot;
2217 } else {
2218 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL);
2219 }
2220
2221 // Use local copy if it is a byval arg.
2222 if (Flags.isByVal())
2223 ArgValue = ByValArgs[j++];
2224
2225 if (VA.isRegLoc()) {
2226 // Queue up the argument copies and emit them at the end.
2227 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
2228 } else {
2229 assert(VA.isMemLoc() && "Argument not register or memory")((VA.isMemLoc() && "Argument not register or memory")
? static_cast<void> (0) : __assert_fail ("VA.isMemLoc() && \"Argument not register or memory\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2229, __PRETTY_FUNCTION__))
;
2230 assert(!IsTailCall && "Tail call not allowed if stack is used "((!IsTailCall && "Tail call not allowed if stack is used "
"for passing parameters") ? static_cast<void> (0) : __assert_fail
("!IsTailCall && \"Tail call not allowed if stack is used \" \"for passing parameters\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2231, __PRETTY_FUNCTION__))
2231 "for passing parameters")((!IsTailCall && "Tail call not allowed if stack is used "
"for passing parameters") ? static_cast<void> (0) : __assert_fail
("!IsTailCall && \"Tail call not allowed if stack is used \" \"for passing parameters\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2231, __PRETTY_FUNCTION__))
;
2232
2233 // Work out the address of the stack slot.
2234 if (!StackPtr.getNode())
2235 StackPtr = DAG.getCopyFromReg(Chain, DL, RISCV::X2, PtrVT);
2236 SDValue Address =
2237 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
2238 DAG.getIntPtrConstant(VA.getLocMemOffset(), DL));
2239
2240 // Emit the store.
2241 MemOpChains.push_back(
2242 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
2243 }
2244 }
2245
2246 // Join the stores, which are independent of one another.
2247 if (!MemOpChains.empty())
2248 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
2249
2250 SDValue Glue;
2251
2252 // Build a sequence of copy-to-reg nodes, chained and glued together.
2253 for (auto &Reg : RegsToPass) {
2254 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
2255 Glue = Chain.getValue(1);
2256 }
2257
2258 // Validate that none of the argument registers have been marked as
2259 // reserved, if so report an error. Do the same for the return address if this
2260 // is not a tailcall.
2261 validateCCReservedRegs(RegsToPass, MF);
2262 if (!IsTailCall &&
2263 MF.getSubtarget<RISCVSubtarget>().isRegisterReservedByUser(RISCV::X1))
2264 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
2265 MF.getFunction(),
2266 "Return address register required, but has been reserved."});
2267
2268 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
2269 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
2270 // split it and then direct call can be matched by PseudoCALL.
2271 if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Callee)) {
2272 const GlobalValue *GV = S->getGlobal();
2273
2274 unsigned OpFlags = RISCVII::MO_CALL;
2275 if (!getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))
2276 OpFlags = RISCVII::MO_PLT;
2277
2278 Callee = DAG.getTargetGlobalAddress(GV, DL, PtrVT, 0, OpFlags);
2279 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
2280 unsigned OpFlags = RISCVII::MO_CALL;
2281
2282 if (!getTargetMachine().shouldAssumeDSOLocal(*MF.getFunction().getParent(),
2283 nullptr))
2284 OpFlags = RISCVII::MO_PLT;
2285
2286 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
2287 }
2288
2289 // The first call operand is the chain and the second is the target address.
2290 SmallVector<SDValue, 8> Ops;
2291 Ops.push_back(Chain);
2292 Ops.push_back(Callee);
2293
2294 // Add argument registers to the end of the list so that they are
2295 // known live into the call.
2296 for (auto &Reg : RegsToPass)
2297 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
2298
2299 if (!IsTailCall) {
2300 // Add a register mask operand representing the call-preserved registers.
2301 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
2302 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
2303 assert(Mask && "Missing call preserved mask for calling convention")((Mask && "Missing call preserved mask for calling convention"
) ? static_cast<void> (0) : __assert_fail ("Mask && \"Missing call preserved mask for calling convention\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2303, __PRETTY_FUNCTION__))
;
2304 Ops.push_back(DAG.getRegisterMask(Mask));
2305 }
2306
2307 // Glue the call to the argument copies, if any.
2308 if (Glue.getNode())
2309 Ops.push_back(Glue);
2310
2311 // Emit the call.
2312 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
2313
2314 if (IsTailCall) {
2315 MF.getFrameInfo().setHasTailCall();
2316 return DAG.getNode(RISCVISD::TAIL, DL, NodeTys, Ops);
2317 }
2318
2319 Chain = DAG.getNode(RISCVISD::CALL, DL, NodeTys, Ops);
2320 Glue = Chain.getValue(1);
2321
2322 // Mark the end of the call, which is glued to the call itself.
2323 Chain = DAG.getCALLSEQ_END(Chain,
2324 DAG.getConstant(NumBytes, DL, PtrVT, true),
2325 DAG.getConstant(0, DL, PtrVT, true),
2326 Glue, DL);
2327 Glue = Chain.getValue(1);
2328
2329 // Assign locations to each value returned by this call.
2330 SmallVector<CCValAssign, 16> RVLocs;
2331 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
2332 analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true);
2333
2334 // Copy all of the result registers out of their specified physreg.
2335 for (auto &VA : RVLocs) {
2336 // Copy the value out
2337 SDValue RetValue =
2338 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
2339 // Glue the RetValue to the end of the call sequence
2340 Chain = RetValue.getValue(1);
2341 Glue = RetValue.getValue(2);
2342
2343 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2344 assert(VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment")((VA.getLocReg() == ArgGPRs[0] && "Unexpected reg assignment"
) ? static_cast<void> (0) : __assert_fail ("VA.getLocReg() == ArgGPRs[0] && \"Unexpected reg assignment\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2344, __PRETTY_FUNCTION__))
;
2345 SDValue RetValue2 =
2346 DAG.getCopyFromReg(Chain, DL, ArgGPRs[1], MVT::i32, Glue);
2347 Chain = RetValue2.getValue(1);
2348 Glue = RetValue2.getValue(2);
2349 RetValue = DAG.getNode(RISCVISD::BuildPairF64, DL, MVT::f64, RetValue,
2350 RetValue2);
2351 }
2352
2353 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
2354
2355 InVals.push_back(RetValue);
2356 }
2357
2358 return Chain;
2359}
2360
2361bool RISCVTargetLowering::CanLowerReturn(
2362 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
2363 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context) const {
2364 SmallVector<CCValAssign, 16> RVLocs;
2365 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
2366 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
2367 MVT VT = Outs[i].VT;
2368 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
2369 RISCVABI::ABI ABI = MF.getSubtarget<RISCVSubtarget>().getTargetABI();
2370 if (CC_RISCV(MF.getDataLayout(), ABI, i, VT, VT, CCValAssign::Full,
2371 ArgFlags, CCInfo, /*IsFixed=*/true, /*IsRet=*/true, nullptr))
2372 return false;
2373 }
2374 return true;
2375}
2376
2377SDValue
2378RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
2379 bool IsVarArg,
2380 const SmallVectorImpl<ISD::OutputArg> &Outs,
2381 const SmallVectorImpl<SDValue> &OutVals,
2382 const SDLoc &DL, SelectionDAG &DAG) const {
2383 const MachineFunction &MF = DAG.getMachineFunction();
2384 const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
2385
2386 // Stores the assignment of the return value to a location.
2387 SmallVector<CCValAssign, 16> RVLocs;
2388
2389 // Info about the registers and stack slot.
2390 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
2391 *DAG.getContext());
2392
2393 analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
2394 nullptr);
2395
2396 SDValue Glue;
2397 SmallVector<SDValue, 4> RetOps(1, Chain);
2398
2399 // Copy the result values into the output registers.
2400 for (unsigned i = 0, e = RVLocs.size(); i < e; ++i) {
2401 SDValue Val = OutVals[i];
2402 CCValAssign &VA = RVLocs[i];
2403 assert(VA.isRegLoc() && "Can only return in registers!")((VA.isRegLoc() && "Can only return in registers!") ?
static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Can only return in registers!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2403, __PRETTY_FUNCTION__))
;
2404
2405 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
2406 // Handle returning f64 on RV32D with a soft float ABI.
2407 assert(VA.isRegLoc() && "Expected return via registers")((VA.isRegLoc() && "Expected return via registers") ?
static_cast<void> (0) : __assert_fail ("VA.isRegLoc() && \"Expected return via registers\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2407, __PRETTY_FUNCTION__))
;
2408 SDValue SplitF64 = DAG.getNode(RISCVISD::SplitF64, DL,
2409 DAG.getVTList(MVT::i32, MVT::i32), Val);
2410 SDValue Lo = SplitF64.getValue(0);
2411 SDValue Hi = SplitF64.getValue(1);
2412 Register RegLo = VA.getLocReg();
2413 assert(RegLo < RISCV::X31 && "Invalid register pair")((RegLo < RISCV::X31 && "Invalid register pair") ?
static_cast<void> (0) : __assert_fail ("RegLo < RISCV::X31 && \"Invalid register pair\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2413, __PRETTY_FUNCTION__))
;
2414 Register RegHi = RegLo + 1;
2415
2416 if (STI.isRegisterReservedByUser(RegLo) ||
2417 STI.isRegisterReservedByUser(RegHi))
2418 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
2419 MF.getFunction(),
2420 "Return value register required, but has been reserved."});
2421
2422 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
2423 Glue = Chain.getValue(1);
2424 RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
2425 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
2426 Glue = Chain.getValue(1);
2427 RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
2428 } else {
2429 // Handle a 'normal' return.
2430 Val = convertValVTToLocVT(DAG, Val, VA, DL);
2431 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
2432
2433 if (STI.isRegisterReservedByUser(VA.getLocReg()))
2434 MF.getFunction().getContext().diagnose(DiagnosticInfoUnsupported{
2435 MF.getFunction(),
2436 "Return value register required, but has been reserved."});
2437
2438 // Guarantee that all emitted copies are stuck together.
2439 Glue = Chain.getValue(1);
2440 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
2441 }
2442 }
2443
2444 RetOps[0] = Chain; // Update chain.
2445
2446 // Add the glue node if we have it.
2447 if (Glue.getNode()) {
2448 RetOps.push_back(Glue);
2449 }
2450
2451 // Interrupt service routines use different return instructions.
2452 const Function &Func = DAG.getMachineFunction().getFunction();
2453 if (Func.hasFnAttribute("interrupt")) {
2454 if (!Func.getReturnType()->isVoidTy())
2455 report_fatal_error(
2456 "Functions with the interrupt attribute must have void return type!");
2457
2458 MachineFunction &MF = DAG.getMachineFunction();
2459 StringRef Kind =
2460 MF.getFunction().getFnAttribute("interrupt").getValueAsString();
2461
2462 unsigned RetOpc;
2463 if (Kind == "user")
2464 RetOpc = RISCVISD::URET_FLAG;
2465 else if (Kind == "supervisor")
2466 RetOpc = RISCVISD::SRET_FLAG;
2467 else
2468 RetOpc = RISCVISD::MRET_FLAG;
2469
2470 return DAG.getNode(RetOpc, DL, MVT::Other, RetOps);
2471 }
2472
2473 return DAG.getNode(RISCVISD::RET_FLAG, DL, MVT::Other, RetOps);
2474}
2475
2476void RISCVTargetLowering::validateCCReservedRegs(
2477 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
2478 MachineFunction &MF) const {
2479 const Function &F = MF.getFunction();
2480 const RISCVSubtarget &STI = MF.getSubtarget<RISCVSubtarget>();
2481
2482 if (std::any_of(std::begin(Regs), std::end(Regs), [&STI](auto Reg) {
2483 return STI.isRegisterReservedByUser(Reg.first);
2484 }))
2485 F.getContext().diagnose(DiagnosticInfoUnsupported{
2486 F, "Argument register required, but has been reserved."});
2487}
2488
2489const char *RISCVTargetLowering::getTargetNodeName(unsigned Opcode) const {
2490 switch ((RISCVISD::NodeType)Opcode) {
2491 case RISCVISD::FIRST_NUMBER:
2492 break;
2493 case RISCVISD::RET_FLAG:
2494 return "RISCVISD::RET_FLAG";
2495 case RISCVISD::URET_FLAG:
2496 return "RISCVISD::URET_FLAG";
2497 case RISCVISD::SRET_FLAG:
2498 return "RISCVISD::SRET_FLAG";
2499 case RISCVISD::MRET_FLAG:
2500 return "RISCVISD::MRET_FLAG";
2501 case RISCVISD::CALL:
2502 return "RISCVISD::CALL";
2503 case RISCVISD::SELECT_CC:
2504 return "RISCVISD::SELECT_CC";
2505 case RISCVISD::BuildPairF64:
2506 return "RISCVISD::BuildPairF64";
2507 case RISCVISD::SplitF64:
2508 return "RISCVISD::SplitF64";
2509 case RISCVISD::TAIL:
2510 return "RISCVISD::TAIL";
2511 case RISCVISD::SLLW:
2512 return "RISCVISD::SLLW";
2513 case RISCVISD::SRAW:
2514 return "RISCVISD::SRAW";
2515 case RISCVISD::SRLW:
2516 return "RISCVISD::SRLW";
2517 case RISCVISD::DIVW:
2518 return "RISCVISD::DIVW";
2519 case RISCVISD::DIVUW:
2520 return "RISCVISD::DIVUW";
2521 case RISCVISD::REMUW:
2522 return "RISCVISD::REMUW";
2523 case RISCVISD::FMV_W_X_RV64:
2524 return "RISCVISD::FMV_W_X_RV64";
2525 case RISCVISD::FMV_X_ANYEXTW_RV64:
2526 return "RISCVISD::FMV_X_ANYEXTW_RV64";
2527 case RISCVISD::READ_CYCLE_WIDE:
2528 return "RISCVISD::READ_CYCLE_WIDE";
2529 }
2530 return nullptr;
2531}
2532
2533/// getConstraintType - Given a constraint letter, return the type of
2534/// constraint it is for this target.
2535RISCVTargetLowering::ConstraintType
2536RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
2537 if (Constraint.size() == 1) {
2538 switch (Constraint[0]) {
2539 default:
2540 break;
2541 case 'f':
2542 return C_RegisterClass;
2543 case 'I':
2544 case 'J':
2545 case 'K':
2546 return C_Immediate;
2547 case 'A':
2548 return C_Memory;
2549 }
2550 }
2551 return TargetLowering::getConstraintType(Constraint);
2552}
2553
2554std::pair<unsigned, const TargetRegisterClass *>
2555RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
2556 StringRef Constraint,
2557 MVT VT) const {
2558 // First, see if this is a constraint that directly corresponds to a
2559 // RISCV register class.
2560 if (Constraint.size() == 1) {
2561 switch (Constraint[0]) {
2562 case 'r':
2563 return std::make_pair(0U, &RISCV::GPRRegClass);
2564 case 'f':
2565 if (Subtarget.hasStdExtF() && VT == MVT::f32)
2566 return std::make_pair(0U, &RISCV::FPR32RegClass);
2567 if (Subtarget.hasStdExtD() && VT == MVT::f64)
2568 return std::make_pair(0U, &RISCV::FPR64RegClass);
2569 break;
2570 default:
2571 break;
2572 }
2573 }
2574
2575 // Clang will correctly decode the usage of register name aliases into their
2576 // official names. However, other frontends like `rustc` do not. This allows
2577 // users of these frontends to use the ABI names for registers in LLVM-style
2578 // register constraints.
2579 Register XRegFromAlias = StringSwitch<Register>(Constraint.lower())
2580 .Case("{zero}", RISCV::X0)
2581 .Case("{ra}", RISCV::X1)
2582 .Case("{sp}", RISCV::X2)
2583 .Case("{gp}", RISCV::X3)
2584 .Case("{tp}", RISCV::X4)
2585 .Case("{t0}", RISCV::X5)
2586 .Case("{t1}", RISCV::X6)
2587 .Case("{t2}", RISCV::X7)
2588 .Cases("{s0}", "{fp}", RISCV::X8)
2589 .Case("{s1}", RISCV::X9)
2590 .Case("{a0}", RISCV::X10)
2591 .Case("{a1}", RISCV::X11)
2592 .Case("{a2}", RISCV::X12)
2593 .Case("{a3}", RISCV::X13)
2594 .Case("{a4}", RISCV::X14)
2595 .Case("{a5}", RISCV::X15)
2596 .Case("{a6}", RISCV::X16)
2597 .Case("{a7}", RISCV::X17)
2598 .Case("{s2}", RISCV::X18)
2599 .Case("{s3}", RISCV::X19)
2600 .Case("{s4}", RISCV::X20)
2601 .Case("{s5}", RISCV::X21)
2602 .Case("{s6}", RISCV::X22)
2603 .Case("{s7}", RISCV::X23)
2604 .Case("{s8}", RISCV::X24)
2605 .Case("{s9}", RISCV::X25)
2606 .Case("{s10}", RISCV::X26)
2607 .Case("{s11}", RISCV::X27)
2608 .Case("{t3}", RISCV::X28)
2609 .Case("{t4}", RISCV::X29)
2610 .Case("{t5}", RISCV::X30)
2611 .Case("{t6}", RISCV::X31)
2612 .Default(RISCV::NoRegister);
2613 if (XRegFromAlias != RISCV::NoRegister)
2614 return std::make_pair(XRegFromAlias, &RISCV::GPRRegClass);
2615
2616 // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
2617 // TableGen record rather than the AsmName to choose registers for InlineAsm
2618 // constraints, plus we want to match those names to the widest floating point
2619 // register type available, manually select floating point registers here.
2620 //
2621 // The second case is the ABI name of the register, so that frontends can also
2622 // use the ABI names in register constraint lists.
2623 if (Subtarget.hasStdExtF() || Subtarget.hasStdExtD()) {
2624 std::pair<Register, Register> FReg =
2625 StringSwitch<std::pair<Register, Register>>(Constraint.lower())
2626 .Cases("{f0}", "{ft0}", {RISCV::F0_F, RISCV::F0_D})
2627 .Cases("{f1}", "{ft1}", {RISCV::F1_F, RISCV::F1_D})
2628 .Cases("{f2}", "{ft2}", {RISCV::F2_F, RISCV::F2_D})
2629 .Cases("{f3}", "{ft3}", {RISCV::F3_F, RISCV::F3_D})
2630 .Cases("{f4}", "{ft4}", {RISCV::F4_F, RISCV::F4_D})
2631 .Cases("{f5}", "{ft5}", {RISCV::F5_F, RISCV::F5_D})
2632 .Cases("{f6}", "{ft6}", {RISCV::F6_F, RISCV::F6_D})
2633 .Cases("{f7}", "{ft7}", {RISCV::F7_F, RISCV::F7_D})
2634 .Cases("{f8}", "{fs0}", {RISCV::F8_F, RISCV::F8_D})
2635 .Cases("{f9}", "{fs1}", {RISCV::F9_F, RISCV::F9_D})
2636 .Cases("{f10}", "{fa0}", {RISCV::F10_F, RISCV::F10_D})
2637 .Cases("{f11}", "{fa1}", {RISCV::F11_F, RISCV::F11_D})
2638 .Cases("{f12}", "{fa2}", {RISCV::F12_F, RISCV::F12_D})
2639 .Cases("{f13}", "{fa3}", {RISCV::F13_F, RISCV::F13_D})
2640 .Cases("{f14}", "{fa4}", {RISCV::F14_F, RISCV::F14_D})
2641 .Cases("{f15}", "{fa5}", {RISCV::F15_F, RISCV::F15_D})
2642 .Cases("{f16}", "{fa6}", {RISCV::F16_F, RISCV::F16_D})
2643 .Cases("{f17}", "{fa7}", {RISCV::F17_F, RISCV::F17_D})
2644 .Cases("{f18}", "{fs2}", {RISCV::F18_F, RISCV::F18_D})
2645 .Cases("{f19}", "{fs3}", {RISCV::F19_F, RISCV::F19_D})
2646 .Cases("{f20}", "{fs4}", {RISCV::F20_F, RISCV::F20_D})
2647 .Cases("{f21}", "{fs5}", {RISCV::F21_F, RISCV::F21_D})
2648 .Cases("{f22}", "{fs6}", {RISCV::F22_F, RISCV::F22_D})
2649 .Cases("{f23}", "{fs7}", {RISCV::F23_F, RISCV::F23_D})
2650 .Cases("{f24}", "{fs8}", {RISCV::F24_F, RISCV::F24_D})
2651 .Cases("{f25}", "{fs9}", {RISCV::F25_F, RISCV::F25_D})
2652 .Cases("{f26}", "{fs10}", {RISCV::F26_F, RISCV::F26_D})
2653 .Cases("{f27}", "{fs11}", {RISCV::F27_F, RISCV::F27_D})
2654 .Cases("{f28}", "{ft8}", {RISCV::F28_F, RISCV::F28_D})
2655 .Cases("{f29}", "{ft9}", {RISCV::F29_F, RISCV::F29_D})
2656 .Cases("{f30}", "{ft10}", {RISCV::F30_F, RISCV::F30_D})
2657 .Cases("{f31}", "{ft11}", {RISCV::F31_F, RISCV::F31_D})
2658 .Default({RISCV::NoRegister, RISCV::NoRegister});
2659 if (FReg.first != RISCV::NoRegister)
2660 return Subtarget.hasStdExtD()
2661 ? std::make_pair(FReg.second, &RISCV::FPR64RegClass)
2662 : std::make_pair(FReg.first, &RISCV::FPR32RegClass);
2663 }
2664
2665 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
2666}
2667
2668unsigned
2669RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
2670 // Currently only support length 1 constraints.
2671 if (ConstraintCode.size() == 1) {
2672 switch (ConstraintCode[0]) {
2673 case 'A':
2674 return InlineAsm::Constraint_A;
2675 default:
2676 break;
2677 }
2678 }
2679
2680 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
2681}
2682
2683void RISCVTargetLowering::LowerAsmOperandForConstraint(
2684 SDValue Op, std::string &Constraint, std::vector<SDValue> &Ops,
2685 SelectionDAG &DAG) const {
2686 // Currently only support length 1 constraints.
2687 if (Constraint.length() == 1) {
2688 switch (Constraint[0]) {
2689 case 'I':
2690 // Validate & create a 12-bit signed immediate operand.
2691 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2692 uint64_t CVal = C->getSExtValue();
2693 if (isInt<12>(CVal))
2694 Ops.push_back(
2695 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2696 }
2697 return;
2698 case 'J':
2699 // Validate & create an integer zero operand.
2700 if (auto *C = dyn_cast<ConstantSDNode>(Op))
2701 if (C->getZExtValue() == 0)
2702 Ops.push_back(
2703 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getXLenVT()));
2704 return;
2705 case 'K':
2706 // Validate & create a 5-bit unsigned immediate operand.
2707 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
2708 uint64_t CVal = C->getZExtValue();
2709 if (isUInt<5>(CVal))
2710 Ops.push_back(
2711 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getXLenVT()));
2712 }
2713 return;
2714 default:
2715 break;
2716 }
2717 }
2718 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
2719}
2720
2721Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilder<> &Builder,
2722 Instruction *Inst,
2723 AtomicOrdering Ord) const {
2724 if (isa<LoadInst>(Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
2725 return Builder.CreateFence(Ord);
2726 if (isa<StoreInst>(Inst) && isReleaseOrStronger(Ord))
2727 return Builder.CreateFence(AtomicOrdering::Release);
2728 return nullptr;
2729}
2730
2731Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilder<> &Builder,
2732 Instruction *Inst,
2733 AtomicOrdering Ord) const {
2734 if (isa<LoadInst>(Inst) && isAcquireOrStronger(Ord))
2735 return Builder.CreateFence(AtomicOrdering::Acquire);
2736 return nullptr;
2737}
2738
2739TargetLowering::AtomicExpansionKind
2740RISCVTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
2741 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
2742 // point operations can't be used in an lr/sc sequence without breaking the
2743 // forward-progress guarantee.
2744 if (AI->isFloatingPointOperation())
2745 return AtomicExpansionKind::CmpXChg;
2746
2747 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
2748 if (Size == 8 || Size == 16)
2749 return AtomicExpansionKind::MaskedIntrinsic;
2750 return AtomicExpansionKind::None;
2751}
2752
2753static Intrinsic::ID
2754getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
2755 if (XLen == 32) {
2756 switch (BinOp) {
2757 default:
2758 llvm_unreachable("Unexpected AtomicRMW BinOp")::llvm::llvm_unreachable_internal("Unexpected AtomicRMW BinOp"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2758)
;
2759 case AtomicRMWInst::Xchg:
2760 return Intrinsic::riscv_masked_atomicrmw_xchg_i32;
2761 case AtomicRMWInst::Add:
2762 return Intrinsic::riscv_masked_atomicrmw_add_i32;
2763 case AtomicRMWInst::Sub:
2764 return Intrinsic::riscv_masked_atomicrmw_sub_i32;
2765 case AtomicRMWInst::Nand:
2766 return Intrinsic::riscv_masked_atomicrmw_nand_i32;
2767 case AtomicRMWInst::Max:
2768 return Intrinsic::riscv_masked_atomicrmw_max_i32;
2769 case AtomicRMWInst::Min:
2770 return Intrinsic::riscv_masked_atomicrmw_min_i32;
2771 case AtomicRMWInst::UMax:
2772 return Intrinsic::riscv_masked_atomicrmw_umax_i32;
2773 case AtomicRMWInst::UMin:
2774 return Intrinsic::riscv_masked_atomicrmw_umin_i32;
2775 }
2776 }
2777
2778 if (XLen == 64) {
2779 switch (BinOp) {
2780 default:
2781 llvm_unreachable("Unexpected AtomicRMW BinOp")::llvm::llvm_unreachable_internal("Unexpected AtomicRMW BinOp"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2781)
;
2782 case AtomicRMWInst::Xchg:
2783 return Intrinsic::riscv_masked_atomicrmw_xchg_i64;
2784 case AtomicRMWInst::Add:
2785 return Intrinsic::riscv_masked_atomicrmw_add_i64;
2786 case AtomicRMWInst::Sub:
2787 return Intrinsic::riscv_masked_atomicrmw_sub_i64;
2788 case AtomicRMWInst::Nand:
2789 return Intrinsic::riscv_masked_atomicrmw_nand_i64;
2790 case AtomicRMWInst::Max:
2791 return Intrinsic::riscv_masked_atomicrmw_max_i64;
2792 case AtomicRMWInst::Min:
2793 return Intrinsic::riscv_masked_atomicrmw_min_i64;
2794 case AtomicRMWInst::UMax:
2795 return Intrinsic::riscv_masked_atomicrmw_umax_i64;
2796 case AtomicRMWInst::UMin:
2797 return Intrinsic::riscv_masked_atomicrmw_umin_i64;
2798 }
2799 }
2800
2801 llvm_unreachable("Unexpected XLen\n")::llvm::llvm_unreachable_internal("Unexpected XLen\n", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/lib/Target/RISCV/RISCVISelLowering.cpp"
, 2801)
;
2802}
2803
2804Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
2805 IRBuilder<> &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
2806 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
2807 unsigned XLen = Subtarget.getXLen();
2808 Value *Ordering =
2809 Builder.getIntN(XLen, static_cast<uint64_t>(AI->getOrdering()));
2810 Type *Tys[] = {AlignedAddr->getType()};
2811 Function *LrwOpScwLoop = Intrinsic::getDeclaration(
2812 AI->getModule(),
2813 getIntrinsicForMaskedAtomicRMWBinOp(XLen, AI->getOperation()), Tys);
2814
2815 if (XLen == 64) {
2816 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
2817 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2818 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
2819 }
2820
2821 Value *Result;
2822
2823 // Must pass the shift amount needed to sign extend the loaded value prior
2824 // to performing a signed comparison for min/max. ShiftAmt is the number of
2825 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
2826 // is the number of bits to left+right shift the value in order to
2827 // sign-extend.
2828 if (AI->getOperation() == AtomicRMWInst::Min ||
2829 AI->getOperation() == AtomicRMWInst::Max) {
2830 const DataLayout &DL = AI->getModule()->getDataLayout();
2831 unsigned ValWidth =
2832 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
2833 Value *SextShamt =
2834 Builder.CreateSub(Builder.getIntN(XLen, XLen - ValWidth), ShiftAmt);
2835 Result = Builder.CreateCall(LrwOpScwLoop,
2836 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
2837 } else {
2838 Result =
2839 Builder.CreateCall(LrwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
2840 }
2841
2842 if (XLen == 64)
2843 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2844 return Result;
2845}
2846
2847TargetLowering::AtomicExpansionKind
2848RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
2849 AtomicCmpXchgInst *CI) const {
2850 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
2851 if (Size == 8 || Size == 16)
2852 return AtomicExpansionKind::MaskedIntrinsic;
2853 return AtomicExpansionKind::None;
2854}
2855
2856Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
2857 IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
2858 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
2859 unsigned XLen = Subtarget.getXLen();
2860 Value *Ordering = Builder.getIntN(XLen, static_cast<uint64_t>(Ord));
2861 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i32;
2862 if (XLen == 64) {
2863 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
2864 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
2865 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
2866 CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg_i64;
2867 }
2868 Type *Tys[] = {AlignedAddr->getType()};
2869 Function *MaskedCmpXchg =
2870 Intrinsic::getDeclaration(CI->getModule(), CmpXchgIntrID, Tys);
2871 Value *Result = Builder.CreateCall(
2872 MaskedCmpXchg, {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
2873 if (XLen == 64)
2874 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
2875 return Result;
2876}
2877
2878unsigned RISCVTargetLowering::getExceptionPointerRegister(
2879 const Constant *PersonalityFn) const {
2880 return RISCV::X10;
2881}
2882
2883unsigned RISCVTargetLowering::getExceptionSelectorRegister(
2884 const Constant *PersonalityFn) const {
2885 return RISCV::X11;
2886}
2887
2888bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
2889 // Return false to suppress the unnecessary extensions if the LibCall
2890 // arguments or return value is f32 type for LP64 ABI.
2891 RISCVABI::ABI ABI = Subtarget.getTargetABI();
2892 if (ABI == RISCVABI::ABI_LP64 && (Type == MVT::f32))
2893 return false;
2894
2895 return true;
2896}
2897
2898#define GET_REGISTER_MATCHER
2899#include "RISCVGenAsmMatcher.inc"
2900
2901Register
2902RISCVTargetLowering::getRegisterByName(const char *RegName, EVT VT,
2903 const MachineFunction &MF) const {
2904 Register Reg = MatchRegisterAltName(RegName);
2905 if (Reg == RISCV::NoRegister)
2906 Reg = MatchRegisterName(RegName);
2907 if (Reg == RISCV::NoRegister)
2908 report_fatal_error(
2909 Twine("Invalid register name \"" + StringRef(RegName) + "\"."));
2910 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
2911 if (!ReservedRegs.test(Reg) && !Subtarget.isRegisterReservedByUser(Reg))
2912 report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
2913 StringRef(RegName) + "\"."));
2914 return Reg;
2915}

/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h

1//===- llvm/CodeGen/TargetLowering.h - Target Lowering Info -----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file describes how to lower LLVM code to machine code. This has two
11/// main components:
12///
13/// 1. Which ValueTypes are natively supported by the target.
14/// 2. Which operations are supported for supported ValueTypes.
15/// 3. Cost thresholds for alternative implementations of certain operations.
16///
17/// In addition it has a few other components, like information about FP
18/// immediates.
19///
20//===----------------------------------------------------------------------===//
21
22#ifndef LLVM_CODEGEN_TARGETLOWERING_H
23#define LLVM_CODEGEN_TARGETLOWERING_H
24
25#include "llvm/ADT/APInt.h"
26#include "llvm/ADT/ArrayRef.h"
27#include "llvm/ADT/DenseMap.h"
28#include "llvm/ADT/STLExtras.h"
29#include "llvm/ADT/SmallVector.h"
30#include "llvm/ADT/StringRef.h"
31#include "llvm/Analysis/ProfileSummaryInfo.h"
32#include "llvm/CodeGen/DAGCombine.h"
33#include "llvm/CodeGen/ISDOpcodes.h"
34#include "llvm/CodeGen/RuntimeLibcalls.h"
35#include "llvm/CodeGen/SelectionDAG.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/TargetCallingConv.h"
38#include "llvm/CodeGen/ValueTypes.h"
39#include "llvm/IR/Attributes.h"
40#include "llvm/IR/CallSite.h"
41#include "llvm/IR/CallingConv.h"
42#include "llvm/IR/DataLayout.h"
43#include "llvm/IR/DerivedTypes.h"
44#include "llvm/IR/Function.h"
45#include "llvm/IR/IRBuilder.h"
46#include "llvm/IR/InlineAsm.h"
47#include "llvm/IR/Instruction.h"
48#include "llvm/IR/Instructions.h"
49#include "llvm/IR/Type.h"
50#include "llvm/MC/MCRegisterInfo.h"
51#include "llvm/Support/Alignment.h"
52#include "llvm/Support/AtomicOrdering.h"
53#include "llvm/Support/Casting.h"
54#include "llvm/Support/ErrorHandling.h"
55#include "llvm/Support/MachineValueType.h"
56#include "llvm/Target/TargetMachine.h"
57#include "llvm/Transforms/Utils/SizeOpts.h"
58#include <algorithm>
59#include <cassert>
60#include <climits>
61#include <cstdint>
62#include <iterator>
63#include <map>
64#include <string>
65#include <utility>
66#include <vector>
67
68namespace llvm {
69
70class BranchProbability;
71class CCState;
72class CCValAssign;
73class Constant;
74class FastISel;
75class FunctionLoweringInfo;
76class GlobalValue;
77class GISelKnownBits;
78class IntrinsicInst;
79struct KnownBits;
80class LegacyDivergenceAnalysis;
81class LLVMContext;
82class MachineBasicBlock;
83class MachineFunction;
84class MachineInstr;
85class MachineJumpTableInfo;
86class MachineLoop;
87class MachineRegisterInfo;
88class MCContext;
89class MCExpr;
90class Module;
91class TargetRegisterClass;
92class TargetLibraryInfo;
93class TargetRegisterInfo;
94class Value;
95
96namespace Sched {
97
98 enum Preference {
99 None, // No preference
100 Source, // Follow source order.
101 RegPressure, // Scheduling for lowest register pressure.
102 Hybrid, // Scheduling for both latency and register pressure.
103 ILP, // Scheduling for ILP in low register pressure mode.
104 VLIW // Scheduling for VLIW targets.
105 };
106
107} // end namespace Sched
108
109/// This base class for TargetLowering contains the SelectionDAG-independent
110/// parts that can be used from the rest of CodeGen.
111class TargetLoweringBase {
112public:
113 /// This enum indicates whether operations are valid for a target, and if not,
114 /// what action should be used to make them valid.
115 enum LegalizeAction : uint8_t {
116 Legal, // The target natively supports this operation.
117 Promote, // This operation should be executed in a larger type.
118 Expand, // Try to expand this to other ops, otherwise use a libcall.
119 LibCall, // Don't try to expand this to other ops, always use a libcall.
120 Custom // Use the LowerOperation hook to implement custom lowering.
121 };
122
123 /// This enum indicates whether a types are legal for a target, and if not,
124 /// what action should be used to make them valid.
125 enum LegalizeTypeAction : uint8_t {
126 TypeLegal, // The target natively supports this type.
127 TypePromoteInteger, // Replace this integer with a larger one.
128 TypeExpandInteger, // Split this integer into two of half the size.
129 TypeSoftenFloat, // Convert this float to a same size integer type.
130 TypeExpandFloat, // Split this float into two of half the size.
131 TypeScalarizeVector, // Replace this one-element vector with its element.
132 TypeSplitVector, // Split this vector into two of half the size.
133 TypeWidenVector, // This vector should be widened into a larger vector.
134 TypePromoteFloat // Replace this float with a larger one.
135 };
136
137 /// LegalizeKind holds the legalization kind that needs to happen to EVT
138 /// in order to type-legalize it.
139 using LegalizeKind = std::pair<LegalizeTypeAction, EVT>;
140
141 /// Enum that describes how the target represents true/false values.
142 enum BooleanContent {
143 UndefinedBooleanContent, // Only bit 0 counts, the rest can hold garbage.
144 ZeroOrOneBooleanContent, // All bits zero except for bit 0.
145 ZeroOrNegativeOneBooleanContent // All bits equal to bit 0.
146 };
147
148 /// Enum that describes what type of support for selects the target has.
149 enum SelectSupportKind {
150 ScalarValSelect, // The target supports scalar selects (ex: cmov).
151 ScalarCondVectorVal, // The target supports selects with a scalar condition
152 // and vector values (ex: cmov).
153 VectorMaskSelect // The target supports vector selects with a vector
154 // mask (ex: x86 blends).
155 };
156
157 /// Enum that specifies what an atomic load/AtomicRMWInst is expanded
158 /// to, if at all. Exists because different targets have different levels of
159 /// support for these atomic instructions, and also have different options
160 /// w.r.t. what they should expand to.
161 enum class AtomicExpansionKind {
162 None, // Don't expand the instruction.
163 LLSC, // Expand the instruction into loadlinked/storeconditional; used
164 // by ARM/AArch64.
165 LLOnly, // Expand the (load) instruction into just a load-linked, which has
166 // greater atomic guarantees than a normal load.
167 CmpXChg, // Expand the instruction into cmpxchg; used by at least X86.
168 MaskedIntrinsic, // Use a target-specific intrinsic for the LL/SC loop.
169 };
170
171 /// Enum that specifies when a multiplication should be expanded.
172 enum class MulExpansionKind {
173 Always, // Always expand the instruction.
174 OnlyLegalOrCustom, // Only expand when the resulting instructions are legal
175 // or custom.
176 };
177
178 class ArgListEntry {
179 public:
180 Value *Val = nullptr;
181 SDValue Node = SDValue();
182 Type *Ty = nullptr;
183 bool IsSExt : 1;
184 bool IsZExt : 1;
185 bool IsInReg : 1;
186 bool IsSRet : 1;
187 bool IsNest : 1;
188 bool IsByVal : 1;
189 bool IsInAlloca : 1;
190 bool IsReturned : 1;
191 bool IsSwiftSelf : 1;
192 bool IsSwiftError : 1;
193 bool IsCFGuardTarget : 1;
194 uint16_t Alignment = 0;
195 Type *ByValType = nullptr;
196
197 ArgListEntry()
198 : IsSExt(false), IsZExt(false), IsInReg(false), IsSRet(false),
199 IsNest(false), IsByVal(false), IsInAlloca(false), IsReturned(false),
200 IsSwiftSelf(false), IsSwiftError(false), IsCFGuardTarget(false) {}
201
202 void setAttributes(const CallBase *Call, unsigned ArgIdx);
203
204 void setAttributes(ImmutableCallSite *CS, unsigned ArgIdx) {
205 return setAttributes(cast<CallBase>(CS->getInstruction()), ArgIdx);
206 }
207 };
208 using ArgListTy = std::vector<ArgListEntry>;
209
210 virtual void markLibCallAttributes(MachineFunction *MF, unsigned CC,
211 ArgListTy &Args) const {};
212
213 static ISD::NodeType getExtendForContent(BooleanContent Content) {
214 switch (Content) {
215 case UndefinedBooleanContent:
216 // Extend by adding rubbish bits.
217 return ISD::ANY_EXTEND;
218 case ZeroOrOneBooleanContent:
219 // Extend by adding zero bits.
220 return ISD::ZERO_EXTEND;
221 case ZeroOrNegativeOneBooleanContent:
222 // Extend by copying the sign bit.
223 return ISD::SIGN_EXTEND;
224 }
225 llvm_unreachable("Invalid content kind")::llvm::llvm_unreachable_internal("Invalid content kind", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 225)
;
226 }
227
228 /// NOTE: The TargetMachine owns TLOF.
229 explicit TargetLoweringBase(const TargetMachine &TM);
230 TargetLoweringBase(const TargetLoweringBase &) = delete;
231 TargetLoweringBase &operator=(const TargetLoweringBase &) = delete;
232 virtual ~TargetLoweringBase() = default;
233
234protected:
235 /// Initialize all of the actions to default values.
236 void initActions();
237
238public:
239 const TargetMachine &getTargetMachine() const { return TM; }
240
241 virtual bool useSoftFloat() const { return false; }
242
243 /// Return the pointer type for the given address space, defaults to
244 /// the pointer type from the data layout.
245 /// FIXME: The default needs to be removed once all the code is updated.
246 virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS = 0) const {
247 return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
248 }
249
250 /// Return the in-memory pointer type for the given address space, defaults to
251 /// the pointer type from the data layout. FIXME: The default needs to be
252 /// removed once all the code is updated.
253 MVT getPointerMemTy(const DataLayout &DL, uint32_t AS = 0) const {
254 return MVT::getIntegerVT(DL.getPointerSizeInBits(AS));
255 }
256
257 /// Return the type for frame index, which is determined by
258 /// the alloca address space specified through the data layout.
259 MVT getFrameIndexTy(const DataLayout &DL) const {
260 return getPointerTy(DL, DL.getAllocaAddrSpace());
261 }
262
263 /// Return the type for operands of fence.
264 /// TODO: Let fence operands be of i32 type and remove this.
265 virtual MVT getFenceOperandTy(const DataLayout &DL) const {
266 return getPointerTy(DL);
267 }
268
269 /// EVT is not used in-tree, but is used by out-of-tree target.
270 /// A documentation for this function would be nice...
271 virtual MVT getScalarShiftAmountTy(const DataLayout &, EVT) const;
272
273 EVT getShiftAmountTy(EVT LHSTy, const DataLayout &DL,
274 bool LegalTypes = true) const;
275
276 /// Returns the type to be used for the index operand of:
277 /// ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
278 /// ISD::INSERT_SUBVECTOR, and ISD::EXTRACT_SUBVECTOR
279 virtual MVT getVectorIdxTy(const DataLayout &DL) const {
280 return getPointerTy(DL);
281 }
282
283 virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
284 return true;
285 }
286
287 /// Return true if it is profitable to convert a select of FP constants into
288 /// a constant pool load whose address depends on the select condition. The
289 /// parameter may be used to differentiate a select with FP compare from
290 /// integer compare.
291 virtual bool reduceSelectOfFPConstantLoads(EVT CmpOpVT) const {
292 return true;
293 }
294
295 /// Return true if multiple condition registers are available.
296 bool hasMultipleConditionRegisters() const {
297 return HasMultipleConditionRegisters;
298 }
299
300 /// Return true if the target has BitExtract instructions.
301 bool hasExtractBitsInsn() const { return HasExtractBitsInsn; }
302
303 /// Return the preferred vector type legalization action.
304 virtual TargetLoweringBase::LegalizeTypeAction
305 getPreferredVectorAction(MVT VT) const {
306 // The default action for one element vectors is to scalarize
307 if (VT.getVectorNumElements() == 1)
308 return TypeScalarizeVector;
309 // The default action for an odd-width vector is to widen.
310 if (!VT.isPow2VectorType())
311 return TypeWidenVector;
312 // The default action for other vectors is to promote
313 return TypePromoteInteger;
314 }
315
316 // There are two general methods for expanding a BUILD_VECTOR node:
317 // 1. Use SCALAR_TO_VECTOR on the defined scalar values and then shuffle
318 // them together.
319 // 2. Build the vector on the stack and then load it.
320 // If this function returns true, then method (1) will be used, subject to
321 // the constraint that all of the necessary shuffles are legal (as determined
322 // by isShuffleMaskLegal). If this function returns false, then method (2) is
323 // always used. The vector type, and the number of defined values, are
324 // provided.
325 virtual bool
326 shouldExpandBuildVectorWithShuffles(EVT /* VT */,
327 unsigned DefinedValues) const {
328 return DefinedValues < 3;
329 }
330
331 /// Return true if integer divide is usually cheaper than a sequence of
332 /// several shifts, adds, and multiplies for this target.
333 /// The definition of "cheaper" may depend on whether we're optimizing
334 /// for speed or for size.
335 virtual bool isIntDivCheap(EVT VT, AttributeList Attr) const { return false; }
336
337 /// Return true if the target can handle a standalone remainder operation.
338 virtual bool hasStandaloneRem(EVT VT) const {
339 return true;
340 }
341
342 /// Return true if SQRT(X) shouldn't be replaced with X*RSQRT(X).
343 virtual bool isFsqrtCheap(SDValue X, SelectionDAG &DAG) const {
344 // Default behavior is to replace SQRT(X) with X*RSQRT(X).
345 return false;
346 }
347
348 /// Reciprocal estimate status values used by the functions below.
349 enum ReciprocalEstimate : int {
350 Unspecified = -1,
351 Disabled = 0,
352 Enabled = 1
353 };
354
355 /// Return a ReciprocalEstimate enum value for a square root of the given type
356 /// based on the function's attributes. If the operation is not overridden by
357 /// the function's attributes, "Unspecified" is returned and target defaults
358 /// are expected to be used for instruction selection.
359 int getRecipEstimateSqrtEnabled(EVT VT, MachineFunction &MF) const;
360
361 /// Return a ReciprocalEstimate enum value for a division of the given type
362 /// based on the function's attributes. If the operation is not overridden by
363 /// the function's attributes, "Unspecified" is returned and target defaults
364 /// are expected to be used for instruction selection.
365 int getRecipEstimateDivEnabled(EVT VT, MachineFunction &MF) const;
366
367 /// Return the refinement step count for a square root of the given type based
368 /// on the function's attributes. If the operation is not overridden by
369 /// the function's attributes, "Unspecified" is returned and target defaults
370 /// are expected to be used for instruction selection.
371 int getSqrtRefinementSteps(EVT VT, MachineFunction &MF) const;
372
373 /// Return the refinement step count for a division of the given type based
374 /// on the function's attributes. If the operation is not overridden by
375 /// the function's attributes, "Unspecified" is returned and target defaults
376 /// are expected to be used for instruction selection.
377 int getDivRefinementSteps(EVT VT, MachineFunction &MF) const;
378
379 /// Returns true if target has indicated at least one type should be bypassed.
380 bool isSlowDivBypassed() const { return !BypassSlowDivWidths.empty(); }
381
382 /// Returns map of slow types for division or remainder with corresponding
383 /// fast types
384 const DenseMap<unsigned int, unsigned int> &getBypassSlowDivWidths() const {
385 return BypassSlowDivWidths;
386 }
387
388 /// Return true if Flow Control is an expensive operation that should be
389 /// avoided.
390 bool isJumpExpensive() const { return JumpIsExpensive; }
391
392 /// Return true if selects are only cheaper than branches if the branch is
393 /// unlikely to be predicted right.
394 bool isPredictableSelectExpensive() const {
395 return PredictableSelectIsExpensive;
396 }
397
398 /// If a branch or a select condition is skewed in one direction by more than
399 /// this factor, it is very likely to be predicted correctly.
400 virtual BranchProbability getPredictableBranchThreshold() const;
401
402 /// Return true if the following transform is beneficial:
403 /// fold (conv (load x)) -> (load (conv*)x)
404 /// On architectures that don't natively support some vector loads
405 /// efficiently, casting the load to a smaller vector of larger types and
406 /// loading is more efficient, however, this can be undone by optimizations in
407 /// dag combiner.
408 virtual bool isLoadBitCastBeneficial(EVT LoadVT, EVT BitcastVT,
409 const SelectionDAG &DAG,
410 const MachineMemOperand &MMO) const {
411 // Don't do if we could do an indexed load on the original type, but not on
412 // the new one.
413 if (!LoadVT.isSimple() || !BitcastVT.isSimple())
414 return true;
415
416 MVT LoadMVT = LoadVT.getSimpleVT();
417
418 // Don't bother doing this if it's just going to be promoted again later, as
419 // doing so might interfere with other combines.
420 if (getOperationAction(ISD::LOAD, LoadMVT) == Promote &&
421 getTypeToPromoteTo(ISD::LOAD, LoadMVT) == BitcastVT.getSimpleVT())
422 return false;
423
424 bool Fast = false;
425 return allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), BitcastVT,
426 MMO, &Fast) && Fast;
427 }
428
429 /// Return true if the following transform is beneficial:
430 /// (store (y (conv x)), y*)) -> (store x, (x*))
431 virtual bool isStoreBitCastBeneficial(EVT StoreVT, EVT BitcastVT,
432 const SelectionDAG &DAG,
433 const MachineMemOperand &MMO) const {
434 // Default to the same logic as loads.
435 return isLoadBitCastBeneficial(StoreVT, BitcastVT, DAG, MMO);
436 }
437
438 /// Return true if it is expected to be cheaper to do a store of a non-zero
439 /// vector constant with the given size and type for the address space than to
440 /// store the individual scalar element constants.
441 virtual bool storeOfVectorConstantIsCheap(EVT MemVT,
442 unsigned NumElem,
443 unsigned AddrSpace) const {
444 return false;
445 }
446
447 /// Allow store merging for the specified type after legalization in addition
448 /// to before legalization. This may transform stores that do not exist
449 /// earlier (for example, stores created from intrinsics).
450 virtual bool mergeStoresAfterLegalization(EVT MemVT) const {
451 return true;
452 }
453
454 /// Returns if it's reasonable to merge stores to MemVT size.
455 virtual bool canMergeStoresTo(unsigned AS, EVT MemVT,
456 const SelectionDAG &DAG) const {
457 return true;
458 }
459
460 /// Return true if it is cheap to speculate a call to intrinsic cttz.
461 virtual bool isCheapToSpeculateCttz() const {
462 return false;
463 }
464
465 /// Return true if it is cheap to speculate a call to intrinsic ctlz.
466 virtual bool isCheapToSpeculateCtlz() const {
467 return false;
468 }
469
470 /// Return true if ctlz instruction is fast.
471 virtual bool isCtlzFast() const {
472 return false;
473 }
474
475 /// Return true if instruction generated for equality comparison is folded
476 /// with instruction generated for signed comparison.
477 virtual bool isEqualityCmpFoldedWithSignedCmp() const { return true; }
478
479 /// Return true if it is safe to transform an integer-domain bitwise operation
480 /// into the equivalent floating-point operation. This should be set to true
481 /// if the target has IEEE-754-compliant fabs/fneg operations for the input
482 /// type.
483 virtual bool hasBitPreservingFPLogic(EVT VT) const {
484 return false;
485 }
486
487 /// Return true if it is cheaper to split the store of a merged int val
488 /// from a pair of smaller values into multiple stores.
489 virtual bool isMultiStoresCheaperThanBitsMerge(EVT LTy, EVT HTy) const {
490 return false;
491 }
492
493 /// Return if the target supports combining a
494 /// chain like:
495 /// \code
496 /// %andResult = and %val1, #mask
497 /// %icmpResult = icmp %andResult, 0
498 /// \endcode
499 /// into a single machine instruction of a form like:
500 /// \code
501 /// cc = test %register, #mask
502 /// \endcode
503 virtual bool isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
504 return false;
505 }
506
507 /// Use bitwise logic to make pairs of compares more efficient. For example:
508 /// and (seteq A, B), (seteq C, D) --> seteq (or (xor A, B), (xor C, D)), 0
509 /// This should be true when it takes more than one instruction to lower
510 /// setcc (cmp+set on x86 scalar), when bitwise ops are faster than logic on
511 /// condition bits (crand on PowerPC), and/or when reducing cmp+br is a win.
512 virtual bool convertSetCCLogicToBitwiseLogic(EVT VT) const {
513 return false;
514 }
515
516 /// Return the preferred operand type if the target has a quick way to compare
517 /// integer values of the given size. Assume that any legal integer type can
518 /// be compared efficiently. Targets may override this to allow illegal wide
519 /// types to return a vector type if there is support to compare that type.
520 virtual MVT hasFastEqualityCompare(unsigned NumBits) const {
521 MVT VT = MVT::getIntegerVT(NumBits);
522 return isTypeLegal(VT) ? VT : MVT::INVALID_SIMPLE_VALUE_TYPE;
523 }
524
525 /// Return true if the target should transform:
526 /// (X & Y) == Y ---> (~X & Y) == 0
527 /// (X & Y) != Y ---> (~X & Y) != 0
528 ///
529 /// This may be profitable if the target has a bitwise and-not operation that
530 /// sets comparison flags. A target may want to limit the transformation based
531 /// on the type of Y or if Y is a constant.
532 ///
533 /// Note that the transform will not occur if Y is known to be a power-of-2
534 /// because a mask and compare of a single bit can be handled by inverting the
535 /// predicate, for example:
536 /// (X & 8) == 8 ---> (X & 8) != 0
537 virtual bool hasAndNotCompare(SDValue Y) const {
538 return false;
539 }
540
541 /// Return true if the target has a bitwise and-not operation:
542 /// X = ~A & B
543 /// This can be used to simplify select or other instructions.
544 virtual bool hasAndNot(SDValue X) const {
545 // If the target has the more complex version of this operation, assume that
546 // it has this operation too.
547 return hasAndNotCompare(X);
548 }
549
550 /// Return true if the target has a bit-test instruction:
551 /// (X & (1 << Y)) ==/!= 0
552 /// This knowledge can be used to prevent breaking the pattern,
553 /// or creating it if it could be recognized.
554 virtual bool hasBitTest(SDValue X, SDValue Y) const { return false; }
555
556 /// There are two ways to clear extreme bits (either low or high):
557 /// Mask: x & (-1 << y) (the instcombine canonical form)
558 /// Shifts: x >> y << y
559 /// Return true if the variant with 2 variable shifts is preferred.
560 /// Return false if there is no preference.
561 virtual bool shouldFoldMaskToVariableShiftPair(SDValue X) const {
562 // By default, let's assume that no one prefers shifts.
563 return false;
564 }
565
566 /// Return true if it is profitable to fold a pair of shifts into a mask.
567 /// This is usually true on most targets. But some targets, like Thumb1,
568 /// have immediate shift instructions, but no immediate "and" instruction;
569 /// this makes the fold unprofitable.
570 virtual bool shouldFoldConstantShiftPairToMask(const SDNode *N,
571 CombineLevel Level) const {
572 return true;
573 }
574
575 /// Should we tranform the IR-optimal check for whether given truncation
576 /// down into KeptBits would be truncating or not:
577 /// (add %x, (1 << (KeptBits-1))) srccond (1 << KeptBits)
578 /// Into it's more traditional form:
579 /// ((%x << C) a>> C) dstcond %x
580 /// Return true if we should transform.
581 /// Return false if there is no preference.
582 virtual bool shouldTransformSignedTruncationCheck(EVT XVT,
583 unsigned KeptBits) const {
584 // By default, let's assume that no one prefers shifts.
585 return false;
586 }
587
588 /// Given the pattern
589 /// (X & (C l>>/<< Y)) ==/!= 0
590 /// return true if it should be transformed into:
591 /// ((X <</l>> Y) & C) ==/!= 0
592 /// WARNING: if 'X' is a constant, the fold may deadlock!
593 /// FIXME: we could avoid passing XC, but we can't use isConstOrConstSplat()
594 /// here because it can end up being not linked in.
595 virtual bool shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
596 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
597 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
598 SelectionDAG &DAG) const {
599 if (hasBitTest(X, Y)) {
600 // One interesting pattern that we'd want to form is 'bit test':
601 // ((1 << Y) & C) ==/!= 0
602 // But we also need to be careful not to try to reverse that fold.
603
604 // Is this '1 << Y' ?
605 if (OldShiftOpcode == ISD::SHL && CC->isOne())
606 return false; // Keep the 'bit test' pattern.
607
608 // Will it be '1 << Y' after the transform ?
609 if (XC && NewShiftOpcode == ISD::SHL && XC->isOne())
610 return true; // Do form the 'bit test' pattern.
611 }
612
613 // If 'X' is a constant, and we transform, then we will immediately
614 // try to undo the fold, thus causing endless combine loop.
615 // So by default, let's assume everyone prefers the fold
616 // iff 'X' is not a constant.
617 return !XC;
618 }
619
620 /// These two forms are equivalent:
621 /// sub %y, (xor %x, -1)
622 /// add (add %x, 1), %y
623 /// The variant with two add's is IR-canonical.
624 /// Some targets may prefer one to the other.
625 virtual bool preferIncOfAddToSubOfNot(EVT VT) const {
626 // By default, let's assume that everyone prefers the form with two add's.
627 return true;
628 }
629
630 /// Return true if the target wants to use the optimization that
631 /// turns ext(promotableInst1(...(promotableInstN(load)))) into
632 /// promotedInst1(...(promotedInstN(ext(load)))).
633 bool enableExtLdPromotion() const { return EnableExtLdPromotion; }
634
635 /// Return true if the target can combine store(extractelement VectorTy,
636 /// Idx).
637 /// \p Cost[out] gives the cost of that transformation when this is true.
638 virtual bool canCombineStoreAndExtract(Type *VectorTy, Value *Idx,
639 unsigned &Cost) const {
640 return false;
641 }
642
643 /// Return true if inserting a scalar into a variable element of an undef
644 /// vector is more efficiently handled by splatting the scalar instead.
645 virtual bool shouldSplatInsEltVarIndex(EVT) const {
646 return false;
647 }
648
649 /// Return true if target always beneficiates from combining into FMA for a
650 /// given value type. This must typically return false on targets where FMA
651 /// takes more cycles to execute than FADD.
652 virtual bool enableAggressiveFMAFusion(EVT VT) const {
653 return false;
654 }
655
656 /// Return the ValueType of the result of SETCC operations.
657 virtual EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
658 EVT VT) const;
659
660 /// Return the ValueType for comparison libcalls. Comparions libcalls include
661 /// floating point comparion calls, and Ordered/Unordered check calls on
662 /// floating point numbers.
663 virtual
664 MVT::SimpleValueType getCmpLibcallReturnType() const;
665
666 /// For targets without i1 registers, this gives the nature of the high-bits
667 /// of boolean values held in types wider than i1.
668 ///
669 /// "Boolean values" are special true/false values produced by nodes like
670 /// SETCC and consumed (as the condition) by nodes like SELECT and BRCOND.
671 /// Not to be confused with general values promoted from i1. Some cpus
672 /// distinguish between vectors of boolean and scalars; the isVec parameter
673 /// selects between the two kinds. For example on X86 a scalar boolean should
674 /// be zero extended from i1, while the elements of a vector of booleans
675 /// should be sign extended from i1.
676 ///
677 /// Some cpus also treat floating point types the same way as they treat
678 /// vectors instead of the way they treat scalars.
679 BooleanContent getBooleanContents(bool isVec, bool isFloat) const {
680 if (isVec)
681 return BooleanVectorContents;
682 return isFloat ? BooleanFloatContents : BooleanContents;
683 }
684
685 BooleanContent getBooleanContents(EVT Type) const {
686 return getBooleanContents(Type.isVector(), Type.isFloatingPoint());
687 }
688
689 /// Return target scheduling preference.
690 Sched::Preference getSchedulingPreference() const {
691 return SchedPreferenceInfo;
692 }
693
694 /// Some scheduler, e.g. hybrid, can switch to different scheduling heuristics
695 /// for different nodes. This function returns the preference (or none) for
696 /// the given node.
697 virtual Sched::Preference getSchedulingPreference(SDNode *) const {
698 return Sched::None;
699 }
700
701 /// Return the register class that should be used for the specified value
702 /// type.
703 virtual const TargetRegisterClass *getRegClassFor(MVT VT, bool isDivergent = false) const {
704 (void)isDivergent;
705 const TargetRegisterClass *RC = RegClassForVT[VT.SimpleTy];
706 assert(RC && "This value type is not natively supported!")((RC && "This value type is not natively supported!")
? static_cast<void> (0) : __assert_fail ("RC && \"This value type is not natively supported!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 706, __PRETTY_FUNCTION__))
;
707 return RC;
708 }
709
710 /// Allows target to decide about the register class of the
711 /// specific value that is live outside the defining block.
712 /// Returns true if the value needs uniform register class.
713 virtual bool requiresUniformRegister(MachineFunction &MF,
714 const Value *) const {
715 return false;
716 }
717
718 /// Return the 'representative' register class for the specified value
719 /// type.
720 ///
721 /// The 'representative' register class is the largest legal super-reg
722 /// register class for the register class of the value type. For example, on
723 /// i386 the rep register class for i8, i16, and i32 are GR32; while the rep
724 /// register class is GR64 on x86_64.
725 virtual const TargetRegisterClass *getRepRegClassFor(MVT VT) const {
726 const TargetRegisterClass *RC = RepRegClassForVT[VT.SimpleTy];
727 return RC;
728 }
729
730 /// Return the cost of the 'representative' register class for the specified
731 /// value type.
732 virtual uint8_t getRepRegClassCostFor(MVT VT) const {
733 return RepRegClassCostForVT[VT.SimpleTy];
734 }
735
736 /// Return true if SHIFT instructions should be expanded to SHIFT_PARTS
737 /// instructions, and false if a library call is preferred (e.g for code-size
738 /// reasons).
739 virtual bool shouldExpandShift(SelectionDAG &DAG, SDNode *N) const {
740 return true;
741 }
742
743 /// Return true if the target has native support for the specified value type.
744 /// This means that it has a register that directly holds it without
745 /// promotions or expansions.
746 bool isTypeLegal(EVT VT) const {
747 assert(!VT.isSimple() ||((!VT.isSimple() || (unsigned)VT.getSimpleVT().SimpleTy < array_lengthof
(RegClassForVT)) ? static_cast<void> (0) : __assert_fail
("!VT.isSimple() || (unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegClassForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 748, __PRETTY_FUNCTION__))
748 (unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegClassForVT))((!VT.isSimple() || (unsigned)VT.getSimpleVT().SimpleTy < array_lengthof
(RegClassForVT)) ? static_cast<void> (0) : __assert_fail
("!VT.isSimple() || (unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegClassForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 748, __PRETTY_FUNCTION__))
;
749 return VT.isSimple() && RegClassForVT[VT.getSimpleVT().SimpleTy] != nullptr;
750 }
751
752 class ValueTypeActionImpl {
753 /// ValueTypeActions - For each value type, keep a LegalizeTypeAction enum
754 /// that indicates how instruction selection should deal with the type.
755 LegalizeTypeAction ValueTypeActions[MVT::LAST_VALUETYPE];
756
757 public:
758 ValueTypeActionImpl() {
759 std::fill(std::begin(ValueTypeActions), std::end(ValueTypeActions),
760 TypeLegal);
761 }
762
763 LegalizeTypeAction getTypeAction(MVT VT) const {
764 return ValueTypeActions[VT.SimpleTy];
765 }
766
767 void setTypeAction(MVT VT, LegalizeTypeAction Action) {
768 ValueTypeActions[VT.SimpleTy] = Action;
769 }
770 };
771
772 const ValueTypeActionImpl &getValueTypeActions() const {
773 return ValueTypeActions;
774 }
775
776 /// Return how we should legalize values of this type, either it is already
777 /// legal (return 'Legal') or we need to promote it to a larger type (return
778 /// 'Promote'), or we need to expand it into multiple registers of smaller
779 /// integer type (return 'Expand'). 'Custom' is not an option.
780 LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const {
781 return getTypeConversion(Context, VT).first;
782 }
783 LegalizeTypeAction getTypeAction(MVT VT) const {
784 return ValueTypeActions.getTypeAction(VT);
785 }
786
787 /// For types supported by the target, this is an identity function. For
788 /// types that must be promoted to larger types, this returns the larger type
789 /// to promote to. For integer types that are larger than the largest integer
790 /// register, this contains one step in the expansion to get to the smaller
791 /// register. For illegal floating point types, this returns the integer type
792 /// to transform to.
793 EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const {
794 return getTypeConversion(Context, VT).second;
795 }
796
797 /// For types supported by the target, this is an identity function. For
798 /// types that must be expanded (i.e. integer types that are larger than the
799 /// largest integer register or illegal floating point types), this returns
800 /// the largest legal type it will be expanded to.
801 EVT getTypeToExpandTo(LLVMContext &Context, EVT VT) const {
802 assert(!VT.isVector())((!VT.isVector()) ? static_cast<void> (0) : __assert_fail
("!VT.isVector()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 802, __PRETTY_FUNCTION__))
;
803 while (true) {
804 switch (getTypeAction(Context, VT)) {
805 case TypeLegal:
806 return VT;
807 case TypeExpandInteger:
808 VT = getTypeToTransformTo(Context, VT);
809 break;
810 default:
811 llvm_unreachable("Type is not legal nor is it to be expanded!")::llvm::llvm_unreachable_internal("Type is not legal nor is it to be expanded!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 811)
;
812 }
813 }
814 }
815
816 /// Vector types are broken down into some number of legal first class types.
817 /// For example, EVT::v8f32 maps to 2 EVT::v4f32 with Altivec or SSE1, or 8
818 /// promoted EVT::f64 values with the X86 FP stack. Similarly, EVT::v2i64
819 /// turns into 4 EVT::i32 values with both PPC and X86.
820 ///
821 /// This method returns the number of registers needed, and the VT for each
822 /// register. It also returns the VT and quantity of the intermediate values
823 /// before they are promoted/expanded.
824 unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
825 EVT &IntermediateVT,
826 unsigned &NumIntermediates,
827 MVT &RegisterVT) const;
828
829 /// Certain targets such as MIPS require that some types such as vectors are
830 /// always broken down into scalars in some contexts. This occurs even if the
831 /// vector type is legal.
832 virtual unsigned getVectorTypeBreakdownForCallingConv(
833 LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
834 unsigned &NumIntermediates, MVT &RegisterVT) const {
835 return getVectorTypeBreakdown(Context, VT, IntermediateVT, NumIntermediates,
836 RegisterVT);
837 }
838
839 struct IntrinsicInfo {
840 unsigned opc = 0; // target opcode
841 EVT memVT; // memory VT
842
843 // value representing memory location
844 PointerUnion<const Value *, const PseudoSourceValue *> ptrVal;
845
846 int offset = 0; // offset off of ptrVal
847 uint64_t size = 0; // the size of the memory location
848 // (taken from memVT if zero)
849 MaybeAlign align = Align::None(); // alignment
850
851 MachineMemOperand::Flags flags = MachineMemOperand::MONone;
852 IntrinsicInfo() = default;
853 };
854
855 /// Given an intrinsic, checks if on the target the intrinsic will need to map
856 /// to a MemIntrinsicNode (touches memory). If this is the case, it returns
857 /// true and store the intrinsic information into the IntrinsicInfo that was
858 /// passed to the function.
859 virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &,
860 MachineFunction &,
861 unsigned /*Intrinsic*/) const {
862 return false;
863 }
864
865 /// Returns true if the target can instruction select the specified FP
866 /// immediate natively. If false, the legalizer will materialize the FP
867 /// immediate as a load from a constant pool.
868 virtual bool isFPImmLegal(const APFloat & /*Imm*/, EVT /*VT*/,
869 bool ForCodeSize = false) const {
870 return false;
871 }
872
873 /// Targets can use this to indicate that they only support *some*
874 /// VECTOR_SHUFFLE operations, those with specific masks. By default, if a
875 /// target supports the VECTOR_SHUFFLE node, all mask values are assumed to be
876 /// legal.
877 virtual bool isShuffleMaskLegal(ArrayRef<int> /*Mask*/, EVT /*VT*/) const {
878 return true;
879 }
880
881 /// Returns true if the operation can trap for the value type.
882 ///
883 /// VT must be a legal type. By default, we optimistically assume most
884 /// operations don't trap except for integer divide and remainder.
885 virtual bool canOpTrap(unsigned Op, EVT VT) const;
886
887 /// Similar to isShuffleMaskLegal. Targets can use this to indicate if there
888 /// is a suitable VECTOR_SHUFFLE that can be used to replace a VAND with a
889 /// constant pool entry.
890 virtual bool isVectorClearMaskLegal(ArrayRef<int> /*Mask*/,
891 EVT /*VT*/) const {
892 return false;
893 }
894
895 /// Return how this operation should be treated: either it is legal, needs to
896 /// be promoted to a larger size, needs to be expanded to some other code
897 /// sequence, or the target has a custom expander for it.
898 LegalizeAction getOperationAction(unsigned Op, EVT VT) const {
899 if (VT.isExtended()) return Expand;
900 // If a target-specific SDNode requires legalization, require the target
901 // to provide custom legalization for it.
902 if (Op >= array_lengthof(OpActions[0])) return Custom;
903 return OpActions[(unsigned)VT.getSimpleVT().SimpleTy][Op];
904 }
905
906 /// Custom method defined by each target to indicate if an operation which
907 /// may require a scale is supported natively by the target.
908 /// If not, the operation is illegal.
909 virtual bool isSupportedFixedPointOperation(unsigned Op, EVT VT,
910 unsigned Scale) const {
911 return false;
912 }
913
914 /// Some fixed point operations may be natively supported by the target but
915 /// only for specific scales. This method allows for checking
916 /// if the width is supported by the target for a given operation that may
917 /// depend on scale.
918 LegalizeAction getFixedPointOperationAction(unsigned Op, EVT VT,
919 unsigned Scale) const {
920 auto Action = getOperationAction(Op, VT);
921 if (Action != Legal)
922 return Action;
923
924 // This operation is supported in this type but may only work on specific
925 // scales.
926 bool Supported;
927 switch (Op) {
928 default:
929 llvm_unreachable("Unexpected fixed point operation.")::llvm::llvm_unreachable_internal("Unexpected fixed point operation."
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 929)
;
930 case ISD::SMULFIX:
931 case ISD::SMULFIXSAT:
932 case ISD::UMULFIX:
933 case ISD::UMULFIXSAT:
934 Supported = isSupportedFixedPointOperation(Op, VT, Scale);
935 break;
936 }
937
938 return Supported ? Action : Expand;
939 }
940
941 // If Op is a strict floating-point operation, return the result
942 // of getOperationAction for the equivalent non-strict operation.
943 LegalizeAction getStrictFPOperationAction(unsigned Op, EVT VT) const {
944 unsigned EqOpc;
945 switch (Op) {
946 default: llvm_unreachable("Unexpected FP pseudo-opcode")::llvm::llvm_unreachable_internal("Unexpected FP pseudo-opcode"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 946)
;
947 case ISD::STRICT_FADD: EqOpc = ISD::FADD; break;
948 case ISD::STRICT_FSUB: EqOpc = ISD::FSUB; break;
949 case ISD::STRICT_FMUL: EqOpc = ISD::FMUL; break;
950 case ISD::STRICT_FDIV: EqOpc = ISD::FDIV; break;
951 case ISD::STRICT_FREM: EqOpc = ISD::FREM; break;
952 case ISD::STRICT_FSQRT: EqOpc = ISD::FSQRT; break;
953 case ISD::STRICT_FPOW: EqOpc = ISD::FPOW; break;
954 case ISD::STRICT_FPOWI: EqOpc = ISD::FPOWI; break;
955 case ISD::STRICT_FMA: EqOpc = ISD::FMA; break;
956 case ISD::STRICT_FSIN: EqOpc = ISD::FSIN; break;
957 case ISD::STRICT_FCOS: EqOpc = ISD::FCOS; break;
958 case ISD::STRICT_FEXP: EqOpc = ISD::FEXP; break;
959 case ISD::STRICT_FEXP2: EqOpc = ISD::FEXP2; break;
960 case ISD::STRICT_FLOG: EqOpc = ISD::FLOG; break;
961 case ISD::STRICT_FLOG10: EqOpc = ISD::FLOG10; break;
962 case ISD::STRICT_FLOG2: EqOpc = ISD::FLOG2; break;
963 case ISD::STRICT_LRINT: EqOpc = ISD::LRINT; break;
964 case ISD::STRICT_LLRINT: EqOpc = ISD::LLRINT; break;
965 case ISD::STRICT_FRINT: EqOpc = ISD::FRINT; break;
966 case ISD::STRICT_FNEARBYINT: EqOpc = ISD::FNEARBYINT; break;
967 case ISD::STRICT_FMAXNUM: EqOpc = ISD::FMAXNUM; break;
968 case ISD::STRICT_FMINNUM: EqOpc = ISD::FMINNUM; break;
969 case ISD::STRICT_FCEIL: EqOpc = ISD::FCEIL; break;
970 case ISD::STRICT_FFLOOR: EqOpc = ISD::FFLOOR; break;
971 case ISD::STRICT_LROUND: EqOpc = ISD::LROUND; break;
972 case ISD::STRICT_LLROUND: EqOpc = ISD::LLROUND; break;
973 case ISD::STRICT_FROUND: EqOpc = ISD::FROUND; break;
974 case ISD::STRICT_FTRUNC: EqOpc = ISD::FTRUNC; break;
975 case ISD::STRICT_FP_TO_SINT: EqOpc = ISD::FP_TO_SINT; break;
976 case ISD::STRICT_FP_TO_UINT: EqOpc = ISD::FP_TO_UINT; break;
977 case ISD::STRICT_FP_ROUND: EqOpc = ISD::FP_ROUND; break;
978 case ISD::STRICT_FP_EXTEND: EqOpc = ISD::FP_EXTEND; break;
979 }
980
981 return getOperationAction(EqOpc, VT);
982 }
983
984 /// Return true if the specified operation is legal on this target or can be
985 /// made legal with custom lowering. This is used to help guide high-level
986 /// lowering decisions.
987 bool isOperationLegalOrCustom(unsigned Op, EVT VT) const {
988 return (VT == MVT::Other || isTypeLegal(VT)) &&
989 (getOperationAction(Op, VT) == Legal ||
990 getOperationAction(Op, VT) == Custom);
991 }
992
993 /// Return true if the specified operation is legal on this target or can be
994 /// made legal using promotion. This is used to help guide high-level lowering
995 /// decisions.
996 bool isOperationLegalOrPromote(unsigned Op, EVT VT) const {
997 return (VT == MVT::Other || isTypeLegal(VT)) &&
998 (getOperationAction(Op, VT) == Legal ||
999 getOperationAction(Op, VT) == Promote);
1000 }
1001
1002 /// Return true if the specified operation is legal on this target or can be
1003 /// made legal with custom lowering or using promotion. This is used to help
1004 /// guide high-level lowering decisions.
1005 bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT) const {
1006 return (VT == MVT::Other || isTypeLegal(VT)) &&
1007 (getOperationAction(Op, VT) == Legal ||
1008 getOperationAction(Op, VT) == Custom ||
1009 getOperationAction(Op, VT) == Promote);
1010 }
1011
1012 /// Return true if the operation uses custom lowering, regardless of whether
1013 /// the type is legal or not.
1014 bool isOperationCustom(unsigned Op, EVT VT) const {
1015 return getOperationAction(Op, VT) == Custom;
1016 }
1017
1018 /// Return true if lowering to a jump table is allowed.
1019 virtual bool areJTsAllowed(const Function *Fn) const {
1020 if (Fn->getFnAttribute("no-jump-tables").getValueAsString() == "true")
1021 return false;
1022
1023 return isOperationLegalOrCustom(ISD::BR_JT, MVT::Other) ||
1024 isOperationLegalOrCustom(ISD::BRIND, MVT::Other);
1025 }
1026
1027 /// Check whether the range [Low,High] fits in a machine word.
1028 bool rangeFitsInWord(const APInt &Low, const APInt &High,
1029 const DataLayout &DL) const {
1030 // FIXME: Using the pointer type doesn't seem ideal.
1031 uint64_t BW = DL.getIndexSizeInBits(0u);
1032 uint64_t Range = (High - Low).getLimitedValue(UINT64_MAX(18446744073709551615UL) - 1) + 1;
1033 return Range <= BW;
1034 }
1035
1036 /// Return true if lowering to a jump table is suitable for a set of case
1037 /// clusters which may contain \p NumCases cases, \p Range range of values.
1038 virtual bool isSuitableForJumpTable(const SwitchInst *SI, uint64_t NumCases,
1039 uint64_t Range, ProfileSummaryInfo *PSI,
1040 BlockFrequencyInfo *BFI) const;
1041
1042 /// Return true if lowering to a bit test is suitable for a set of case
1043 /// clusters which contains \p NumDests unique destinations, \p Low and
1044 /// \p High as its lowest and highest case values, and expects \p NumCmps
1045 /// case value comparisons. Check if the number of destinations, comparison
1046 /// metric, and range are all suitable.
1047 bool isSuitableForBitTests(unsigned NumDests, unsigned NumCmps,
1048 const APInt &Low, const APInt &High,
1049 const DataLayout &DL) const {
1050 // FIXME: I don't think NumCmps is the correct metric: a single case and a
1051 // range of cases both require only one branch to lower. Just looking at the
1052 // number of clusters and destinations should be enough to decide whether to
1053 // build bit tests.
1054
1055 // To lower a range with bit tests, the range must fit the bitwidth of a
1056 // machine word.
1057 if (!rangeFitsInWord(Low, High, DL))
1058 return false;
1059
1060 // Decide whether it's profitable to lower this range with bit tests. Each
1061 // destination requires a bit test and branch, and there is an overall range
1062 // check branch. For a small number of clusters, separate comparisons might
1063 // be cheaper, and for many destinations, splitting the range might be
1064 // better.
1065 return (NumDests == 1 && NumCmps >= 3) || (NumDests == 2 && NumCmps >= 5) ||
1066 (NumDests == 3 && NumCmps >= 6);
1067 }
1068
1069 /// Return true if the specified operation is illegal on this target or
1070 /// unlikely to be made legal with custom lowering. This is used to help guide
1071 /// high-level lowering decisions.
1072 bool isOperationExpand(unsigned Op, EVT VT) const {
1073 return (!isTypeLegal(VT) || getOperationAction(Op, VT) == Expand);
1074 }
1075
1076 /// Return true if the specified operation is legal on this target.
1077 bool isOperationLegal(unsigned Op, EVT VT) const {
1078 return (VT == MVT::Other || isTypeLegal(VT)) &&
1079 getOperationAction(Op, VT) == Legal;
1080 }
1081
1082 /// Return how this load with extension should be treated: either it is legal,
1083 /// needs to be promoted to a larger size, needs to be expanded to some other
1084 /// code sequence, or the target has a custom expander for it.
1085 LegalizeAction getLoadExtAction(unsigned ExtType, EVT ValVT,
1086 EVT MemVT) const {
1087 if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
1088 unsigned ValI = (unsigned) ValVT.getSimpleVT().SimpleTy;
1089 unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy;
1090 assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::LAST_VALUETYPE &&((ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT
::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE &&
"Table isn't big enough!") ? static_cast<void> (0) : __assert_fail
("ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1091, __PRETTY_FUNCTION__))
1091 MemI < MVT::LAST_VALUETYPE && "Table isn't big enough!")((ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT
::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE &&
"Table isn't big enough!") ? static_cast<void> (0) : __assert_fail
("ExtType < ISD::LAST_LOADEXT_TYPE && ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1091, __PRETTY_FUNCTION__))
;
1092 unsigned Shift = 4 * ExtType;
1093 return (LegalizeAction)((LoadExtActions[ValI][MemI] >> Shift) & 0xf);
1094 }
1095
1096 /// Return true if the specified load with extension is legal on this target.
1097 bool isLoadExtLegal(unsigned ExtType, EVT ValVT, EVT MemVT) const {
1098 return getLoadExtAction(ExtType, ValVT, MemVT) == Legal;
1099 }
1100
1101 /// Return true if the specified load with extension is legal or custom
1102 /// on this target.
1103 bool isLoadExtLegalOrCustom(unsigned ExtType, EVT ValVT, EVT MemVT) const {
1104 return getLoadExtAction(ExtType, ValVT, MemVT) == Legal ||
1105 getLoadExtAction(ExtType, ValVT, MemVT) == Custom;
1106 }
1107
1108 /// Return how this store with truncation should be treated: either it is
1109 /// legal, needs to be promoted to a larger size, needs to be expanded to some
1110 /// other code sequence, or the target has a custom expander for it.
1111 LegalizeAction getTruncStoreAction(EVT ValVT, EVT MemVT) const {
1112 if (ValVT.isExtended() || MemVT.isExtended()) return Expand;
1113 unsigned ValI = (unsigned) ValVT.getSimpleVT().SimpleTy;
1114 unsigned MemI = (unsigned) MemVT.getSimpleVT().SimpleTy;
1115 assert(ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE &&((ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE
&& "Table isn't big enough!") ? static_cast<void>
(0) : __assert_fail ("ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1116, __PRETTY_FUNCTION__))
1116 "Table isn't big enough!")((ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE
&& "Table isn't big enough!") ? static_cast<void>
(0) : __assert_fail ("ValI < MVT::LAST_VALUETYPE && MemI < MVT::LAST_VALUETYPE && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1116, __PRETTY_FUNCTION__))
;
1117 return TruncStoreActions[ValI][MemI];
1118 }
1119
1120 /// Return true if the specified store with truncation is legal on this
1121 /// target.
1122 bool isTruncStoreLegal(EVT ValVT, EVT MemVT) const {
1123 return isTypeLegal(ValVT) && getTruncStoreAction(ValVT, MemVT) == Legal;
1124 }
1125
1126 /// Return true if the specified store with truncation has solution on this
1127 /// target.
1128 bool isTruncStoreLegalOrCustom(EVT ValVT, EVT MemVT) const {
1129 return isTypeLegal(ValVT) &&
1130 (getTruncStoreAction(ValVT, MemVT) == Legal ||
1131 getTruncStoreAction(ValVT, MemVT) == Custom);
1132 }
1133
1134 /// Return how the indexed load should be treated: either it is legal, needs
1135 /// to be promoted to a larger size, needs to be expanded to some other code
1136 /// sequence, or the target has a custom expander for it.
1137 LegalizeAction
1138 getIndexedLoadAction(unsigned IdxMode, MVT VT) const {
1139 assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() &&((IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid()
&& "Table isn't big enough!") ? static_cast<void>
(0) : __assert_fail ("IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1140, __PRETTY_FUNCTION__))
1140 "Table isn't big enough!")((IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid()
&& "Table isn't big enough!") ? static_cast<void>
(0) : __assert_fail ("IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1140, __PRETTY_FUNCTION__))
;
1141 unsigned Ty = (unsigned)VT.SimpleTy;
1142 return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] & 0xf0) >> 4);
1143 }
1144
1145 /// Return true if the specified indexed load is legal on this target.
1146 bool isIndexedLoadLegal(unsigned IdxMode, EVT VT) const {
1147 return VT.isSimple() &&
1148 (getIndexedLoadAction(IdxMode, VT.getSimpleVT()) == Legal ||
1149 getIndexedLoadAction(IdxMode, VT.getSimpleVT()) == Custom);
1150 }
1151
1152 /// Return how the indexed store should be treated: either it is legal, needs
1153 /// to be promoted to a larger size, needs to be expanded to some other code
1154 /// sequence, or the target has a custom expander for it.
1155 LegalizeAction
1156 getIndexedStoreAction(unsigned IdxMode, MVT VT) const {
1157 assert(IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() &&((IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid()
&& "Table isn't big enough!") ? static_cast<void>
(0) : __assert_fail ("IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1158, __PRETTY_FUNCTION__))
1158 "Table isn't big enough!")((IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid()
&& "Table isn't big enough!") ? static_cast<void>
(0) : __assert_fail ("IdxMode < ISD::LAST_INDEXED_MODE && VT.isValid() && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1158, __PRETTY_FUNCTION__))
;
1159 unsigned Ty = (unsigned)VT.SimpleTy;
1160 return (LegalizeAction)(IndexedModeActions[Ty][IdxMode] & 0x0f);
1161 }
1162
1163 /// Return true if the specified indexed load is legal on this target.
1164 bool isIndexedStoreLegal(unsigned IdxMode, EVT VT) const {
1165 return VT.isSimple() &&
1166 (getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Legal ||
1167 getIndexedStoreAction(IdxMode, VT.getSimpleVT()) == Custom);
1168 }
1169
1170 /// Return how the condition code should be treated: either it is legal, needs
1171 /// to be expanded to some other code sequence, or the target has a custom
1172 /// expander for it.
1173 LegalizeAction
1174 getCondCodeAction(ISD::CondCode CC, MVT VT) const {
1175 assert((unsigned)CC < array_lengthof(CondCodeActions) &&(((unsigned)CC < array_lengthof(CondCodeActions) &&
((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions
[0]) && "Table isn't big enough!") ? static_cast<void
> (0) : __assert_fail ("(unsigned)CC < array_lengthof(CondCodeActions) && ((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions[0]) && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1177, __PRETTY_FUNCTION__))
1176 ((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions[0]) &&(((unsigned)CC < array_lengthof(CondCodeActions) &&
((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions
[0]) && "Table isn't big enough!") ? static_cast<void
> (0) : __assert_fail ("(unsigned)CC < array_lengthof(CondCodeActions) && ((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions[0]) && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1177, __PRETTY_FUNCTION__))
1177 "Table isn't big enough!")(((unsigned)CC < array_lengthof(CondCodeActions) &&
((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions
[0]) && "Table isn't big enough!") ? static_cast<void
> (0) : __assert_fail ("(unsigned)CC < array_lengthof(CondCodeActions) && ((unsigned)VT.SimpleTy >> 3) < array_lengthof(CondCodeActions[0]) && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1177, __PRETTY_FUNCTION__))
;
1178 // See setCondCodeAction for how this is encoded.
1179 uint32_t Shift = 4 * (VT.SimpleTy & 0x7);
1180 uint32_t Value = CondCodeActions[CC][VT.SimpleTy >> 3];
1181 LegalizeAction Action = (LegalizeAction) ((Value >> Shift) & 0xF);
1182 assert(Action != Promote && "Can't promote condition code!")((Action != Promote && "Can't promote condition code!"
) ? static_cast<void> (0) : __assert_fail ("Action != Promote && \"Can't promote condition code!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1182, __PRETTY_FUNCTION__))
;
1183 return Action;
1184 }
1185
1186 /// Return true if the specified condition code is legal on this target.
1187 bool isCondCodeLegal(ISD::CondCode CC, MVT VT) const {
1188 return getCondCodeAction(CC, VT) == Legal;
1189 }
1190
1191 /// Return true if the specified condition code is legal or custom on this
1192 /// target.
1193 bool isCondCodeLegalOrCustom(ISD::CondCode CC, MVT VT) const {
1194 return getCondCodeAction(CC, VT) == Legal ||
1195 getCondCodeAction(CC, VT) == Custom;
1196 }
1197
1198 /// If the action for this operation is to promote, this method returns the
1199 /// ValueType to promote to.
1200 MVT getTypeToPromoteTo(unsigned Op, MVT VT) const {
1201 assert(getOperationAction(Op, VT) == Promote &&((getOperationAction(Op, VT) == Promote && "This operation isn't promoted!"
) ? static_cast<void> (0) : __assert_fail ("getOperationAction(Op, VT) == Promote && \"This operation isn't promoted!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1202, __PRETTY_FUNCTION__))
1202 "This operation isn't promoted!")((getOperationAction(Op, VT) == Promote && "This operation isn't promoted!"
) ? static_cast<void> (0) : __assert_fail ("getOperationAction(Op, VT) == Promote && \"This operation isn't promoted!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1202, __PRETTY_FUNCTION__))
;
1203
1204 // See if this has an explicit type specified.
1205 std::map<std::pair<unsigned, MVT::SimpleValueType>,
1206 MVT::SimpleValueType>::const_iterator PTTI =
1207 PromoteToType.find(std::make_pair(Op, VT.SimpleTy));
1208 if (PTTI != PromoteToType.end()) return PTTI->second;
1209
1210 assert((VT.isInteger() || VT.isFloatingPoint()) &&(((VT.isInteger() || VT.isFloatingPoint()) && "Cannot autopromote this type, add it with AddPromotedToType."
) ? static_cast<void> (0) : __assert_fail ("(VT.isInteger() || VT.isFloatingPoint()) && \"Cannot autopromote this type, add it with AddPromotedToType.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1211, __PRETTY_FUNCTION__))
1211 "Cannot autopromote this type, add it with AddPromotedToType.")(((VT.isInteger() || VT.isFloatingPoint()) && "Cannot autopromote this type, add it with AddPromotedToType."
) ? static_cast<void> (0) : __assert_fail ("(VT.isInteger() || VT.isFloatingPoint()) && \"Cannot autopromote this type, add it with AddPromotedToType.\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1211, __PRETTY_FUNCTION__))
;
1212
1213 MVT NVT = VT;
1214 do {
1215 NVT = (MVT::SimpleValueType)(NVT.SimpleTy+1);
1216 assert(NVT.isInteger() == VT.isInteger() && NVT != MVT::isVoid &&((NVT.isInteger() == VT.isInteger() && NVT != MVT::isVoid
&& "Didn't find type to promote to!") ? static_cast<
void> (0) : __assert_fail ("NVT.isInteger() == VT.isInteger() && NVT != MVT::isVoid && \"Didn't find type to promote to!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1217, __PRETTY_FUNCTION__))
1217 "Didn't find type to promote to!")((NVT.isInteger() == VT.isInteger() && NVT != MVT::isVoid
&& "Didn't find type to promote to!") ? static_cast<
void> (0) : __assert_fail ("NVT.isInteger() == VT.isInteger() && NVT != MVT::isVoid && \"Didn't find type to promote to!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1217, __PRETTY_FUNCTION__))
;
1218 } while (!isTypeLegal(NVT) ||
1219 getOperationAction(Op, NVT) == Promote);
1220 return NVT;
1221 }
1222
1223 /// Return the EVT corresponding to this LLVM type. This is fixed by the LLVM
1224 /// operations except for the pointer size. If AllowUnknown is true, this
1225 /// will return MVT::Other for types with no EVT counterpart (e.g. structs),
1226 /// otherwise it will assert.
1227 EVT getValueType(const DataLayout &DL, Type *Ty,
1228 bool AllowUnknown = false) const {
1229 // Lower scalar pointers to native pointer types.
1230 if (auto *PTy = dyn_cast<PointerType>(Ty))
1231 return getPointerTy(DL, PTy->getAddressSpace());
1232
1233 if (auto *VTy = dyn_cast<VectorType>(Ty)) {
1234 Type *EltTy = VTy->getElementType();
1235 // Lower vectors of pointers to native pointer types.
1236 if (auto *PTy = dyn_cast<PointerType>(EltTy)) {
1237 EVT PointerTy(getPointerTy(DL, PTy->getAddressSpace()));
1238 EltTy = PointerTy.getTypeForEVT(Ty->getContext());
1239 }
1240 return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(EltTy, false),
1241 VTy->getElementCount());
1242 }
1243
1244 return EVT::getEVT(Ty, AllowUnknown);
1245 }
1246
1247 EVT getMemValueType(const DataLayout &DL, Type *Ty,
1248 bool AllowUnknown = false) const {
1249 // Lower scalar pointers to native pointer types.
1250 if (PointerType *PTy = dyn_cast<PointerType>(Ty))
1251 return getPointerMemTy(DL, PTy->getAddressSpace());
1252 else if (VectorType *VTy = dyn_cast<VectorType>(Ty)) {
1253 Type *Elm = VTy->getElementType();
1254 if (PointerType *PT = dyn_cast<PointerType>(Elm)) {
1255 EVT PointerTy(getPointerMemTy(DL, PT->getAddressSpace()));
1256 Elm = PointerTy.getTypeForEVT(Ty->getContext());
1257 }
1258 return EVT::getVectorVT(Ty->getContext(), EVT::getEVT(Elm, false),
1259 VTy->getNumElements());
1260 }
1261
1262 return getValueType(DL, Ty, AllowUnknown);
1263 }
1264
1265
1266 /// Return the MVT corresponding to this LLVM type. See getValueType.
1267 MVT getSimpleValueType(const DataLayout &DL, Type *Ty,
1268 bool AllowUnknown = false) const {
1269 return getValueType(DL, Ty, AllowUnknown).getSimpleVT();
1270 }
1271
1272 /// Return the desired alignment for ByVal or InAlloca aggregate function
1273 /// arguments in the caller parameter area. This is the actual alignment, not
1274 /// its logarithm.
1275 virtual unsigned getByValTypeAlignment(Type *Ty, const DataLayout &DL) const;
1276
1277 /// Return the type of registers that this ValueType will eventually require.
1278 MVT getRegisterType(MVT VT) const {
1279 assert((unsigned)VT.SimpleTy < array_lengthof(RegisterTypeForVT))(((unsigned)VT.SimpleTy < array_lengthof(RegisterTypeForVT
)) ? static_cast<void> (0) : __assert_fail ("(unsigned)VT.SimpleTy < array_lengthof(RegisterTypeForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1279, __PRETTY_FUNCTION__))
;
1280 return RegisterTypeForVT[VT.SimpleTy];
1281 }
1282
1283 /// Return the type of registers that this ValueType will eventually require.
1284 MVT getRegisterType(LLVMContext &Context, EVT VT) const {
1285 if (VT.isSimple()) {
1286 assert((unsigned)VT.getSimpleVT().SimpleTy <(((unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegisterTypeForVT
)) ? static_cast<void> (0) : __assert_fail ("(unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegisterTypeForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1287, __PRETTY_FUNCTION__))
1287 array_lengthof(RegisterTypeForVT))(((unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegisterTypeForVT
)) ? static_cast<void> (0) : __assert_fail ("(unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(RegisterTypeForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1287, __PRETTY_FUNCTION__))
;
1288 return RegisterTypeForVT[VT.getSimpleVT().SimpleTy];
1289 }
1290 if (VT.isVector()) {
1291 EVT VT1;
1292 MVT RegisterVT;
1293 unsigned NumIntermediates;
1294 (void)getVectorTypeBreakdown(Context, VT, VT1,
1295 NumIntermediates, RegisterVT);
1296 return RegisterVT;
1297 }
1298 if (VT.isInteger()) {
1299 return getRegisterType(Context, getTypeToTransformTo(Context, VT));
1300 }
1301 llvm_unreachable("Unsupported extended type!")::llvm::llvm_unreachable_internal("Unsupported extended type!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1301)
;
1302 }
1303
1304 /// Return the number of registers that this ValueType will eventually
1305 /// require.
1306 ///
1307 /// This is one for any types promoted to live in larger registers, but may be
1308 /// more than one for types (like i64) that are split into pieces. For types
1309 /// like i140, which are first promoted then expanded, it is the number of
1310 /// registers needed to hold all the bits of the original type. For an i140
1311 /// on a 32 bit machine this means 5 registers.
1312 unsigned getNumRegisters(LLVMContext &Context, EVT VT) const {
1313 if (VT.isSimple()) {
1314 assert((unsigned)VT.getSimpleVT().SimpleTy <(((unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(NumRegistersForVT
)) ? static_cast<void> (0) : __assert_fail ("(unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(NumRegistersForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1315, __PRETTY_FUNCTION__))
1315 array_lengthof(NumRegistersForVT))(((unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(NumRegistersForVT
)) ? static_cast<void> (0) : __assert_fail ("(unsigned)VT.getSimpleVT().SimpleTy < array_lengthof(NumRegistersForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1315, __PRETTY_FUNCTION__))
;
1316 return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
1317 }
1318 if (VT.isVector()) {
1319 EVT VT1;
1320 MVT VT2;
1321 unsigned NumIntermediates;
1322 return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2);
1323 }
1324 if (VT.isInteger()) {
1325 unsigned BitWidth = VT.getSizeInBits();
1326 unsigned RegWidth = getRegisterType(Context, VT).getSizeInBits();
1327 return (BitWidth + RegWidth - 1) / RegWidth;
1328 }
1329 llvm_unreachable("Unsupported extended type!")::llvm::llvm_unreachable_internal("Unsupported extended type!"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1329)
;
1330 }
1331
1332 /// Certain combinations of ABIs, Targets and features require that types
1333 /// are legal for some operations and not for other operations.
1334 /// For MIPS all vector types must be passed through the integer register set.
1335 virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context,
1336 CallingConv::ID CC, EVT VT) const {
1337 return getRegisterType(Context, VT);
1338 }
1339
1340 /// Certain targets require unusual breakdowns of certain types. For MIPS,
1341 /// this occurs when a vector type is used, as vector are passed through the
1342 /// integer register set.
1343 virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context,
1344 CallingConv::ID CC,
1345 EVT VT) const {
1346 return getNumRegisters(Context, VT);
1347 }
1348
1349 /// Certain targets have context senstive alignment requirements, where one
1350 /// type has the alignment requirement of another type.
1351 virtual Align getABIAlignmentForCallingConv(Type *ArgTy,
1352 DataLayout DL) const {
1353 return Align(DL.getABITypeAlignment(ArgTy));
1354 }
1355
1356 /// If true, then instruction selection should seek to shrink the FP constant
1357 /// of the specified type to a smaller type in order to save space and / or
1358 /// reduce runtime.
1359 virtual bool ShouldShrinkFPConstant(EVT) const { return true; }
1360
1361 /// Return true if it is profitable to reduce a load to a smaller type.
1362 /// Example: (i16 (trunc (i32 (load x))) -> i16 load x
1363 virtual bool shouldReduceLoadWidth(SDNode *Load, ISD::LoadExtType ExtTy,
1364 EVT NewVT) const {
1365 // By default, assume that it is cheaper to extract a subvector from a wide
1366 // vector load rather than creating multiple narrow vector loads.
1367 if (NewVT.isVector() && !Load->hasOneUse())
1368 return false;
1369
1370 return true;
1371 }
1372
1373 /// When splitting a value of the specified type into parts, does the Lo
1374 /// or Hi part come first? This usually follows the endianness, except
1375 /// for ppcf128, where the Hi part always comes first.
1376 bool hasBigEndianPartOrdering(EVT VT, const DataLayout &DL) const {
1377 return DL.isBigEndian() || VT == MVT::ppcf128;
1378 }
1379
1380 /// If true, the target has custom DAG combine transformations that it can
1381 /// perform for the specified node.
1382 bool hasTargetDAGCombine(ISD::NodeType NT) const {
1383 assert(unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray))((unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray
)) ? static_cast<void> (0) : __assert_fail ("unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1383, __PRETTY_FUNCTION__))
;
1384 return TargetDAGCombineArray[NT >> 3] & (1 << (NT&7));
1385 }
1386
1387 unsigned getGatherAllAliasesMaxDepth() const {
1388 return GatherAllAliasesMaxDepth;
1389 }
1390
1391 /// Returns the size of the platform's va_list object.
1392 virtual unsigned getVaListSizeInBits(const DataLayout &DL) const {
1393 return getPointerTy(DL).getSizeInBits();
1394 }
1395
1396 /// Get maximum # of store operations permitted for llvm.memset
1397 ///
1398 /// This function returns the maximum number of store operations permitted
1399 /// to replace a call to llvm.memset. The value is set by the target at the
1400 /// performance threshold for such a replacement. If OptSize is true,
1401 /// return the limit for functions that have OptSize attribute.
1402 unsigned getMaxStoresPerMemset(bool OptSize) const {
1403 return OptSize ? MaxStoresPerMemsetOptSize : MaxStoresPerMemset;
1404 }
1405
1406 /// Get maximum # of store operations permitted for llvm.memcpy
1407 ///
1408 /// This function returns the maximum number of store operations permitted
1409 /// to replace a call to llvm.memcpy. The value is set by the target at the
1410 /// performance threshold for such a replacement. If OptSize is true,
1411 /// return the limit for functions that have OptSize attribute.
1412 unsigned getMaxStoresPerMemcpy(bool OptSize) const {
1413 return OptSize ? MaxStoresPerMemcpyOptSize : MaxStoresPerMemcpy;
1414 }
1415
1416 /// \brief Get maximum # of store operations to be glued together
1417 ///
1418 /// This function returns the maximum number of store operations permitted
1419 /// to glue together during lowering of llvm.memcpy. The value is set by
1420 // the target at the performance threshold for such a replacement.
1421 virtual unsigned getMaxGluedStoresPerMemcpy() const {
1422 return MaxGluedStoresPerMemcpy;
1423 }
1424
1425 /// Get maximum # of load operations permitted for memcmp
1426 ///
1427 /// This function returns the maximum number of load operations permitted
1428 /// to replace a call to memcmp. The value is set by the target at the
1429 /// performance threshold for such a replacement. If OptSize is true,
1430 /// return the limit for functions that have OptSize attribute.
1431 unsigned getMaxExpandSizeMemcmp(bool OptSize) const {
1432 return OptSize ? MaxLoadsPerMemcmpOptSize : MaxLoadsPerMemcmp;
1433 }
1434
1435 /// Get maximum # of store operations permitted for llvm.memmove
1436 ///
1437 /// This function returns the maximum number of store operations permitted
1438 /// to replace a call to llvm.memmove. The value is set by the target at the
1439 /// performance threshold for such a replacement. If OptSize is true,
1440 /// return the limit for functions that have OptSize attribute.
1441 unsigned getMaxStoresPerMemmove(bool OptSize) const {
1442 return OptSize ? MaxStoresPerMemmoveOptSize : MaxStoresPerMemmove;
1443 }
1444
1445 /// Determine if the target supports unaligned memory accesses.
1446 ///
1447 /// This function returns true if the target allows unaligned memory accesses
1448 /// of the specified type in the given address space. If true, it also returns
1449 /// whether the unaligned memory access is "fast" in the last argument by
1450 /// reference. This is used, for example, in situations where an array
1451 /// copy/move/set is converted to a sequence of store operations. Its use
1452 /// helps to ensure that such replacements don't generate code that causes an
1453 /// alignment error (trap) on the target machine.
1454 virtual bool allowsMisalignedMemoryAccesses(
1455 EVT, unsigned AddrSpace = 0, unsigned Align = 1,
1456 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
1457 bool * /*Fast*/ = nullptr) const {
1458 return false;
1459 }
1460
1461 /// LLT handling variant.
1462 virtual bool allowsMisalignedMemoryAccesses(
1463 LLT, unsigned AddrSpace = 0, unsigned Align = 1,
1464 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
1465 bool * /*Fast*/ = nullptr) const {
1466 return false;
1467 }
1468
1469 /// This function returns true if the memory access is aligned or if the
1470 /// target allows this specific unaligned memory access. If the access is
1471 /// allowed, the optional final parameter returns if the access is also fast
1472 /// (as defined by the target).
1473 bool allowsMemoryAccessForAlignment(
1474 LLVMContext &Context, const DataLayout &DL, EVT VT,
1475 unsigned AddrSpace = 0, unsigned Alignment = 1,
1476 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
1477 bool *Fast = nullptr) const;
1478
1479 /// Return true if the memory access of this type is aligned or if the target
1480 /// allows this specific unaligned access for the given MachineMemOperand.
1481 /// If the access is allowed, the optional final parameter returns if the
1482 /// access is also fast (as defined by the target).
1483 bool allowsMemoryAccessForAlignment(LLVMContext &Context,
1484 const DataLayout &DL, EVT VT,
1485 const MachineMemOperand &MMO,
1486 bool *Fast = nullptr) const;
1487
1488 /// Return true if the target supports a memory access of this type for the
1489 /// given address space and alignment. If the access is allowed, the optional
1490 /// final parameter returns if the access is also fast (as defined by the
1491 /// target).
1492 virtual bool
1493 allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
1494 unsigned AddrSpace = 0, unsigned Alignment = 1,
1495 MachineMemOperand::Flags Flags = MachineMemOperand::MONone,
1496 bool *Fast = nullptr) const;
1497
1498 /// Return true if the target supports a memory access of this type for the
1499 /// given MachineMemOperand. If the access is allowed, the optional
1500 /// final parameter returns if the access is also fast (as defined by the
1501 /// target).
1502 bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT,
1503 const MachineMemOperand &MMO,
1504 bool *Fast = nullptr) const;
1505
1506 /// Returns the target specific optimal type for load and store operations as
1507 /// a result of memset, memcpy, and memmove lowering.
1508 ///
1509 /// If DstAlign is zero that means it's safe to destination alignment can
1510 /// satisfy any constraint. Similarly if SrcAlign is zero it means there isn't
1511 /// a need to check it against alignment requirement, probably because the
1512 /// source does not need to be loaded. If 'IsMemset' is true, that means it's
1513 /// expanding a memset. If 'ZeroMemset' is true, that means it's a memset of
1514 /// zero. 'MemcpyStrSrc' indicates whether the memcpy source is constant so it
1515 /// does not need to be loaded. It returns EVT::Other if the type should be
1516 /// determined using generic target-independent logic.
1517 virtual EVT
1518 getOptimalMemOpType(uint64_t /*Size*/, unsigned /*DstAlign*/,
1519 unsigned /*SrcAlign*/, bool /*IsMemset*/,
1520 bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/,
1521 const AttributeList & /*FuncAttributes*/) const {
1522 return MVT::Other;
1523 }
1524
1525
1526 /// LLT returning variant.
1527 virtual LLT
1528 getOptimalMemOpLLT(uint64_t /*Size*/, unsigned /*DstAlign*/,
1529 unsigned /*SrcAlign*/, bool /*IsMemset*/,
1530 bool /*ZeroMemset*/, bool /*MemcpyStrSrc*/,
1531 const AttributeList & /*FuncAttributes*/) const {
1532 return LLT();
1533 }
1534
1535 /// Returns true if it's safe to use load / store of the specified type to
1536 /// expand memcpy / memset inline.
1537 ///
1538 /// This is mostly true for all types except for some special cases. For
1539 /// example, on X86 targets without SSE2 f64 load / store are done with fldl /
1540 /// fstpl which also does type conversion. Note the specified type doesn't
1541 /// have to be legal as the hook is used before type legalization.
1542 virtual bool isSafeMemOpType(MVT /*VT*/) const { return true; }
1543
1544 /// Determine if we should use _setjmp or setjmp to implement llvm.setjmp.
1545 bool usesUnderscoreSetJmp() const {
1546 return UseUnderscoreSetJmp;
1547 }
1548
1549 /// Determine if we should use _longjmp or longjmp to implement llvm.longjmp.
1550 bool usesUnderscoreLongJmp() const {
1551 return UseUnderscoreLongJmp;
1552 }
1553
1554 /// Return lower limit for number of blocks in a jump table.
1555 virtual unsigned getMinimumJumpTableEntries() const;
1556
1557 /// Return lower limit of the density in a jump table.
1558 unsigned getMinimumJumpTableDensity(bool OptForSize) const;
1559
1560 /// Return upper limit for number of entries in a jump table.
1561 /// Zero if no limit.
1562 unsigned getMaximumJumpTableSize() const;
1563
1564 virtual bool isJumpTableRelative() const {
1565 return TM.isPositionIndependent();
1566 }
1567
1568 /// If a physical register, this specifies the register that
1569 /// llvm.savestack/llvm.restorestack should save and restore.
1570 unsigned getStackPointerRegisterToSaveRestore() const {
1571 return StackPointerRegisterToSaveRestore;
1572 }
1573
1574 /// If a physical register, this returns the register that receives the
1575 /// exception address on entry to an EH pad.
1576 virtual unsigned
1577 getExceptionPointerRegister(const Constant *PersonalityFn) const {
1578 // 0 is guaranteed to be the NoRegister value on all targets
1579 return 0;
1580 }
1581
1582 /// If a physical register, this returns the register that receives the
1583 /// exception typeid on entry to a landing pad.
1584 virtual unsigned
1585 getExceptionSelectorRegister(const Constant *PersonalityFn) const {
1586 // 0 is guaranteed to be the NoRegister value on all targets
1587 return 0;
1588 }
1589
1590 virtual bool needsFixedCatchObjects() const {
1591 report_fatal_error("Funclet EH is not implemented for this target");
1592 }
1593
1594 /// Return the minimum stack alignment of an argument.
1595 Align getMinStackArgumentAlignment() const {
1596 return MinStackArgumentAlignment;
1597 }
1598
1599 /// Return the minimum function alignment.
1600 Align getMinFunctionAlignment() const { return MinFunctionAlignment; }
1601
1602 /// Return the preferred function alignment.
1603 Align getPrefFunctionAlignment() const { return PrefFunctionAlignment; }
1604
1605 /// Return the preferred loop alignment.
1606 virtual Align getPrefLoopAlignment(MachineLoop *ML = nullptr) const {
1607 return PrefLoopAlignment;
1608 }
1609
1610 /// Should loops be aligned even when the function is marked OptSize (but not
1611 /// MinSize).
1612 virtual bool alignLoopsWithOptSize() const {
1613 return false;
1614 }
1615
1616 /// If the target has a standard location for the stack protector guard,
1617 /// returns the address of that location. Otherwise, returns nullptr.
1618 /// DEPRECATED: please override useLoadStackGuardNode and customize
1619 /// LOAD_STACK_GUARD, or customize \@llvm.stackguard().
1620 virtual Value *getIRStackGuard(IRBuilder<> &IRB) const;
1621
1622 /// Inserts necessary declarations for SSP (stack protection) purpose.
1623 /// Should be used only when getIRStackGuard returns nullptr.
1624 virtual void insertSSPDeclarations(Module &M) const;
1625
1626 /// Return the variable that's previously inserted by insertSSPDeclarations,
1627 /// if any, otherwise return nullptr. Should be used only when
1628 /// getIRStackGuard returns nullptr.
1629 virtual Value *getSDagStackGuard(const Module &M) const;
1630
1631 /// If this function returns true, stack protection checks should XOR the
1632 /// frame pointer (or whichever pointer is used to address locals) into the
1633 /// stack guard value before checking it. getIRStackGuard must return nullptr
1634 /// if this returns true.
1635 virtual bool useStackGuardXorFP() const { return false; }
1636
1637 /// If the target has a standard stack protection check function that
1638 /// performs validation and error handling, returns the function. Otherwise,
1639 /// returns nullptr. Must be previously inserted by insertSSPDeclarations.
1640 /// Should be used only when getIRStackGuard returns nullptr.
1641 virtual Function *getSSPStackGuardCheck(const Module &M) const;
1642
1643protected:
1644 Value *getDefaultSafeStackPointerLocation(IRBuilder<> &IRB,
1645 bool UseTLS) const;
1646
1647public:
1648 /// Returns the target-specific address of the unsafe stack pointer.
1649 virtual Value *getSafeStackPointerLocation(IRBuilder<> &IRB) const;
1650
1651 /// Returns the name of the symbol used to emit stack probes or the empty
1652 /// string if not applicable.
1653 virtual StringRef getStackProbeSymbolName(MachineFunction &MF) const {
1654 return "";
1655 }
1656
1657 /// Returns true if a cast between SrcAS and DestAS is a noop.
1658 virtual bool isNoopAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
1659 return false;
1660 }
1661
1662 /// Returns true if a cast from SrcAS to DestAS is "cheap", such that e.g. we
1663 /// are happy to sink it into basic blocks. A cast may be free, but not
1664 /// necessarily a no-op. e.g. a free truncate from a 64-bit to 32-bit pointer.
1665 virtual bool isFreeAddrSpaceCast(unsigned SrcAS, unsigned DestAS) const {
1666 return isNoopAddrSpaceCast(SrcAS, DestAS);
1667 }
1668
1669 /// Return true if the pointer arguments to CI should be aligned by aligning
1670 /// the object whose address is being passed. If so then MinSize is set to the
1671 /// minimum size the object must be to be aligned and PrefAlign is set to the
1672 /// preferred alignment.
1673 virtual bool shouldAlignPointerArgs(CallInst * /*CI*/, unsigned & /*MinSize*/,
1674 unsigned & /*PrefAlign*/) const {
1675 return false;
1676 }
1677
1678 //===--------------------------------------------------------------------===//
1679 /// \name Helpers for TargetTransformInfo implementations
1680 /// @{
1681
1682 /// Get the ISD node that corresponds to the Instruction class opcode.
1683 int InstructionOpcodeToISD(unsigned Opcode) const;
1684
1685 /// Estimate the cost of type-legalization and the legalized type.
1686 std::pair<int, MVT> getTypeLegalizationCost(const DataLayout &DL,
1687 Type *Ty) const;
1688
1689 /// @}
1690
1691 //===--------------------------------------------------------------------===//
1692 /// \name Helpers for atomic expansion.
1693 /// @{
1694
1695 /// Returns the maximum atomic operation size (in bits) supported by
1696 /// the backend. Atomic operations greater than this size (as well
1697 /// as ones that are not naturally aligned), will be expanded by
1698 /// AtomicExpandPass into an __atomic_* library call.
1699 unsigned getMaxAtomicSizeInBitsSupported() const {
1700 return MaxAtomicSizeInBitsSupported;
1701 }
1702
1703 /// Returns the size of the smallest cmpxchg or ll/sc instruction
1704 /// the backend supports. Any smaller operations are widened in
1705 /// AtomicExpandPass.
1706 ///
1707 /// Note that *unlike* operations above the maximum size, atomic ops
1708 /// are still natively supported below the minimum; they just
1709 /// require a more complex expansion.
1710 unsigned getMinCmpXchgSizeInBits() const { return MinCmpXchgSizeInBits; }
1711
1712 /// Whether the target supports unaligned atomic operations.
1713 bool supportsUnalignedAtomics() const { return SupportsUnalignedAtomics; }
1714
1715 /// Whether AtomicExpandPass should automatically insert fences and reduce
1716 /// ordering for this atomic. This should be true for most architectures with
1717 /// weak memory ordering. Defaults to false.
1718 virtual bool shouldInsertFencesForAtomic(const Instruction *I) const {
1719 return false;
1720 }
1721
1722 /// Perform a load-linked operation on Addr, returning a "Value *" with the
1723 /// corresponding pointee type. This may entail some non-trivial operations to
1724 /// truncate or reconstruct types that will be illegal in the backend. See
1725 /// ARMISelLowering for an example implementation.
1726 virtual Value *emitLoadLinked(IRBuilder<> &Builder, Value *Addr,
1727 AtomicOrdering Ord) const {
1728 llvm_unreachable("Load linked unimplemented on this target")::llvm::llvm_unreachable_internal("Load linked unimplemented on this target"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1728)
;
1729 }
1730
1731 /// Perform a store-conditional operation to Addr. Return the status of the
1732 /// store. This should be 0 if the store succeeded, non-zero otherwise.
1733 virtual Value *emitStoreConditional(IRBuilder<> &Builder, Value *Val,
1734 Value *Addr, AtomicOrdering Ord) const {
1735 llvm_unreachable("Store conditional unimplemented on this target")::llvm::llvm_unreachable_internal("Store conditional unimplemented on this target"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1735)
;
1736 }
1737
1738 /// Perform a masked atomicrmw using a target-specific intrinsic. This
1739 /// represents the core LL/SC loop which will be lowered at a late stage by
1740 /// the backend.
1741 virtual Value *emitMaskedAtomicRMWIntrinsic(IRBuilder<> &Builder,
1742 AtomicRMWInst *AI,
1743 Value *AlignedAddr, Value *Incr,
1744 Value *Mask, Value *ShiftAmt,
1745 AtomicOrdering Ord) const {
1746 llvm_unreachable("Masked atomicrmw expansion unimplemented on this target")::llvm::llvm_unreachable_internal("Masked atomicrmw expansion unimplemented on this target"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1746)
;
1747 }
1748
1749 /// Perform a masked cmpxchg using a target-specific intrinsic. This
1750 /// represents the core LL/SC loop which will be lowered at a late stage by
1751 /// the backend.
1752 virtual Value *emitMaskedAtomicCmpXchgIntrinsic(
1753 IRBuilder<> &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
1754 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
1755 llvm_unreachable("Masked cmpxchg expansion unimplemented on this target")::llvm::llvm_unreachable_internal("Masked cmpxchg expansion unimplemented on this target"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 1755)
;
1756 }
1757
1758 /// Inserts in the IR a target-specific intrinsic specifying a fence.
1759 /// It is called by AtomicExpandPass before expanding an
1760 /// AtomicRMW/AtomicCmpXchg/AtomicStore/AtomicLoad
1761 /// if shouldInsertFencesForAtomic returns true.
1762 ///
1763 /// Inst is the original atomic instruction, prior to other expansions that
1764 /// may be performed.
1765 ///
1766 /// This function should either return a nullptr, or a pointer to an IR-level
1767 /// Instruction*. Even complex fence sequences can be represented by a
1768 /// single Instruction* through an intrinsic to be lowered later.
1769 /// Backends should override this method to produce target-specific intrinsic
1770 /// for their fences.
1771 /// FIXME: Please note that the default implementation here in terms of
1772 /// IR-level fences exists for historical/compatibility reasons and is
1773 /// *unsound* ! Fences cannot, in general, be used to restore sequential
1774 /// consistency. For example, consider the following example:
1775 /// atomic<int> x = y = 0;
1776 /// int r1, r2, r3, r4;
1777 /// Thread 0:
1778 /// x.store(1);
1779 /// Thread 1:
1780 /// y.store(1);
1781 /// Thread 2:
1782 /// r1 = x.load();
1783 /// r2 = y.load();
1784 /// Thread 3:
1785 /// r3 = y.load();
1786 /// r4 = x.load();
1787 /// r1 = r3 = 1 and r2 = r4 = 0 is impossible as long as the accesses are all
1788 /// seq_cst. But if they are lowered to monotonic accesses, no amount of
1789 /// IR-level fences can prevent it.
1790 /// @{
1791 virtual Instruction *emitLeadingFence(IRBuilder<> &Builder, Instruction *Inst,
1792 AtomicOrdering Ord) const {
1793 if (isReleaseOrStronger(Ord) && Inst->hasAtomicStore())
1794 return Builder.CreateFence(Ord);
1795 else
1796 return nullptr;
1797 }
1798
1799 virtual Instruction *emitTrailingFence(IRBuilder<> &Builder,
1800 Instruction *Inst,
1801 AtomicOrdering Ord) const {
1802 if (isAcquireOrStronger(Ord))
1803 return Builder.CreateFence(Ord);
1804 else
1805 return nullptr;
1806 }
1807 /// @}
1808
1809 // Emits code that executes when the comparison result in the ll/sc
1810 // expansion of a cmpxchg instruction is such that the store-conditional will
1811 // not execute. This makes it possible to balance out the load-linked with
1812 // a dedicated instruction, if desired.
1813 // E.g., on ARM, if ldrex isn't followed by strex, the exclusive monitor would
1814 // be unnecessarily held, except if clrex, inserted by this hook, is executed.
1815 virtual void emitAtomicCmpXchgNoStoreLLBalance(IRBuilder<> &Builder) const {}
1816
1817 /// Returns true if the given (atomic) store should be expanded by the
1818 /// IR-level AtomicExpand pass into an "atomic xchg" which ignores its input.
1819 virtual bool shouldExpandAtomicStoreInIR(StoreInst *SI) const {
1820 return false;
1821 }
1822
1823 /// Returns true if arguments should be sign-extended in lib calls.
1824 virtual bool shouldSignExtendTypeInLibCall(EVT Type, bool IsSigned) const {
1825 return IsSigned;
1826 }
1827
1828 /// Returns true if arguments should be extended in lib calls.
1829 virtual bool shouldExtendTypeInLibCall(EVT Type) const {
1830 return true;
1831 }
1832
1833 /// Returns how the given (atomic) load should be expanded by the
1834 /// IR-level AtomicExpand pass.
1835 virtual AtomicExpansionKind shouldExpandAtomicLoadInIR(LoadInst *LI) const {
1836 return AtomicExpansionKind::None;
1837 }
1838
1839 /// Returns how the given atomic cmpxchg should be expanded by the IR-level
1840 /// AtomicExpand pass.
1841 virtual AtomicExpansionKind
1842 shouldExpandAtomicCmpXchgInIR(AtomicCmpXchgInst *AI) const {
1843 return AtomicExpansionKind::None;
1844 }
1845
1846 /// Returns how the IR-level AtomicExpand pass should expand the given
1847 /// AtomicRMW, if at all. Default is to never expand.
1848 virtual AtomicExpansionKind shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
1849 return RMW->isFloatingPointOperation() ?
1850 AtomicExpansionKind::CmpXChg : AtomicExpansionKind::None;
1851 }
1852
1853 /// On some platforms, an AtomicRMW that never actually modifies the value
1854 /// (such as fetch_add of 0) can be turned into a fence followed by an
1855 /// atomic load. This may sound useless, but it makes it possible for the
1856 /// processor to keep the cacheline shared, dramatically improving
1857 /// performance. And such idempotent RMWs are useful for implementing some
1858 /// kinds of locks, see for example (justification + benchmarks):
1859 /// http://www.hpl.hp.com/techreports/2012/HPL-2012-68.pdf
1860 /// This method tries doing that transformation, returning the atomic load if
1861 /// it succeeds, and nullptr otherwise.
1862 /// If shouldExpandAtomicLoadInIR returns true on that load, it will undergo
1863 /// another round of expansion.
1864 virtual LoadInst *
1865 lowerIdempotentRMWIntoFencedLoad(AtomicRMWInst *RMWI) const {
1866 return nullptr;
1867 }
1868
1869 /// Returns how the platform's atomic operations are extended (ZERO_EXTEND,
1870 /// SIGN_EXTEND, or ANY_EXTEND).
1871 virtual ISD::NodeType getExtendForAtomicOps() const {
1872 return ISD::ZERO_EXTEND;
1873 }
1874
1875 /// @}
1876
1877 /// Returns true if we should normalize
1878 /// select(N0&N1, X, Y) => select(N0, select(N1, X, Y), Y) and
1879 /// select(N0|N1, X, Y) => select(N0, select(N1, X, Y, Y)) if it is likely
1880 /// that it saves us from materializing N0 and N1 in an integer register.
1881 /// Targets that are able to perform and/or on flags should return false here.
1882 virtual bool shouldNormalizeToSelectSequence(LLVMContext &Context,
1883 EVT VT) const {
1884 // If a target has multiple condition registers, then it likely has logical
1885 // operations on those registers.
1886 if (hasMultipleConditionRegisters())
1887 return false;
1888 // Only do the transform if the value won't be split into multiple
1889 // registers.
1890 LegalizeTypeAction Action = getTypeAction(Context, VT);
1891 return Action != TypeExpandInteger && Action != TypeExpandFloat &&
1892 Action != TypeSplitVector;
1893 }
1894
1895 virtual bool isProfitableToCombineMinNumMaxNum(EVT VT) const { return true; }
1896
1897 /// Return true if a select of constants (select Cond, C1, C2) should be
1898 /// transformed into simple math ops with the condition value. For example:
1899 /// select Cond, C1, C1-1 --> add (zext Cond), C1-1
1900 virtual bool convertSelectOfConstantsToMath(EVT VT) const {
1901 return false;
1902 }
1903
1904 /// Return true if it is profitable to transform an integer
1905 /// multiplication-by-constant into simpler operations like shifts and adds.
1906 /// This may be true if the target does not directly support the
1907 /// multiplication operation for the specified type or the sequence of simpler
1908 /// ops is faster than the multiply.
1909 virtual bool decomposeMulByConstant(LLVMContext &Context,
1910 EVT VT, SDValue C) const {
1911 return false;
1912 }
1913
1914 /// Return true if it is more correct/profitable to use strict FP_TO_INT
1915 /// conversion operations - canonicalizing the FP source value instead of
1916 /// converting all cases and then selecting based on value.
1917 /// This may be true if the target throws exceptions for out of bounds
1918 /// conversions or has fast FP CMOV.
1919 virtual bool shouldUseStrictFP_TO_INT(EVT FpVT, EVT IntVT,
1920 bool IsSigned) const {
1921 return false;
1922 }
1923
1924 //===--------------------------------------------------------------------===//
1925 // TargetLowering Configuration Methods - These methods should be invoked by
1926 // the derived class constructor to configure this object for the target.
1927 //
1928protected:
1929 /// Specify how the target extends the result of integer and floating point
1930 /// boolean values from i1 to a wider type. See getBooleanContents.
1931 void setBooleanContents(BooleanContent Ty) {
1932 BooleanContents = Ty;
1933 BooleanFloatContents = Ty;
1934 }
1935
1936 /// Specify how the target extends the result of integer and floating point
1937 /// boolean values from i1 to a wider type. See getBooleanContents.
1938 void setBooleanContents(BooleanContent IntTy, BooleanContent FloatTy) {
1939 BooleanContents = IntTy;
1940 BooleanFloatContents = FloatTy;
1941 }
1942
1943 /// Specify how the target extends the result of a vector boolean value from a
1944 /// vector of i1 to a wider type. See getBooleanContents.
1945 void setBooleanVectorContents(BooleanContent Ty) {
1946 BooleanVectorContents = Ty;
1947 }
1948
1949 /// Specify the target scheduling preference.
1950 void setSchedulingPreference(Sched::Preference Pref) {
1951 SchedPreferenceInfo = Pref;
1952 }
1953
1954 /// Indicate whether this target prefers to use _setjmp to implement
1955 /// llvm.setjmp or the version without _. Defaults to false.
1956 void setUseUnderscoreSetJmp(bool Val) {
1957 UseUnderscoreSetJmp = Val;
1958 }
1959
1960 /// Indicate whether this target prefers to use _longjmp to implement
1961 /// llvm.longjmp or the version without _. Defaults to false.
1962 void setUseUnderscoreLongJmp(bool Val) {
1963 UseUnderscoreLongJmp = Val;
1964 }
1965
1966 /// Indicate the minimum number of blocks to generate jump tables.
1967 void setMinimumJumpTableEntries(unsigned Val);
1968
1969 /// Indicate the maximum number of entries in jump tables.
1970 /// Set to zero to generate unlimited jump tables.
1971 void setMaximumJumpTableSize(unsigned);
1972
1973 /// If set to a physical register, this specifies the register that
1974 /// llvm.savestack/llvm.restorestack should save and restore.
1975 void setStackPointerRegisterToSaveRestore(unsigned R) {
1976 StackPointerRegisterToSaveRestore = R;
1977 }
1978
1979 /// Tells the code generator that the target has multiple (allocatable)
1980 /// condition registers that can be used to store the results of comparisons
1981 /// for use by selects and conditional branches. With multiple condition
1982 /// registers, the code generator will not aggressively sink comparisons into
1983 /// the blocks of their users.
1984 void setHasMultipleConditionRegisters(bool hasManyRegs = true) {
1985 HasMultipleConditionRegisters = hasManyRegs;
1986 }
1987
1988 /// Tells the code generator that the target has BitExtract instructions.
1989 /// The code generator will aggressively sink "shift"s into the blocks of
1990 /// their users if the users will generate "and" instructions which can be
1991 /// combined with "shift" to BitExtract instructions.
1992 void setHasExtractBitsInsn(bool hasExtractInsn = true) {
1993 HasExtractBitsInsn = hasExtractInsn;
1994 }
1995
1996 /// Tells the code generator not to expand logic operations on comparison
1997 /// predicates into separate sequences that increase the amount of flow
1998 /// control.
1999 void setJumpIsExpensive(bool isExpensive = true);
2000
2001 /// Tells the code generator which bitwidths to bypass.
2002 void addBypassSlowDiv(unsigned int SlowBitWidth, unsigned int FastBitWidth) {
2003 BypassSlowDivWidths[SlowBitWidth] = FastBitWidth;
2004 }
2005
2006 /// Add the specified register class as an available regclass for the
2007 /// specified value type. This indicates the selector can handle values of
2008 /// that class natively.
2009 void addRegisterClass(MVT VT, const TargetRegisterClass *RC) {
2010 assert((unsigned)VT.SimpleTy < array_lengthof(RegClassForVT))(((unsigned)VT.SimpleTy < array_lengthof(RegClassForVT)) ?
static_cast<void> (0) : __assert_fail ("(unsigned)VT.SimpleTy < array_lengthof(RegClassForVT)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2010, __PRETTY_FUNCTION__))
;
2011 RegClassForVT[VT.SimpleTy] = RC;
2012 }
2013
2014 /// Return the largest legal super-reg register class of the register class
2015 /// for the specified type and its associated "cost".
2016 virtual std::pair<const TargetRegisterClass *, uint8_t>
2017 findRepresentativeClass(const TargetRegisterInfo *TRI, MVT VT) const;
2018
2019 /// Once all of the register classes are added, this allows us to compute
2020 /// derived properties we expose.
2021 void computeRegisterProperties(const TargetRegisterInfo *TRI);
2022
2023 /// Indicate that the specified operation does not work with the specified
2024 /// type and indicate what to do about it. Note that VT may refer to either
2025 /// the type of a result or that of an operand of Op.
2026 void setOperationAction(unsigned Op, MVT VT,
2027 LegalizeAction Action) {
2028 assert(Op < array_lengthof(OpActions[0]) && "Table isn't big enough!")((Op < array_lengthof(OpActions[0]) && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("Op < array_lengthof(OpActions[0]) && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2028, __PRETTY_FUNCTION__))
;
2029 OpActions[(unsigned)VT.SimpleTy][Op] = Action;
2030 }
2031
2032 /// Indicate that the specified load with extension does not work with the
2033 /// specified type and indicate what to do about it.
2034 void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT,
2035 LegalizeAction Action) {
2036 assert(ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() &&((ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid
() && MemVT.isValid() && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() && MemVT.isValid() && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2037, __PRETTY_FUNCTION__))
2037 MemVT.isValid() && "Table isn't big enough!")((ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid
() && MemVT.isValid() && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("ExtType < ISD::LAST_LOADEXT_TYPE && ValVT.isValid() && MemVT.isValid() && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2037, __PRETTY_FUNCTION__))
;
2038 assert((unsigned)Action < 0x10 && "too many bits for bitfield array")(((unsigned)Action < 0x10 && "too many bits for bitfield array"
) ? static_cast<void> (0) : __assert_fail ("(unsigned)Action < 0x10 && \"too many bits for bitfield array\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2038, __PRETTY_FUNCTION__))
;
2039 unsigned Shift = 4 * ExtType;
2040 LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] &= ~((uint16_t)0xF << Shift);
2041 LoadExtActions[ValVT.SimpleTy][MemVT.SimpleTy] |= (uint16_t)Action << Shift;
2042 }
2043
2044 /// Indicate that the specified truncating store does not work with the
2045 /// specified type and indicate what to do about it.
2046 void setTruncStoreAction(MVT ValVT, MVT MemVT,
2047 LegalizeAction Action) {
2048 assert(ValVT.isValid() && MemVT.isValid() && "Table isn't big enough!")((ValVT.isValid() && MemVT.isValid() && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("ValVT.isValid() && MemVT.isValid() && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2048, __PRETTY_FUNCTION__))
;
2049 TruncStoreActions[(unsigned)ValVT.SimpleTy][MemVT.SimpleTy] = Action;
2050 }
2051
2052 /// Indicate that the specified indexed load does or does not work with the
2053 /// specified type and indicate what to do abort it.
2054 ///
2055 /// NOTE: All indexed mode loads are initialized to Expand in
2056 /// TargetLowering.cpp
2057 void setIndexedLoadAction(unsigned IdxMode, MVT VT,
2058 LegalizeAction Action) {
2059 assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE &&((VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE
&& (unsigned)Action < 0xf && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE && (unsigned)Action < 0xf && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2060, __PRETTY_FUNCTION__))
2060 (unsigned)Action < 0xf && "Table isn't big enough!")((VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE
&& (unsigned)Action < 0xf && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE && (unsigned)Action < 0xf && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2060, __PRETTY_FUNCTION__))
;
2061 // Load action are kept in the upper half.
2062 IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] &= ~0xf0;
2063 IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] |= ((uint8_t)Action) <<4;
2064 }
2065
2066 /// Indicate that the specified indexed store does or does not work with the
2067 /// specified type and indicate what to do about it.
2068 ///
2069 /// NOTE: All indexed mode stores are initialized to Expand in
2070 /// TargetLowering.cpp
2071 void setIndexedStoreAction(unsigned IdxMode, MVT VT,
2072 LegalizeAction Action) {
2073 assert(VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE &&((VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE
&& (unsigned)Action < 0xf && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE && (unsigned)Action < 0xf && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2074, __PRETTY_FUNCTION__))
2074 (unsigned)Action < 0xf && "Table isn't big enough!")((VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE
&& (unsigned)Action < 0xf && "Table isn't big enough!"
) ? static_cast<void> (0) : __assert_fail ("VT.isValid() && IdxMode < ISD::LAST_INDEXED_MODE && (unsigned)Action < 0xf && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2074, __PRETTY_FUNCTION__))
;
2075 // Store action are kept in the lower half.
2076 IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] &= ~0x0f;
2077 IndexedModeActions[(unsigned)VT.SimpleTy][IdxMode] |= ((uint8_t)Action);
2078 }
2079
2080 /// Indicate that the specified condition code is or isn't supported on the
2081 /// target and indicate what to do about it.
2082 void setCondCodeAction(ISD::CondCode CC, MVT VT,
2083 LegalizeAction Action) {
2084 assert(VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions) &&((VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions
) && "Table isn't big enough!") ? static_cast<void
> (0) : __assert_fail ("VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions) && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2085, __PRETTY_FUNCTION__))
2085 "Table isn't big enough!")((VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions
) && "Table isn't big enough!") ? static_cast<void
> (0) : __assert_fail ("VT.isValid() && (unsigned)CC < array_lengthof(CondCodeActions) && \"Table isn't big enough!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2085, __PRETTY_FUNCTION__))
;
2086 assert((unsigned)Action < 0x10 && "too many bits for bitfield array")(((unsigned)Action < 0x10 && "too many bits for bitfield array"
) ? static_cast<void> (0) : __assert_fail ("(unsigned)Action < 0x10 && \"too many bits for bitfield array\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2086, __PRETTY_FUNCTION__))
;
2087 /// The lower 3 bits of the SimpleTy index into Nth 4bit set from the 32-bit
2088 /// value and the upper 29 bits index into the second dimension of the array
2089 /// to select what 32-bit value to use.
2090 uint32_t Shift = 4 * (VT.SimpleTy & 0x7);
2091 CondCodeActions[CC][VT.SimpleTy >> 3] &= ~((uint32_t)0xF << Shift);
2092 CondCodeActions[CC][VT.SimpleTy >> 3] |= (uint32_t)Action << Shift;
2093 }
2094
2095 /// If Opc/OrigVT is specified as being promoted, the promotion code defaults
2096 /// to trying a larger integer/fp until it can find one that works. If that
2097 /// default is insufficient, this method can be used by the target to override
2098 /// the default.
2099 void AddPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT) {
2100 PromoteToType[std::make_pair(Opc, OrigVT.SimpleTy)] = DestVT.SimpleTy;
2101 }
2102
2103 /// Convenience method to set an operation to Promote and specify the type
2104 /// in a single call.
2105 void setOperationPromotedToType(unsigned Opc, MVT OrigVT, MVT DestVT) {
2106 setOperationAction(Opc, OrigVT, Promote);
2107 AddPromotedToType(Opc, OrigVT, DestVT);
2108 }
2109
2110 /// Targets should invoke this method for each target independent node that
2111 /// they want to provide a custom DAG combiner for by implementing the
2112 /// PerformDAGCombine virtual method.
2113 void setTargetDAGCombine(ISD::NodeType NT) {
2114 assert(unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray))((unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray
)) ? static_cast<void> (0) : __assert_fail ("unsigned(NT >> 3) < array_lengthof(TargetDAGCombineArray)"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2114, __PRETTY_FUNCTION__))
;
2115 TargetDAGCombineArray[NT >> 3] |= 1 << (NT&7);
2116 }
2117
2118 /// Set the target's minimum function alignment.
2119 void setMinFunctionAlignment(Align Alignment) {
2120 MinFunctionAlignment = Alignment;
2121 }
2122
2123 /// Set the target's preferred function alignment. This should be set if
2124 /// there is a performance benefit to higher-than-minimum alignment
2125 void setPrefFunctionAlignment(Align Alignment) {
2126 PrefFunctionAlignment = Alignment;
2127 }
2128
2129 /// Set the target's preferred loop alignment. Default alignment is one, it
2130 /// means the target does not care about loop alignment. The target may also
2131 /// override getPrefLoopAlignment to provide per-loop values.
2132 void setPrefLoopAlignment(Align Alignment) { PrefLoopAlignment = Alignment; }
2133
2134 /// Set the minimum stack alignment of an argument.
2135 void setMinStackArgumentAlignment(Align Alignment) {
2136 MinStackArgumentAlignment = Alignment;
2137 }
2138
2139 /// Set the maximum atomic operation size supported by the
2140 /// backend. Atomic operations greater than this size (as well as
2141 /// ones that are not naturally aligned), will be expanded by
2142 /// AtomicExpandPass into an __atomic_* library call.
2143 void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits) {
2144 MaxAtomicSizeInBitsSupported = SizeInBits;
2145 }
2146
2147 /// Sets the minimum cmpxchg or ll/sc size supported by the backend.
2148 void setMinCmpXchgSizeInBits(unsigned SizeInBits) {
2149 MinCmpXchgSizeInBits = SizeInBits;
2150 }
2151
2152 /// Sets whether unaligned atomic operations are supported.
2153 void setSupportsUnalignedAtomics(bool UnalignedSupported) {
2154 SupportsUnalignedAtomics = UnalignedSupported;
2155 }
2156
2157public:
2158 //===--------------------------------------------------------------------===//
2159 // Addressing mode description hooks (used by LSR etc).
2160 //
2161
2162 /// CodeGenPrepare sinks address calculations into the same BB as Load/Store
2163 /// instructions reading the address. This allows as much computation as
2164 /// possible to be done in the address mode for that operand. This hook lets
2165 /// targets also pass back when this should be done on intrinsics which
2166 /// load/store.
2167 virtual bool getAddrModeArguments(IntrinsicInst * /*I*/,
2168 SmallVectorImpl<Value*> &/*Ops*/,
2169 Type *&/*AccessTy*/) const {
2170 return false;
2171 }
2172
2173 /// This represents an addressing mode of:
2174 /// BaseGV + BaseOffs + BaseReg + Scale*ScaleReg
2175 /// If BaseGV is null, there is no BaseGV.
2176 /// If BaseOffs is zero, there is no base offset.
2177 /// If HasBaseReg is false, there is no base register.
2178 /// If Scale is zero, there is no ScaleReg. Scale of 1 indicates a reg with
2179 /// no scale.
2180 struct AddrMode {
2181 GlobalValue *BaseGV = nullptr;
2182 int64_t BaseOffs = 0;
2183 bool HasBaseReg = false;
2184 int64_t Scale = 0;
2185 AddrMode() = default;
2186 };
2187
2188 /// Return true if the addressing mode represented by AM is legal for this
2189 /// target, for a load/store of the specified type.
2190 ///
2191 /// The type may be VoidTy, in which case only return true if the addressing
2192 /// mode is legal for a load/store of any legal type. TODO: Handle
2193 /// pre/postinc as well.
2194 ///
2195 /// If the address space cannot be determined, it will be -1.
2196 ///
2197 /// TODO: Remove default argument
2198 virtual bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM,
2199 Type *Ty, unsigned AddrSpace,
2200 Instruction *I = nullptr) const;
2201
2202 /// Return the cost of the scaling factor used in the addressing mode
2203 /// represented by AM for this target, for a load/store of the specified type.
2204 ///
2205 /// If the AM is supported, the return value must be >= 0.
2206 /// If the AM is not supported, it returns a negative value.
2207 /// TODO: Handle pre/postinc as well.
2208 /// TODO: Remove default argument
2209 virtual int getScalingFactorCost(const DataLayout &DL, const AddrMode &AM,
2210 Type *Ty, unsigned AS = 0) const {
2211 // Default: assume that any scaling factor used in a legal AM is free.
2212 if (isLegalAddressingMode(DL, AM, Ty, AS))
2213 return 0;
2214 return -1;
2215 }
2216
2217 /// Return true if the specified immediate is legal icmp immediate, that is
2218 /// the target has icmp instructions which can compare a register against the
2219 /// immediate without having to materialize the immediate into a register.
2220 virtual bool isLegalICmpImmediate(int64_t) const {
2221 return true;
2222 }
2223
2224 /// Return true if the specified immediate is legal add immediate, that is the
2225 /// target has add instructions which can add a register with the immediate
2226 /// without having to materialize the immediate into a register.
2227 virtual bool isLegalAddImmediate(int64_t) const {
2228 return true;
2229 }
2230
2231 /// Return true if the specified immediate is legal for the value input of a
2232 /// store instruction.
2233 virtual bool isLegalStoreImmediate(int64_t Value) const {
2234 // Default implementation assumes that at least 0 works since it is likely
2235 // that a zero register exists or a zero immediate is allowed.
2236 return Value == 0;
2237 }
2238
2239 /// Return true if it's significantly cheaper to shift a vector by a uniform
2240 /// scalar than by an amount which will vary across each lane. On x86, for
2241 /// example, there is a "psllw" instruction for the former case, but no simple
2242 /// instruction for a general "a << b" operation on vectors.
2243 virtual bool isVectorShiftByScalarCheap(Type *Ty) const {
2244 return false;
2245 }
2246
2247 /// Returns true if the opcode is a commutative binary operation.
2248 virtual bool isCommutativeBinOp(unsigned Opcode) const {
2249 // FIXME: This should get its info from the td file.
2250 switch (Opcode) {
2251 case ISD::ADD:
2252 case ISD::SMIN:
2253 case ISD::SMAX:
2254 case ISD::UMIN:
2255 case ISD::UMAX:
2256 case ISD::MUL:
2257 case ISD::MULHU:
2258 case ISD::MULHS:
2259 case ISD::SMUL_LOHI:
2260 case ISD::UMUL_LOHI:
2261 case ISD::FADD:
2262 case ISD::FMUL:
2263 case ISD::AND:
2264 case ISD::OR:
2265 case ISD::XOR:
2266 case ISD::SADDO:
2267 case ISD::UADDO:
2268 case ISD::ADDC:
2269 case ISD::ADDE:
2270 case ISD::SADDSAT:
2271 case ISD::UADDSAT:
2272 case ISD::FMINNUM:
2273 case ISD::FMAXNUM:
2274 case ISD::FMINNUM_IEEE:
2275 case ISD::FMAXNUM_IEEE:
2276 case ISD::FMINIMUM:
2277 case ISD::FMAXIMUM:
2278 return true;
2279 default: return false;
2280 }
2281 }
2282
2283 /// Return true if the node is a math/logic binary operator.
2284 virtual bool isBinOp(unsigned Opcode) const {
2285 // A commutative binop must be a binop.
2286 if (isCommutativeBinOp(Opcode))
2287 return true;
2288 // These are non-commutative binops.
2289 switch (Opcode) {
2290 case ISD::SUB:
2291 case ISD::SHL:
2292 case ISD::SRL:
2293 case ISD::SRA:
2294 case ISD::SDIV:
2295 case ISD::UDIV:
2296 case ISD::SREM:
2297 case ISD::UREM:
2298 case ISD::FSUB:
2299 case ISD::FDIV:
2300 case ISD::FREM:
2301 return true;
2302 default:
2303 return false;
2304 }
2305 }
2306
2307 /// Return true if it's free to truncate a value of type FromTy to type
2308 /// ToTy. e.g. On x86 it's free to truncate a i32 value in register EAX to i16
2309 /// by referencing its sub-register AX.
2310 /// Targets must return false when FromTy <= ToTy.
2311 virtual bool isTruncateFree(Type *FromTy, Type *ToTy) const {
2312 return false;
2313 }
2314
2315 /// Return true if a truncation from FromTy to ToTy is permitted when deciding
2316 /// whether a call is in tail position. Typically this means that both results
2317 /// would be assigned to the same register or stack slot, but it could mean
2318 /// the target performs adequate checks of its own before proceeding with the
2319 /// tail call. Targets must return false when FromTy <= ToTy.
2320 virtual bool allowTruncateForTailCall(Type *FromTy, Type *ToTy) const {
2321 return false;
2322 }
2323
2324 virtual bool isTruncateFree(EVT FromVT, EVT ToVT) const {
2325 return false;
2326 }
2327
2328 virtual bool isProfitableToHoist(Instruction *I) const { return true; }
2329
2330 /// Return true if the extension represented by \p I is free.
2331 /// Unlikely the is[Z|FP]ExtFree family which is based on types,
2332 /// this method can use the context provided by \p I to decide
2333 /// whether or not \p I is free.
2334 /// This method extends the behavior of the is[Z|FP]ExtFree family.
2335 /// In other words, if is[Z|FP]Free returns true, then this method
2336 /// returns true as well. The converse is not true.
2337 /// The target can perform the adequate checks by overriding isExtFreeImpl.
2338 /// \pre \p I must be a sign, zero, or fp extension.
2339 bool isExtFree(const Instruction *I) const {
2340 switch (I->getOpcode()) {
2341 case Instruction::FPExt:
2342 if (isFPExtFree(EVT::getEVT(I->getType()),
2343 EVT::getEVT(I->getOperand(0)->getType())))
2344 return true;
2345 break;
2346 case Instruction::ZExt:
2347 if (isZExtFree(I->getOperand(0)->getType(), I->getType()))
2348 return true;
2349 break;
2350 case Instruction::SExt:
2351 break;
2352 default:
2353 llvm_unreachable("Instruction is not an extension")::llvm::llvm_unreachable_internal("Instruction is not an extension"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2353)
;
2354 }
2355 return isExtFreeImpl(I);
2356 }
2357
2358 /// Return true if \p Load and \p Ext can form an ExtLoad.
2359 /// For example, in AArch64
2360 /// %L = load i8, i8* %ptr
2361 /// %E = zext i8 %L to i32
2362 /// can be lowered into one load instruction
2363 /// ldrb w0, [x0]
2364 bool isExtLoad(const LoadInst *Load, const Instruction *Ext,
2365 const DataLayout &DL) const {
2366 EVT VT = getValueType(DL, Ext->getType());
2367 EVT LoadVT = getValueType(DL, Load->getType());
2368
2369 // If the load has other users and the truncate is not free, the ext
2370 // probably isn't free.
2371 if (!Load->hasOneUse() && (isTypeLegal(LoadVT) || !isTypeLegal(VT)) &&
2372 !isTruncateFree(Ext->getType(), Load->getType()))
2373 return false;
2374
2375 // Check whether the target supports casts folded into loads.
2376 unsigned LType;
2377 if (isa<ZExtInst>(Ext))
2378 LType = ISD::ZEXTLOAD;
2379 else {
2380 assert(isa<SExtInst>(Ext) && "Unexpected ext type!")((isa<SExtInst>(Ext) && "Unexpected ext type!")
? static_cast<void> (0) : __assert_fail ("isa<SExtInst>(Ext) && \"Unexpected ext type!\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2380, __PRETTY_FUNCTION__))
;
2381 LType = ISD::SEXTLOAD;
2382 }
2383
2384 return isLoadExtLegal(LType, VT, LoadVT);
2385 }
2386
2387 /// Return true if any actual instruction that defines a value of type FromTy
2388 /// implicitly zero-extends the value to ToTy in the result register.
2389 ///
2390 /// The function should return true when it is likely that the truncate can
2391 /// be freely folded with an instruction defining a value of FromTy. If
2392 /// the defining instruction is unknown (because you're looking at a
2393 /// function argument, PHI, etc.) then the target may require an
2394 /// explicit truncate, which is not necessarily free, but this function
2395 /// does not deal with those cases.
2396 /// Targets must return false when FromTy >= ToTy.
2397 virtual bool isZExtFree(Type *FromTy, Type *ToTy) const {
2398 return false;
2399 }
2400
2401 virtual bool isZExtFree(EVT FromTy, EVT ToTy) const {
2402 return false;
2403 }
2404
2405 /// Return true if sign-extension from FromTy to ToTy is cheaper than
2406 /// zero-extension.
2407 virtual bool isSExtCheaperThanZExt(EVT FromTy, EVT ToTy) const {
2408 return false;
2409 }
2410
2411 /// Return true if sinking I's operands to the same basic block as I is
2412 /// profitable, e.g. because the operands can be folded into a target
2413 /// instruction during instruction selection. After calling the function
2414 /// \p Ops contains the Uses to sink ordered by dominance (dominating users
2415 /// come first).
2416 virtual bool shouldSinkOperands(Instruction *I,
2417 SmallVectorImpl<Use *> &Ops) const {
2418 return false;
2419 }
2420
2421 /// Return true if the target supplies and combines to a paired load
2422 /// two loaded values of type LoadedType next to each other in memory.
2423 /// RequiredAlignment gives the minimal alignment constraints that must be met
2424 /// to be able to select this paired load.
2425 ///
2426 /// This information is *not* used to generate actual paired loads, but it is
2427 /// used to generate a sequence of loads that is easier to combine into a
2428 /// paired load.
2429 /// For instance, something like this:
2430 /// a = load i64* addr
2431 /// b = trunc i64 a to i32
2432 /// c = lshr i64 a, 32
2433 /// d = trunc i64 c to i32
2434 /// will be optimized into:
2435 /// b = load i32* addr1
2436 /// d = load i32* addr2
2437 /// Where addr1 = addr2 +/- sizeof(i32).
2438 ///
2439 /// In other words, unless the target performs a post-isel load combining,
2440 /// this information should not be provided because it will generate more
2441 /// loads.
2442 virtual bool hasPairedLoad(EVT /*LoadedType*/,
2443 unsigned & /*RequiredAlignment*/) const {
2444 return false;
2445 }
2446
2447 /// Return true if the target has a vector blend instruction.
2448 virtual bool hasVectorBlend() const { return false; }
2449
2450 /// Get the maximum supported factor for interleaved memory accesses.
2451 /// Default to be the minimum interleave factor: 2.
2452 virtual unsigned getMaxSupportedInterleaveFactor() const { return 2; }
2453
2454 /// Lower an interleaved load to target specific intrinsics. Return
2455 /// true on success.
2456 ///
2457 /// \p LI is the vector load instruction.
2458 /// \p Shuffles is the shufflevector list to DE-interleave the loaded vector.
2459 /// \p Indices is the corresponding indices for each shufflevector.
2460 /// \p Factor is the interleave factor.
2461 virtual bool lowerInterleavedLoad(LoadInst *LI,
2462 ArrayRef<ShuffleVectorInst *> Shuffles,
2463 ArrayRef<unsigned> Indices,
2464 unsigned Factor) const {
2465 return false;
2466 }
2467
2468 /// Lower an interleaved store to target specific intrinsics. Return
2469 /// true on success.
2470 ///
2471 /// \p SI is the vector store instruction.
2472 /// \p SVI is the shufflevector to RE-interleave the stored vector.
2473 /// \p Factor is the interleave factor.
2474 virtual bool lowerInterleavedStore(StoreInst *SI, ShuffleVectorInst *SVI,
2475 unsigned Factor) const {
2476 return false;
2477 }
2478
2479 /// Return true if zero-extending the specific node Val to type VT2 is free
2480 /// (either because it's implicitly zero-extended such as ARM ldrb / ldrh or
2481 /// because it's folded such as X86 zero-extending loads).
2482 virtual bool isZExtFree(SDValue Val, EVT VT2) const {
2483 return isZExtFree(Val.getValueType(), VT2);
5
Calling 'SDValue::getValueType'
2484 }
2485
2486 /// Return true if an fpext operation is free (for instance, because
2487 /// single-precision floating-point numbers are implicitly extended to
2488 /// double-precision).
2489 virtual bool isFPExtFree(EVT DestVT, EVT SrcVT) const {
2490 assert(SrcVT.isFloatingPoint() && DestVT.isFloatingPoint() &&((SrcVT.isFloatingPoint() && DestVT.isFloatingPoint()
&& "invalid fpext types") ? static_cast<void> (
0) : __assert_fail ("SrcVT.isFloatingPoint() && DestVT.isFloatingPoint() && \"invalid fpext types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2491, __PRETTY_FUNCTION__))
2491 "invalid fpext types")((SrcVT.isFloatingPoint() && DestVT.isFloatingPoint()
&& "invalid fpext types") ? static_cast<void> (
0) : __assert_fail ("SrcVT.isFloatingPoint() && DestVT.isFloatingPoint() && \"invalid fpext types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2491, __PRETTY_FUNCTION__))
;
2492 return false;
2493 }
2494
2495 /// Return true if an fpext operation input to an \p Opcode operation is free
2496 /// (for instance, because half-precision floating-point numbers are
2497 /// implicitly extended to float-precision) for an FMA instruction.
2498 virtual bool isFPExtFoldable(const SelectionDAG &DAG, unsigned Opcode,
2499 EVT DestVT, EVT SrcVT) const {
2500 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&((DestVT.isFloatingPoint() && SrcVT.isFloatingPoint()
&& "invalid fpext types") ? static_cast<void> (
0) : __assert_fail ("DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && \"invalid fpext types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2501, __PRETTY_FUNCTION__))
2501 "invalid fpext types")((DestVT.isFloatingPoint() && SrcVT.isFloatingPoint()
&& "invalid fpext types") ? static_cast<void> (
0) : __assert_fail ("DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() && \"invalid fpext types\""
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2501, __PRETTY_FUNCTION__))
;
2502 return isFPExtFree(DestVT, SrcVT);
2503 }
2504
2505 /// Return true if folding a vector load into ExtVal (a sign, zero, or any
2506 /// extend node) is profitable.
2507 virtual bool isVectorLoadExtDesirable(SDValue ExtVal) const { return false; }
2508
2509 /// Return true if an fneg operation is free to the point where it is never
2510 /// worthwhile to replace it with a bitwise operation.
2511 virtual bool isFNegFree(EVT VT) const {
2512 assert(VT.isFloatingPoint())((VT.isFloatingPoint()) ? static_cast<void> (0) : __assert_fail
("VT.isFloatingPoint()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2512, __PRETTY_FUNCTION__))
;
2513 return false;
2514 }
2515
2516 /// Return true if an fabs operation is free to the point where it is never
2517 /// worthwhile to replace it with a bitwise operation.
2518 virtual bool isFAbsFree(EVT VT) const {
2519 assert(VT.isFloatingPoint())((VT.isFloatingPoint()) ? static_cast<void> (0) : __assert_fail
("VT.isFloatingPoint()", "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2519, __PRETTY_FUNCTION__))
;
2520 return false;
2521 }
2522
2523 /// Return true if an FMA operation is faster than a pair of fmul and fadd
2524 /// instructions. fmuladd intrinsics will be expanded to FMAs when this method
2525 /// returns true, otherwise fmuladd is expanded to fmul + fadd.
2526 ///
2527 /// NOTE: This may be called before legalization on types for which FMAs are
2528 /// not legal, but should return true if those types will eventually legalize
2529 /// to types that support FMAs. After legalization, it will only be called on
2530 /// types that support FMAs (via Legal or Custom actions)
2531 virtual bool isFMAFasterThanFMulAndFAdd(EVT) const {
2532 return false;
2533 }
2534
2535 /// Returns true if the FADD or FSUB node passed could legally be combined with
2536 /// an fmul to form an ISD::FMAD.
2537 virtual bool isFMADLegalForFAddFSub(const SelectionDAG &DAG,
2538 const SDNode *N) const {
2539 assert(N->getOpcode() == ISD::FADD || N->getOpcode() == ISD::FSUB)((N->getOpcode() == ISD::FADD || N->getOpcode() == ISD::
FSUB) ? static_cast<void> (0) : __assert_fail ("N->getOpcode() == ISD::FADD || N->getOpcode() == ISD::FSUB"
, "/build/llvm-toolchain-snapshot-10~+201911111502510600c19528f1809/llvm/include/llvm/CodeGen/TargetLowering.h"
, 2539, __PRETTY_FUNCTION__))
;
2540 return isOperationLegal(ISD::FMAD, N->getValueType(0));
2541 }
2542
2543 /// Return true if it's profitable to narrow operations of type VT1 to
2544 /// VT2. e.g. on x86, it's profitable to narrow from i32 to i8 but not from
2545 /// i32 to i16.
2546 virtual bool isNarrowingProfitable(EVT /*VT1*/, EVT /*VT2*/) const {
2547 return false;
2548 }
2549
2550 /// Return true if it is beneficial to convert a load of a constant to
2551 /// just the constant itself.
2552 /// On some targets it might be more efficient to use a combination of
2553 /// arithmetic instructions to materialize the constant instead of loading it
2554 /// from a constant pool.
2555 virtual bool shouldConvertConstantLoadToIntImm(const APInt &Imm,
2556 Type *Ty) const {
2557 return false;
2558 }
2559
2560 /// Return true if EXTRACT_SUBVECTOR is cheap for extracting this result type
2561 /// from this source type with this index. This is needed because
2562 /// EXTRACT_SUBVECTOR usually has custom lowering that depends on the index of
2563 /// the first element, and only the target knows which lowering is cheap.
2564 virtual bool isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2565 unsigned Index) const {
2566 return false;
2567 }
2568
2569 /// Try to convert an extract element of a vector binary operation into an
2570 /// extract element followed by a scalar operation.
2571 virtual bool shouldScalarizeBinop(SDValue VecOp) const {
2572 return false;
2573 }
2574
2575 /// Return true if extraction of a scalar element from the given vector type
2576 /// at the given index is cheap. For example, if scalar operations occur on
2577 /// the same register file as vector operations, then an extract element may
2578 /// be a sub-register rename rather than an actual instruction.
2579 virtual bool isExtractVecEltCheap(EVT VT, unsigned Index) const {
2580 return false;
2581 }
2582
2583 /// Try to convert math with an overflow comparison into the corresponding DAG
2584 /// node operation. Targets may want to override this independently of whether
2585 /// the operation is legal/custom for the given type because it may obscure
2586 /// matching of other patterns.
2587 virtual bool shouldFormOverflowOp(unsigned Opcode, EVT VT) const {
2588 // TODO: The default logic is inherited from code in CodeGenPrepare.
2589 // The opcode should not make a difference by default?
2590 if (Opcode != ISD::UADDO)
2591 return false;
2592
2593 // Allow the transform as long as we have an integer type that is not
2594 // obviously illegal and unsupported.
2595 if (VT.isVector())
2596 return false;
2597 return VT.isSimple() || !isOperationExpand(Opcode, VT);
2598 }
2599
2600 // Return true if it is profitable to use a scalar input to a BUILD_VECTOR
2601 // even if the vector itself has multiple uses.
2602 virtual bool aggressivelyPreferBuildVectorSources(EVT VecVT) const {
2603 return false;
2604 }
2605
2606 // Return true if CodeGenPrepare should consider splitting large offset of a
2607 // GEP to make the GEP fit into the addressing mode and can be sunk into the
2608 // same blocks of its users.
2609 virtual bool shouldConsiderGEPOffsetSplit() const { return false; }
2610
2611 // Return the shift amount threshold for profitable transforms into shifts.
2612 // Transforms creating shifts above the returned value will be avoided.
2613 virtual unsigned getShiftAmountThreshold(EVT VT) const {
2614 return VT.getScalarSizeInBits();
2615 }
2616
2617 //===--------------------------------------------------------------------===//
2618 // Runtime Library hooks
2619 //
2620
2621 /// Rename the default libcall routine name for the specified libcall.
2622 void setLibcallName(RTLIB::Libcall Call, const char *Name) {
2623 LibcallRoutineNames[Call] = Name;
2624 }
2625
2626 /// Get the libcall routine name for the specified libcall.
2627 const char *getLibcallName(RTLIB::Libcall Call) const {
2628 return LibcallRoutineNames[Call];
2629 }
2630
2631 /// Override the default CondCode to be used to test the result of the
2632 /// comparison libcall against zero.
2633 void setCmpLibcallCC(RTLIB::Libcall Call, ISD::CondCode CC) {
2634 CmpLibcallCCs[Call] = CC;
2635 }
2636
2637 /// Get the CondCode that's to be used to test the result of the comparison
2638 /// libcall against zero.
2639 ISD::CondCode getCmpLibcallCC(RTLIB::Libcall Call) const {
2640 return CmpLibcallCCs[Call];
2641 }
2642
2643 /// Set the CallingConv that should be used for the specified libcall.
2644 void setLibcallCallingConv(RTLIB::Libcall Call, CallingConv::ID CC) {
2645 LibcallCallingConvs[Call] = CC;
2646 }
2647
2648 /// Get the CallingConv that should be used for the specified libcall.
2649 CallingConv::ID getLibcallCallingConv(RTLIB::Libcall Call) const {
2650 return LibcallCallingConvs[Call];
2651 }
2652
2653 /// Execute target specific actions to finalize target lowering.
2654 /// This is used to set extra flags in MachineFrameInformation and freezing
2655 /// the set of reserved registers.
2656 /// The default implementation just freezes the set of reserved registers.
2657 virtual void finalizeLowering(MachineFunction &MF) const;
2658
2659private:
2660 const TargetMachine &TM;
2661
2662 /// Tells the code generator that the target has multiple (allocatable)
2663 /// condition registers that can be used to store the results of comparisons
2664 /// for use by selects and conditional branches. With multiple condition
2665 /// registers, the code generator will not aggressively sink comparisons into
2666 /// the blocks of their users.
2667 bool HasMultipleConditionRegisters;
2668
2669 /// Tells the code generator that the target has BitExtract instructions.
2670 /// The code generator will aggressively sink "shift"s into the blocks of
2671 /// their users if the users will generate "and" instructions which can be
2672 /// combined with "shift" to BitExtract instructions.
2673 bool HasExtractBitsInsn;
2674
2675 /// Tells the code generator to bypass slow divide or remainder
2676 /// instructions. For example, BypassSlowDivWidths[32,8] tells the code
2677 /// generator to bypass 32-bit integer div/rem with an 8-bit unsigned integer
2678 /// div/rem when the operands are positive and less than 256.
2679 DenseMap <unsigned int, unsigned int> BypassSlowDivWidths;
2680
2681 /// Tells the code generator that it shouldn't generate extra flow control
2682 /// instructions and should attempt to combine flow control instructions via
2683 /// predication.
2684 bool JumpIsExpensive;
2685
2686 /// This target prefers to use _setjmp to implement llvm.setjmp.
2687 ///
2688 /// Defaults to false.
2689 bool UseUnderscoreSetJmp;
2690
2691 /// This target prefers to use _longjmp to implement llvm.longjmp.
2692 ///
2693 /// Defaults to false.
2694 bool UseUnderscoreLongJmp;
2695
2696 /// Information about the contents of the high-bits in boolean values held in
2697 /// a type wider than i1. See getBooleanContents.
2698 BooleanContent BooleanContents;
2699
2700 /// Information about the contents of the high-bits in boolean values held in
2701 /// a type wider than i1. See getBooleanContents.
2702 BooleanContent BooleanFloatContents;
2703
2704 /// Information about the contents of the high-bits in boolean vector values
2705 /// when the element type is wider than i1. See getBooleanContents.
2706 BooleanContent BooleanVectorContents;
2707
2708 /// The target scheduling preference: shortest possible total cycles or lowest
2709 /// register usage.
2710 Sched::Preference SchedPreferenceInfo;
2711
2712 /// The minimum alignment that any argument on the stack needs to have.
2713 Align MinStackArgumentAlignment;
2714
2715 /// The minimum function alignment (used when optimizing for size, and to
2716 /// prevent explicitly provided alignment from leading to incorrect code).
2717 Align MinFunctionAlignment;
2718
2719 /// The preferred function alignment (used when alignment unspecified and
2720 /// optimizing for speed).
2721 Align PrefFunctionAlignment;
2722
2723 /// The preferred loop alignment (in log2 bot in bytes).
2724 Align PrefLoopAlignment;
2725
2726 /// Size in bits of the maximum atomics size the backend supports.
2727 /// Accesses larger than this will be expanded by AtomicExpandPass.
2728 unsigned MaxAtomicSizeInBitsSupported;
2729
2730 /// Size in bits of the minimum cmpxchg or ll/sc operation the
2731 /// backend supports.
2732 unsigned MinCmpXchgSizeInBits;
2733
2734 /// This indicates if the target supports unaligned atomic operations.
2735 bool SupportsUnalignedAtomics;
2736
2737 /// If set to a physical register, this specifies the register that
2738 /// llvm.savestack/llvm.restorestack should save and restore.
2739 unsigned StackPointerRegisterToSaveRestore;
2740
2741 /// This indicates the default register class to use for each ValueType the
2742 /// target supports natively.
2743 const TargetRegisterClass *RegClassForVT[MVT::LAST_VALUETYPE];
2744 unsigned char NumRegistersForVT[MVT::LAST_VALUETYPE];
2745 MVT RegisterTypeForVT[MVT::LAST_VALUETYPE];
2746
2747 /// This indicates the "representative" register class to use for each
2748 /// ValueType the target supports natively. This information is used by the
2749 /// scheduler to track register pressure. By default, the representative
2750 /// register class is the largest legal super-reg register class of the
2751 /// register class of the specified type. e.g. On x86, i8, i16, and i32's
2752 /// representative class would be GR32.
2753 const TargetRegisterClass *RepRegClassForVT[MVT::LAST_VALUETYPE];
2754
2755 /// This indicates the "cost" of the "representative" register class for each
2756 /// ValueType. The cost is used by the scheduler to approximate register
2757 /// pressure.
2758 uint8_t RepRegClassCostForVT[MVT::LAST_VALUETYPE];
2759
2760 /// For any value types we are promoting or expanding, this contains the value
2761 /// type that we are changing to. For Expanded types, this contains one step
2762 /// of the expand (e.g. i64 -> i32), even if there are multiple steps required
2763 /// (e.g. i64 -> i16). For types natively supported by the system, this holds
2764 /// the same type (e.g. i32 -> i32).
2765 MVT TransformToType[MVT::LAST_VALUETYPE];
2766
2767 /// For each operation and each value type, keep a LegalizeAction that
2768 /// indicates how instruction selection should deal with the operation. Most
2769 /// operations are Legal (aka, supported natively by the target), but
2770 /// operations that are not should be described. Note that operations on
2771 /// non-legal value types are not described here.
2772 LegalizeAction OpActions[MVT::LAST_VALUETYPE][ISD::BUILTIN_OP_END];
2773
2774 /// For each load extension type and each value type, keep a LegalizeAction
2775 /// that indicates how instruction selection should deal with a load of a
2776 /// specific value type and extension type. Uses 4-bits to store the action
2777 /// for each of the 4 load ext types.
2778 uint16_t LoadExtActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE];
2779
2780 /// For each value type pair keep a LegalizeAction that indicates whether a
2781 /// truncating store of a specific value type and truncating type is legal.
2782 LegalizeAction TruncStoreActions[MVT::LAST_VALUETYPE][MVT::LAST_VALUETYPE];
2783
2784 /// For each indexed mode and each value type, keep a pair of LegalizeAction
2785 /// that indicates how instruction selection should deal with the load /
2786 /// store.
2787 ///
2788 /// The first dimension is the value_type for the reference. The second
2789 /// dimension represents the various modes for load store.
2790 uint8_t IndexedModeActions[MVT::LAST_VALUETYPE][ISD::LAST_INDEXED_MODE];
2791
2792 /// For each condition code (ISD::CondCode) keep a LegalizeAction that
2793 /// indicates how instruction selection should deal with the condition code.
2794 ///
2795 /// Because each CC action takes up 4 bits, we need to have the array size be
2796 /// large enough to fit all of the value types. This can be done by rounding
2797 /// up the MVT::LAST_VALUETYPE value to the next multiple of 8.
2798 uint32_t CondCodeActions[ISD::SETCC_INVALID][(MVT::LAST_VALUETYPE + 7) / 8];
2799
2800 ValueTypeActionImpl ValueTypeActions;
2801
2802private:
2803 LegalizeKind getTypeConversion(LLVMContext &Context, EVT VT) const;
2804
2805 /// Targets can specify ISD nodes that they would like PerformDAGCombine
2806 /// callbacks for by calling setTargetDAGCombine(), which sets a bit in this
2807 /// array.
2808 unsigned char
2809 TargetDAGCombineArray[(ISD::BUILTIN_OP_END+CHAR_BIT8-1)/CHAR_BIT8];
2810
2811 /// For operations that must be promoted to a specific type, this holds the
2812 /// destination type. This map should be sparse, so don't hold it as an
2813 /// array.
2814 ///
2815 /// Targets add entries to this map with AddPromotedToType(..), clients access
2816 /// this with getTypeToPromoteTo(..).
2817 std::map<std::pair<unsigned, MVT::SimpleValueType>, MVT::SimpleValueType>
2818 PromoteToType;
2819
2820 /// Stores the name each libcall.
2821 const char *LibcallRoutineNames[RTLIB::UNKNOWN_LIBCALL + 1];
2822
2823 /// The ISD::CondCode that should be used to test the result of each of the
2824 /// comparison libcall against zero.
2825 ISD::CondCode CmpLibcallCCs[RTLIB::UNKNOWN_LIBCALL];
2826
2827 /// Stores the CallingConv that should be used for each libcall.