Bug Summary

File:llvm/include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1150, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name WebAssemblyISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -fno-rounding-math -masm-verbose -mconstructor-aliases -munwind-tables -target-cpu x86-64 -dwarf-column-info -fno-split-dwarf-inlining -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/include -I /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/build-llvm/lib/Target/WebAssembly -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2020-01-13-084841-49055-1 -x c++ /build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "WebAssemblyMachineFunctionInfo.h"
17#include "WebAssemblySubtarget.h"
18#include "WebAssemblyTargetMachine.h"
19#include "llvm/CodeGen/Analysis.h"
20#include "llvm/CodeGen/CallingConvLower.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineModuleInfo.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/WasmEHFuncInfo.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/DiagnosticPrinter.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/IR/IntrinsicsWebAssembly.h"
32#include "llvm/Support/Debug.h"
33#include "llvm/Support/ErrorHandling.h"
34#include "llvm/Support/raw_ostream.h"
35#include "llvm/Target/TargetOptions.h"
36using namespace llvm;
37
38#define DEBUG_TYPE"wasm-lower" "wasm-lower"
39
40WebAssemblyTargetLowering::WebAssemblyTargetLowering(
41 const TargetMachine &TM, const WebAssemblySubtarget &STI)
42 : TargetLowering(TM), Subtarget(&STI) {
43 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
44
45 // Booleans always contain 0 or 1.
46 setBooleanContents(ZeroOrOneBooleanContent);
47 // Except in SIMD vectors
48 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
49 // We don't know the microarchitecture here, so just reduce register pressure.
50 setSchedulingPreference(Sched::RegPressure);
51 // Tell ISel that we have a stack pointer.
52 setStackPointerRegisterToSaveRestore(
53 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
54 // Set up the register classes.
55 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
56 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
57 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
58 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
59 if (Subtarget->hasSIMD128()) {
60 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
61 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
62 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
63 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
64 }
65 if (Subtarget->hasUnimplementedSIMD128()) {
66 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
67 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
68 }
69 // Compute derived properties from the register classes.
70 computeRegisterProperties(Subtarget->getRegisterInfo());
71
72 setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
73 setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
74 setOperationAction(ISD::JumpTable, MVTPtr, Custom);
75 setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
76 setOperationAction(ISD::BRIND, MVT::Other, Custom);
77
78 // Take the default expansion for va_arg, va_copy, and va_end. There is no
79 // default action for va_start, so we do that custom.
80 setOperationAction(ISD::VASTART, MVT::Other, Custom);
81 setOperationAction(ISD::VAARG, MVT::Other, Expand);
82 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
83 setOperationAction(ISD::VAEND, MVT::Other, Expand);
84
85 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
86 // Don't expand the floating-point types to constant pools.
87 setOperationAction(ISD::ConstantFP, T, Legal);
88 // Expand floating-point comparisons.
89 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
90 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
91 setCondCodeAction(CC, T, Expand);
92 // Expand floating-point library function operators.
93 for (auto Op :
94 {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
95 setOperationAction(Op, T, Expand);
96 // Note supported floating-point library function operators that otherwise
97 // default to expand.
98 for (auto Op :
99 {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
100 setOperationAction(Op, T, Legal);
101 // Support minimum and maximum, which otherwise default to expand.
102 setOperationAction(ISD::FMINIMUM, T, Legal);
103 setOperationAction(ISD::FMAXIMUM, T, Legal);
104 // WebAssembly currently has no builtin f16 support.
105 setOperationAction(ISD::FP16_TO_FP, T, Expand);
106 setOperationAction(ISD::FP_TO_FP16, T, Expand);
107 setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
108 setTruncStoreAction(T, MVT::f16, Expand);
109 }
110
111 // Expand unavailable integer operations.
112 for (auto Op :
113 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
114 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
115 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
116 for (auto T : {MVT::i32, MVT::i64})
117 setOperationAction(Op, T, Expand);
118 if (Subtarget->hasSIMD128())
119 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
120 setOperationAction(Op, T, Expand);
121 if (Subtarget->hasUnimplementedSIMD128())
122 setOperationAction(Op, MVT::v2i64, Expand);
123 }
124
125 // SIMD-specific configuration
126 if (Subtarget->hasSIMD128()) {
127 // Support saturating add for i8x16 and i16x8
128 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
129 for (auto T : {MVT::v16i8, MVT::v8i16})
130 setOperationAction(Op, T, Legal);
131
132 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
133 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
134 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
135 if (Subtarget->hasUnimplementedSIMD128())
136 for (auto T : {MVT::v2i64, MVT::v2f64})
137 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
138
139 // We have custom shuffle lowering to expose the shuffle mask
140 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
141 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
142 if (Subtarget->hasUnimplementedSIMD128())
143 for (auto T: {MVT::v2i64, MVT::v2f64})
144 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
145
146 // Custom lowering since wasm shifts must have a scalar shift amount
147 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL}) {
148 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
149 setOperationAction(Op, T, Custom);
150 if (Subtarget->hasUnimplementedSIMD128())
151 setOperationAction(Op, MVT::v2i64, Custom);
152 }
153
154 // Custom lower lane accesses to expand out variable indices
155 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}) {
156 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
157 setOperationAction(Op, T, Custom);
158 if (Subtarget->hasUnimplementedSIMD128())
159 for (auto T : {MVT::v2i64, MVT::v2f64})
160 setOperationAction(Op, T, Custom);
161 }
162
163 // There is no i64x2.mul instruction
164 setOperationAction(ISD::MUL, MVT::v2i64, Expand);
165
166 // There are no vector select instructions
167 for (auto Op : {ISD::VSELECT, ISD::SELECT_CC, ISD::SELECT}) {
168 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
169 setOperationAction(Op, T, Expand);
170 if (Subtarget->hasUnimplementedSIMD128())
171 for (auto T : {MVT::v2i64, MVT::v2f64})
172 setOperationAction(Op, T, Expand);
173 }
174
175 // Expand integer operations supported for scalars but not SIMD
176 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP, ISD::SDIV, ISD::UDIV,
177 ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR}) {
178 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
179 setOperationAction(Op, T, Expand);
180 if (Subtarget->hasUnimplementedSIMD128())
181 setOperationAction(Op, MVT::v2i64, Expand);
182 }
183
184 // But we do have integer min and max operations
185 if (Subtarget->hasUnimplementedSIMD128()) {
186 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
187 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
188 setOperationAction(Op, T, Legal);
189 }
190
191 // Expand float operations supported for scalars but not SIMD
192 for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
193 ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
194 ISD::FEXP, ISD::FEXP2, ISD::FRINT}) {
195 setOperationAction(Op, MVT::v4f32, Expand);
196 if (Subtarget->hasUnimplementedSIMD128())
197 setOperationAction(Op, MVT::v2f64, Expand);
198 }
199
200 // Expand operations not supported for i64x2 vectors
201 if (Subtarget->hasUnimplementedSIMD128())
202 for (unsigned CC = 0; CC < ISD::SETCC_INVALID; ++CC)
203 setCondCodeAction(static_cast<ISD::CondCode>(CC), MVT::v2i64, Custom);
204
205 // Expand additional SIMD ops that V8 hasn't implemented yet
206 if (!Subtarget->hasUnimplementedSIMD128()) {
207 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
208 setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
209 }
210 }
211
212 // As a special case, these operators use the type to mean the type to
213 // sign-extend from.
214 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
215 if (!Subtarget->hasSignExt()) {
216 // Sign extends are legal only when extending a vector extract
217 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
218 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
219 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
220 }
221 for (auto T : MVT::integer_fixedlen_vector_valuetypes())
222 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
223
224 // Dynamic stack allocation: use the default expansion.
225 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
226 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
227 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);
228
229 setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
230 setOperationAction(ISD::CopyToReg, MVT::Other, Custom);
231
232 // Expand these forms; we pattern-match the forms that we can handle in isel.
233 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
234 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
235 setOperationAction(Op, T, Expand);
236
237 // We have custom switch handling.
238 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
239
240 // WebAssembly doesn't have:
241 // - Floating-point extending loads.
242 // - Floating-point truncating stores.
243 // - i1 extending loads.
244 // - truncating SIMD stores and most extending loads
245 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
246 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
247 for (auto T : MVT::integer_valuetypes())
248 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
249 setLoadExtAction(Ext, T, MVT::i1, Promote);
250 if (Subtarget->hasSIMD128()) {
251 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
252 MVT::v2f64}) {
253 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
254 if (MVT(T) != MemT) {
255 setTruncStoreAction(T, MemT, Expand);
256 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
257 setLoadExtAction(Ext, T, MemT, Expand);
258 }
259 }
260 }
261 // But some vector extending loads are legal
262 if (Subtarget->hasUnimplementedSIMD128()) {
263 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
264 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
265 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
266 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
267 }
268 }
269 }
270
271 // Don't do anything clever with build_pairs
272 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
273
274 // Trap lowers to wasm unreachable
275 setOperationAction(ISD::TRAP, MVT::Other, Legal);
276
277 // Exception handling intrinsics
278 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
279 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
280
281 setMaxAtomicSizeInBitsSupported(64);
282
283 // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
284 // consistent with the f64 and f128 names.
285 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
286 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
287
288 // Define the emscripten name for return address helper.
289 // TODO: when implementing other WASM backends, make this generic or only do
290 // this on emscripten depending on what they end up doing.
291 setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
292
293 // Always convert switches to br_tables unless there is only one case, which
294 // is equivalent to a simple branch. This reduces code size for wasm, and we
295 // defer possible jump table optimizations to the VM.
296 setMinimumJumpTableEntries(2);
297}
298
299TargetLowering::AtomicExpansionKind
300WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
301 // We have wasm instructions for these
302 switch (AI->getOperation()) {
303 case AtomicRMWInst::Add:
304 case AtomicRMWInst::Sub:
305 case AtomicRMWInst::And:
306 case AtomicRMWInst::Or:
307 case AtomicRMWInst::Xor:
308 case AtomicRMWInst::Xchg:
309 return AtomicExpansionKind::None;
310 default:
311 break;
312 }
313 return AtomicExpansionKind::CmpXChg;
314}
315
316FastISel *WebAssemblyTargetLowering::createFastISel(
317 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
318 return WebAssembly::createFastISel(FuncInfo, LibInfo);
319}
320
321MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
322 EVT VT) const {
323 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
324 if (BitWidth > 1 && BitWidth < 8)
325 BitWidth = 8;
326
327 if (BitWidth > 64) {
328 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
329 // the count to be an i32.
330 BitWidth = 32;
331 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&((BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && "32-bit shift counts ought to be enough for anyone"
) ? static_cast<void> (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 332, __PRETTY_FUNCTION__))
332 "32-bit shift counts ought to be enough for anyone")((BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && "32-bit shift counts ought to be enough for anyone"
) ? static_cast<void> (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 332, __PRETTY_FUNCTION__))
;
333 }
334
335 MVT Result = MVT::getIntegerVT(BitWidth);
336 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&((Result != MVT::INVALID_SIMPLE_VALUE_TYPE && "Unable to represent scalar shift amount type"
) ? static_cast<void> (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 337, __PRETTY_FUNCTION__))
337 "Unable to represent scalar shift amount type")((Result != MVT::INVALID_SIMPLE_VALUE_TYPE && "Unable to represent scalar shift amount type"
) ? static_cast<void> (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 337, __PRETTY_FUNCTION__))
;
338 return Result;
339}
340
341// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
342// undefined result on invalid/overflow, to the WebAssembly opcode, which
343// traps on invalid/overflow.
344static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
345 MachineBasicBlock *BB,
346 const TargetInstrInfo &TII,
347 bool IsUnsigned, bool Int64,
348 bool Float64, unsigned LoweredOpcode) {
349 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
350
351 Register OutReg = MI.getOperand(0).getReg();
352 Register InReg = MI.getOperand(1).getReg();
353
354 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
355 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
356 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
357 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
358 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
359 unsigned Eqz = WebAssembly::EQZ_I32;
360 unsigned And = WebAssembly::AND_I32;
361 int64_t Limit = Int64 ? INT64_MIN(-9223372036854775807L -1) : INT32_MIN(-2147483647-1);
362 int64_t Substitute = IsUnsigned ? 0 : Limit;
363 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
364 auto &Context = BB->getParent()->getFunction().getContext();
365 Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);
366
367 const BasicBlock *LLVMBB = BB->getBasicBlock();
368 MachineFunction *F = BB->getParent();
369 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
370 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
371 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
372
373 MachineFunction::iterator It = ++BB->getIterator();
374 F->insert(It, FalseMBB);
375 F->insert(It, TrueMBB);
376 F->insert(It, DoneMBB);
377
378 // Transfer the remainder of BB and its successor edges to DoneMBB.
379 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
380 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
381
382 BB->addSuccessor(TrueMBB);
383 BB->addSuccessor(FalseMBB);
384 TrueMBB->addSuccessor(DoneMBB);
385 FalseMBB->addSuccessor(DoneMBB);
386
387 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
388 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
389 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
390 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
391 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
392 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
393 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
394
395 MI.eraseFromParent();
396 // For signed numbers, we can do a single comparison to determine whether
397 // fabs(x) is within range.
398 if (IsUnsigned) {
399 Tmp0 = InReg;
400 } else {
401 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
402 }
403 BuildMI(BB, DL, TII.get(FConst), Tmp1)
404 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
405 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
406
407 // For unsigned numbers, we have to do a separate comparison with zero.
408 if (IsUnsigned) {
409 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
410 Register SecondCmpReg =
411 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
412 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
413 BuildMI(BB, DL, TII.get(FConst), Tmp1)
414 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
415 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
416 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
417 CmpReg = AndReg;
418 }
419
420 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
421
422 // Create the CFG diamond to select between doing the conversion or using
423 // the substitute value.
424 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
425 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
426 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
427 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
428 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
429 .addReg(FalseReg)
430 .addMBB(FalseMBB)
431 .addReg(TrueReg)
432 .addMBB(TrueMBB);
433
434 return DoneMBB;
435}
436
437MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
438 MachineInstr &MI, MachineBasicBlock *BB) const {
439 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
440 DebugLoc DL = MI.getDebugLoc();
441
442 switch (MI.getOpcode()) {
443 default:
444 llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 444)
;
445 case WebAssembly::FP_TO_SINT_I32_F32:
446 return LowerFPToInt(MI, DL, BB, TII, false, false, false,
447 WebAssembly::I32_TRUNC_S_F32);
448 case WebAssembly::FP_TO_UINT_I32_F32:
449 return LowerFPToInt(MI, DL, BB, TII, true, false, false,
450 WebAssembly::I32_TRUNC_U_F32);
451 case WebAssembly::FP_TO_SINT_I64_F32:
452 return LowerFPToInt(MI, DL, BB, TII, false, true, false,
453 WebAssembly::I64_TRUNC_S_F32);
454 case WebAssembly::FP_TO_UINT_I64_F32:
455 return LowerFPToInt(MI, DL, BB, TII, true, true, false,
456 WebAssembly::I64_TRUNC_U_F32);
457 case WebAssembly::FP_TO_SINT_I32_F64:
458 return LowerFPToInt(MI, DL, BB, TII, false, false, true,
459 WebAssembly::I32_TRUNC_S_F64);
460 case WebAssembly::FP_TO_UINT_I32_F64:
461 return LowerFPToInt(MI, DL, BB, TII, true, false, true,
462 WebAssembly::I32_TRUNC_U_F64);
463 case WebAssembly::FP_TO_SINT_I64_F64:
464 return LowerFPToInt(MI, DL, BB, TII, false, true, true,
465 WebAssembly::I64_TRUNC_S_F64);
466 case WebAssembly::FP_TO_UINT_I64_F64:
467 return LowerFPToInt(MI, DL, BB, TII, true, true, true,
468 WebAssembly::I64_TRUNC_U_F64);
469 llvm_unreachable("Unexpected instruction to emit with custom inserter")::llvm::llvm_unreachable_internal("Unexpected instruction to emit with custom inserter"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 469)
;
470 }
471}
472
473const char *
474WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
475 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
476 case WebAssemblyISD::FIRST_NUMBER:
477 case WebAssemblyISD::FIRST_MEM_OPCODE:
478 break;
479#define HANDLE_NODETYPE(NODE) \
480 case WebAssemblyISD::NODE: \
481 return "WebAssemblyISD::" #NODE;
482#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
483#include "WebAssemblyISD.def"
484#undef HANDLE_MEM_NODETYPE
485#undef HANDLE_NODETYPE
486 }
487 return nullptr;
488}
489
490std::pair<unsigned, const TargetRegisterClass *>
491WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
492 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
493 // First, see if this is a constraint that directly corresponds to a
494 // WebAssembly register class.
495 if (Constraint.size() == 1) {
496 switch (Constraint[0]) {
497 case 'r':
498 assert(VT != MVT::iPTR && "Pointer MVT not expected here")((VT != MVT::iPTR && "Pointer MVT not expected here")
? static_cast<void> (0) : __assert_fail ("VT != MVT::iPTR && \"Pointer MVT not expected here\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 498, __PRETTY_FUNCTION__))
;
499 if (Subtarget->hasSIMD128() && VT.isVector()) {
500 if (VT.getSizeInBits() == 128)
501 return std::make_pair(0U, &WebAssembly::V128RegClass);
502 }
503 if (VT.isInteger() && !VT.isVector()) {
504 if (VT.getSizeInBits() <= 32)
505 return std::make_pair(0U, &WebAssembly::I32RegClass);
506 if (VT.getSizeInBits() <= 64)
507 return std::make_pair(0U, &WebAssembly::I64RegClass);
508 }
509 break;
510 default:
511 break;
512 }
513 }
514
515 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
516}
517
518bool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const {
519 // Assume ctz is a relatively cheap operation.
520 return true;
521}
522
523bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const {
524 // Assume clz is a relatively cheap operation.
525 return true;
526}
527
528bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
529 const AddrMode &AM,
530 Type *Ty, unsigned AS,
531 Instruction *I) const {
532 // WebAssembly offsets are added as unsigned without wrapping. The
533 // isLegalAddressingMode gives us no way to determine if wrapping could be
534 // happening, so we approximate this by accepting only non-negative offsets.
535 if (AM.BaseOffs < 0)
536 return false;
537
538 // WebAssembly has no scale register operands.
539 if (AM.Scale != 0)
540 return false;
541
542 // Everything else is legal.
543 return true;
544}
545
546bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
547 EVT /*VT*/, unsigned /*AddrSpace*/, unsigned /*Align*/,
548 MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
549 // WebAssembly supports unaligned accesses, though it should be declared
550 // with the p2align attribute on loads and stores which do so, and there
551 // may be a performance impact. We tell LLVM they're "fast" because
552 // for the kinds of things that LLVM uses this for (merging adjacent stores
553 // of constants, etc.), WebAssembly implementations will either want the
554 // unaligned access or they'll split anyway.
555 if (Fast)
556 *Fast = true;
557 return true;
558}
559
560bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
561 AttributeList Attr) const {
562 // The current thinking is that wasm engines will perform this optimization,
563 // so we can save on code size.
564 return true;
565}
566
567bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
568 if (!Subtarget->hasUnimplementedSIMD128())
569 return false;
570 MVT ExtT = ExtVal.getSimpleValueType();
571 MVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getSimpleValueType(0);
572 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
573 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
574 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
575}
576
577EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
578 LLVMContext &C,
579 EVT VT) const {
580 if (VT.isVector())
581 return VT.changeVectorElementTypeToInteger();
582
583 return TargetLowering::getSetCCResultType(DL, C, VT);
584}
585
586bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
587 const CallInst &I,
588 MachineFunction &MF,
589 unsigned Intrinsic) const {
590 switch (Intrinsic) {
591 case Intrinsic::wasm_atomic_notify:
592 Info.opc = ISD::INTRINSIC_W_CHAIN;
593 Info.memVT = MVT::i32;
594 Info.ptrVal = I.getArgOperand(0);
595 Info.offset = 0;
596 Info.align = Align(4);
597 // atomic.notify instruction does not really load the memory specified with
598 // this argument, but MachineMemOperand should either be load or store, so
599 // we set this to a load.
600 // FIXME Volatile isn't really correct, but currently all LLVM atomic
601 // instructions are treated as volatiles in the backend, so we should be
602 // consistent. The same applies for wasm_atomic_wait intrinsics too.
603 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
604 return true;
605 case Intrinsic::wasm_atomic_wait_i32:
606 Info.opc = ISD::INTRINSIC_W_CHAIN;
607 Info.memVT = MVT::i32;
608 Info.ptrVal = I.getArgOperand(0);
609 Info.offset = 0;
610 Info.align = Align(4);
611 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
612 return true;
613 case Intrinsic::wasm_atomic_wait_i64:
614 Info.opc = ISD::INTRINSIC_W_CHAIN;
615 Info.memVT = MVT::i64;
616 Info.ptrVal = I.getArgOperand(0);
617 Info.offset = 0;
618 Info.align = Align(8);
619 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
620 return true;
621 default:
622 return false;
623 }
624}
625
626//===----------------------------------------------------------------------===//
627// WebAssembly Lowering private implementation.
628//===----------------------------------------------------------------------===//
629
630//===----------------------------------------------------------------------===//
631// Lowering Code
632//===----------------------------------------------------------------------===//
633
634static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
635 MachineFunction &MF = DAG.getMachineFunction();
636 DAG.getContext()->diagnose(
637 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
638}
639
640// Test whether the given calling convention is supported.
641static bool callingConvSupported(CallingConv::ID CallConv) {
642 // We currently support the language-independent target-independent
643 // conventions. We don't yet have a way to annotate calls with properties like
644 // "cold", and we don't have any call-clobbered registers, so these are mostly
645 // all handled the same.
646 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
647 CallConv == CallingConv::Cold ||
648 CallConv == CallingConv::PreserveMost ||
649 CallConv == CallingConv::PreserveAll ||
650 CallConv == CallingConv::CXX_FAST_TLS ||
651 CallConv == CallingConv::WASM_EmscriptenInvoke;
652}
653
654SDValue
655WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
656 SmallVectorImpl<SDValue> &InVals) const {
657 SelectionDAG &DAG = CLI.DAG;
658 SDLoc DL = CLI.DL;
659 SDValue Chain = CLI.Chain;
660 SDValue Callee = CLI.Callee;
661 MachineFunction &MF = DAG.getMachineFunction();
662 auto Layout = MF.getDataLayout();
663
664 CallingConv::ID CallConv = CLI.CallConv;
665 if (!callingConvSupported(CallConv))
666 fail(DL, DAG,
667 "WebAssembly doesn't support language-specific or target-specific "
668 "calling conventions yet");
669 if (CLI.IsPatchPoint)
670 fail(DL, DAG, "WebAssembly doesn't support patch point yet");
671
672 if (CLI.IsTailCall) {
673 bool MustTail = CLI.CS && CLI.CS.isMustTailCall();
674 if (Subtarget->hasTailCall() && !CLI.IsVarArg) {
675 // Do not tail call unless caller and callee return types match
676 const Function &F = MF.getFunction();
677 const TargetMachine &TM = getTargetMachine();
678 Type *RetTy = F.getReturnType();
679 SmallVector<MVT, 4> CallerRetTys;
680 SmallVector<MVT, 4> CalleeRetTys;
681 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
682 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
683 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
684 std::equal(CallerRetTys.begin(), CallerRetTys.end(),
685 CalleeRetTys.begin());
686 if (!TypesMatch) {
687 // musttail in this case would be an LLVM IR validation failure
688 assert(!MustTail)((!MustTail) ? static_cast<void> (0) : __assert_fail ("!MustTail"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 688, __PRETTY_FUNCTION__))
;
689 CLI.IsTailCall = false;
690 }
691 } else {
692 CLI.IsTailCall = false;
693 if (MustTail) {
694 if (CLI.IsVarArg) {
695 // The return would pop the argument buffer
696 fail(DL, DAG, "WebAssembly does not support varargs tail calls");
697 } else {
698 fail(DL, DAG, "WebAssembly 'tail-call' feature not enabled");
699 }
700 }
701 }
702 }
703
704 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
705 if (Ins.size() > 1)
706 fail(DL, DAG, "WebAssembly doesn't support more than 1 returned value yet");
707
708 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
709 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
710
711 // The generic code may have added an sret argument. If we're lowering an
712 // invoke function, the ABI requires that the function pointer be the first
713 // argument, so we may have to swap the arguments.
714 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
715 Outs[0].Flags.isSRet()) {
716 std::swap(Outs[0], Outs[1]);
717 std::swap(OutVals[0], OutVals[1]);
718 }
719
720 unsigned NumFixedArgs = 0;
721 for (unsigned I = 0; I < Outs.size(); ++I) {
722 const ISD::OutputArg &Out = Outs[I];
723 SDValue &OutVal = OutVals[I];
724 if (Out.Flags.isNest())
725 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
726 if (Out.Flags.isInAlloca())
727 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
728 if (Out.Flags.isInConsecutiveRegs())
729 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
730 if (Out.Flags.isInConsecutiveRegsLast())
731 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
732 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
733 auto &MFI = MF.getFrameInfo();
734 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
735 Out.Flags.getByValAlign(),
736 /*isSS=*/false);
737 SDValue SizeNode =
738 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
739 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
740 Chain = DAG.getMemcpy(
741 Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getByValAlign(),
742 /*isVolatile*/ false, /*AlwaysInline=*/false,
743 /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
744 OutVal = FINode;
745 }
746 // Count the number of fixed args *after* legalization.
747 NumFixedArgs += Out.IsFixed;
748 }
749
750 bool IsVarArg = CLI.IsVarArg;
751 auto PtrVT = getPointerTy(Layout);
752
753 // Analyze operands of the call, assigning locations to each operand.
754 SmallVector<CCValAssign, 16> ArgLocs;
755 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
756
757 if (IsVarArg) {
758 // Outgoing non-fixed arguments are placed in a buffer. First
759 // compute their offsets and the total amount of buffer space needed.
760 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
761 const ISD::OutputArg &Out = Outs[I];
762 SDValue &Arg = OutVals[I];
763 EVT VT = Arg.getValueType();
764 assert(VT != MVT::iPTR && "Legalized args should be concrete")((VT != MVT::iPTR && "Legalized args should be concrete"
) ? static_cast<void> (0) : __assert_fail ("VT != MVT::iPTR && \"Legalized args should be concrete\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 764, __PRETTY_FUNCTION__))
;
765 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
766 unsigned Align = std::max(Out.Flags.getOrigAlign(),
767 Layout.getABITypeAlignment(Ty));
768 unsigned Offset = CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty),
769 Align);
770 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
771 Offset, VT.getSimpleVT(),
772 CCValAssign::Full));
773 }
774 }
775
776 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
777
778 SDValue FINode;
779 if (IsVarArg && NumBytes) {
780 // For non-fixed arguments, next emit stores to store the argument values
781 // to the stack buffer at the offsets computed above.
782 int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
783 Layout.getStackAlignment(),
784 /*isSS=*/false);
785 unsigned ValNo = 0;
786 SmallVector<SDValue, 8> Chains;
787 for (SDValue Arg :
788 make_range(OutVals.begin() + NumFixedArgs, OutVals.end())) {
789 assert(ArgLocs[ValNo].getValNo() == ValNo &&((ArgLocs[ValNo].getValNo() == ValNo && "ArgLocs should remain in order and only hold varargs args"
) ? static_cast<void> (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 790, __PRETTY_FUNCTION__))
790 "ArgLocs should remain in order and only hold varargs args")((ArgLocs[ValNo].getValNo() == ValNo && "ArgLocs should remain in order and only hold varargs args"
) ? static_cast<void> (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 790, __PRETTY_FUNCTION__))
;
791 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
792 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
793 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
794 DAG.getConstant(Offset, DL, PtrVT));
795 Chains.push_back(
796 DAG.getStore(Chain, DL, Arg, Add,
797 MachinePointerInfo::getFixedStack(MF, FI, Offset), 0));
798 }
799 if (!Chains.empty())
800 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
801 } else if (IsVarArg) {
802 FINode = DAG.getIntPtrConstant(0, DL);
803 }
804
805 if (Callee->getOpcode() == ISD::GlobalAddress) {
806 // If the callee is a GlobalAddress node (quite common, every direct call
807 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
808 // doesn't at MO_GOT which is not needed for direct calls.
809 GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee);
810 Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
811 getPointerTy(DAG.getDataLayout()),
812 GA->getOffset());
813 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
814 getPointerTy(DAG.getDataLayout()), Callee);
815 }
816
817 // Compute the operands for the CALLn node.
818 SmallVector<SDValue, 16> Ops;
819 Ops.push_back(Chain);
820 Ops.push_back(Callee);
821
822 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
823 // isn't reliable.
824 Ops.append(OutVals.begin(),
825 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
826 // Add a pointer to the vararg buffer.
827 if (IsVarArg)
828 Ops.push_back(FINode);
829
830 SmallVector<EVT, 8> InTys;
831 for (const auto &In : Ins) {
832 assert(!In.Flags.isByVal() && "byval is not valid for return values")((!In.Flags.isByVal() && "byval is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!In.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 832, __PRETTY_FUNCTION__))
;
833 assert(!In.Flags.isNest() && "nest is not valid for return values")((!In.Flags.isNest() && "nest is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!In.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 833, __PRETTY_FUNCTION__))
;
834 if (In.Flags.isInAlloca())
835 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
836 if (In.Flags.isInConsecutiveRegs())
837 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
838 if (In.Flags.isInConsecutiveRegsLast())
839 fail(DL, DAG,
840 "WebAssembly hasn't implemented cons regs last return values");
841 // Ignore In.getOrigAlign() because all our arguments are passed in
842 // registers.
843 InTys.push_back(In.VT);
844 }
845
846 if (CLI.IsTailCall) {
847 // ret_calls do not return values to the current frame
848 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
849 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
850 }
851
852 InTys.push_back(MVT::Other);
853 SDVTList InTyList = DAG.getVTList(InTys);
854 SDValue Res =
855 DAG.getNode(Ins.empty() ? WebAssemblyISD::CALL0 : WebAssemblyISD::CALL1,
856 DL, InTyList, Ops);
857 if (Ins.empty()) {
858 Chain = Res;
859 } else {
860 InVals.push_back(Res);
861 Chain = Res.getValue(1);
862 }
863
864 return Chain;
865}
866
867bool WebAssemblyTargetLowering::CanLowerReturn(
868 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
869 const SmallVectorImpl<ISD::OutputArg> &Outs,
870 LLVMContext & /*Context*/) const {
871 // WebAssembly can only handle returning tuples with multivalue enabled
872 return Subtarget->hasMultivalue() || Outs.size() <= 1;
873}
874
875SDValue WebAssemblyTargetLowering::LowerReturn(
876 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
877 const SmallVectorImpl<ISD::OutputArg> &Outs,
878 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
879 SelectionDAG &DAG) const {
880 assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&(((Subtarget->hasMultivalue() || Outs.size() <= 1) &&
"MVP WebAssembly can only return up to one value") ? static_cast
<void> (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 881, __PRETTY_FUNCTION__))
881 "MVP WebAssembly can only return up to one value")(((Subtarget->hasMultivalue() || Outs.size() <= 1) &&
"MVP WebAssembly can only return up to one value") ? static_cast
<void> (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 881, __PRETTY_FUNCTION__))
;
882 if (!callingConvSupported(CallConv))
883 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
884
885 SmallVector<SDValue, 4> RetOps(1, Chain);
886 RetOps.append(OutVals.begin(), OutVals.end());
887 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
888
889 // Record the number and types of the return values.
890 for (const ISD::OutputArg &Out : Outs) {
891 assert(!Out.Flags.isByVal() && "byval is not valid for return values")((!Out.Flags.isByVal() && "byval is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!Out.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 891, __PRETTY_FUNCTION__))
;
892 assert(!Out.Flags.isNest() && "nest is not valid for return values")((!Out.Flags.isNest() && "nest is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!Out.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 892, __PRETTY_FUNCTION__))
;
893 assert(Out.IsFixed && "non-fixed return value is not valid")((Out.IsFixed && "non-fixed return value is not valid"
) ? static_cast<void> (0) : __assert_fail ("Out.IsFixed && \"non-fixed return value is not valid\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 893, __PRETTY_FUNCTION__))
;
894 if (Out.Flags.isInAlloca())
895 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
896 if (Out.Flags.isInConsecutiveRegs())
897 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
898 if (Out.Flags.isInConsecutiveRegsLast())
899 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
900 }
901
902 return Chain;
903}
904
905SDValue WebAssemblyTargetLowering::LowerFormalArguments(
906 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
907 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
908 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
909 if (!callingConvSupported(CallConv))
910 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
911
912 MachineFunction &MF = DAG.getMachineFunction();
913 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
914
915 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
916 // of the incoming values before they're represented by virtual registers.
917 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
918
919 for (const ISD::InputArg &In : Ins) {
920 if (In.Flags.isInAlloca())
921 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
922 if (In.Flags.isNest())
923 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
924 if (In.Flags.isInConsecutiveRegs())
925 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
926 if (In.Flags.isInConsecutiveRegsLast())
927 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
928 // Ignore In.getOrigAlign() because all our arguments are passed in
929 // registers.
930 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
931 DAG.getTargetConstant(InVals.size(),
932 DL, MVT::i32))
933 : DAG.getUNDEF(In.VT));
934
935 // Record the number and types of arguments.
936 MFI->addParam(In.VT);
937 }
938
939 // Varargs are copied into a buffer allocated by the caller, and a pointer to
940 // the buffer is passed as an argument.
941 if (IsVarArg) {
942 MVT PtrVT = getPointerTy(MF.getDataLayout());
943 Register VarargVreg =
944 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
945 MFI->setVarargBufferVreg(VarargVreg);
946 Chain = DAG.getCopyToReg(
947 Chain, DL, VarargVreg,
948 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
949 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
950 MFI->addParam(PtrVT);
951 }
952
953 // Record the number and types of arguments and results.
954 SmallVector<MVT, 4> Params;
955 SmallVector<MVT, 4> Results;
956 computeSignatureVTs(MF.getFunction().getFunctionType(), MF.getFunction(),
957 DAG.getTarget(), Params, Results);
958 for (MVT VT : Results)
959 MFI->addResult(VT);
960 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
961 // the param logic here with ComputeSignatureVTs
962 assert(MFI->getParams().size() == Params.size() &&((MFI->getParams().size() == Params.size() && std::
equal(MFI->getParams().begin(), MFI->getParams().end(),
Params.begin())) ? static_cast<void> (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 964, __PRETTY_FUNCTION__))
963 std::equal(MFI->getParams().begin(), MFI->getParams().end(),((MFI->getParams().size() == Params.size() && std::
equal(MFI->getParams().begin(), MFI->getParams().end(),
Params.begin())) ? static_cast<void> (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 964, __PRETTY_FUNCTION__))
964 Params.begin()))((MFI->getParams().size() == Params.size() && std::
equal(MFI->getParams().begin(), MFI->getParams().end(),
Params.begin())) ? static_cast<void> (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 964, __PRETTY_FUNCTION__))
;
965
966 return Chain;
967}
968
969void WebAssemblyTargetLowering::ReplaceNodeResults(
970 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
971 switch (N->getOpcode()) {
972 case ISD::SIGN_EXTEND_INREG:
973 // Do not add any results, signifying that N should not be custom lowered
974 // after all. This happens because simd128 turns on custom lowering for
975 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
976 // illegal type.
977 break;
978 default:
979 llvm_unreachable(::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 980)
980 "ReplaceNodeResults not implemented for this op for WebAssembly!")::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 980)
;
981 }
982}
983
984//===----------------------------------------------------------------------===//
985// Custom lowering hooks.
986//===----------------------------------------------------------------------===//
987
988SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
989 SelectionDAG &DAG) const {
990 SDLoc DL(Op);
991 switch (Op.getOpcode()) {
1
Control jumps to 'case BUILD_VECTOR:' at line 1026
992 default:
993 llvm_unreachable("unimplemented operation lowering")::llvm::llvm_unreachable_internal("unimplemented operation lowering"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 993)
;
994 return SDValue();
995 case ISD::FrameIndex:
996 return LowerFrameIndex(Op, DAG);
997 case ISD::GlobalAddress:
998 return LowerGlobalAddress(Op, DAG);
999 case ISD::ExternalSymbol:
1000 return LowerExternalSymbol(Op, DAG);
1001 case ISD::JumpTable:
1002 return LowerJumpTable(Op, DAG);
1003 case ISD::BR_JT:
1004 return LowerBR_JT(Op, DAG);
1005 case ISD::VASTART:
1006 return LowerVASTART(Op, DAG);
1007 case ISD::BlockAddress:
1008 case ISD::BRIND:
1009 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
1010 return SDValue();
1011 case ISD::RETURNADDR:
1012 return LowerRETURNADDR(Op, DAG);
1013 case ISD::FRAMEADDR:
1014 return LowerFRAMEADDR(Op, DAG);
1015 case ISD::CopyToReg:
1016 return LowerCopyToReg(Op, DAG);
1017 case ISD::EXTRACT_VECTOR_ELT:
1018 case ISD::INSERT_VECTOR_ELT:
1019 return LowerAccessVectorElement(Op, DAG);
1020 case ISD::INTRINSIC_VOID:
1021 case ISD::INTRINSIC_WO_CHAIN:
1022 case ISD::INTRINSIC_W_CHAIN:
1023 return LowerIntrinsic(Op, DAG);
1024 case ISD::SIGN_EXTEND_INREG:
1025 return LowerSIGN_EXTEND_INREG(Op, DAG);
1026 case ISD::BUILD_VECTOR:
1027 return LowerBUILD_VECTOR(Op, DAG);
2
Calling 'WebAssemblyTargetLowering::LowerBUILD_VECTOR'
1028 case ISD::VECTOR_SHUFFLE:
1029 return LowerVECTOR_SHUFFLE(Op, DAG);
1030 case ISD::SETCC:
1031 return LowerSETCC(Op, DAG);
1032 case ISD::SHL:
1033 case ISD::SRA:
1034 case ISD::SRL:
1035 return LowerShift(Op, DAG);
1036 }
1037}
1038
1039SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1040 SelectionDAG &DAG) const {
1041 SDValue Src = Op.getOperand(2);
1042 if (isa<FrameIndexSDNode>(Src.getNode())) {
1043 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1044 // the FI to some LEA-like instruction, but since we don't have that, we
1045 // need to insert some kind of instruction that can take an FI operand and
1046 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1047 // local.copy between Op and its FI operand.
1048 SDValue Chain = Op.getOperand(0);
1049 SDLoc DL(Op);
1050 unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
1051 EVT VT = Src.getValueType();
1052 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1053 : WebAssembly::COPY_I64,
1054 DL, VT, Src),
1055 0);
1056 return Op.getNode()->getNumValues() == 1
1057 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
1058 : DAG.getCopyToReg(Chain, DL, Reg, Copy,
1059 Op.getNumOperands() == 4 ? Op.getOperand(3)
1060 : SDValue());
1061 }
1062 return SDValue();
1063}
1064
1065SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1066 SelectionDAG &DAG) const {
1067 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
1068 return DAG.getTargetFrameIndex(FI, Op.getValueType());
1069}
1070
1071SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1072 SelectionDAG &DAG) const {
1073 SDLoc DL(Op);
1074
1075 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1076 fail(DL, DAG,
1077 "Non-Emscripten WebAssembly hasn't implemented "
1078 "__builtin_return_address");
1079 return SDValue();
1080 }
1081
1082 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1083 return SDValue();
1084
1085 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1086 MakeLibCallOptions CallOptions;
1087 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
1088 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
1089 .first;
1090}
1091
1092SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1093 SelectionDAG &DAG) const {
1094 // Non-zero depths are not supported by WebAssembly currently. Use the
1095 // legalizer's default expansion, which is to return 0 (what this function is
1096 // documented to do).
1097 if (Op.getConstantOperandVal(0) > 0)
1098 return SDValue();
1099
1100 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
1101 EVT VT = Op.getValueType();
1102 Register FP =
1103 Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
1104 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1105}
1106
1107SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1108 SelectionDAG &DAG) const {
1109 SDLoc DL(Op);
1110 const auto *GA = cast<GlobalAddressSDNode>(Op);
1111 EVT VT = Op.getValueType();
1112 assert(GA->getTargetFlags() == 0 &&((GA->getTargetFlags() == 0 && "Unexpected target flags on generic GlobalAddressSDNode"
) ? static_cast<void> (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1113, __PRETTY_FUNCTION__))
1113 "Unexpected target flags on generic GlobalAddressSDNode")((GA->getTargetFlags() == 0 && "Unexpected target flags on generic GlobalAddressSDNode"
) ? static_cast<void> (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1113, __PRETTY_FUNCTION__))
;
1114 if (GA->getAddressSpace() != 0)
1115 fail(DL, DAG, "WebAssembly only expects the 0 address space");
1116
1117 unsigned OperandFlags = 0;
1118 if (isPositionIndependent()) {
1119 const GlobalValue *GV = GA->getGlobal();
1120 if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
1121 MachineFunction &MF = DAG.getMachineFunction();
1122 MVT PtrVT = getPointerTy(MF.getDataLayout());
1123 const char *BaseName;
1124 if (GV->getValueType()->isFunctionTy()) {
1125 BaseName = MF.createExternalSymbolName("__table_base");
1126 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
1127 }
1128 else {
1129 BaseName = MF.createExternalSymbolName("__memory_base");
1130 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
1131 }
1132 SDValue BaseAddr =
1133 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1134 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1135
1136 SDValue SymAddr = DAG.getNode(
1137 WebAssemblyISD::WrapperPIC, DL, VT,
1138 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
1139 OperandFlags));
1140
1141 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
1142 } else {
1143 OperandFlags = WebAssemblyII::MO_GOT;
1144 }
1145 }
1146
1147 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1148 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1149 GA->getOffset(), OperandFlags));
1150}
1151
1152SDValue
1153WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
1154 SelectionDAG &DAG) const {
1155 SDLoc DL(Op);
1156 const auto *ES = cast<ExternalSymbolSDNode>(Op);
1157 EVT VT = Op.getValueType();
1158 assert(ES->getTargetFlags() == 0 &&((ES->getTargetFlags() == 0 && "Unexpected target flags on generic ExternalSymbolSDNode"
) ? static_cast<void> (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1159, __PRETTY_FUNCTION__))
1159 "Unexpected target flags on generic ExternalSymbolSDNode")((ES->getTargetFlags() == 0 && "Unexpected target flags on generic ExternalSymbolSDNode"
) ? static_cast<void> (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1159, __PRETTY_FUNCTION__))
;
1160 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1161 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1162}
1163
1164SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1165 SelectionDAG &DAG) const {
1166 // There's no need for a Wrapper node because we always incorporate a jump
1167 // table operand into a BR_TABLE instruction, rather than ever
1168 // materializing it in a register.
1169 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
1170 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
1171 JT->getTargetFlags());
1172}
1173
1174SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1175 SelectionDAG &DAG) const {
1176 SDLoc DL(Op);
1177 SDValue Chain = Op.getOperand(0);
1178 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
1179 SDValue Index = Op.getOperand(2);
1180 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags")((JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags"
) ? static_cast<void> (0) : __assert_fail ("JT->getTargetFlags() == 0 && \"WebAssembly doesn't set target flags\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1180, __PRETTY_FUNCTION__))
;
1181
1182 SmallVector<SDValue, 8> Ops;
1183 Ops.push_back(Chain);
1184 Ops.push_back(Index);
1185
1186 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
1187 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
1188
1189 // Add an operand for each case.
1190 for (auto MBB : MBBs)
1191 Ops.push_back(DAG.getBasicBlock(MBB));
1192
1193 // TODO: For now, we just pick something arbitrary for a default case for now.
1194 // We really want to sniff out the guard and put in the real default case (and
1195 // delete the guard).
1196 Ops.push_back(DAG.getBasicBlock(MBBs[0]));
1197
1198 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1199}
1200
1201SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1202 SelectionDAG &DAG) const {
1203 SDLoc DL(Op);
1204 EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());
1205
1206 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
1207 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
1208
1209 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
1210 MFI->getVarargBufferVreg(), PtrVT);
1211 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
1212 MachinePointerInfo(SV), 0);
1213}
1214
1215SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1216 SelectionDAG &DAG) const {
1217 MachineFunction &MF = DAG.getMachineFunction();
1218 unsigned IntNo;
1219 switch (Op.getOpcode()) {
1220 case ISD::INTRINSIC_VOID:
1221 case ISD::INTRINSIC_W_CHAIN:
1222 IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
1223 break;
1224 case ISD::INTRINSIC_WO_CHAIN:
1225 IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1226 break;
1227 default:
1228 llvm_unreachable("Invalid intrinsic")::llvm::llvm_unreachable_internal("Invalid intrinsic", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1228)
;
1229 }
1230 SDLoc DL(Op);
1231
1232 switch (IntNo) {
1233 default:
1234 return SDValue(); // Don't custom lower most intrinsics.
1235
1236 case Intrinsic::wasm_lsda: {
1237 EVT VT = Op.getValueType();
1238 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1239 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
1240 auto &Context = MF.getMMI().getContext();
1241 MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
1242 Twine(MF.getFunctionNumber()));
1243 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1244 DAG.getMCSymbol(S, PtrVT));
1245 }
1246
1247 case Intrinsic::wasm_throw: {
1248 // We only support C++ exceptions for now
1249 int Tag = cast<ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
1250 if (Tag != CPP_EXCEPTION)
1251 llvm_unreachable("Invalid tag!")::llvm::llvm_unreachable_internal("Invalid tag!", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1251)
;
1252 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1253 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
1254 const char *SymName = MF.createExternalSymbolName("__cpp_exception");
1255 SDValue SymNode = DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1256 DAG.getTargetExternalSymbol(SymName, PtrVT));
1257 return DAG.getNode(WebAssemblyISD::THROW, DL,
1258 MVT::Other, // outchain type
1259 {
1260 Op.getOperand(0), // inchain
1261 SymNode, // exception symbol
1262 Op.getOperand(3) // thrown value
1263 });
1264 }
1265 }
1266}
1267
1268SDValue
1269WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1270 SelectionDAG &DAG) const {
1271 SDLoc DL(Op);
1272 // If sign extension operations are disabled, allow sext_inreg only if operand
1273 // is a vector extract. SIMD does not depend on sign extension operations, but
1274 // allowing sext_inreg in this context lets us have simple patterns to select
1275 // extract_lane_s instructions. Expanding sext_inreg everywhere would be
1276 // simpler in this file, but would necessitate large and brittle patterns to
1277 // undo the expansion and select extract_lane_s instructions.
1278 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128())((!Subtarget->hasSignExt() && Subtarget->hasSIMD128
()) ? static_cast<void> (0) : __assert_fail ("!Subtarget->hasSignExt() && Subtarget->hasSIMD128()"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1278, __PRETTY_FUNCTION__))
;
1279 if (Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1280 const SDValue &Extract = Op.getOperand(0);
1281 MVT VecT = Extract.getOperand(0).getSimpleValueType();
1282 MVT ExtractedLaneT = static_cast<VTSDNode *>(Op.getOperand(1).getNode())
1283 ->getVT()
1284 .getSimpleVT();
1285 MVT ExtractedVecT =
1286 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
1287 if (ExtractedVecT == VecT)
1288 return Op;
1289 // Bitcast vector to appropriate type to ensure ISel pattern coverage
1290 const SDValue &Index = Extract.getOperand(1);
1291 unsigned IndexVal =
1292 static_cast<ConstantSDNode *>(Index.getNode())->getZExtValue();
1293 unsigned Scale =
1294 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
1295 assert(Scale > 1)((Scale > 1) ? static_cast<void> (0) : __assert_fail
("Scale > 1", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1295, __PRETTY_FUNCTION__))
;
1296 SDValue NewIndex =
1297 DAG.getConstant(IndexVal * Scale, DL, Index.getValueType());
1298 SDValue NewExtract = DAG.getNode(
1299 ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
1300 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
1301 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(),
1302 NewExtract, Op.getOperand(1));
1303 }
1304 // Otherwise expand
1305 return SDValue();
1306}
1307
1308SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1309 SelectionDAG &DAG) const {
1310 SDLoc DL(Op);
1311 const EVT VecT = Op.getValueType();
1312 const EVT LaneT = Op.getOperand(0).getValueType();
1313 const size_t Lanes = Op.getNumOperands();
1314 bool CanSwizzle = Subtarget->hasUnimplementedSIMD128() && VecT == MVT::v16i8;
1315
1316 // BUILD_VECTORs are lowered to the instruction that initializes the highest
1317 // possible number of lanes at once followed by a sequence of replace_lane
1318 // instructions to individually initialize any remaining lanes.
1319
1320 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
1321 // swizzled lanes should be given greater weight.
1322
1323 // TODO: Investigate building vectors by shuffling together vectors built by
1324 // separately specialized means.
1325
1326 auto IsConstant = [](const SDValue &V) {
1327 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
1328 };
1329
1330 // Returns the source vector and index vector pair if they exist. Checks for:
1331 // (extract_vector_elt
1332 // $src,
1333 // (sign_extend_inreg (extract_vector_elt $indices, $i))
1334 // )
1335 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
1336 auto Bail = std::make_pair(SDValue(), SDValue());
1337 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1338 return Bail;
1339 const SDValue &SwizzleSrc = Lane->getOperand(0);
1340 const SDValue &IndexExt = Lane->getOperand(1);
1341 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
1342 return Bail;
1343 const SDValue &Index = IndexExt->getOperand(0);
1344 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1345 return Bail;
1346 const SDValue &SwizzleIndices = Index->getOperand(0);
1347 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
1348 SwizzleIndices.getValueType() != MVT::v16i8 ||
1349 Index->getOperand(1)->getOpcode() != ISD::Constant ||
1350 Index->getConstantOperandVal(1) != I)
1351 return Bail;
1352 return std::make_pair(SwizzleSrc, SwizzleIndices);
1353 };
1354
1355 using ValueEntry = std::pair<SDValue, size_t>;
1356 SmallVector<ValueEntry, 16> SplatValueCounts;
1357
1358 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
1359 SmallVector<SwizzleEntry, 16> SwizzleCounts;
1360
1361 auto AddCount = [](auto &Counts, const auto &Val) {
1362 auto CountIt = std::find_if(Counts.begin(), Counts.end(),
1363 [&Val](auto E) { return E.first == Val; });
1364 if (CountIt == Counts.end()) {
1365 Counts.emplace_back(Val, 1);
1366 } else {
1367 CountIt->second++;
1368 }
1369 };
1370
1371 auto GetMostCommon = [](auto &Counts) {
1372 auto CommonIt =
1373 std::max_element(Counts.begin(), Counts.end(),
1374 [](auto A, auto B) { return A.second < B.second; });
1375 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector")((CommonIt != Counts.end() && "Unexpected all-undef build_vector"
) ? static_cast<void> (0) : __assert_fail ("CommonIt != Counts.end() && \"Unexpected all-undef build_vector\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1375, __PRETTY_FUNCTION__))
;
1376 return *CommonIt;
1377 };
1378
1379 size_t NumConstantLanes = 0;
1380
1381 // Count eligible lanes for each type of vector creation op
1382 for (size_t I = 0; I
2.1
'I' is < 'Lanes'
2.1
'I' is < 'Lanes'
2.1
'I' is < 'Lanes'
< Lanes
; ++I) {
3
Loop condition is true. Entering loop body
6
Assuming 'I' is >= 'Lanes'
7
Loop condition is false. Execution continues on line 1398
1383 const SDValue &Lane = Op->getOperand(I);
1384 if (Lane.isUndef())
4
Taking true branch
1385 continue;
5
Execution continues on line 1382
1386
1387 AddCount(SplatValueCounts, Lane);
1388
1389 if (IsConstant(Lane)) {
1390 NumConstantLanes++;
1391 } else if (CanSwizzle) {
1392 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
1393 if (SwizzleSrcs.first)
1394 AddCount(SwizzleCounts, SwizzleSrcs);
1395 }
1396 }
1397
1398 SDValue SplatValue;
1399 size_t NumSplatLanes;
1400 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
1401
1402 SDValue SwizzleSrc;
1403 SDValue SwizzleIndices;
1404 size_t NumSwizzleLanes = 0;
1405 if (SwizzleCounts.size())
8
Assuming the condition is false
9
Taking false branch
1406 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
1407 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
1408
1409 // Predicate returning true if the lane is properly initialized by the
1410 // original instruction
1411 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
1412 SDValue Result;
1413 if (Subtarget->hasUnimplementedSIMD128()) {
10
Taking true branch
1414 // Prefer swizzles over vector consts over splats
1415 if (NumSwizzleLanes >= NumSplatLanes &&
11
Assuming 'NumSwizzleLanes' is < 'NumSplatLanes'
1416 NumSwizzleLanes >= NumConstantLanes) {
1417 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
1418 SwizzleIndices);
1419 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
1420 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
1421 return Swizzled == GetSwizzleSrcs(I, Lane);
1422 };
1423 } else if (NumConstantLanes
11.1
'NumConstantLanes' is < 'NumSplatLanes'
11.1
'NumConstantLanes' is < 'NumSplatLanes'
11.1
'NumConstantLanes' is < 'NumSplatLanes'
>= NumSplatLanes) {
12
Taking false branch
1424 SmallVector<SDValue, 16> ConstLanes;
1425 for (const SDValue &Lane : Op->op_values()) {
1426 if (IsConstant(Lane)) {
1427 ConstLanes.push_back(Lane);
1428 } else if (LaneT.isFloatingPoint()) {
1429 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
1430 } else {
1431 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
1432 }
1433 }
1434 Result = DAG.getBuildVector(VecT, DL, ConstLanes);
1435 IsLaneConstructed = [&](size_t _, const SDValue &Lane) {
1436 return IsConstant(Lane);
1437 };
1438 }
1439 }
1440 if (!Result) {
13
Taking true branch
1441 // Use a splat, but possibly a load_splat
1442 LoadSDNode *SplattedLoad;
1443 if (Subtarget->hasUnimplementedSIMD128() &&
15
Assuming pointer value is null
16
Taking false branch
1444 (SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
14
Assuming 'SplattedLoad' is null
1445 SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
1446 Result = DAG.getMemIntrinsicNode(
1447 WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT),
1448 {SplattedLoad->getChain(), SplattedLoad->getBasePtr(),
1449 SplattedLoad->getOffset()},
1450 SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand());
1451 } else {
1452 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
17
Value assigned to 'Op.Node'
18
Calling 'SelectionDAG::getSplatBuildVector'
1453 }
1454 IsLaneConstructed = [&](size_t _, const SDValue &Lane) {
1455 return Lane == SplatValue;
1456 };
1457 }
1458
1459 // Add replace_lane instructions for any unhandled values
1460 for (size_t I = 0; I < Lanes; ++I) {
1461 const SDValue &Lane = Op->getOperand(I);
1462 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
1463 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
1464 DAG.getConstant(I, DL, MVT::i32));
1465 }
1466
1467 return Result;
1468}
1469
1470SDValue
1471WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
1472 SelectionDAG &DAG) const {
1473 SDLoc DL(Op);
1474 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
1475 MVT VecType = Op.getOperand(0).getSimpleValueType();
1476 assert(VecType.is128BitVector() && "Unexpected shuffle vector type")((VecType.is128BitVector() && "Unexpected shuffle vector type"
) ? static_cast<void> (0) : __assert_fail ("VecType.is128BitVector() && \"Unexpected shuffle vector type\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1476, __PRETTY_FUNCTION__))
;
1477 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
1478
1479 // Space for two vector args and sixteen mask indices
1480 SDValue Ops[18];
1481 size_t OpIdx = 0;
1482 Ops[OpIdx++] = Op.getOperand(0);
1483 Ops[OpIdx++] = Op.getOperand(1);
1484
1485 // Expand mask indices to byte indices and materialize them as operands
1486 for (int M : Mask) {
1487 for (size_t J = 0; J < LaneBytes; ++J) {
1488 // Lower undefs (represented by -1 in mask) to zero
1489 uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J;
1490 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
1491 }
1492 }
1493
1494 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
1495}
1496
1497SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
1498 SelectionDAG &DAG) const {
1499 SDLoc DL(Op);
1500 // The legalizer does not know how to expand the comparison modes of i64x2
1501 // vectors because no comparison modes are supported. We could solve this by
1502 // expanding all i64x2 SETCC nodes, but that seems to expand f64x2 SETCC nodes
1503 // (which return i64x2 results) as well. So instead we manually unroll i64x2
1504 // comparisons here.
1505 assert(Subtarget->hasUnimplementedSIMD128())((Subtarget->hasUnimplementedSIMD128()) ? static_cast<void
> (0) : __assert_fail ("Subtarget->hasUnimplementedSIMD128()"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1505, __PRETTY_FUNCTION__))
;
1506 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64)((Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64
) ? static_cast<void> (0) : __assert_fail ("Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1506, __PRETTY_FUNCTION__))
;
1507 SmallVector<SDValue, 2> LHS, RHS;
1508 DAG.ExtractVectorElements(Op->getOperand(0), LHS);
1509 DAG.ExtractVectorElements(Op->getOperand(1), RHS);
1510 const SDValue &CC = Op->getOperand(2);
1511 auto MakeLane = [&](unsigned I) {
1512 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
1513 DAG.getConstant(uint64_t(-1), DL, MVT::i64),
1514 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
1515 };
1516 return DAG.getBuildVector(Op->getValueType(0), DL,
1517 {MakeLane(0), MakeLane(1)});
1518}
1519
1520SDValue
1521WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
1522 SelectionDAG &DAG) const {
1523 // Allow constant lane indices, expand variable lane indices
1524 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
1525 if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
1526 return Op;
1527 else
1528 // Perform default expansion
1529 return SDValue();
1530}
1531
1532static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
1533 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
1534 // 32-bit and 64-bit unrolled shifts will have proper semantics
1535 if (LaneT.bitsGE(MVT::i32))
1536 return DAG.UnrollVectorOp(Op.getNode());
1537 // Otherwise mask the shift value to get proper semantics from 32-bit shift
1538 SDLoc DL(Op);
1539 SDValue ShiftVal = Op.getOperand(1);
1540 uint64_t MaskVal = LaneT.getSizeInBits() - 1;
1541 SDValue MaskedShiftVal = DAG.getNode(
1542 ISD::AND, // mask opcode
1543 DL, ShiftVal.getValueType(), // masked value type
1544 ShiftVal, // original shift value operand
1545 DAG.getConstant(MaskVal, DL, ShiftVal.getValueType()) // mask operand
1546 );
1547
1548 return DAG.UnrollVectorOp(
1549 DAG.getNode(Op.getOpcode(), // original shift opcode
1550 DL, Op.getValueType(), // original return type
1551 Op.getOperand(0), // original vector operand,
1552 MaskedShiftVal // new masked shift value operand
1553 )
1554 .getNode());
1555}
1556
1557SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
1558 SelectionDAG &DAG) const {
1559 SDLoc DL(Op);
1560
1561 // Only manually lower vector shifts
1562 assert(Op.getSimpleValueType().isVector())((Op.getSimpleValueType().isVector()) ? static_cast<void>
(0) : __assert_fail ("Op.getSimpleValueType().isVector()", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1562, __PRETTY_FUNCTION__))
;
1563
1564 // Unroll non-splat vector shifts
1565 BuildVectorSDNode *ShiftVec;
1566 SDValue SplatVal;
1567 if (!(ShiftVec = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) ||
1568 !(SplatVal = ShiftVec->getSplatValue()))
1569 return unrollVectorShift(Op, DAG);
1570
1571 // All splats except i64x2 const splats are handled by patterns
1572 auto *SplatConst = dyn_cast<ConstantSDNode>(SplatVal);
1573 if (!SplatConst || Op.getSimpleValueType() != MVT::v2i64)
1574 return Op;
1575
1576 // i64x2 const splats are custom lowered to avoid unnecessary wraps
1577 unsigned Opcode;
1578 switch (Op.getOpcode()) {
1579 case ISD::SHL:
1580 Opcode = WebAssemblyISD::VEC_SHL;
1581 break;
1582 case ISD::SRA:
1583 Opcode = WebAssemblyISD::VEC_SHR_S;
1584 break;
1585 case ISD::SRL:
1586 Opcode = WebAssemblyISD::VEC_SHR_U;
1587 break;
1588 default:
1589 llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1589)
;
1590 }
1591 APInt Shift = SplatConst->getAPIntValue().zextOrTrunc(32);
1592 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0),
1593 DAG.getConstant(Shift, DL, MVT::i32));
1594}
1595
1596//===----------------------------------------------------------------------===//
1597// WebAssembly Optimization Hooks
1598//===----------------------------------------------------------------------===//

/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h

1//===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SelectionDAG class, and transitively defines the
10// SDNode class and subclasses.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CODEGEN_SELECTIONDAG_H
15#define LLVM_CODEGEN_SELECTIONDAG_H
16
17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/APInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/FoldingSet.h"
23#include "llvm/ADT/SetVector.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/ilist.h"
27#include "llvm/ADT/iterator.h"
28#include "llvm/ADT/iterator_range.h"
29#include "llvm/CodeGen/DAGCombine.h"
30#include "llvm/CodeGen/FunctionLoweringInfo.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineFunction.h"
33#include "llvm/CodeGen/MachineMemOperand.h"
34#include "llvm/CodeGen/SelectionDAGNodes.h"
35#include "llvm/CodeGen/ValueTypes.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/Support/Allocator.h"
40#include "llvm/Support/ArrayRecycler.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/CodeGen.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/RecyclingAllocator.h"
47#include <algorithm>
48#include <cassert>
49#include <cstdint>
50#include <functional>
51#include <map>
52#include <string>
53#include <tuple>
54#include <utility>
55#include <vector>
56
57namespace llvm {
58
59class AAResults;
60class BlockAddress;
61class BlockFrequencyInfo;
62class Constant;
63class ConstantFP;
64class ConstantInt;
65class DataLayout;
66struct fltSemantics;
67class GlobalValue;
68struct KnownBits;
69class LegacyDivergenceAnalysis;
70class LLVMContext;
71class MachineBasicBlock;
72class MachineConstantPoolValue;
73class MCSymbol;
74class OptimizationRemarkEmitter;
75class ProfileSummaryInfo;
76class SDDbgValue;
77class SDDbgLabel;
78class SelectionDAG;
79class SelectionDAGTargetInfo;
80class TargetLibraryInfo;
81class TargetLowering;
82class TargetMachine;
83class TargetSubtargetInfo;
84class Value;
85
86class SDVTListNode : public FoldingSetNode {
87 friend struct FoldingSetTrait<SDVTListNode>;
88
89 /// A reference to an Interned FoldingSetNodeID for this node.
90 /// The Allocator in SelectionDAG holds the data.
91 /// SDVTList contains all types which are frequently accessed in SelectionDAG.
92 /// The size of this list is not expected to be big so it won't introduce
93 /// a memory penalty.
94 FoldingSetNodeIDRef FastID;
95 const EVT *VTs;
96 unsigned int NumVTs;
97 /// The hash value for SDVTList is fixed, so cache it to avoid
98 /// hash calculation.
99 unsigned HashValue;
100
101public:
102 SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) :
103 FastID(ID), VTs(VT), NumVTs(Num) {
104 HashValue = ID.ComputeHash();
105 }
106
107 SDVTList getSDVTList() {
108 SDVTList result = {VTs, NumVTs};
109 return result;
110 }
111};
112
113/// Specialize FoldingSetTrait for SDVTListNode
114/// to avoid computing temp FoldingSetNodeID and hash value.
115template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> {
116 static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) {
117 ID = X.FastID;
118 }
119
120 static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID,
121 unsigned IDHash, FoldingSetNodeID &TempID) {
122 if (X.HashValue != IDHash)
123 return false;
124 return ID == X.FastID;
125 }
126
127 static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) {
128 return X.HashValue;
129 }
130};
131
132template <> struct ilist_alloc_traits<SDNode> {
133 static void deleteNode(SDNode *) {
134 llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!")::llvm::llvm_unreachable_internal("ilist_traits<SDNode> shouldn't see a deleteNode call!"
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 134)
;
135 }
136};
137
138/// Keeps track of dbg_value information through SDISel. We do
139/// not build SDNodes for these so as not to perturb the generated code;
140/// instead the info is kept off to the side in this structure. Each SDNode may
141/// have one or more associated dbg_value entries. This information is kept in
142/// DbgValMap.
143/// Byval parameters are handled separately because they don't use alloca's,
144/// which busts the normal mechanism. There is good reason for handling all
145/// parameters separately: they may not have code generated for them, they
146/// should always go at the beginning of the function regardless of other code
147/// motion, and debug info for them is potentially useful even if the parameter
148/// is unused. Right now only byval parameters are handled separately.
149class SDDbgInfo {
150 BumpPtrAllocator Alloc;
151 SmallVector<SDDbgValue*, 32> DbgValues;
152 SmallVector<SDDbgValue*, 32> ByvalParmDbgValues;
153 SmallVector<SDDbgLabel*, 4> DbgLabels;
154 using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>;
155 DbgValMapType DbgValMap;
156
157public:
158 SDDbgInfo() = default;
159 SDDbgInfo(const SDDbgInfo &) = delete;
160 SDDbgInfo &operator=(const SDDbgInfo &) = delete;
161
162 void add(SDDbgValue *V, const SDNode *Node, bool isParameter) {
163 if (isParameter) {
164 ByvalParmDbgValues.push_back(V);
165 } else DbgValues.push_back(V);
166 if (Node)
167 DbgValMap[Node].push_back(V);
168 }
169
170 void add(SDDbgLabel *L) {
171 DbgLabels.push_back(L);
172 }
173
174 /// Invalidate all DbgValues attached to the node and remove
175 /// it from the Node-to-DbgValues map.
176 void erase(const SDNode *Node);
177
178 void clear() {
179 DbgValMap.clear();
180 DbgValues.clear();
181 ByvalParmDbgValues.clear();
182 DbgLabels.clear();
183 Alloc.Reset();
184 }
185
186 BumpPtrAllocator &getAlloc() { return Alloc; }
187
188 bool empty() const {
189 return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty();
190 }
191
192 ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const {
193 auto I = DbgValMap.find(Node);
194 if (I != DbgValMap.end())
195 return I->second;
196 return ArrayRef<SDDbgValue*>();
197 }
198
199 using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator;
200 using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator;
201
202 DbgIterator DbgBegin() { return DbgValues.begin(); }
203 DbgIterator DbgEnd() { return DbgValues.end(); }
204 DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); }
205 DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); }
206 DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); }
207 DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); }
208};
209
210void checkForCycles(const SelectionDAG *DAG, bool force = false);
211
212/// This is used to represent a portion of an LLVM function in a low-level
213/// Data Dependence DAG representation suitable for instruction selection.
214/// This DAG is constructed as the first step of instruction selection in order
215/// to allow implementation of machine specific optimizations
216/// and code simplifications.
217///
218/// The representation used by the SelectionDAG is a target-independent
219/// representation, which has some similarities to the GCC RTL representation,
220/// but is significantly more simple, powerful, and is a graph form instead of a
221/// linear form.
222///
223class SelectionDAG {
224 const TargetMachine &TM;
225 const SelectionDAGTargetInfo *TSI = nullptr;
226 const TargetLowering *TLI = nullptr;
227 const TargetLibraryInfo *LibInfo = nullptr;
228 MachineFunction *MF;
229 Pass *SDAGISelPass = nullptr;
230 LLVMContext *Context;
231 CodeGenOpt::Level OptLevel;
232
233 LegacyDivergenceAnalysis * DA = nullptr;
234 FunctionLoweringInfo * FLI = nullptr;
235
236 /// The function-level optimization remark emitter. Used to emit remarks
237 /// whenever manipulating the DAG.
238 OptimizationRemarkEmitter *ORE;
239
240 ProfileSummaryInfo *PSI = nullptr;
241 BlockFrequencyInfo *BFI = nullptr;
242
243 /// The starting token.
244 SDNode EntryNode;
245
246 /// The root of the entire DAG.
247 SDValue Root;
248
249 /// A linked list of nodes in the current DAG.
250 ilist<SDNode> AllNodes;
251
252 /// The AllocatorType for allocating SDNodes. We use
253 /// pool allocation with recycling.
254 using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode,
255 sizeof(LargestSDNode),
256 alignof(MostAlignedSDNode)>;
257
258 /// Pool allocation for nodes.
259 NodeAllocatorType NodeAllocator;
260
261 /// This structure is used to memoize nodes, automatically performing
262 /// CSE with existing nodes when a duplicate is requested.
263 FoldingSet<SDNode> CSEMap;
264
265 /// Pool allocation for machine-opcode SDNode operands.
266 BumpPtrAllocator OperandAllocator;
267 ArrayRecycler<SDUse> OperandRecycler;
268
269 /// Pool allocation for misc. objects that are created once per SelectionDAG.
270 BumpPtrAllocator Allocator;
271
272 /// Tracks dbg_value and dbg_label information through SDISel.
273 SDDbgInfo *DbgInfo;
274
275 using CallSiteInfo = MachineFunction::CallSiteInfo;
276 using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl;
277
278 struct CallSiteDbgInfo {
279 CallSiteInfo CSInfo;
280 MDNode *HeapAllocSite = nullptr;
281 };
282
283 DenseMap<const SDNode *, CallSiteDbgInfo> SDCallSiteDbgInfo;
284
285 uint16_t NextPersistentId = 0;
286
287public:
288 /// Clients of various APIs that cause global effects on
289 /// the DAG can optionally implement this interface. This allows the clients
290 /// to handle the various sorts of updates that happen.
291 ///
292 /// A DAGUpdateListener automatically registers itself with DAG when it is
293 /// constructed, and removes itself when destroyed in RAII fashion.
294 struct DAGUpdateListener {
295 DAGUpdateListener *const Next;
296 SelectionDAG &DAG;
297
298 explicit DAGUpdateListener(SelectionDAG &D)
299 : Next(D.UpdateListeners), DAG(D) {
300 DAG.UpdateListeners = this;
301 }
302
303 virtual ~DAGUpdateListener() {
304 assert(DAG.UpdateListeners == this &&((DAG.UpdateListeners == this && "DAGUpdateListeners must be destroyed in LIFO order"
) ? static_cast<void> (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 305, __PRETTY_FUNCTION__))
305 "DAGUpdateListeners must be destroyed in LIFO order")((DAG.UpdateListeners == this && "DAGUpdateListeners must be destroyed in LIFO order"
) ? static_cast<void> (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 305, __PRETTY_FUNCTION__))
;
306 DAG.UpdateListeners = Next;
307 }
308
309 /// The node N that was deleted and, if E is not null, an
310 /// equivalent node E that replaced it.
311 virtual void NodeDeleted(SDNode *N, SDNode *E);
312
313 /// The node N that was updated.
314 virtual void NodeUpdated(SDNode *N);
315
316 /// The node N that was inserted.
317 virtual void NodeInserted(SDNode *N);
318 };
319
320 struct DAGNodeDeletedListener : public DAGUpdateListener {
321 std::function<void(SDNode *, SDNode *)> Callback;
322
323 DAGNodeDeletedListener(SelectionDAG &DAG,
324 std::function<void(SDNode *, SDNode *)> Callback)
325 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {}
326
327 void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); }
328
329 private:
330 virtual void anchor();
331 };
332
333 /// When true, additional steps are taken to
334 /// ensure that getConstant() and similar functions return DAG nodes that
335 /// have legal types. This is important after type legalization since
336 /// any illegally typed nodes generated after this point will not experience
337 /// type legalization.
338 bool NewNodesMustHaveLegalTypes = false;
339
340private:
341 /// DAGUpdateListener is a friend so it can manipulate the listener stack.
342 friend struct DAGUpdateListener;
343
344 /// Linked list of registered DAGUpdateListener instances.
345 /// This stack is maintained by DAGUpdateListener RAII.
346 DAGUpdateListener *UpdateListeners = nullptr;
347
348 /// Implementation of setSubgraphColor.
349 /// Return whether we had to truncate the search.
350 bool setSubgraphColorHelper(SDNode *N, const char *Color,
351 DenseSet<SDNode *> &visited,
352 int level, bool &printed);
353
354 template <typename SDNodeT, typename... ArgTypes>
355 SDNodeT *newSDNode(ArgTypes &&... Args) {
356 return new (NodeAllocator.template Allocate<SDNodeT>())
357 SDNodeT(std::forward<ArgTypes>(Args)...);
358 }
359
360 /// Build a synthetic SDNodeT with the given args and extract its subclass
361 /// data as an integer (e.g. for use in a folding set).
362 ///
363 /// The args to this function are the same as the args to SDNodeT's
364 /// constructor, except the second arg (assumed to be a const DebugLoc&) is
365 /// omitted.
366 template <typename SDNodeT, typename... ArgTypes>
367 static uint16_t getSyntheticNodeSubclassData(unsigned IROrder,
368 ArgTypes &&... Args) {
369 // The compiler can reduce this expression to a constant iff we pass an
370 // empty DebugLoc. Thankfully, the debug location doesn't have any bearing
371 // on the subclass data.
372 return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...)
373 .getRawSubclassData();
374 }
375
376 template <typename SDNodeTy>
377 static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order,
378 SDVTList VTs, EVT MemoryVT,
379 MachineMemOperand *MMO) {
380 return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO)
381 .getRawSubclassData();
382 }
383
384 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals);
385
386 void removeOperands(SDNode *Node) {
387 if (!Node->OperandList)
388 return;
389 OperandRecycler.deallocate(
390 ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands),
391 Node->OperandList);
392 Node->NumOperands = 0;
393 Node->OperandList = nullptr;
394 }
395 void CreateTopologicalOrder(std::vector<SDNode*>& Order);
396
397public:
398 // Maximum depth for recursive analysis such as computeKnownBits, etc.
399 static constexpr unsigned MaxRecursionDepth = 6;
400
401 explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level);
402 SelectionDAG(const SelectionDAG &) = delete;
403 SelectionDAG &operator=(const SelectionDAG &) = delete;
404 ~SelectionDAG();
405
406 /// Prepare this SelectionDAG to process code in the given MachineFunction.
407 void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
408 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
409 LegacyDivergenceAnalysis * Divergence,
410 ProfileSummaryInfo *PSIin, BlockFrequencyInfo *BFIin);
411
412 void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) {
413 FLI = FuncInfo;
414 }
415
416 /// Clear state and free memory necessary to make this
417 /// SelectionDAG ready to process a new block.
418 void clear();
419
420 MachineFunction &getMachineFunction() const { return *MF; }
421 const Pass *getPass() const { return SDAGISelPass; }
422
423 const DataLayout &getDataLayout() const { return MF->getDataLayout(); }
424 const TargetMachine &getTarget() const { return TM; }
425 const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); }
426 const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
427 const TargetLibraryInfo &getLibInfo() const { return *LibInfo; }
428 const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; }
429 const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; }
430 LLVMContext *getContext() const { return Context; }
431 OptimizationRemarkEmitter &getORE() const { return *ORE; }
432 ProfileSummaryInfo *getPSI() const { return PSI; }
433 BlockFrequencyInfo *getBFI() const { return BFI; }
434
435 /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
436 void viewGraph(const std::string &Title);
437 void viewGraph();
438
439#ifndef NDEBUG
440 std::map<const SDNode *, std::string> NodeGraphAttrs;
441#endif
442
443 /// Clear all previously defined node graph attributes.
444 /// Intended to be used from a debugging tool (eg. gdb).
445 void clearGraphAttrs();
446
447 /// Set graph attributes for a node. (eg. "color=red".)
448 void setGraphAttrs(const SDNode *N, const char *Attrs);
449
450 /// Get graph attributes for a node. (eg. "color=red".)
451 /// Used from getNodeAttributes.
452 const std::string getGraphAttrs(const SDNode *N) const;
453
454 /// Convenience for setting node color attribute.
455 void setGraphColor(const SDNode *N, const char *Color);
456
457 /// Convenience for setting subgraph color attribute.
458 void setSubgraphColor(SDNode *N, const char *Color);
459
460 using allnodes_const_iterator = ilist<SDNode>::const_iterator;
461
462 allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); }
463 allnodes_const_iterator allnodes_end() const { return AllNodes.end(); }
464
465 using allnodes_iterator = ilist<SDNode>::iterator;
466
467 allnodes_iterator allnodes_begin() { return AllNodes.begin(); }
468 allnodes_iterator allnodes_end() { return AllNodes.end(); }
469
470 ilist<SDNode>::size_type allnodes_size() const {
471 return AllNodes.size();
472 }
473
474 iterator_range<allnodes_iterator> allnodes() {
475 return make_range(allnodes_begin(), allnodes_end());
476 }
477 iterator_range<allnodes_const_iterator> allnodes() const {
478 return make_range(allnodes_begin(), allnodes_end());
479 }
480
481 /// Return the root tag of the SelectionDAG.
482 const SDValue &getRoot() const { return Root; }
483
484 /// Return the token chain corresponding to the entry of the function.
485 SDValue getEntryNode() const {
486 return SDValue(const_cast<SDNode *>(&EntryNode), 0);
487 }
488
489 /// Set the current root tag of the SelectionDAG.
490 ///
491 const SDValue &setRoot(SDValue N) {
492 assert((!N.getNode() || N.getValueType() == MVT::Other) &&(((!N.getNode() || N.getValueType() == MVT::Other) &&
"DAG root value is not a chain!") ? static_cast<void> (
0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 493, __PRETTY_FUNCTION__))
493 "DAG root value is not a chain!")(((!N.getNode() || N.getValueType() == MVT::Other) &&
"DAG root value is not a chain!") ? static_cast<void> (
0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 493, __PRETTY_FUNCTION__))
;
494 if (N.getNode())
495 checkForCycles(N.getNode(), this);
496 Root = N;
497 if (N.getNode())
498 checkForCycles(this);
499 return Root;
500 }
501
502#ifndef NDEBUG
503 void VerifyDAGDiverence();
504#endif
505
506 /// This iterates over the nodes in the SelectionDAG, folding
507 /// certain types of nodes together, or eliminating superfluous nodes. The
508 /// Level argument controls whether Combine is allowed to produce nodes and
509 /// types that are illegal on the target.
510 void Combine(CombineLevel Level, AAResults *AA,
511 CodeGenOpt::Level OptLevel);
512
513 /// This transforms the SelectionDAG into a SelectionDAG that
514 /// only uses types natively supported by the target.
515 /// Returns "true" if it made any changes.
516 ///
517 /// Note that this is an involved process that may invalidate pointers into
518 /// the graph.
519 bool LegalizeTypes();
520
521 /// This transforms the SelectionDAG into a SelectionDAG that is
522 /// compatible with the target instruction selector, as indicated by the
523 /// TargetLowering object.
524 ///
525 /// Note that this is an involved process that may invalidate pointers into
526 /// the graph.
527 void Legalize();
528
529 /// Transforms a SelectionDAG node and any operands to it into a node
530 /// that is compatible with the target instruction selector, as indicated by
531 /// the TargetLowering object.
532 ///
533 /// \returns true if \c N is a valid, legal node after calling this.
534 ///
535 /// This essentially runs a single recursive walk of the \c Legalize process
536 /// over the given node (and its operands). This can be used to incrementally
537 /// legalize the DAG. All of the nodes which are directly replaced,
538 /// potentially including N, are added to the output parameter \c
539 /// UpdatedNodes so that the delta to the DAG can be understood by the
540 /// caller.
541 ///
542 /// When this returns false, N has been legalized in a way that make the
543 /// pointer passed in no longer valid. It may have even been deleted from the
544 /// DAG, and so it shouldn't be used further. When this returns true, the
545 /// N passed in is a legal node, and can be immediately processed as such.
546 /// This may still have done some work on the DAG, and will still populate
547 /// UpdatedNodes with any new nodes replacing those originally in the DAG.
548 bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes);
549
550 /// This transforms the SelectionDAG into a SelectionDAG
551 /// that only uses vector math operations supported by the target. This is
552 /// necessary as a separate step from Legalize because unrolling a vector
553 /// operation can introduce illegal types, which requires running
554 /// LegalizeTypes again.
555 ///
556 /// This returns true if it made any changes; in that case, LegalizeTypes
557 /// is called again before Legalize.
558 ///
559 /// Note that this is an involved process that may invalidate pointers into
560 /// the graph.
561 bool LegalizeVectors();
562
563 /// This method deletes all unreachable nodes in the SelectionDAG.
564 void RemoveDeadNodes();
565
566 /// Remove the specified node from the system. This node must
567 /// have no referrers.
568 void DeleteNode(SDNode *N);
569
570 /// Return an SDVTList that represents the list of values specified.
571 SDVTList getVTList(EVT VT);
572 SDVTList getVTList(EVT VT1, EVT VT2);
573 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3);
574 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4);
575 SDVTList getVTList(ArrayRef<EVT> VTs);
576
577 //===--------------------------------------------------------------------===//
578 // Node creation methods.
579
580 /// Create a ConstantSDNode wrapping a constant value.
581 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
582 ///
583 /// If only legal types can be produced, this does the necessary
584 /// transformations (e.g., if the vector element type is illegal).
585 /// @{
586 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
587 bool isTarget = false, bool isOpaque = false);
588 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
589 bool isTarget = false, bool isOpaque = false);
590
591 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
592 bool IsOpaque = false) {
593 return getConstant(APInt::getAllOnesValue(VT.getScalarSizeInBits()), DL,
594 VT, IsTarget, IsOpaque);
595 }
596
597 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
598 bool isTarget = false, bool isOpaque = false);
599 SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL,
600 bool isTarget = false);
601 SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL,
602 bool LegalTypes = true);
603
604 SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT,
605 bool isOpaque = false) {
606 return getConstant(Val, DL, VT, true, isOpaque);
607 }
608 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT,
609 bool isOpaque = false) {
610 return getConstant(Val, DL, VT, true, isOpaque);
611 }
612 SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
613 bool isOpaque = false) {
614 return getConstant(Val, DL, VT, true, isOpaque);
615 }
616
617 /// Create a true or false constant of type \p VT using the target's
618 /// BooleanContent for type \p OpVT.
619 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT);
620 /// @}
621
622 /// Create a ConstantFPSDNode wrapping a constant value.
623 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
624 ///
625 /// If only legal types can be produced, this does the necessary
626 /// transformations (e.g., if the vector element type is illegal).
627 /// The forms that take a double should only be used for simple constants
628 /// that can be exactly represented in VT. No checks are made.
629 /// @{
630 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT,
631 bool isTarget = false);
632 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT,
633 bool isTarget = false);
634 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT,
635 bool isTarget = false);
636 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) {
637 return getConstantFP(Val, DL, VT, true);
638 }
639 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) {
640 return getConstantFP(Val, DL, VT, true);
641 }
642 SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) {
643 return getConstantFP(Val, DL, VT, true);
644 }
645 /// @}
646
647 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
648 int64_t offset = 0, bool isTargetGA = false,
649 unsigned TargetFlags = 0);
650 SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
651 int64_t offset = 0, unsigned TargetFlags = 0) {
652 return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags);
653 }
654 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false);
655 SDValue getTargetFrameIndex(int FI, EVT VT) {
656 return getFrameIndex(FI, VT, true);
657 }
658 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false,
659 unsigned TargetFlags = 0);
660 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) {
661 return getJumpTable(JTI, VT, true, TargetFlags);
662 }
663 SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align = 0,
664 int Offs = 0, bool isT = false,
665 unsigned TargetFlags = 0);
666 SDValue getTargetConstantPool(const Constant *C, EVT VT, unsigned Align = 0,
667 int Offset = 0, unsigned TargetFlags = 0) {
668 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
669 }
670 SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT,
671 unsigned Align = 0, int Offs = 0, bool isT=false,
672 unsigned TargetFlags = 0);
673 SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT,
674 unsigned Align = 0, int Offset = 0,
675 unsigned TargetFlags = 0) {
676 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
677 }
678 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0,
679 unsigned TargetFlags = 0);
680 // When generating a branch to a BB, we don't in general know enough
681 // to provide debug info for the BB at that time, so keep this one around.
682 SDValue getBasicBlock(MachineBasicBlock *MBB);
683 SDValue getBasicBlock(MachineBasicBlock *MBB, SDLoc dl);
684 SDValue getExternalSymbol(const char *Sym, EVT VT);
685 SDValue getExternalSymbol(const char *Sym, const SDLoc &dl, EVT VT);
686 SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
687 unsigned TargetFlags = 0);
688 SDValue getMCSymbol(MCSymbol *Sym, EVT VT);
689
690 SDValue getValueType(EVT);
691 SDValue getRegister(unsigned Reg, EVT VT);
692 SDValue getRegisterMask(const uint32_t *RegMask);
693 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label);
694 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root,
695 MCSymbol *Label);
696 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0,
697 bool isTarget = false, unsigned TargetFlags = 0);
698 SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT,
699 int64_t Offset = 0, unsigned TargetFlags = 0) {
700 return getBlockAddress(BA, VT, Offset, true, TargetFlags);
701 }
702
703 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg,
704 SDValue N) {
705 return getNode(ISD::CopyToReg, dl, MVT::Other, Chain,
706 getRegister(Reg, N.getValueType()), N);
707 }
708
709 // This version of the getCopyToReg method takes an extra operand, which
710 // indicates that there is potentially an incoming glue value (if Glue is not
711 // null) and that there should be a glue result.
712 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N,
713 SDValue Glue) {
714 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
715 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue };
716 return getNode(ISD::CopyToReg, dl, VTs,
717 makeArrayRef(Ops, Glue.getNode() ? 4 : 3));
718 }
719
720 // Similar to last getCopyToReg() except parameter Reg is a SDValue
721 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
722 SDValue Glue) {
723 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
724 SDValue Ops[] = { Chain, Reg, N, Glue };
725 return getNode(ISD::CopyToReg, dl, VTs,
726 makeArrayRef(Ops, Glue.getNode() ? 4 : 3));
727 }
728
729 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) {
730 SDVTList VTs = getVTList(VT, MVT::Other);
731 SDValue Ops[] = { Chain, getRegister(Reg, VT) };
732 return getNode(ISD::CopyFromReg, dl, VTs, Ops);
733 }
734
735 // This version of the getCopyFromReg method takes an extra operand, which
736 // indicates that there is potentially an incoming glue value (if Glue is not
737 // null) and that there should be a glue result.
738 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT,
739 SDValue Glue) {
740 SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue);
741 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue };
742 return getNode(ISD::CopyFromReg, dl, VTs,
743 makeArrayRef(Ops, Glue.getNode() ? 3 : 2));
744 }
745
746 SDValue getCondCode(ISD::CondCode Cond);
747
748 /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT,
749 /// which must be a vector type, must match the number of mask elements
750 /// NumElts. An integer mask element equal to -1 is treated as undefined.
751 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
752 ArrayRef<int> Mask);
753
754 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
755 /// which must be a vector type, must match the number of operands in Ops.
756 /// The operands must have the same type as (or, for integers, a type wider
757 /// than) VT's element type.
758 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) {
759 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
760 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
761 }
762
763 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
764 /// which must be a vector type, must match the number of operands in Ops.
765 /// The operands must have the same type as (or, for integers, a type wider
766 /// than) VT's element type.
767 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) {
768 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
769 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
770 }
771
772 /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all
773 /// elements. VT must be a vector type. Op's type must be the same as (or,
774 /// for integers, a type wider than) VT's element type.
775 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) {
776 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
777 if (Op.getOpcode() == ISD::UNDEF) {
19
Calling 'SDValue::getOpcode'
778 assert((VT.getVectorElementType() == Op.getValueType() ||(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 782, __PRETTY_FUNCTION__))
779 (VT.isInteger() &&(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 782, __PRETTY_FUNCTION__))
780 VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 782, __PRETTY_FUNCTION__))
781 "A splatted value must have a width equal or (for integers) "(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 782, __PRETTY_FUNCTION__))
782 "greater than the vector element type!")(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 782, __PRETTY_FUNCTION__))
;
783 return getNode(ISD::UNDEF, SDLoc(), VT);
784 }
785
786 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op);
787 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
788 }
789
790 // Return a splat ISD::SPLAT_VECTOR node, consisting of Op splatted to all
791 // elements.
792 SDValue getSplatVector(EVT VT, const SDLoc &DL, SDValue Op) {
793 if (Op.getOpcode() == ISD::UNDEF) {
794 assert((VT.getVectorElementType() == Op.getValueType() ||(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 798, __PRETTY_FUNCTION__))
795 (VT.isInteger() &&(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 798, __PRETTY_FUNCTION__))
796 VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 798, __PRETTY_FUNCTION__))
797 "A splatted value must have a width equal or (for integers) "(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 798, __PRETTY_FUNCTION__))
798 "greater than the vector element type!")(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 798, __PRETTY_FUNCTION__))
;
799 return getNode(ISD::UNDEF, SDLoc(), VT);
800 }
801 return getNode(ISD::SPLAT_VECTOR, DL, VT, Op);
802 }
803
804 /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to
805 /// the shuffle node in input but with swapped operands.
806 ///
807 /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3>
808 SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV);
809
810 /// Convert Op, which must be of float type, to the
811 /// float type VT, by either extending or rounding (by truncation).
812 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT);
813
814 /// Convert Op, which must be a STRICT operation of float type, to the
815 /// float type VT, by either extending or rounding (by truncation).
816 std::pair<SDValue, SDValue>
817 getStrictFPExtendOrRound(SDValue Op, SDValue Chain, const SDLoc &DL, EVT VT);
818
819 /// Convert Op, which must be of integer type, to the
820 /// integer type VT, by either any-extending or truncating it.
821 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
822
823 /// Convert Op, which must be of integer type, to the
824 /// integer type VT, by either sign-extending or truncating it.
825 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
826
827 /// Convert Op, which must be of integer type, to the
828 /// integer type VT, by either zero-extending or truncating it.
829 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
830
831 /// Return the expression required to zero extend the Op
832 /// value assuming it was the smaller SrcTy value.
833 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
834
835 /// Convert Op, which must be of integer type, to the integer type VT, by
836 /// either truncating it or performing either zero or sign extension as
837 /// appropriate extension for the pointer's semantics.
838 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
839
840 /// Return the expression required to extend the Op as a pointer value
841 /// assuming it was the smaller SrcTy value. This may be either a zero extend
842 /// or a sign extend.
843 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
844
845 /// Convert Op, which must be of integer type, to the integer type VT,
846 /// by using an extension appropriate for the target's
847 /// BooleanContent for type OpVT or truncating it.
848 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT);
849
850 /// Create a bitwise NOT operation as (XOR Val, -1).
851 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT);
852
853 /// Create a logical NOT operation as (XOR Val, BooleanOne).
854 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT);
855
856 /// Returns sum of the base pointer and offset.
857 /// Unlike getObjectPtrOffset this does not set NoUnsignedWrap by default.
858 SDValue getMemBasePlusOffset(SDValue Base, int64_t Offset, const SDLoc &DL,
859 const SDNodeFlags Flags = SDNodeFlags());
860 SDValue getMemBasePlusOffset(SDValue Base, SDValue Offset, const SDLoc &DL,
861 const SDNodeFlags Flags = SDNodeFlags());
862
863 /// Create an add instruction with appropriate flags when used for
864 /// addressing some offset of an object. i.e. if a load is split into multiple
865 /// components, create an add nuw from the base pointer to the offset.
866 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, int64_t Offset) {
867 SDNodeFlags Flags;
868 Flags.setNoUnsignedWrap(true);
869 return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
870 }
871
872 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Ptr, SDValue Offset) {
873 // The object itself can't wrap around the address space, so it shouldn't be
874 // possible for the adds of the offsets to the split parts to overflow.
875 SDNodeFlags Flags;
876 Flags.setNoUnsignedWrap(true);
877 return getMemBasePlusOffset(Ptr, Offset, SL, Flags);
878 }
879
880 /// Return a new CALLSEQ_START node, that starts new call frame, in which
881 /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and
882 /// OutSize specifies part of the frame set up prior to the sequence.
883 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize,
884 const SDLoc &DL) {
885 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
886 SDValue Ops[] = { Chain,
887 getIntPtrConstant(InSize, DL, true),
888 getIntPtrConstant(OutSize, DL, true) };
889 return getNode(ISD::CALLSEQ_START, DL, VTs, Ops);
890 }
891
892 /// Return a new CALLSEQ_END node, which always must have a
893 /// glue result (to ensure it's not CSE'd).
894 /// CALLSEQ_END does not have a useful SDLoc.
895 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
896 SDValue InGlue, const SDLoc &DL) {
897 SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue);
898 SmallVector<SDValue, 4> Ops;
899 Ops.push_back(Chain);
900 Ops.push_back(Op1);
901 Ops.push_back(Op2);
902 if (InGlue.getNode())
903 Ops.push_back(InGlue);
904 return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops);
905 }
906
907 /// Return true if the result of this operation is always undefined.
908 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops);
909
910 /// Return an UNDEF node. UNDEF does not have a useful SDLoc.
911 SDValue getUNDEF(EVT VT) {
912 return getNode(ISD::UNDEF, SDLoc(), VT);
913 }
914
915 /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
916 SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
917 return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);
918 }
919
920 /// Gets or creates the specified node.
921 ///
922 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
923 ArrayRef<SDUse> Ops);
924 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
925 ArrayRef<SDValue> Ops, const SDNodeFlags Flags = SDNodeFlags());
926 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys,
927 ArrayRef<SDValue> Ops);
928 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
929 ArrayRef<SDValue> Ops);
930
931 // Specialize based on number of operands.
932 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT);
933 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand,
934 const SDNodeFlags Flags = SDNodeFlags());
935 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
936 SDValue N2, const SDNodeFlags Flags = SDNodeFlags());
937 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
938 SDValue N2, SDValue N3,
939 const SDNodeFlags Flags = SDNodeFlags());
940 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
941 SDValue N2, SDValue N3, SDValue N4);
942 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
943 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
944
945 // Specialize again based on number of operands for nodes with a VTList
946 // rather than a single VT.
947 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList);
948 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N);
949 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
950 SDValue N2);
951 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
952 SDValue N2, SDValue N3);
953 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
954 SDValue N2, SDValue N3, SDValue N4);
955 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
956 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
957
958 /// Compute a TokenFactor to force all the incoming stack arguments to be
959 /// loaded from the stack. This is used in tail call lowering to protect
960 /// stack arguments from being clobbered.
961 SDValue getStackArgumentTokenFactor(SDValue Chain);
962
963 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
964 SDValue Size, unsigned Align, bool isVol, bool AlwaysInline,
965 bool isTailCall, MachinePointerInfo DstPtrInfo,
966 MachinePointerInfo SrcPtrInfo);
967
968 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
969 SDValue Size, unsigned Align, bool isVol, bool isTailCall,
970 MachinePointerInfo DstPtrInfo,
971 MachinePointerInfo SrcPtrInfo);
972
973 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
974 SDValue Size, unsigned Align, bool isVol, bool isTailCall,
975 MachinePointerInfo DstPtrInfo);
976
977 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
978 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
979 SDValue Size, Type *SizeTy, unsigned ElemSz,
980 bool isTailCall, MachinePointerInfo DstPtrInfo,
981 MachinePointerInfo SrcPtrInfo);
982
983 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
984 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
985 SDValue Size, Type *SizeTy, unsigned ElemSz,
986 bool isTailCall, MachinePointerInfo DstPtrInfo,
987 MachinePointerInfo SrcPtrInfo);
988
989 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
990 unsigned DstAlign, SDValue Value, SDValue Size,
991 Type *SizeTy, unsigned ElemSz, bool isTailCall,
992 MachinePointerInfo DstPtrInfo);
993
994 /// Helper function to make it easier to build SetCC's if you just have an
995 /// ISD::CondCode instead of an SDValue.
996 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
997 ISD::CondCode Cond, SDValue Chain = SDValue(),
998 bool IsSignaling = false) {
999 assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&((LHS.getValueType().isVector() == RHS.getValueType().isVector
() && "Cannot compare scalars to vectors") ? static_cast
<void> (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1000, __PRETTY_FUNCTION__))
1000 "Cannot compare scalars to vectors")((LHS.getValueType().isVector() == RHS.getValueType().isVector
() && "Cannot compare scalars to vectors") ? static_cast
<void> (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1000, __PRETTY_FUNCTION__))
;
1001 assert(LHS.getValueType().isVector() == VT.isVector() &&((LHS.getValueType().isVector() == VT.isVector() && "Cannot compare scalars to vectors"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1002, __PRETTY_FUNCTION__))
1002 "Cannot compare scalars to vectors")((LHS.getValueType().isVector() == VT.isVector() && "Cannot compare scalars to vectors"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1002, __PRETTY_FUNCTION__))
;
1003 assert(Cond != ISD::SETCC_INVALID &&((Cond != ISD::SETCC_INVALID && "Cannot create a setCC of an invalid node."
) ? static_cast<void> (0) : __assert_fail ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1004, __PRETTY_FUNCTION__))
1004 "Cannot create a setCC of an invalid node.")((Cond != ISD::SETCC_INVALID && "Cannot create a setCC of an invalid node."
) ? static_cast<void> (0) : __assert_fail ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1004, __PRETTY_FUNCTION__))
;
1005 if (Chain)
1006 return getNode(IsSignaling ? ISD::STRICT_FSETCCS : ISD::STRICT_FSETCC, DL,
1007 {VT, MVT::Other}, {Chain, LHS, RHS, getCondCode(Cond)});
1008 return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
1009 }
1010
1011 /// Helper function to make it easier to build Select's if you just have
1012 /// operands and don't want to check for vector.
1013 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
1014 SDValue RHS) {
1015 assert(LHS.getValueType() == RHS.getValueType() &&((LHS.getValueType() == RHS.getValueType() && "Cannot use select on differing types"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1016, __PRETTY_FUNCTION__))
1016 "Cannot use select on differing types")((LHS.getValueType() == RHS.getValueType() && "Cannot use select on differing types"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1016, __PRETTY_FUNCTION__))
;
1017 assert(VT.isVector() == LHS.getValueType().isVector() &&((VT.isVector() == LHS.getValueType().isVector() && "Cannot mix vectors and scalars"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1018, __PRETTY_FUNCTION__))
1018 "Cannot mix vectors and scalars")((VT.isVector() == LHS.getValueType().isVector() && "Cannot mix vectors and scalars"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1018, __PRETTY_FUNCTION__))
;
1019 auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
1020 return getNode(Opcode, DL, VT, Cond, LHS, RHS);
1021 }
1022
1023 /// Helper function to make it easier to build SelectCC's if you just have an
1024 /// ISD::CondCode instead of an SDValue.
1025 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
1026 SDValue False, ISD::CondCode Cond) {
1027 return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True,
1028 False, getCondCode(Cond));
1029 }
1030
1031 /// Try to simplify a select/vselect into 1 of its operands or a constant.
1032 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);
1033
1034 /// Try to simplify a shift into 1 of its operands or a constant.
1035 SDValue simplifyShift(SDValue X, SDValue Y);
1036
1037 /// Try to simplify a floating-point binary operation into 1 of its operands
1038 /// or a constant.
1039 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y);
1040
1041 /// VAArg produces a result and token chain, and takes a pointer
1042 /// and a source value as input.
1043 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1044 SDValue SV, unsigned Align);
1045
1046 /// Gets a node for an atomic cmpxchg op. There are two
1047 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a
1048 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
1049 /// a success flag (initially i1), and a chain.
1050 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1051 SDVTList VTs, SDValue Chain, SDValue Ptr,
1052 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
1053
1054 /// Gets a node for an atomic op, produces result (if relevant)
1055 /// and chain and takes 2 operands.
1056 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
1057 SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
1058
1059 /// Gets a node for an atomic op, produces result and chain and
1060 /// takes 1 operand.
1061 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
1062 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);
1063
1064 /// Gets a node for an atomic op, produces result and chain and takes N
1065 /// operands.
1066 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1067 SDVTList VTList, ArrayRef<SDValue> Ops,
1068 MachineMemOperand *MMO);
1069
1070 /// Creates a MemIntrinsicNode that may produce a
1071 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
1072 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
1073 /// less than FIRST_TARGET_MEMORY_OPCODE.
1074 SDValue getMemIntrinsicNode(
1075 unsigned Opcode, const SDLoc &dl, SDVTList VTList,
1076 ArrayRef<SDValue> Ops, EVT MemVT,
1077 MachinePointerInfo PtrInfo,
1078 unsigned Align = 0,
1079 MachineMemOperand::Flags Flags
1080 = MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
1081 uint64_t Size = 0,
1082 const AAMDNodes &AAInfo = AAMDNodes());
1083
1084 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
1085 ArrayRef<SDValue> Ops, EVT MemVT,
1086 MachineMemOperand *MMO);
1087
1088 /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends
1089 /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between
1090 /// offsets `Offset` and `Offset + Size`.
1091 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
1092 int FrameIndex, int64_t Size, int64_t Offset = -1);
1093
1094 /// Create a MERGE_VALUES node from the given operands.
1095 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl);
1096
1097 /// Loads are not normal binary operators: their result type is not
1098 /// determined by their operands, and they produce a value AND a token chain.
1099 ///
1100 /// This function will set the MOLoad flag on MMOFlags, but you can set it if
1101 /// you want. The MOStore flag must not be set.
1102 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1103 MachinePointerInfo PtrInfo, unsigned Alignment = 0,
1104 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1105 const AAMDNodes &AAInfo = AAMDNodes(),
1106 const MDNode *Ranges = nullptr);
1107 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1108 MachineMemOperand *MMO);
1109 SDValue
1110 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
1111 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
1112 unsigned Alignment = 0,
1113 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1114 const AAMDNodes &AAInfo = AAMDNodes());
1115 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
1116 SDValue Chain, SDValue Ptr, EVT MemVT,
1117 MachineMemOperand *MMO);
1118 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1119 SDValue Offset, ISD::MemIndexedMode AM);
1120 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1121 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1122 MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment = 0,
1123 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1124 const AAMDNodes &AAInfo = AAMDNodes(),
1125 const MDNode *Ranges = nullptr);
1126 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1127 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1128 EVT MemVT, MachineMemOperand *MMO);
1129
1130 /// Helper function to build ISD::STORE nodes.
1131 ///
1132 /// This function will set the MOStore flag on MMOFlags, but you can set it if
1133 /// you want. The MOLoad and MOInvariant flags must not be set.
1134 SDValue
1135 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1136 MachinePointerInfo PtrInfo, unsigned Alignment = 0,
1137 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1138 const AAMDNodes &AAInfo = AAMDNodes());
1139 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1140 MachineMemOperand *MMO);
1141 SDValue
1142 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1143 MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment = 0,
1144 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1145 const AAMDNodes &AAInfo = AAMDNodes());
1146 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1147 SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
1148 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
1149 SDValue Offset, ISD::MemIndexedMode AM);
1150
1151 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Base,
1152 SDValue Offset, SDValue Mask, SDValue Src0, EVT MemVT,
1153 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
1154 ISD::LoadExtType, bool IsExpanding = false);
1155 SDValue getIndexedMaskedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1156 SDValue Offset, ISD::MemIndexedMode AM);
1157 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1158 SDValue Base, SDValue Offset, SDValue Mask, EVT MemVT,
1159 MachineMemOperand *MMO, ISD::MemIndexedMode AM,
1160 bool IsTruncating = false, bool IsCompressing = false);
1161 SDValue getIndexedMaskedStore(SDValue OrigStore, const SDLoc &dl,
1162 SDValue Base, SDValue Offset,
1163 ISD::MemIndexedMode AM);
1164 SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
1165 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1166 ISD::MemIndexType IndexType);
1167 SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
1168 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1169 ISD::MemIndexType IndexType);
1170
1171 /// Return (create a new or find existing) a target-specific node.
1172 /// TargetMemSDNode should be derived class from MemSDNode.
1173 template <class TargetMemSDNode>
1174 SDValue getTargetMemSDNode(SDVTList VTs, ArrayRef<SDValue> Ops,
1175 const SDLoc &dl, EVT MemVT,
1176 MachineMemOperand *MMO);
1177
1178 /// Construct a node to track a Value* through the backend.
1179 SDValue getSrcValue(const Value *v);
1180
1181 /// Return an MDNodeSDNode which holds an MDNode.
1182 SDValue getMDNode(const MDNode *MD);
1183
1184 /// Return a bitcast using the SDLoc of the value operand, and casting to the
1185 /// provided type. Use getNode to set a custom SDLoc.
1186 SDValue getBitcast(EVT VT, SDValue V);
1187
1188 /// Return an AddrSpaceCastSDNode.
1189 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
1190 unsigned DestAS);
1191
1192 /// Return the specified value casted to
1193 /// the target's desired shift amount type.
1194 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
1195
1196 /// Expand the specified \c ISD::VAARG node as the Legalize pass would.
1197 SDValue expandVAArg(SDNode *Node);
1198
1199 /// Expand the specified \c ISD::VACOPY node as the Legalize pass would.
1200 SDValue expandVACopy(SDNode *Node);
1201
1202 /// Returs an GlobalAddress of the function from the current module with
1203 /// name matching the given ExternalSymbol. Additionally can provide the
1204 /// matched function.
1205 /// Panics the function doesn't exists.
1206 SDValue getSymbolFunctionGlobalAddress(SDValue Op,
1207 Function **TargetFunction = nullptr);
1208
1209 /// *Mutate* the specified node in-place to have the
1210 /// specified operands. If the resultant node already exists in the DAG,
1211 /// this does not modify the specified node, instead it returns the node that
1212 /// already exists. If the resultant node does not exist in the DAG, the
1213 /// input node is returned. As a degenerate case, if you specify the same
1214 /// input operands as the node already has, the input node is returned.
1215 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op);
1216 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2);
1217 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1218 SDValue Op3);
1219 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1220 SDValue Op3, SDValue Op4);
1221 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1222 SDValue Op3, SDValue Op4, SDValue Op5);
1223 SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);
1224
1225 /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k
1226 /// values or more, move values into new TokenFactors in 64k-1 blocks, until
1227 /// the final TokenFactor has less than 64k operands.
1228 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals);
1229
1230 /// *Mutate* the specified machine node's memory references to the provided
1231 /// list.
1232 void setNodeMemRefs(MachineSDNode *N,
1233 ArrayRef<MachineMemOperand *> NewMemRefs);
1234
1235 // Propagates the change in divergence to users
1236 void updateDivergence(SDNode * N);
1237
1238 /// These are used for target selectors to *mutate* the
1239 /// specified node to have the specified return type, Target opcode, and
1240 /// operands. Note that target opcodes are stored as
1241 /// ~TargetOpcode in the node opcode field. The resultant node is returned.
1242 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT);
1243 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1);
1244 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1245 SDValue Op1, SDValue Op2);
1246 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1247 SDValue Op1, SDValue Op2, SDValue Op3);
1248 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1249 ArrayRef<SDValue> Ops);
1250 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2);
1251 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1252 EVT VT2, ArrayRef<SDValue> Ops);
1253 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1254 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1255 SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
1256 EVT VT2, SDValue Op1);
1257 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1258 EVT VT2, SDValue Op1, SDValue Op2);
1259 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs,
1260 ArrayRef<SDValue> Ops);
1261
1262 /// This *mutates* the specified node to have the specified
1263 /// return type, opcode, and operands.
1264 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
1265 ArrayRef<SDValue> Ops);
1266
1267 /// Mutate the specified strict FP node to its non-strict equivalent,
1268 /// unlinking the node from its chain and dropping the metadata arguments.
1269 /// The node must be a strict FP node.
1270 SDNode *mutateStrictFPToFP(SDNode *Node);
1271
1272 /// These are used for target selectors to create a new node
1273 /// with specified return type(s), MachineInstr opcode, and operands.
1274 ///
1275 /// Note that getMachineNode returns the resultant node. If there is already
1276 /// a node of the specified opcode and operands, it returns that node instead
1277 /// of the current one.
1278 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT);
1279 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1280 SDValue Op1);
1281 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1282 SDValue Op1, SDValue Op2);
1283 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1284 SDValue Op1, SDValue Op2, SDValue Op3);
1285 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1286 ArrayRef<SDValue> Ops);
1287 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1288 EVT VT2, SDValue Op1, SDValue Op2);
1289 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1290 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
1291 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1292 EVT VT2, ArrayRef<SDValue> Ops);
1293 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1294 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2);
1295 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1296 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2,
1297 SDValue Op3);
1298 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1299 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1300 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl,
1301 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops);
1302 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs,
1303 ArrayRef<SDValue> Ops);
1304
1305 /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
1306 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1307 SDValue Operand);
1308
1309 /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
1310 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1311 SDValue Operand, SDValue Subreg);
1312
1313 /// Get the specified node if it's already available, or else return NULL.
1314 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops,
1315 const SDNodeFlags Flags = SDNodeFlags());
1316
1317 /// Creates a SDDbgValue node.
1318 SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N,
1319 unsigned R, bool IsIndirect, const DebugLoc &DL,
1320 unsigned O);
1321
1322 /// Creates a constant SDDbgValue node.
1323 SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr,
1324 const Value *C, const DebugLoc &DL,
1325 unsigned O);
1326
1327 /// Creates a FrameIndex SDDbgValue node.
1328 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr,
1329 unsigned FI, bool IsIndirect,
1330 const DebugLoc &DL, unsigned O);
1331
1332 /// Creates a VReg SDDbgValue node.
1333 SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
1334 unsigned VReg, bool IsIndirect,
1335 const DebugLoc &DL, unsigned O);
1336
1337 /// Creates a SDDbgLabel node.
1338 SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O);
1339
1340 /// Transfer debug values from one node to another, while optionally
1341 /// generating fragment expressions for split-up values. If \p InvalidateDbg
1342 /// is set, debug values are invalidated after they are transferred.
1343 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0,
1344 unsigned SizeInBits = 0, bool InvalidateDbg = true);
1345
1346 /// Remove the specified node from the system. If any of its
1347 /// operands then becomes dead, remove them as well. Inform UpdateListener
1348 /// for each node deleted.
1349 void RemoveDeadNode(SDNode *N);
1350
1351 /// This method deletes the unreachable nodes in the
1352 /// given list, and any nodes that become unreachable as a result.
1353 void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes);
1354
1355 /// Modify anything using 'From' to use 'To' instead.
1356 /// This can cause recursive merging of nodes in the DAG. Use the first
1357 /// version if 'From' is known to have a single result, use the second
1358 /// if you have two nodes with identical results (or if 'To' has a superset
1359 /// of the results of 'From'), use the third otherwise.
1360 ///
1361 /// These methods all take an optional UpdateListener, which (if not null) is
1362 /// informed about nodes that are deleted and modified due to recursive
1363 /// changes in the dag.
1364 ///
1365 /// These functions only replace all existing uses. It's possible that as
1366 /// these replacements are being performed, CSE may cause the From node
1367 /// to be given new uses. These new uses of From are left in place, and
1368 /// not automatically transferred to To.
1369 ///
1370 void ReplaceAllUsesWith(SDValue From, SDValue To);
1371 void ReplaceAllUsesWith(SDNode *From, SDNode *To);
1372 void ReplaceAllUsesWith(SDNode *From, const SDValue *To);
1373
1374 /// Replace any uses of From with To, leaving
1375 /// uses of other values produced by From.getNode() alone.
1376 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);
1377
1378 /// Like ReplaceAllUsesOfValueWith, but for multiple values at once.
1379 /// This correctly handles the case where
1380 /// there is an overlap between the From values and the To values.
1381 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
1382 unsigned Num);
1383
1384 /// If an existing load has uses of its chain, create a token factor node with
1385 /// that chain and the new memory node's chain and update users of the old
1386 /// chain to the token factor. This ensures that the new memory node will have
1387 /// the same relative memory dependency position as the old load. Returns the
1388 /// new merged load chain.
1389 SDValue makeEquivalentMemoryOrdering(LoadSDNode *Old, SDValue New);
1390
1391 /// Topological-sort the AllNodes list and a
1392 /// assign a unique node id for each node in the DAG based on their
1393 /// topological order. Returns the number of nodes.
1394 unsigned AssignTopologicalOrder();
1395
1396 /// Move node N in the AllNodes list to be immediately
1397 /// before the given iterator Position. This may be used to update the
1398 /// topological ordering when the list of nodes is modified.
1399 void RepositionNode(allnodes_iterator Position, SDNode *N) {
1400 AllNodes.insert(Position, AllNodes.remove(N));
1401 }
1402
1403 /// Returns an APFloat semantics tag appropriate for the given type. If VT is
1404 /// a vector type, the element semantics are returned.
1405 static const fltSemantics &EVTToAPFloatSemantics(EVT VT) {
1406 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
1407 default: llvm_unreachable("Unknown FP format")::llvm::llvm_unreachable_internal("Unknown FP format", "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAG.h"
, 1407)
;
1408 case MVT::f16: return APFloat::IEEEhalf();
1409 case MVT::f32: return APFloat::IEEEsingle();
1410 case MVT::f64: return APFloat::IEEEdouble();
1411 case MVT::f80: return APFloat::x87DoubleExtended();
1412 case MVT::f128: return APFloat::IEEEquad();
1413 case MVT::ppcf128: return APFloat::PPCDoubleDouble();
1414 }
1415 }
1416
1417 /// Add a dbg_value SDNode. If SD is non-null that means the
1418 /// value is produced by SD.
1419 void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter);
1420
1421 /// Add a dbg_label SDNode.
1422 void AddDbgLabel(SDDbgLabel *DB);
1423
1424 /// Get the debug values which reference the given SDNode.
1425 ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const {
1426 return DbgInfo->getSDDbgValues(SD);
1427 }
1428
1429public:
1430 /// Return true if there are any SDDbgValue nodes associated
1431 /// with this SelectionDAG.
1432 bool hasDebugValues() const { return !DbgInfo->empty(); }
1433
1434 SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); }
1435 SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); }
1436
1437 SDDbgInfo::DbgIterator ByvalParmDbgBegin() const {
1438 return DbgInfo->ByvalParmDbgBegin();
1439 }
1440 SDDbgInfo::DbgIterator ByvalParmDbgEnd() const {
1441 return DbgInfo->ByvalParmDbgEnd();
1442 }
1443
1444 SDDbgInfo::DbgLabelIterator DbgLabelBegin() const {
1445 return DbgInfo->DbgLabelBegin();
1446 }
1447 SDDbgInfo::DbgLabelIterator DbgLabelEnd() const {
1448 return DbgInfo->DbgLabelEnd();
1449 }
1450
1451 /// To be invoked on an SDNode that is slated to be erased. This
1452 /// function mirrors \c llvm::salvageDebugInfo.
1453 void salvageDebugInfo(SDNode &N);
1454
1455 void dump() const;
1456
1457 /// Create a stack temporary, suitable for holding the specified value type.
1458 /// If minAlign is specified, the slot size will have at least that alignment.
1459 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1);
1460
1461 /// Create a stack temporary suitable for holding either of the specified
1462 /// value types.
1463 SDValue CreateStackTemporary(EVT VT1, EVT VT2);
1464
1465 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT,
1466 const GlobalAddressSDNode *GA,
1467 const SDNode *N2);
1468
1469 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1470 SDNode *N1, SDNode *N2);
1471
1472 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1473 const ConstantSDNode *C1,
1474 const ConstantSDNode *C2);
1475
1476 SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1477 ArrayRef<SDValue> Ops,
1478 const SDNodeFlags Flags = SDNodeFlags());
1479
1480 /// Fold floating-point operations with 2 operands when both operands are
1481 /// constants and/or undefined.
1482 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
1483 SDValue N1, SDValue N2);
1484
1485 /// Constant fold a setcc to true or false.
1486 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
1487 const SDLoc &dl);
1488
1489 /// See if the specified operand can be simplified with the knowledge that
1490 /// only the bits specified by DemandedBits are used. If so, return the
1491 /// simpler operand, otherwise return a null SDValue.
1492 ///
1493 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
1494 /// simplify nodes with multiple uses more aggressively.)
1495 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits);
1496
1497 /// See if the specified operand can be simplified with the knowledge that
1498 /// only the bits specified by DemandedBits are used in the elements specified
1499 /// by DemandedElts. If so, return the simpler operand, otherwise return a
1500 /// null SDValue.
1501 ///
1502 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
1503 /// simplify nodes with multiple uses more aggressively.)
1504 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits,
1505 const APInt &DemandedElts);
1506
1507 /// Return true if the sign bit of Op is known to be zero.
1508 /// We use this predicate to simplify operations downstream.
1509 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const;
1510
1511 /// Return true if 'Op & Mask' is known to be zero. We
1512 /// use this predicate to simplify operations downstream. Op and Mask are
1513 /// known to be the same type.
1514 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1515 unsigned Depth = 0) const;
1516
1517 /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We
1518 /// use this predicate to simplify operations downstream. Op and Mask are
1519 /// known to be the same type.
1520 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1521 const APInt &DemandedElts, unsigned Depth = 0) const;
1522
1523 /// Return true if '(Op & Mask) == Mask'.
1524 /// Op and Mask are known to be the same type.
1525 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask,
1526 unsigned Depth = 0) const;
1527
1528 /// Determine which bits of Op are known to be either zero or one and return
1529 /// them in Known. For vectors, the known bits are those that are shared by
1530 /// every vector element.
1531 /// Targets can implement the computeKnownBitsForTargetNode method in the
1532 /// TargetLowering class to allow target nodes to be understood.
1533 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const;
1534
1535 /// Determine which bits of Op are known to be either zero or one and return
1536 /// them in Known. The DemandedElts argument allows us to only collect the
1537 /// known bits that are shared by the requested vector elements.
1538 /// Targets can implement the computeKnownBitsForTargetNode method in the
1539 /// TargetLowering class to allow target nodes to be understood.
1540 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts,
1541 unsigned Depth = 0) const;
1542
1543 /// Used to represent the possible overflow behavior of an operation.
1544 /// Never: the operation cannot overflow.
1545 /// Always: the operation will always overflow.
1546 /// Sometime: the operation may or may not overflow.
1547 enum OverflowKind {
1548 OFK_Never,
1549 OFK_Sometime,
1550 OFK_Always,
1551 };
1552
1553 /// Determine if the result of the addition of 2 node can overflow.
1554 OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const;
1555
1556 /// Test if the given value is known to have exactly one bit set. This differs
1557 /// from computeKnownBits in that it doesn't necessarily determine which bit
1558 /// is set.
1559 bool isKnownToBeAPowerOfTwo(SDValue Val) const;
1560
1561 /// Return the number of times the sign bit of the register is replicated into
1562 /// the other bits. We know that at least 1 bit is always equal to the sign
1563 /// bit (itself), but other cases can give us information. For example,
1564 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
1565 /// to each other, so we return 3. Targets can implement the
1566 /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow
1567 /// target nodes to be understood.
1568 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const;
1569
1570 /// Return the number of times the sign bit of the register is replicated into
1571 /// the other bits. We know that at least 1 bit is always equal to the sign
1572 /// bit (itself), but other cases can give us information. For example,
1573 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
1574 /// to each other, so we return 3. The DemandedElts argument allows
1575 /// us to only collect the minimum sign bits of the requested vector elements.
1576 /// Targets can implement the ComputeNumSignBitsForTarget method in the
1577 /// TargetLowering class to allow target nodes to be understood.
1578 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
1579 unsigned Depth = 0) const;
1580
1581 /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode
1582 /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that
1583 /// is guaranteed to have the same semantics as an ADD. This handles the
1584 /// equivalence:
1585 /// X|Cst == X+Cst iff X&Cst = 0.
1586 bool isBaseWithConstantOffset(SDValue Op) const;
1587
1588 /// Test whether the given SDValue is known to never be NaN. If \p SNaN is
1589 /// true, returns if \p Op is known to never be a signaling NaN (it may still
1590 /// be a qNaN).
1591 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
1592
1593 /// \returns true if \p Op is known to never be a signaling NaN.
1594 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
1595 return isKnownNeverNaN(Op, true, Depth);
1596 }
1597
1598 /// Test whether the given floating point SDValue is known to never be
1599 /// positive or negative zero.
1600 bool isKnownNeverZeroFloat(SDValue Op) const;
1601
1602 /// Test whether the given SDValue is known to contain non-zero value(s).
1603 bool isKnownNeverZero(SDValue Op) const;
1604
1605 /// Test whether two SDValues are known to compare equal. This
1606 /// is true if they are the same value, or if one is negative zero and the
1607 /// other positive zero.
1608 bool isEqualTo(SDValue A, SDValue B) const;
1609
1610 /// Return true if A and B have no common bits set. As an example, this can
1611 /// allow an 'add' to be transformed into an 'or'.
1612 bool haveNoCommonBitsSet(SDValue A, SDValue B) const;
1613
1614 /// Test whether \p V has a splatted value for all the demanded elements.
1615 ///
1616 /// On success \p UndefElts will indicate the elements that have UNDEF
1617 /// values instead of the splat value, this is only guaranteed to be correct
1618 /// for \p DemandedElts.
1619 ///
1620 /// NOTE: The function will return true for a demanded splat of UNDEF values.
1621 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts);
1622
1623 /// Test whether \p V has a splatted value.
1624 bool isSplatValue(SDValue V, bool AllowUndefs = false);
1625
1626 /// If V is a splatted value, return the source vector and its splat index.
1627 SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
1628
1629 /// If V is a splat vector, return its scalar source operand by extracting
1630 /// that element from the source vector.
1631 SDValue getSplatValue(SDValue V);
1632
1633 /// Match a binop + shuffle pyramid that represents a horizontal reduction
1634 /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p
1635 /// Extract. The reduction must use one of the opcodes listed in /p
1636 /// CandidateBinOps and on success /p BinOp will contain the matching opcode.
1637 /// Returns the vector that is being reduced on, or SDValue() if a reduction
1638 /// was not matched. If \p AllowPartials is set then in the case of a
1639 /// reduction pattern that only matches the first few stages, the extracted
1640 /// subvector of the start of the reduction is returned.
1641 SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
1642 ArrayRef<ISD::NodeType> CandidateBinOps,
1643 bool AllowPartials = false);
1644
1645 /// Utility function used by legalize and lowering to
1646 /// "unroll" a vector operation by splitting out the scalars and operating
1647 /// on each element individually. If the ResNE is 0, fully unroll the vector
1648 /// op. If ResNE is less than the width of the vector op, unroll up to ResNE.
1649 /// If the ResNE is greater than the width of the vector op, unroll the
1650 /// vector op and fill the end of the resulting vector with UNDEFS.
1651 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
1652
1653 /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
1654 /// This is a separate function because those opcodes have two results.
1655 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
1656 unsigned ResNE = 0);
1657
1658 /// Return true if loads are next to each other and can be
1659 /// merged. Check that both are nonvolatile and if LD is loading
1660 /// 'Bytes' bytes from a location that is 'Dist' units away from the
1661 /// location that the 'Base' load is loading from.
1662 bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base,
1663 unsigned Bytes, int Dist) const;
1664
1665 /// Infer alignment of a load / store address. Return 0 if
1666 /// it cannot be inferred.
1667 unsigned InferPtrAlignment(SDValue Ptr) const;
1668
1669 /// Compute the VTs needed for the low/hi parts of a type
1670 /// which is split (or expanded) into two not necessarily identical pieces.
1671 std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const;
1672
1673 /// Split the vector with EXTRACT_SUBVECTOR using the provides
1674 /// VTs and return the low/high part.
1675 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
1676 const EVT &LoVT, const EVT &HiVT);
1677
1678 /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part.
1679 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
1680 EVT LoVT, HiVT;
1681 std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
1682 return SplitVector(N, DL, LoVT, HiVT);
1683 }
1684
1685 /// Split the node's operand with EXTRACT_SUBVECTOR and
1686 /// return the low/high part.
1687 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
1688 {
1689 return SplitVector(N->getOperand(OpNo), SDLoc(N));
1690 }
1691
1692 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
1693 SDValue WidenVector(const SDValue &N, const SDLoc &DL);
1694
1695 /// Append the extracted elements from Start to Count out of the vector Op
1696 /// in Args. If Count is 0, all of the elements will be extracted.
1697 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
1698 unsigned Start = 0, unsigned Count = 0);
1699
1700 /// Compute the default alignment value for the given type.
1701 unsigned getEVTAlignment(EVT MemoryVT) const;
1702
1703 /// Test whether the given value is a constant int or similar node.
1704 SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N);
1705
1706 /// Test whether the given value is a constant FP or similar node.
1707 SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N);
1708
1709 /// \returns true if \p N is any kind of constant or build_vector of
1710 /// constants, int or float. If a vector, it may not necessarily be a splat.
1711 inline bool isConstantValueOfAnyType(SDValue N) {
1712 return isConstantIntBuildVectorOrConstantInt(N) ||
1713 isConstantFPBuildVectorOrConstantFP(N);
1714 }
1715
1716 void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) {
1717 SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo);
1718 }
1719
1720 CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) {
1721 auto I = SDCallSiteDbgInfo.find(CallNode);
1722 if (I != SDCallSiteDbgInfo.end())
1723 return std::move(I->second).CSInfo;
1724 return CallSiteInfo();
1725 }
1726
1727 void addHeapAllocSite(const SDNode *Node, MDNode *MD) {
1728 SDCallSiteDbgInfo[Node].HeapAllocSite = MD;
1729 }
1730
1731 /// Return the HeapAllocSite type associated with the SDNode, if it exists.
1732 MDNode *getHeapAllocSite(const SDNode *Node) {
1733 auto It = SDCallSiteDbgInfo.find(Node);
1734 if (It == SDCallSiteDbgInfo.end())
1735 return nullptr;
1736 return It->second.HeapAllocSite;
1737 }
1738
1739 /// Return the current function's default denormal handling kind for the given
1740 /// floating point type.
1741 DenormalMode getDenormalMode(EVT VT) const {
1742 return MF->getDenormalMode(EVTToAPFloatSemantics(VT));
1743 }
1744
1745 bool shouldOptForSize() const;
1746
1747private:
1748 void InsertNode(SDNode *N);
1749 bool RemoveNodeFromCSEMaps(SDNode *N);
1750 void AddModifiedNodeToCSEMaps(SDNode *N);
1751 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
1752 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1753 void *&InsertPos);
1754 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1755 void *&InsertPos);
1756 SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc);
1757
1758 void DeleteNodeNotInCSEMaps(SDNode *N);
1759 void DeallocateNode(SDNode *N);
1760
1761 void allnodes_clear();
1762
1763 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
1764 /// not, return the insertion token that will make insertion faster. This
1765 /// overload is for nodes other than Constant or ConstantFP, use the other one
1766 /// for those.
1767 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos);
1768
1769 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
1770 /// not, return the insertion token that will make insertion faster. Performs
1771 /// additional processing for constant nodes.
1772 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL,
1773 void *&InsertPos);
1774
1775 /// List of non-single value types.
1776 FoldingSet<SDVTListNode> VTListMap;
1777
1778 /// Maps to auto-CSE operations.
1779 std::vector<CondCodeSDNode*> CondCodeNodes;
1780
1781 std::vector<SDNode*> ValueTypeNodes;
1782 std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes;
1783 StringMap<SDNode*> ExternalSymbols;
1784
1785 std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols;
1786 DenseMap<MCSymbol *, SDNode *> MCSymbols;
1787};
1788
1789template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> {
1790 using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>;
1791
1792 static nodes_iterator nodes_begin(SelectionDAG *G) {
1793 return nodes_iterator(G->allnodes_begin());
1794 }
1795
1796 static nodes_iterator nodes_end(SelectionDAG *G) {
1797 return nodes_iterator(G->allnodes_end());
1798 }
1799};
1800
1801template <class TargetMemSDNode>
1802SDValue SelectionDAG::getTargetMemSDNode(SDVTList VTs,
1803 ArrayRef<SDValue> Ops,
1804 const SDLoc &dl, EVT MemVT,
1805 MachineMemOperand *MMO) {
1806 /// Compose node ID and try to find an existing node.
1807 FoldingSetNodeID ID;
1808 unsigned Opcode =
1809 TargetMemSDNode(dl.getIROrder(), DebugLoc(), VTs, MemVT, MMO).getOpcode();
1810 ID.AddInteger(Opcode);
1811 ID.AddPointer(VTs.VTs);
1812 for (auto& Op : Ops) {
1813 ID.AddPointer(Op.getNode());
1814 ID.AddInteger(Op.getResNo());
1815 }
1816 ID.AddInteger(MemVT.getRawBits());
1817 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
1818 ID.AddInteger(getSyntheticNodeSubclassData<TargetMemSDNode>(
1819 dl.getIROrder(), VTs, MemVT, MMO));
1820
1821 void *IP = nullptr;
1822 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
1823 cast<TargetMemSDNode>(E)->refineAlignment(MMO);
1824 return SDValue(E, 0);
1825 }
1826
1827 /// Existing node was not found. Create a new one.
1828 auto *N = newSDNode<TargetMemSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
1829 MemVT, MMO);
1830 createOperands(N, Ops);
1831 CSEMap.InsertNode(N, IP);
1832 InsertNode(N);
1833 return SDValue(N, 0);
1834}
1835
1836} // end namespace llvm
1837
1838#endif // LLVM_CODEGEN_SELECTIONDAG_H

/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/ValueTypes.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/Support/AlignOf.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MachineValueType.h"
45#include "llvm/Support/TypeSize.h"
46#include <algorithm>
47#include <cassert>
48#include <climits>
49#include <cstddef>
50#include <cstdint>
51#include <cstring>
52#include <iterator>
53#include <string>
54#include <tuple>
55
56namespace llvm {
57
58class APInt;
59class Constant;
60template <typename T> struct DenseMapInfo;
61class GlobalValue;
62class MachineBasicBlock;
63class MachineConstantPoolValue;
64class MCSymbol;
65class raw_ostream;
66class SDNode;
67class SelectionDAG;
68class Type;
69class Value;
70
71void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
72 bool force = false);
73
74/// This represents a list of ValueType's that has been intern'd by
75/// a SelectionDAG. Instances of this simple value class are returned by
76/// SelectionDAG::getVTList(...).
77///
78struct SDVTList {
79 const EVT *VTs;
80 unsigned int NumVTs;
81};
82
83namespace ISD {
84
85 /// Node predicates
86
87 /// If N is a BUILD_VECTOR node whose elements are all the same constant or
88 /// undefined, return true and return the constant value in \p SplatValue.
89 bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
90
91 /// Return true if the specified node is a BUILD_VECTOR where all of the
92 /// elements are ~0 or undef.
93 bool isBuildVectorAllOnes(const SDNode *N);
94
95 /// Return true if the specified node is a BUILD_VECTOR where all of the
96 /// elements are 0 or undef.
97 bool isBuildVectorAllZeros(const SDNode *N);
98
99 /// Return true if the specified node is a BUILD_VECTOR node of all
100 /// ConstantSDNode or undef.
101 bool isBuildVectorOfConstantSDNodes(const SDNode *N);
102
103 /// Return true if the specified node is a BUILD_VECTOR node of all
104 /// ConstantFPSDNode or undef.
105 bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
106
107 /// Return true if the node has at least one operand and all operands of the
108 /// specified node are ISD::UNDEF.
109 bool allOperandsUndef(const SDNode *N);
110
111} // end namespace ISD
112
113//===----------------------------------------------------------------------===//
114/// Unlike LLVM values, Selection DAG nodes may return multiple
115/// values as the result of a computation. Many nodes return multiple values,
116/// from loads (which define a token and a return value) to ADDC (which returns
117/// a result and a carry value), to calls (which may return an arbitrary number
118/// of values).
119///
120/// As such, each use of a SelectionDAG computation must indicate the node that
121/// computes it as well as which return value to use from that node. This pair
122/// of information is represented with the SDValue value type.
123///
124class SDValue {
125 friend struct DenseMapInfo<SDValue>;
126
127 SDNode *Node = nullptr; // The node defining the value we are using.
128 unsigned ResNo = 0; // Which return value of the node we are using.
129
130public:
131 SDValue() = default;
132 SDValue(SDNode *node, unsigned resno);
133
134 /// get the index which selects a specific result in the SDNode
135 unsigned getResNo() const { return ResNo; }
136
137 /// get the SDNode which holds the desired result
138 SDNode *getNode() const { return Node; }
139
140 /// set the SDNode
141 void setNode(SDNode *N) { Node = N; }
142
143 inline SDNode *operator->() const { return Node; }
144
145 bool operator==(const SDValue &O) const {
146 return Node == O.Node && ResNo == O.ResNo;
147 }
148 bool operator!=(const SDValue &O) const {
149 return !operator==(O);
150 }
151 bool operator<(const SDValue &O) const {
152 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
153 }
154 explicit operator bool() const {
155 return Node != nullptr;
156 }
157
158 SDValue getValue(unsigned R) const {
159 return SDValue(Node, R);
160 }
161
162 /// Return true if this node is an operand of N.
163 bool isOperandOf(const SDNode *N) const;
164
165 /// Return the ValueType of the referenced return value.
166 inline EVT getValueType() const;
167
168 /// Return the simple ValueType of the referenced return value.
169 MVT getSimpleValueType() const {
170 return getValueType().getSimpleVT();
171 }
172
173 /// Returns the size of the value in bits.
174 ///
175 /// If the value type is a scalable vector type, the scalable property will
176 /// be set and the runtime size will be a positive integer multiple of the
177 /// base size.
178 TypeSize getValueSizeInBits() const {
179 return getValueType().getSizeInBits();
180 }
181
182 TypeSize getScalarValueSizeInBits() const {
183 return getValueType().getScalarType().getSizeInBits();
184 }
185
186 // Forwarding methods - These forward to the corresponding methods in SDNode.
187 inline unsigned getOpcode() const;
188 inline unsigned getNumOperands() const;
189 inline const SDValue &getOperand(unsigned i) const;
190 inline uint64_t getConstantOperandVal(unsigned i) const;
191 inline const APInt &getConstantOperandAPInt(unsigned i) const;
192 inline bool isTargetMemoryOpcode() const;
193 inline bool isTargetOpcode() const;
194 inline bool isMachineOpcode() const;
195 inline bool isUndef() const;
196 inline unsigned getMachineOpcode() const;
197 inline const DebugLoc &getDebugLoc() const;
198 inline void dump() const;
199 inline void dump(const SelectionDAG *G) const;
200 inline void dumpr() const;
201 inline void dumpr(const SelectionDAG *G) const;
202
203 /// Return true if this operand (which must be a chain) reaches the
204 /// specified operand without crossing any side-effecting instructions.
205 /// In practice, this looks through token factors and non-volatile loads.
206 /// In order to remain efficient, this only
207 /// looks a couple of nodes in, it does not do an exhaustive search.
208 bool reachesChainWithoutSideEffects(SDValue Dest,
209 unsigned Depth = 2) const;
210
211 /// Return true if there are no nodes using value ResNo of Node.
212 inline bool use_empty() const;
213
214 /// Return true if there is exactly one node using value ResNo of Node.
215 inline bool hasOneUse() const;
216};
217
218template<> struct DenseMapInfo<SDValue> {
219 static inline SDValue getEmptyKey() {
220 SDValue V;
221 V.ResNo = -1U;
222 return V;
223 }
224
225 static inline SDValue getTombstoneKey() {
226 SDValue V;
227 V.ResNo = -2U;
228 return V;
229 }
230
231 static unsigned getHashValue(const SDValue &Val) {
232 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
233 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
234 }
235
236 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
237 return LHS == RHS;
238 }
239};
240
241/// Allow casting operators to work directly on
242/// SDValues as if they were SDNode*'s.
243template<> struct simplify_type<SDValue> {
244 using SimpleType = SDNode *;
245
246 static SimpleType getSimplifiedValue(SDValue &Val) {
247 return Val.getNode();
248 }
249};
250template<> struct simplify_type<const SDValue> {
251 using SimpleType = /*const*/ SDNode *;
252
253 static SimpleType getSimplifiedValue(const SDValue &Val) {
254 return Val.getNode();
255 }
256};
257
258/// Represents a use of a SDNode. This class holds an SDValue,
259/// which records the SDNode being used and the result number, a
260/// pointer to the SDNode using the value, and Next and Prev pointers,
261/// which link together all the uses of an SDNode.
262///
263class SDUse {
264 /// Val - The value being used.
265 SDValue Val;
266 /// User - The user of this value.
267 SDNode *User = nullptr;
268 /// Prev, Next - Pointers to the uses list of the SDNode referred by
269 /// this operand.
270 SDUse **Prev = nullptr;
271 SDUse *Next = nullptr;
272
273public:
274 SDUse() = default;
275 SDUse(const SDUse &U) = delete;
276 SDUse &operator=(const SDUse &) = delete;
277
278 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
279 operator const SDValue&() const { return Val; }
280
281 /// If implicit conversion to SDValue doesn't work, the get() method returns
282 /// the SDValue.
283 const SDValue &get() const { return Val; }
284
285 /// This returns the SDNode that contains this Use.
286 SDNode *getUser() { return User; }
287
288 /// Get the next SDUse in the use list.
289 SDUse *getNext() const { return Next; }
290
291 /// Convenience function for get().getNode().
292 SDNode *getNode() const { return Val.getNode(); }
293 /// Convenience function for get().getResNo().
294 unsigned getResNo() const { return Val.getResNo(); }
295 /// Convenience function for get().getValueType().
296 EVT getValueType() const { return Val.getValueType(); }
297
298 /// Convenience function for get().operator==
299 bool operator==(const SDValue &V) const {
300 return Val == V;
301 }
302
303 /// Convenience function for get().operator!=
304 bool operator!=(const SDValue &V) const {
305 return Val != V;
306 }
307
308 /// Convenience function for get().operator<
309 bool operator<(const SDValue &V) const {
310 return Val < V;
311 }
312
313private:
314 friend class SelectionDAG;
315 friend class SDNode;
316 // TODO: unfriend HandleSDNode once we fix its operand handling.
317 friend class HandleSDNode;
318
319 void setUser(SDNode *p) { User = p; }
320
321 /// Remove this use from its existing use list, assign it the
322 /// given value, and add it to the new value's node's use list.
323 inline void set(const SDValue &V);
324 /// Like set, but only supports initializing a newly-allocated
325 /// SDUse with a non-null value.
326 inline void setInitial(const SDValue &V);
327 /// Like set, but only sets the Node portion of the value,
328 /// leaving the ResNo portion unmodified.
329 inline void setNode(SDNode *N);
330
331 void addToList(SDUse **List) {
332 Next = *List;
333 if (Next) Next->Prev = &Next;
334 Prev = List;
335 *List = this;
336 }
337
338 void removeFromList() {
339 *Prev = Next;
340 if (Next) Next->Prev = Prev;
341 }
342};
343
344/// simplify_type specializations - Allow casting operators to work directly on
345/// SDValues as if they were SDNode*'s.
346template<> struct simplify_type<SDUse> {
347 using SimpleType = SDNode *;
348
349 static SimpleType getSimplifiedValue(SDUse &Val) {
350 return Val.getNode();
351 }
352};
353
354/// These are IR-level optimization flags that may be propagated to SDNodes.
355/// TODO: This data structure should be shared by the IR optimizer and the
356/// the backend.
357struct SDNodeFlags {
358private:
359 // This bit is used to determine if the flags are in a defined state.
360 // Flag bits can only be masked out during intersection if the masking flags
361 // are defined.
362 bool AnyDefined : 1;
363
364 bool NoUnsignedWrap : 1;
365 bool NoSignedWrap : 1;
366 bool Exact : 1;
367 bool NoNaNs : 1;
368 bool NoInfs : 1;
369 bool NoSignedZeros : 1;
370 bool AllowReciprocal : 1;
371 bool VectorReduction : 1;
372 bool AllowContract : 1;
373 bool ApproximateFuncs : 1;
374 bool AllowReassociation : 1;
375
376 // We assume instructions do not raise floating-point exceptions by default,
377 // and only those marked explicitly may do so. We could choose to represent
378 // this via a positive "FPExcept" flags like on the MI level, but having a
379 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
380 // intersection logic more straightforward.
381 bool NoFPExcept : 1;
382
383public:
384 /// Default constructor turns off all optimization flags.
385 SDNodeFlags()
386 : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
387 Exact(false), NoNaNs(false), NoInfs(false),
388 NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
389 AllowContract(false), ApproximateFuncs(false),
390 AllowReassociation(false), NoFPExcept(false) {}
391
392 /// Propagate the fast-math-flags from an IR FPMathOperator.
393 void copyFMF(const FPMathOperator &FPMO) {
394 setNoNaNs(FPMO.hasNoNaNs());
395 setNoInfs(FPMO.hasNoInfs());
396 setNoSignedZeros(FPMO.hasNoSignedZeros());
397 setAllowReciprocal(FPMO.hasAllowReciprocal());
398 setAllowContract(FPMO.hasAllowContract());
399 setApproximateFuncs(FPMO.hasApproxFunc());
400 setAllowReassociation(FPMO.hasAllowReassoc());
401 }
402
403 /// Sets the state of the flags to the defined state.
404 void setDefined() { AnyDefined = true; }
405 /// Returns true if the flags are in a defined state.
406 bool isDefined() const { return AnyDefined; }
407
408 // These are mutators for each flag.
409 void setNoUnsignedWrap(bool b) {
410 setDefined();
411 NoUnsignedWrap = b;
412 }
413 void setNoSignedWrap(bool b) {
414 setDefined();
415 NoSignedWrap = b;
416 }
417 void setExact(bool b) {
418 setDefined();
419 Exact = b;
420 }
421 void setNoNaNs(bool b) {
422 setDefined();
423 NoNaNs = b;
424 }
425 void setNoInfs(bool b) {
426 setDefined();
427 NoInfs = b;
428 }
429 void setNoSignedZeros(bool b) {
430 setDefined();
431 NoSignedZeros = b;
432 }
433 void setAllowReciprocal(bool b) {
434 setDefined();
435 AllowReciprocal = b;
436 }
437 void setVectorReduction(bool b) {
438 setDefined();
439 VectorReduction = b;
440 }
441 void setAllowContract(bool b) {
442 setDefined();
443 AllowContract = b;
444 }
445 void setApproximateFuncs(bool b) {
446 setDefined();
447 ApproximateFuncs = b;
448 }
449 void setAllowReassociation(bool b) {
450 setDefined();
451 AllowReassociation = b;
452 }
453 void setNoFPExcept(bool b) {
454 setDefined();
455 NoFPExcept = b;
456 }
457
458 // These are accessors for each flag.
459 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
460 bool hasNoSignedWrap() const { return NoSignedWrap; }
461 bool hasExact() const { return Exact; }
462 bool hasNoNaNs() const { return NoNaNs; }
463 bool hasNoInfs() const { return NoInfs; }
464 bool hasNoSignedZeros() const { return NoSignedZeros; }
465 bool hasAllowReciprocal() const { return AllowReciprocal; }
466 bool hasVectorReduction() const { return VectorReduction; }
467 bool hasAllowContract() const { return AllowContract; }
468 bool hasApproximateFuncs() const { return ApproximateFuncs; }
469 bool hasAllowReassociation() const { return AllowReassociation; }
470 bool hasNoFPExcept() const { return NoFPExcept; }
471
472 bool isFast() const {
473 return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept &&
474 AllowContract && ApproximateFuncs && AllowReassociation;
475 }
476
477 /// Clear any flags in this flag set that aren't also set in Flags.
478 /// If the given Flags are undefined then don't do anything.
479 void intersectWith(const SDNodeFlags Flags) {
480 if (!Flags.isDefined())
481 return;
482 NoUnsignedWrap &= Flags.NoUnsignedWrap;
483 NoSignedWrap &= Flags.NoSignedWrap;
484 Exact &= Flags.Exact;
485 NoNaNs &= Flags.NoNaNs;
486 NoInfs &= Flags.NoInfs;
487 NoSignedZeros &= Flags.NoSignedZeros;
488 AllowReciprocal &= Flags.AllowReciprocal;
489 VectorReduction &= Flags.VectorReduction;
490 AllowContract &= Flags.AllowContract;
491 ApproximateFuncs &= Flags.ApproximateFuncs;
492 AllowReassociation &= Flags.AllowReassociation;
493 NoFPExcept &= Flags.NoFPExcept;
494 }
495};
496
497/// Represents one node in the SelectionDAG.
498///
499class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
500private:
501 /// The operation that this node performs.
502 int16_t NodeType;
503
504protected:
505 // We define a set of mini-helper classes to help us interpret the bits in our
506 // SubclassData. These are designed to fit within a uint16_t so they pack
507 // with NodeType.
508
509#if defined(_AIX) && (!defined(__GNUC__4) || defined(__ibmxl__))
510// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
511// and give the `pack` pragma push semantics.
512#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
513#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
514#else
515#define BEGIN_TWO_BYTE_PACK()
516#define END_TWO_BYTE_PACK()
517#endif
518
519BEGIN_TWO_BYTE_PACK()
520 class SDNodeBitfields {
521 friend class SDNode;
522 friend class MemIntrinsicSDNode;
523 friend class MemSDNode;
524 friend class SelectionDAG;
525
526 uint16_t HasDebugValue : 1;
527 uint16_t IsMemIntrinsic : 1;
528 uint16_t IsDivergent : 1;
529 };
530 enum { NumSDNodeBits = 3 };
531
532 class ConstantSDNodeBitfields {
533 friend class ConstantSDNode;
534
535 uint16_t : NumSDNodeBits;
536
537 uint16_t IsOpaque : 1;
538 };
539
540 class MemSDNodeBitfields {
541 friend class MemSDNode;
542 friend class MemIntrinsicSDNode;
543 friend class AtomicSDNode;
544
545 uint16_t : NumSDNodeBits;
546
547 uint16_t IsVolatile : 1;
548 uint16_t IsNonTemporal : 1;
549 uint16_t IsDereferenceable : 1;
550 uint16_t IsInvariant : 1;
551 };
552 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
553
554 class LSBaseSDNodeBitfields {
555 friend class LSBaseSDNode;
556 friend class MaskedLoadStoreSDNode;
557 friend class MaskedGatherScatterSDNode;
558
559 uint16_t : NumMemSDNodeBits;
560
561 // This storage is shared between disparate class hierarchies to hold an
562 // enumeration specific to the class hierarchy in use.
563 // LSBaseSDNode => enum ISD::MemIndexedMode
564 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
565 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
566 uint16_t AddressingMode : 3;
567 };
568 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
569
570 class LoadSDNodeBitfields {
571 friend class LoadSDNode;
572 friend class MaskedLoadSDNode;
573
574 uint16_t : NumLSBaseSDNodeBits;
575
576 uint16_t ExtTy : 2; // enum ISD::LoadExtType
577 uint16_t IsExpanding : 1;
578 };
579
580 class StoreSDNodeBitfields {
581 friend class StoreSDNode;
582 friend class MaskedStoreSDNode;
583
584 uint16_t : NumLSBaseSDNodeBits;
585
586 uint16_t IsTruncating : 1;
587 uint16_t IsCompressing : 1;
588 };
589
590 union {
591 char RawSDNodeBits[sizeof(uint16_t)];
592 SDNodeBitfields SDNodeBits;
593 ConstantSDNodeBitfields ConstantSDNodeBits;
594 MemSDNodeBitfields MemSDNodeBits;
595 LSBaseSDNodeBitfields LSBaseSDNodeBits;
596 LoadSDNodeBitfields LoadSDNodeBits;
597 StoreSDNodeBitfields StoreSDNodeBits;
598 };
599END_TWO_BYTE_PACK()
600#undef BEGIN_TWO_BYTE_PACK
601#undef END_TWO_BYTE_PACK
602
603 // RawSDNodeBits must cover the entirety of the union. This means that all of
604 // the union's members must have size <= RawSDNodeBits. We write the RHS as
605 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
606 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
607 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
608 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
609 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
610 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
611 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
612
613private:
614 friend class SelectionDAG;
615 // TODO: unfriend HandleSDNode once we fix its operand handling.
616 friend class HandleSDNode;
617
618 /// Unique id per SDNode in the DAG.
619 int NodeId = -1;
620
621 /// The values that are used by this operation.
622 SDUse *OperandList = nullptr;
623
624 /// The types of the values this node defines. SDNode's may
625 /// define multiple values simultaneously.
626 const EVT *ValueList;
627
628 /// List of uses for this SDNode.
629 SDUse *UseList = nullptr;
630
631 /// The number of entries in the Operand/Value list.
632 unsigned short NumOperands = 0;
633 unsigned short NumValues;
634
635 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
636 // original LLVM instructions.
637 // This is used for turning off scheduling, because we'll forgo
638 // the normal scheduling algorithms and output the instructions according to
639 // this ordering.
640 unsigned IROrder;
641
642 /// Source line information.
643 DebugLoc debugLoc;
644
645 /// Return a pointer to the specified value type.
646 static const EVT *getValueTypeList(EVT VT);
647
648 SDNodeFlags Flags;
649
650public:
651 /// Unique and persistent id per SDNode in the DAG.
652 /// Used for debug printing.
653 uint16_t PersistentId;
654
655 //===--------------------------------------------------------------------===//
656 // Accessors
657 //
658
659 /// Return the SelectionDAG opcode value for this node. For
660 /// pre-isel nodes (those for which isMachineOpcode returns false), these
661 /// are the opcode values in the ISD and <target>ISD namespaces. For
662 /// post-isel opcodes, see getMachineOpcode.
663 unsigned getOpcode() const { return (unsigned short)NodeType; }
664
665 /// Test if this node has a target-specific opcode (in the
666 /// \<target\>ISD namespace).
667 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
668
669 /// Test if this node has a target-specific opcode that may raise
670 /// FP exceptions (in the \<target\>ISD namespace and greater than
671 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
672 /// opcode are currently automatically considered to possibly raise
673 /// FP exceptions as well.
674 bool isTargetStrictFPOpcode() const {
675 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
676 }
677
678 /// Test if this node has a target-specific
679 /// memory-referencing opcode (in the \<target\>ISD namespace and
680 /// greater than FIRST_TARGET_MEMORY_OPCODE).
681 bool isTargetMemoryOpcode() const {
682 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
683 }
684
685 /// Return true if the type of the node type undefined.
686 bool isUndef() const { return NodeType == ISD::UNDEF; }
687
688 /// Test if this node is a memory intrinsic (with valid pointer information).
689 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
690 /// non-memory intrinsics (with chains) that are not really instances of
691 /// MemSDNode. For such nodes, we need some extra state to determine the
692 /// proper classof relationship.
693 bool isMemIntrinsic() const {
694 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
695 NodeType == ISD::INTRINSIC_VOID) &&
696 SDNodeBits.IsMemIntrinsic;
697 }
698
699 /// Test if this node is a strict floating point pseudo-op.
700 bool isStrictFPOpcode() {
701 switch (NodeType) {
702 default:
703 return false;
704#define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
705 case ISD::STRICT_##DAGN:
706#include "llvm/IR/ConstrainedOps.def"
707 return true;
708 }
709 }
710
711 /// Test if this node has a post-isel opcode, directly
712 /// corresponding to a MachineInstr opcode.
713 bool isMachineOpcode() const { return NodeType < 0; }
714
715 /// This may only be called if isMachineOpcode returns
716 /// true. It returns the MachineInstr opcode value that the node's opcode
717 /// corresponds to.
718 unsigned getMachineOpcode() const {
719 assert(isMachineOpcode() && "Not a MachineInstr opcode!")((isMachineOpcode() && "Not a MachineInstr opcode!") ?
static_cast<void> (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 719, __PRETTY_FUNCTION__))
;
720 return ~NodeType;
721 }
722
723 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
724 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
725
726 bool isDivergent() const { return SDNodeBits.IsDivergent; }
727
728 /// Return true if there are no uses of this node.
729 bool use_empty() const { return UseList == nullptr; }
730
731 /// Return true if there is exactly one use of this node.
732 bool hasOneUse() const {
733 return !use_empty() && std::next(use_begin()) == use_end();
734 }
735
736 /// Return the number of uses of this node. This method takes
737 /// time proportional to the number of uses.
738 size_t use_size() const { return std::distance(use_begin(), use_end()); }
739
740 /// Return the unique node id.
741 int getNodeId() const { return NodeId; }
742
743 /// Set unique node id.
744 void setNodeId(int Id) { NodeId = Id; }
745
746 /// Return the node ordering.
747 unsigned getIROrder() const { return IROrder; }
748
749 /// Set the node ordering.
750 void setIROrder(unsigned Order) { IROrder = Order; }
751
752 /// Return the source location info.
753 const DebugLoc &getDebugLoc() const { return debugLoc; }
754
755 /// Set source location info. Try to avoid this, putting
756 /// it in the constructor is preferable.
757 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
758
759 /// This class provides iterator support for SDUse
760 /// operands that use a specific SDNode.
761 class use_iterator
762 : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> {
763 friend class SDNode;
764
765 SDUse *Op = nullptr;
766
767 explicit use_iterator(SDUse *op) : Op(op) {}
768
769 public:
770 using reference = std::iterator<std::forward_iterator_tag,
771 SDUse, ptrdiff_t>::reference;
772 using pointer = std::iterator<std::forward_iterator_tag,
773 SDUse, ptrdiff_t>::pointer;
774
775 use_iterator() = default;
776 use_iterator(const use_iterator &I) : Op(I.Op) {}
777
778 bool operator==(const use_iterator &x) const {
779 return Op == x.Op;
780 }
781 bool operator!=(const use_iterator &x) const {
782 return !operator==(x);
783 }
784
785 /// Return true if this iterator is at the end of uses list.
786 bool atEnd() const { return Op == nullptr; }
787
788 // Iterator traversal: forward iteration only.
789 use_iterator &operator++() { // Preincrement
790 assert(Op && "Cannot increment end iterator!")((Op && "Cannot increment end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 790, __PRETTY_FUNCTION__))
;
791 Op = Op->getNext();
792 return *this;
793 }
794
795 use_iterator operator++(int) { // Postincrement
796 use_iterator tmp = *this; ++*this; return tmp;
797 }
798
799 /// Retrieve a pointer to the current user node.
800 SDNode *operator*() const {
801 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 801, __PRETTY_FUNCTION__))
;
802 return Op->getUser();
803 }
804
805 SDNode *operator->() const { return operator*(); }
806
807 SDUse &getUse() const { return *Op; }
808
809 /// Retrieve the operand # of this use in its user.
810 unsigned getOperandNo() const {
811 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 811, __PRETTY_FUNCTION__))
;
812 return (unsigned)(Op - Op->getUser()->OperandList);
813 }
814 };
815
816 /// Provide iteration support to walk over all uses of an SDNode.
817 use_iterator use_begin() const {
818 return use_iterator(UseList);
819 }
820
821 static use_iterator use_end() { return use_iterator(nullptr); }
822
823 inline iterator_range<use_iterator> uses() {
824 return make_range(use_begin(), use_end());
825 }
826 inline iterator_range<use_iterator> uses() const {
827 return make_range(use_begin(), use_end());
828 }
829
830 /// Return true if there are exactly NUSES uses of the indicated value.
831 /// This method ignores uses of other values defined by this operation.
832 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
833
834 /// Return true if there are any use of the indicated value.
835 /// This method ignores uses of other values defined by this operation.
836 bool hasAnyUseOfValue(unsigned Value) const;
837
838 /// Return true if this node is the only use of N.
839 bool isOnlyUserOf(const SDNode *N) const;
840
841 /// Return true if this node is an operand of N.
842 bool isOperandOf(const SDNode *N) const;
843
844 /// Return true if this node is a predecessor of N.
845 /// NOTE: Implemented on top of hasPredecessor and every bit as
846 /// expensive. Use carefully.
847 bool isPredecessorOf(const SDNode *N) const {
848 return N->hasPredecessor(this);
849 }
850
851 /// Return true if N is a predecessor of this node.
852 /// N is either an operand of this node, or can be reached by recursively
853 /// traversing up the operands.
854 /// NOTE: This is an expensive method. Use it carefully.
855 bool hasPredecessor(const SDNode *N) const;
856
857 /// Returns true if N is a predecessor of any node in Worklist. This
858 /// helper keeps Visited and Worklist sets externally to allow unions
859 /// searches to be performed in parallel, caching of results across
860 /// queries and incremental addition to Worklist. Stops early if N is
861 /// found but will resume. Remember to clear Visited and Worklists
862 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
863 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
864 /// topologically ordered (Operands have strictly smaller node id) and search
865 /// can be pruned leveraging this.
866 static bool hasPredecessorHelper(const SDNode *N,
867 SmallPtrSetImpl<const SDNode *> &Visited,
868 SmallVectorImpl<const SDNode *> &Worklist,
869 unsigned int MaxSteps = 0,
870 bool TopologicalPrune = false) {
871 SmallVector<const SDNode *, 8> DeferredNodes;
872 if (Visited.count(N))
873 return true;
874
875 // Node Id's are assigned in three places: As a topological
876 // ordering (> 0), during legalization (results in values set to
877 // 0), new nodes (set to -1). If N has a topolgical id then we
878 // know that all nodes with ids smaller than it cannot be
879 // successors and we need not check them. Filter out all node
880 // that can't be matches. We add them to the worklist before exit
881 // in case of multiple calls. Note that during selection the topological id
882 // may be violated if a node's predecessor is selected before it. We mark
883 // this at selection negating the id of unselected successors and
884 // restricting topological pruning to positive ids.
885
886 int NId = N->getNodeId();
887 // If we Invalidated the Id, reconstruct original NId.
888 if (NId < -1)
889 NId = -(NId + 1);
890
891 bool Found = false;
892 while (!Worklist.empty()) {
893 const SDNode *M = Worklist.pop_back_val();
894 int MId = M->getNodeId();
895 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
896 (MId > 0) && (MId < NId)) {
897 DeferredNodes.push_back(M);
898 continue;
899 }
900 for (const SDValue &OpV : M->op_values()) {
901 SDNode *Op = OpV.getNode();
902 if (Visited.insert(Op).second)
903 Worklist.push_back(Op);
904 if (Op == N)
905 Found = true;
906 }
907 if (Found)
908 break;
909 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
910 break;
911 }
912 // Push deferred nodes back on worklist.
913 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
914 // If we bailed early, conservatively return found.
915 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
916 return true;
917 return Found;
918 }
919
920 /// Return true if all the users of N are contained in Nodes.
921 /// NOTE: Requires at least one match, but doesn't require them all.
922 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
923
924 /// Return the number of values used by this operation.
925 unsigned getNumOperands() const { return NumOperands; }
926
927 /// Return the maximum number of operands that a SDNode can hold.
928 static constexpr size_t getMaxNumOperands() {
929 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
930 }
931
932 /// Helper method returns the integer value of a ConstantSDNode operand.
933 inline uint64_t getConstantOperandVal(unsigned Num) const;
934
935 /// Helper method returns the APInt of a ConstantSDNode operand.
936 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
937
938 const SDValue &getOperand(unsigned Num) const {
939 assert(Num < NumOperands && "Invalid child # of SDNode!")((Num < NumOperands && "Invalid child # of SDNode!"
) ? static_cast<void> (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 939, __PRETTY_FUNCTION__))
;
940 return OperandList[Num];
941 }
942
943 using op_iterator = SDUse *;
944
945 op_iterator op_begin() const { return OperandList; }
946 op_iterator op_end() const { return OperandList+NumOperands; }
947 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
948
949 /// Iterator for directly iterating over the operand SDValue's.
950 struct value_op_iterator
951 : iterator_adaptor_base<value_op_iterator, op_iterator,
952 std::random_access_iterator_tag, SDValue,
953 ptrdiff_t, value_op_iterator *,
954 value_op_iterator *> {
955 explicit value_op_iterator(SDUse *U = nullptr)
956 : iterator_adaptor_base(U) {}
957
958 const SDValue &operator*() const { return I->get(); }
959 };
960
961 iterator_range<value_op_iterator> op_values() const {
962 return make_range(value_op_iterator(op_begin()),
963 value_op_iterator(op_end()));
964 }
965
966 SDVTList getVTList() const {
967 SDVTList X = { ValueList, NumValues };
968 return X;
969 }
970
971 /// If this node has a glue operand, return the node
972 /// to which the glue operand points. Otherwise return NULL.
973 SDNode *getGluedNode() const {
974 if (getNumOperands() != 0 &&
975 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
976 return getOperand(getNumOperands()-1).getNode();
977 return nullptr;
978 }
979
980 /// If this node has a glue value with a user, return
981 /// the user (there is at most one). Otherwise return NULL.
982 SDNode *getGluedUser() const {
983 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
984 if (UI.getUse().get().getValueType() == MVT::Glue)
985 return *UI;
986 return nullptr;
987 }
988
989 const SDNodeFlags getFlags() const { return Flags; }
990 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
991 bool isFast() { return Flags.isFast(); }
992
993 /// Clear any flags in this node that aren't also set in Flags.
994 /// If Flags is not in a defined state then this has no effect.
995 void intersectFlagsWith(const SDNodeFlags Flags);
996
997 /// Return the number of values defined/returned by this operator.
998 unsigned getNumValues() const { return NumValues; }
999
1000 /// Return the type of a specified result.
1001 EVT getValueType(unsigned ResNo) const {
1002 assert(ResNo < NumValues && "Illegal result number!")((ResNo < NumValues && "Illegal result number!") ?
static_cast<void> (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1002, __PRETTY_FUNCTION__))
;
1003 return ValueList[ResNo];
1004 }
1005
1006 /// Return the type of a specified result as a simple type.
1007 MVT getSimpleValueType(unsigned ResNo) const {
1008 return getValueType(ResNo).getSimpleVT();
1009 }
1010
1011 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
1012 ///
1013 /// If the value type is a scalable vector type, the scalable property will
1014 /// be set and the runtime size will be a positive integer multiple of the
1015 /// base size.
1016 TypeSize getValueSizeInBits(unsigned ResNo) const {
1017 return getValueType(ResNo).getSizeInBits();
1018 }
1019
1020 using value_iterator = const EVT *;
1021
1022 value_iterator value_begin() const { return ValueList; }
1023 value_iterator value_end() const { return ValueList+NumValues; }
1024
1025 /// Return the opcode of this operation for printing.
1026 std::string getOperationName(const SelectionDAG *G = nullptr) const;
1027 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
1028 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
1029 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
1030 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1031 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1032
1033 /// Print a SelectionDAG node and all children down to
1034 /// the leaves. The given SelectionDAG allows target-specific nodes
1035 /// to be printed in human-readable form. Unlike printr, this will
1036 /// print the whole DAG, including children that appear multiple
1037 /// times.
1038 ///
1039 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1040
1041 /// Print a SelectionDAG node and children up to
1042 /// depth "depth." The given SelectionDAG allows target-specific
1043 /// nodes to be printed in human-readable form. Unlike printr, this
1044 /// will print children that appear multiple times wherever they are
1045 /// used.
1046 ///
1047 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1048 unsigned depth = 100) const;
1049
1050 /// Dump this node, for debugging.
1051 void dump() const;
1052
1053 /// Dump (recursively) this node and its use-def subgraph.
1054 void dumpr() const;
1055
1056 /// Dump this node, for debugging.
1057 /// The given SelectionDAG allows target-specific nodes to be printed
1058 /// in human-readable form.
1059 void dump(const SelectionDAG *G) const;
1060
1061 /// Dump (recursively) this node and its use-def subgraph.
1062 /// The given SelectionDAG allows target-specific nodes to be printed
1063 /// in human-readable form.
1064 void dumpr(const SelectionDAG *G) const;
1065
1066 /// printrFull to dbgs(). The given SelectionDAG allows
1067 /// target-specific nodes to be printed in human-readable form.
1068 /// Unlike dumpr, this will print the whole DAG, including children
1069 /// that appear multiple times.
1070 void dumprFull(const SelectionDAG *G = nullptr) const;
1071
1072 /// printrWithDepth to dbgs(). The given
1073 /// SelectionDAG allows target-specific nodes to be printed in
1074 /// human-readable form. Unlike dumpr, this will print children
1075 /// that appear multiple times wherever they are used.
1076 ///
1077 void dumprWithDepth(const SelectionDAG *G = nullptr,
1078 unsigned depth = 100) const;
1079
1080 /// Gather unique data for the node.
1081 void Profile(FoldingSetNodeID &ID) const;
1082
1083 /// This method should only be used by the SDUse class.
1084 void addUse(SDUse &U) { U.addToList(&UseList); }
1085
1086protected:
1087 static SDVTList getSDVTList(EVT VT) {
1088 SDVTList Ret = { getValueTypeList(VT), 1 };
1089 return Ret;
1090 }
1091
1092 /// Create an SDNode.
1093 ///
1094 /// SDNodes are created without any operands, and never own the operand
1095 /// storage. To add operands, see SelectionDAG::createOperands.
1096 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1097 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1098 IROrder(Order), debugLoc(std::move(dl)) {
1099 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1100 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor"
) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1100, __PRETTY_FUNCTION__))
;
1101 assert(NumValues == VTs.NumVTs &&((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1102, __PRETTY_FUNCTION__))
1102 "NumValues wasn't wide enough for its operands!")((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1102, __PRETTY_FUNCTION__))
;
1103 }
1104
1105 /// Release the operands and set this node to have zero operands.
1106 void DropOperands();
1107};
1108
1109/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1110/// into SDNode creation functions.
1111/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1112/// from the original Instruction, and IROrder is the ordinal position of
1113/// the instruction.
1114/// When an SDNode is created after the DAG is being built, both DebugLoc and
1115/// the IROrder are propagated from the original SDNode.
1116/// So SDLoc class provides two constructors besides the default one, one to
1117/// be used by the DAGBuilder, the other to be used by others.
1118class SDLoc {
1119private:
1120 DebugLoc DL;
1121 int IROrder = 0;
1122
1123public:
1124 SDLoc() = default;
1125 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1126 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1127 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1128 assert(Order >= 0 && "bad IROrder")((Order >= 0 && "bad IROrder") ? static_cast<void
> (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1128, __PRETTY_FUNCTION__))
;
1129 if (I)
1130 DL = I->getDebugLoc();
1131 }
1132
1133 unsigned getIROrder() const { return IROrder; }
1134 const DebugLoc &getDebugLoc() const { return DL; }
1135};
1136
1137// Define inline functions from the SDValue class.
1138
1139inline SDValue::SDValue(SDNode *node, unsigned resno)
1140 : Node(node), ResNo(resno) {
1141 // Explicitly check for !ResNo to avoid use-after-free, because there are
1142 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1143 // combines.
1144 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1145, __PRETTY_FUNCTION__))
1145 "Invalid result number for the given node!")(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1145, __PRETTY_FUNCTION__))
;
1146 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? static_cast<void> (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1146, __PRETTY_FUNCTION__))
;
1147}
1148
1149inline unsigned SDValue::getOpcode() const {
1150 return Node->getOpcode();
20
Called C++ object pointer is null
1151}
1152
1153inline EVT SDValue::getValueType() const {
1154 return Node->getValueType(ResNo);
1155}
1156
1157inline unsigned SDValue::getNumOperands() const {
1158 return Node->getNumOperands();
1159}
1160
1161inline const SDValue &SDValue::getOperand(unsigned i) const {
1162 return Node->getOperand(i);
1163}
1164
1165inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1166 return Node->getConstantOperandVal(i);
1167}
1168
1169inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1170 return Node->getConstantOperandAPInt(i);
1171}
1172
1173inline bool SDValue::isTargetOpcode() const {
1174 return Node->isTargetOpcode();
1175}
1176
1177inline bool SDValue::isTargetMemoryOpcode() const {
1178 return Node->isTargetMemoryOpcode();
1179}
1180
1181inline bool SDValue::isMachineOpcode() const {
1182 return Node->isMachineOpcode();
1183}
1184
1185inline unsigned SDValue::getMachineOpcode() const {
1186 return Node->getMachineOpcode();
1187}
1188
1189inline bool SDValue::isUndef() const {
1190 return Node->isUndef();
1191}
1192
1193inline bool SDValue::use_empty() const {
1194 return !Node->hasAnyUseOfValue(ResNo);
1195}
1196
1197inline bool SDValue::hasOneUse() const {
1198 return Node->hasNUsesOfValue(1, ResNo);
1199}
1200
1201inline const DebugLoc &SDValue::getDebugLoc() const {
1202 return Node->getDebugLoc();
1203}
1204
1205inline void SDValue::dump() const {
1206 return Node->dump();
1207}
1208
1209inline void SDValue::dump(const SelectionDAG *G) const {
1210 return Node->dump(G);
1211}
1212
1213inline void SDValue::dumpr() const {
1214 return Node->dumpr();
1215}
1216
1217inline void SDValue::dumpr(const SelectionDAG *G) const {
1218 return Node->dumpr(G);
1219}
1220
1221// Define inline functions from the SDUse class.
1222
1223inline void SDUse::set(const SDValue &V) {
1224 if (Val.getNode()) removeFromList();
1225 Val = V;
1226 if (V.getNode()) V.getNode()->addUse(*this);
1227}
1228
1229inline void SDUse::setInitial(const SDValue &V) {
1230 Val = V;
1231 V.getNode()->addUse(*this);
1232}
1233
1234inline void SDUse::setNode(SDNode *N) {
1235 if (Val.getNode()) removeFromList();
1236 Val.setNode(N);
1237 if (N) N->addUse(*this);
1238}
1239
1240/// This class is used to form a handle around another node that
1241/// is persistent and is updated across invocations of replaceAllUsesWith on its
1242/// operand. This node should be directly created by end-users and not added to
1243/// the AllNodes list.
1244class HandleSDNode : public SDNode {
1245 SDUse Op;
1246
1247public:
1248 explicit HandleSDNode(SDValue X)
1249 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1250 // HandleSDNodes are never inserted into the DAG, so they won't be
1251 // auto-numbered. Use ID 65535 as a sentinel.
1252 PersistentId = 0xffff;
1253
1254 // Manually set up the operand list. This node type is special in that it's
1255 // always stack allocated and SelectionDAG does not manage its operands.
1256 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1257 // be so special.
1258 Op.setUser(this);
1259 Op.setInitial(X);
1260 NumOperands = 1;
1261 OperandList = &Op;
1262 }
1263 ~HandleSDNode();
1264
1265 const SDValue &getValue() const { return Op; }
1266};
1267
1268class AddrSpaceCastSDNode : public SDNode {
1269private:
1270 unsigned SrcAddrSpace;
1271 unsigned DestAddrSpace;
1272
1273public:
1274 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1275 unsigned SrcAS, unsigned DestAS);
1276
1277 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1278 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1279
1280 static bool classof(const SDNode *N) {
1281 return N->getOpcode() == ISD::ADDRSPACECAST;
1282 }
1283};
1284
1285/// This is an abstract virtual class for memory operations.
1286class MemSDNode : public SDNode {
1287private:
1288 // VT of in-memory value.
1289 EVT MemoryVT;
1290
1291protected:
1292 /// Memory reference information.
1293 MachineMemOperand *MMO;
1294
1295public:
1296 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1297 EVT memvt, MachineMemOperand *MMO);
1298
1299 bool readMem() const { return MMO->isLoad(); }
1300 bool writeMem() const { return MMO->isStore(); }
1301
1302 /// Returns alignment and volatility of the memory access
1303 unsigned getOriginalAlignment() const {
1304 return MMO->getBaseAlignment();
1305 }
1306 unsigned getAlignment() const {
1307 return MMO->getAlignment();
1308 }
1309
1310 /// Return the SubclassData value, without HasDebugValue. This contains an
1311 /// encoding of the volatile flag, as well as bits used by subclasses. This
1312 /// function should only be used to compute a FoldingSetNodeID value.
1313 /// The HasDebugValue bit is masked out because CSE map needs to match
1314 /// nodes with debug info with nodes without debug info. Same is about
1315 /// isDivergent bit.
1316 unsigned getRawSubclassData() const {
1317 uint16_t Data;
1318 union {
1319 char RawSDNodeBits[sizeof(uint16_t)];
1320 SDNodeBitfields SDNodeBits;
1321 };
1322 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1323 SDNodeBits.HasDebugValue = 0;
1324 SDNodeBits.IsDivergent = false;
1325 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1326 return Data;
1327 }
1328
1329 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1330 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1331 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1332 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1333
1334 // Returns the offset from the location of the access.
1335 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1336
1337 /// Returns the AA info that describes the dereference.
1338 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1339
1340 /// Returns the Ranges that describes the dereference.
1341 const MDNode *getRanges() const { return MMO->getRanges(); }
1342
1343 /// Returns the synchronization scope ID for this memory operation.
1344 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1345
1346 /// Return the atomic ordering requirements for this memory operation. For
1347 /// cmpxchg atomic operations, return the atomic ordering requirements when
1348 /// store occurs.
1349 AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
1350
1351 /// Return true if the memory operation ordering is Unordered or higher.
1352 bool isAtomic() const { return MMO->isAtomic(); }
1353
1354 /// Returns true if the memory operation doesn't imply any ordering
1355 /// constraints on surrounding memory operations beyond the normal memory
1356 /// aliasing rules.
1357 bool isUnordered() const { return MMO->isUnordered(); }
1358
1359 /// Returns true if the memory operation is neither atomic or volatile.
1360 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1361
1362 /// Return the type of the in-memory value.
1363 EVT getMemoryVT() const { return MemoryVT; }
1364
1365 /// Return a MachineMemOperand object describing the memory
1366 /// reference performed by operation.
1367 MachineMemOperand *getMemOperand() const { return MMO; }
1368
1369 const MachinePointerInfo &getPointerInfo() const {
1370 return MMO->getPointerInfo();
1371 }
1372
1373 /// Return the address space for the associated pointer
1374 unsigned getAddressSpace() const {
1375 return getPointerInfo().getAddrSpace();
1376 }
1377
1378 /// Update this MemSDNode's MachineMemOperand information
1379 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1380 /// This must only be used when the new alignment applies to all users of
1381 /// this MachineMemOperand.
1382 void refineAlignment(const MachineMemOperand *NewMMO) {
1383 MMO->refineAlignment(NewMMO);
1384 }
1385
1386 const SDValue &getChain() const { return getOperand(0); }
1387 const SDValue &getBasePtr() const {
1388 return getOperand(getOpcode() == ISD::STORE ? 2 : 1);
1389 }
1390
1391 // Methods to support isa and dyn_cast
1392 static bool classof(const SDNode *N) {
1393 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1394 // with either an intrinsic or a target opcode.
1395 return N->getOpcode() == ISD::LOAD ||
1396 N->getOpcode() == ISD::STORE ||
1397 N->getOpcode() == ISD::PREFETCH ||
1398 N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1399 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1400 N->getOpcode() == ISD::ATOMIC_SWAP ||
1401 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1402 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1403 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1404 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1405 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1406 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1407 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1408 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1409 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1410 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1411 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1412 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1413 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1414 N->getOpcode() == ISD::ATOMIC_LOAD ||
1415 N->getOpcode() == ISD::ATOMIC_STORE ||
1416 N->getOpcode() == ISD::MLOAD ||
1417 N->getOpcode() == ISD::MSTORE ||
1418 N->getOpcode() == ISD::MGATHER ||
1419 N->getOpcode() == ISD::MSCATTER ||
1420 N->isMemIntrinsic() ||
1421 N->isTargetMemoryOpcode();
1422 }
1423};
1424
1425/// This is an SDNode representing atomic operations.
1426class AtomicSDNode : public MemSDNode {
1427public:
1428 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1429 EVT MemVT, MachineMemOperand *MMO)
1430 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1431 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1432, __PRETTY_FUNCTION__))
1432 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1432, __PRETTY_FUNCTION__))
;
1433 }
1434
1435 const SDValue &getBasePtr() const { return getOperand(1); }
1436 const SDValue &getVal() const { return getOperand(2); }
1437
1438 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1439 /// otherwise.
1440 bool isCompareAndSwap() const {
1441 unsigned Op = getOpcode();
1442 return Op == ISD::ATOMIC_CMP_SWAP ||
1443 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1444 }
1445
1446 /// For cmpxchg atomic operations, return the atomic ordering requirements
1447 /// when store does not occur.
1448 AtomicOrdering getFailureOrdering() const {
1449 assert(isCompareAndSwap() && "Must be cmpxchg operation")((isCompareAndSwap() && "Must be cmpxchg operation") ?
static_cast<void> (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1449, __PRETTY_FUNCTION__))
;
1450 return MMO->getFailureOrdering();
1451 }
1452
1453 // Methods to support isa and dyn_cast
1454 static bool classof(const SDNode *N) {
1455 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1456 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1457 N->getOpcode() == ISD::ATOMIC_SWAP ||
1458 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1459 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1460 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1461 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1462 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1463 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1464 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1465 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1466 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1467 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1468 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1469 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1470 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1471 N->getOpcode() == ISD::ATOMIC_LOAD ||
1472 N->getOpcode() == ISD::ATOMIC_STORE;
1473 }
1474};
1475
1476/// This SDNode is used for target intrinsics that touch
1477/// memory and need an associated MachineMemOperand. Its opcode may be
1478/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1479/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1480class MemIntrinsicSDNode : public MemSDNode {
1481public:
1482 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1483 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1484 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1485 SDNodeBits.IsMemIntrinsic = true;
1486 }
1487
1488 // Methods to support isa and dyn_cast
1489 static bool classof(const SDNode *N) {
1490 // We lower some target intrinsics to their target opcode
1491 // early a node with a target opcode can be of this class
1492 return N->isMemIntrinsic() ||
1493 N->getOpcode() == ISD::PREFETCH ||
1494 N->isTargetMemoryOpcode();
1495 }
1496};
1497
1498/// This SDNode is used to implement the code generator
1499/// support for the llvm IR shufflevector instruction. It combines elements
1500/// from two input vectors into a new input vector, with the selection and
1501/// ordering of elements determined by an array of integers, referred to as
1502/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1503/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1504/// An index of -1 is treated as undef, such that the code generator may put
1505/// any value in the corresponding element of the result.
1506class ShuffleVectorSDNode : public SDNode {
1507 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1508 // is freed when the SelectionDAG object is destroyed.
1509 const int *Mask;
1510
1511protected:
1512 friend class SelectionDAG;
1513
1514 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1515 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1516
1517public:
1518 ArrayRef<int> getMask() const {
1519 EVT VT = getValueType(0);
1520 return makeArrayRef(Mask, VT.getVectorNumElements());
1521 }
1522
1523 int getMaskElt(unsigned Idx) const {
1524 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")((Idx < getValueType(0).getVectorNumElements() && "Idx out of range!"
) ? static_cast<void> (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1524, __PRETTY_FUNCTION__))
;
1525 return Mask[Idx];
1526 }
1527
1528 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1529
1530 int getSplatIndex() const {
1531 assert(isSplat() && "Cannot get splat index for non-splat!")((isSplat() && "Cannot get splat index for non-splat!"
) ? static_cast<void> (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1531, __PRETTY_FUNCTION__))
;
1532 EVT VT = getValueType(0);
1533 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1534 if (Mask[i] >= 0)
1535 return Mask[i];
1536
1537 // We can choose any index value here and be correct because all elements
1538 // are undefined. Return 0 for better potential for callers to simplify.
1539 return 0;
1540 }
1541
1542 static bool isSplatMask(const int *Mask, EVT VT);
1543
1544 /// Change values in a shuffle permute mask assuming
1545 /// the two vector operands have swapped position.
1546 static void commuteMask(MutableArrayRef<int> Mask) {
1547 unsigned NumElems = Mask.size();
1548 for (unsigned i = 0; i != NumElems; ++i) {
1549 int idx = Mask[i];
1550 if (idx < 0)
1551 continue;
1552 else if (idx < (int)NumElems)
1553 Mask[i] = idx + NumElems;
1554 else
1555 Mask[i] = idx - NumElems;
1556 }
1557 }
1558
1559 static bool classof(const SDNode *N) {
1560 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1561 }
1562};
1563
1564class ConstantSDNode : public SDNode {
1565 friend class SelectionDAG;
1566
1567 const ConstantInt *Value;
1568
1569 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1570 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1571 getSDVTList(VT)),
1572 Value(val) {
1573 ConstantSDNodeBits.IsOpaque = isOpaque;
1574 }
1575
1576public:
1577 const ConstantInt *getConstantIntValue() const { return Value; }
1578 const APInt &getAPIntValue() const { return Value->getValue(); }
1579 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1580 int64_t getSExtValue() const { return Value->getSExtValue(); }
1581 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1582 return Value->getLimitedValue(Limit);
1583 }
1584
1585 bool isOne() const { return Value->isOne(); }
1586 bool isNullValue() const { return Value->isZero(); }
1587 bool isAllOnesValue() const { return Value->isMinusOne(); }
1588
1589 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1590
1591 static bool classof(const SDNode *N) {
1592 return N->getOpcode() == ISD::Constant ||
1593 N->getOpcode() == ISD::TargetConstant;
1594 }
1595};
1596
1597uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1598 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1599}
1600
1601const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1602 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1603}
1604
1605class ConstantFPSDNode : public SDNode {
1606 friend class SelectionDAG;
1607
1608 const ConstantFP *Value;
1609
1610 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1611 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1612 DebugLoc(), getSDVTList(VT)),
1613 Value(val) {}
1614
1615public:
1616 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1617 const ConstantFP *getConstantFPValue() const { return Value; }
1618
1619 /// Return true if the value is positive or negative zero.
1620 bool isZero() const { return Value->isZero(); }
1621
1622 /// Return true if the value is a NaN.
1623 bool isNaN() const { return Value->isNaN(); }
1624
1625 /// Return true if the value is an infinity
1626 bool isInfinity() const { return Value->isInfinity(); }
1627
1628 /// Return true if the value is negative.
1629 bool isNegative() const { return Value->isNegative(); }
1630
1631 /// We don't rely on operator== working on double values, as
1632 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1633 /// As such, this method can be used to do an exact bit-for-bit comparison of
1634 /// two floating point values.
1635
1636 /// We leave the version with the double argument here because it's just so
1637 /// convenient to write "2.0" and the like. Without this function we'd
1638 /// have to duplicate its logic everywhere it's called.
1639 bool isExactlyValue(double V) const {
1640 return Value->getValueAPF().isExactlyValue(V);
1641 }
1642 bool isExactlyValue(const APFloat& V) const;
1643
1644 static bool isValueValidForType(EVT VT, const APFloat& Val);
1645
1646 static bool classof(const SDNode *N) {
1647 return N->getOpcode() == ISD::ConstantFP ||
1648 N->getOpcode() == ISD::TargetConstantFP;
1649 }
1650};
1651
1652/// Returns true if \p V is a constant integer zero.
1653bool isNullConstant(SDValue V);
1654
1655/// Returns true if \p V is an FP constant with a value of positive zero.
1656bool isNullFPConstant(SDValue V);
1657
1658/// Returns true if \p V is an integer constant with all bits set.
1659bool isAllOnesConstant(SDValue V);
1660
1661/// Returns true if \p V is a constant integer one.
1662bool isOneConstant(SDValue V);
1663
1664/// Return the non-bitcasted source operand of \p V if it exists.
1665/// If \p V is not a bitcasted value, it is returned as-is.
1666SDValue peekThroughBitcasts(SDValue V);
1667
1668/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1669/// If \p V is not a bitcasted one-use value, it is returned as-is.
1670SDValue peekThroughOneUseBitcasts(SDValue V);
1671
1672/// Return the non-extracted vector source operand of \p V if it exists.
1673/// If \p V is not an extracted subvector, it is returned as-is.
1674SDValue peekThroughExtractSubvectors(SDValue V);
1675
1676/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1677/// constant is canonicalized to be operand 1.
1678bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1679
1680/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1681ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1682 bool AllowTruncation = false);
1683
1684/// Returns the SDNode if it is a demanded constant splat BuildVector or
1685/// constant int.
1686ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1687 bool AllowUndefs = false,
1688 bool AllowTruncation = false);
1689
1690/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1691ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1692
1693/// Returns the SDNode if it is a demanded constant splat BuildVector or
1694/// constant float.
1695ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1696 bool AllowUndefs = false);
1697
1698/// Return true if the value is a constant 0 integer or a splatted vector of
1699/// a constant 0 integer (with no undefs by default).
1700/// Build vector implicit truncation is not an issue for null values.
1701bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1702
1703/// Return true if the value is a constant 1 integer or a splatted vector of a
1704/// constant 1 integer (with no undefs).
1705/// Does not permit build vector implicit truncation.
1706bool isOneOrOneSplat(SDValue V);
1707
1708/// Return true if the value is a constant -1 integer or a splatted vector of a
1709/// constant -1 integer (with no undefs).
1710/// Does not permit build vector implicit truncation.
1711bool isAllOnesOrAllOnesSplat(SDValue V);
1712
1713class GlobalAddressSDNode : public SDNode {
1714 friend class SelectionDAG;
1715
1716 const GlobalValue *TheGlobal;
1717 int64_t Offset;
1718 unsigned TargetFlags;
1719
1720 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1721 const GlobalValue *GA, EVT VT, int64_t o,
1722 unsigned TF);
1723
1724public:
1725 const GlobalValue *getGlobal() const { return TheGlobal; }
1726 int64_t getOffset() const { return Offset; }
1727 unsigned getTargetFlags() const { return TargetFlags; }
1728 // Return the address space this GlobalAddress belongs to.
1729 unsigned getAddressSpace() const;
1730
1731 static bool classof(const SDNode *N) {
1732 return N->getOpcode() == ISD::GlobalAddress ||
1733 N->getOpcode() == ISD::TargetGlobalAddress ||
1734 N->getOpcode() == ISD::GlobalTLSAddress ||
1735 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1736 }
1737};
1738
1739class FrameIndexSDNode : public SDNode {
1740 friend class SelectionDAG;
1741
1742 int FI;
1743
1744 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1745 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1746 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1747 }
1748
1749public:
1750 int getIndex() const { return FI; }
1751
1752 static bool classof(const SDNode *N) {
1753 return N->getOpcode() == ISD::FrameIndex ||
1754 N->getOpcode() == ISD::TargetFrameIndex;
1755 }
1756};
1757
1758/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1759/// the offet and size that are started/ended in the underlying FrameIndex.
1760class LifetimeSDNode : public SDNode {
1761 friend class SelectionDAG;
1762 int64_t Size;
1763 int64_t Offset; // -1 if offset is unknown.
1764
1765 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1766 SDVTList VTs, int64_t Size, int64_t Offset)
1767 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1768public:
1769 int64_t getFrameIndex() const {
1770 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1771 }
1772
1773 bool hasOffset() const { return Offset >= 0; }
1774 int64_t getOffset() const {
1775 assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast<
void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1775, __PRETTY_FUNCTION__))
;
1776 return Offset;
1777 }
1778 int64_t getSize() const {
1779 assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast<
void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1779, __PRETTY_FUNCTION__))
;
1780 return Size;
1781 }
1782
1783 // Methods to support isa and dyn_cast
1784 static bool classof(const SDNode *N) {
1785 return N->getOpcode() == ISD::LIFETIME_START ||
1786 N->getOpcode() == ISD::LIFETIME_END;
1787 }
1788};
1789
1790class JumpTableSDNode : public SDNode {
1791 friend class SelectionDAG;
1792
1793 int JTI;
1794 unsigned TargetFlags;
1795
1796 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1797 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1798 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1799 }
1800
1801public:
1802 int getIndex() const { return JTI; }
1803 unsigned getTargetFlags() const { return TargetFlags; }
1804
1805 static bool classof(const SDNode *N) {
1806 return N->getOpcode() == ISD::JumpTable ||
1807 N->getOpcode() == ISD::TargetJumpTable;
1808 }
1809};
1810
1811class ConstantPoolSDNode : public SDNode {
1812 friend class SelectionDAG;
1813
1814 union {
1815 const Constant *ConstVal;
1816 MachineConstantPoolValue *MachineCPVal;
1817 } Val;
1818 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1819 unsigned Alignment; // Minimum alignment requirement of CP (not log2 value).
1820 unsigned TargetFlags;
1821
1822 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1823 unsigned Align, unsigned TF)
1824 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1825 DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
1826 TargetFlags(TF) {
1827 assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1827, __PRETTY_FUNCTION__))
;
1828 Val.ConstVal = c;
1829 }
1830
1831 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v,
1832 EVT VT, int o, unsigned Align, unsigned TF)
1833 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1834 DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
1835 TargetFlags(TF) {
1836 assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1836, __PRETTY_FUNCTION__))
;
1837 Val.MachineCPVal = v;
1838 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1839 }
1840
1841public:
1842 bool isMachineConstantPoolEntry() const {
1843 return Offset < 0;
1844 }
1845
1846 const Constant *getConstVal() const {
1847 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")((!isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1847, __PRETTY_FUNCTION__))
;
1848 return Val.ConstVal;
1849 }
1850
1851 MachineConstantPoolValue *getMachineCPVal() const {
1852 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")((isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1852, __PRETTY_FUNCTION__))
;
1853 return Val.MachineCPVal;
1854 }
1855
1856 int getOffset() const {
1857 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1858 }
1859
1860 // Return the alignment of this constant pool object, which is either 0 (for
1861 // default alignment) or the desired value.
1862 unsigned getAlignment() const { return Alignment; }
1863 unsigned getTargetFlags() const { return TargetFlags; }
1864
1865 Type *getType() const;
1866
1867 static bool classof(const SDNode *N) {
1868 return N->getOpcode() == ISD::ConstantPool ||
1869 N->getOpcode() == ISD::TargetConstantPool;
1870 }
1871};
1872
1873/// Completely target-dependent object reference.
1874class TargetIndexSDNode : public SDNode {
1875 friend class SelectionDAG;
1876
1877 unsigned TargetFlags;
1878 int Index;
1879 int64_t Offset;
1880
1881public:
1882 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1883 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1884 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1885
1886 unsigned getTargetFlags() const { return TargetFlags; }
1887 int getIndex() const { return Index; }
1888 int64_t getOffset() const { return Offset; }
1889
1890 static bool classof(const SDNode *N) {
1891 return N->getOpcode() == ISD::TargetIndex;
1892 }
1893};
1894
1895class BasicBlockSDNode : public SDNode {
1896 friend class SelectionDAG;
1897
1898 MachineBasicBlock *MBB;
1899
1900 /// Debug info is meaningful and potentially useful here, but we create
1901 /// blocks out of order when they're jumped to, which makes it a bit
1902 /// harder. Let's see if we need it first.
1903 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1904 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1905 {}
1906
1907public:
1908 MachineBasicBlock *getBasicBlock() const { return MBB; }
1909
1910 static bool classof(const SDNode *N) {
1911 return N->getOpcode() == ISD::BasicBlock;
1912 }
1913};
1914
1915/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1916class BuildVectorSDNode : public SDNode {
1917public:
1918 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1919 explicit BuildVectorSDNode() = delete;
1920
1921 /// Check if this is a constant splat, and if so, find the
1922 /// smallest element size that splats the vector. If MinSplatBits is
1923 /// nonzero, the element size must be at least that large. Note that the
1924 /// splat element may be the entire vector (i.e., a one element vector).
1925 /// Returns the splat element value in SplatValue. Any undefined bits in
1926 /// that value are zero, and the corresponding bits in the SplatUndef mask
1927 /// are set. The SplatBitSize value is set to the splat element size in
1928 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1929 /// undefined. isBigEndian describes the endianness of the target.
1930 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1931 unsigned &SplatBitSize, bool &HasAnyUndefs,
1932 unsigned MinSplatBits = 0,
1933 bool isBigEndian = false) const;
1934
1935 /// Returns the demanded splatted value or a null value if this is not a
1936 /// splat.
1937 ///
1938 /// The DemandedElts mask indicates the elements that must be in the splat.
1939 /// If passed a non-null UndefElements bitvector, it will resize it to match
1940 /// the vector width and set the bits where elements are undef.
1941 SDValue getSplatValue(const APInt &DemandedElts,
1942 BitVector *UndefElements = nullptr) const;
1943
1944 /// Returns the splatted value or a null value if this is not a splat.
1945 ///
1946 /// If passed a non-null UndefElements bitvector, it will resize it to match
1947 /// the vector width and set the bits where elements are undef.
1948 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1949
1950 /// Returns the demanded splatted constant or null if this is not a constant
1951 /// splat.
1952 ///
1953 /// The DemandedElts mask indicates the elements that must be in the splat.
1954 /// If passed a non-null UndefElements bitvector, it will resize it to match
1955 /// the vector width and set the bits where elements are undef.
1956 ConstantSDNode *
1957 getConstantSplatNode(const APInt &DemandedElts,
1958 BitVector *UndefElements = nullptr) const;
1959
1960 /// Returns the splatted constant or null if this is not a constant
1961 /// splat.
1962 ///
1963 /// If passed a non-null UndefElements bitvector, it will resize it to match
1964 /// the vector width and set the bits where elements are undef.
1965 ConstantSDNode *
1966 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
1967
1968 /// Returns the demanded splatted constant FP or null if this is not a
1969 /// constant FP splat.
1970 ///
1971 /// The DemandedElts mask indicates the elements that must be in the splat.
1972 /// If passed a non-null UndefElements bitvector, it will resize it to match
1973 /// the vector width and set the bits where elements are undef.
1974 ConstantFPSDNode *
1975 getConstantFPSplatNode(const APInt &DemandedElts,
1976 BitVector *UndefElements = nullptr) const;
1977
1978 /// Returns the splatted constant FP or null if this is not a constant
1979 /// FP splat.
1980 ///
1981 /// If passed a non-null UndefElements bitvector, it will resize it to match
1982 /// the vector width and set the bits where elements are undef.
1983 ConstantFPSDNode *
1984 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
1985
1986 /// If this is a constant FP splat and the splatted constant FP is an
1987 /// exact power or 2, return the log base 2 integer value. Otherwise,
1988 /// return -1.
1989 ///
1990 /// The BitWidth specifies the necessary bit precision.
1991 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
1992 uint32_t BitWidth) const;
1993
1994 bool isConstant() const;
1995
1996 static bool classof(const SDNode *N) {
1997 return N->getOpcode() == ISD::BUILD_VECTOR;
1998 }
1999};
2000
2001/// An SDNode that holds an arbitrary LLVM IR Value. This is
2002/// used when the SelectionDAG needs to make a simple reference to something
2003/// in the LLVM IR representation.
2004///
2005class SrcValueSDNode : public SDNode {
2006 friend class SelectionDAG;
2007
2008 const Value *V;
2009
2010 /// Create a SrcValue for a general value.
2011 explicit SrcValueSDNode(const Value *v)
2012 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2013
2014public:
2015 /// Return the contained Value.
2016 const Value *getValue() const { return V; }
2017
2018 static bool classof(const SDNode *N) {
2019 return N->getOpcode() == ISD::SRCVALUE;
2020 }
2021};
2022
2023class MDNodeSDNode : public SDNode {
2024 friend class SelectionDAG;
2025
2026 const MDNode *MD;
2027
2028 explicit MDNodeSDNode(const MDNode *md)
2029 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2030 {}
2031
2032public:
2033 const MDNode *getMD() const { return MD; }
2034
2035 static bool classof(const SDNode *N) {
2036 return N->getOpcode() == ISD::MDNODE_SDNODE;
2037 }
2038};
2039
2040class RegisterSDNode : public SDNode {
2041 friend class SelectionDAG;
2042
2043 unsigned Reg;
2044
2045 RegisterSDNode(unsigned reg, EVT VT)
2046 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2047
2048public:
2049 unsigned getReg() const { return Reg; }
2050
2051 static bool classof(const SDNode *N) {
2052 return N->getOpcode() == ISD::Register;
2053 }
2054};
2055
2056class RegisterMaskSDNode : public SDNode {
2057 friend class SelectionDAG;
2058
2059 // The memory for RegMask is not owned by the node.
2060 const uint32_t *RegMask;
2061
2062 RegisterMaskSDNode(const uint32_t *mask)
2063 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2064 RegMask(mask) {}
2065
2066public:
2067 const uint32_t *getRegMask() const { return RegMask; }
2068
2069 static bool classof(const SDNode *N) {
2070 return N->getOpcode() == ISD::RegisterMask;
2071 }
2072};
2073
2074class BlockAddressSDNode : public SDNode {
2075 friend class SelectionDAG;
2076
2077 const BlockAddress *BA;
2078 int64_t Offset;
2079 unsigned TargetFlags;
2080
2081 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2082 int64_t o, unsigned Flags)
2083 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2084 BA(ba), Offset(o), TargetFlags(Flags) {}
2085
2086public:
2087 const BlockAddress *getBlockAddress() const { return BA; }
2088 int64_t getOffset() const { return Offset; }
2089 unsigned getTargetFlags() const { return TargetFlags; }
2090
2091 static bool classof(const SDNode *N) {
2092 return N->getOpcode() == ISD::BlockAddress ||
2093 N->getOpcode() == ISD::TargetBlockAddress;
2094 }
2095};
2096
2097class LabelSDNode : public SDNode {
2098 friend class SelectionDAG;
2099
2100 MCSymbol *Label;
2101
2102 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2103 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2104 assert(LabelSDNode::classof(this) && "not a label opcode")((LabelSDNode::classof(this) && "not a label opcode")
? static_cast<void> (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2104, __PRETTY_FUNCTION__))
;
2105 }
2106
2107public:
2108 MCSymbol *getLabel() const { return Label; }
2109
2110 static bool classof(const SDNode *N) {
2111 return N->getOpcode() == ISD::EH_LABEL ||
2112 N->getOpcode() == ISD::ANNOTATION_LABEL;
2113 }
2114};
2115
2116class ExternalSymbolSDNode : public SDNode {
2117 friend class SelectionDAG;
2118
2119 const char *Symbol;
2120 unsigned TargetFlags;
2121
2122 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2123 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2124 DebugLoc(), getSDVTList(VT)),
2125 Symbol(Sym), TargetFlags(TF) {}
2126
2127public:
2128 const char *getSymbol() const { return Symbol; }
2129 unsigned getTargetFlags() const { return TargetFlags; }
2130
2131 static bool classof(const SDNode *N) {
2132 return N->getOpcode() == ISD::ExternalSymbol ||
2133 N->getOpcode() == ISD::TargetExternalSymbol;
2134 }
2135};
2136
2137class MCSymbolSDNode : public SDNode {
2138 friend class SelectionDAG;
2139
2140 MCSymbol *Symbol;
2141
2142 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2143 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2144
2145public:
2146 MCSymbol *getMCSymbol() const { return Symbol; }
2147
2148 static bool classof(const SDNode *N) {
2149 return N->getOpcode() == ISD::MCSymbol;
2150 }
2151};
2152
2153class CondCodeSDNode : public SDNode {
2154 friend class SelectionDAG;
2155
2156 ISD::CondCode Condition;
2157
2158 explicit CondCodeSDNode(ISD::CondCode Cond)
2159 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2160 Condition(Cond) {}
2161
2162public:
2163 ISD::CondCode get() const { return Condition; }
2164
2165 static bool classof(const SDNode *N) {
2166 return N->getOpcode() == ISD::CONDCODE;
2167 }
2168};
2169
2170/// This class is used to represent EVT's, which are used
2171/// to parameterize some operations.
2172class VTSDNode : public SDNode {
2173 friend class SelectionDAG;
2174
2175 EVT ValueType;
2176
2177 explicit VTSDNode(EVT VT)
2178 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2179 ValueType(VT) {}
2180
2181public:
2182 EVT getVT() const { return ValueType; }
2183
2184 static bool classof(const SDNode *N) {
2185 return N->getOpcode() == ISD::VALUETYPE;
2186 }
2187};
2188
2189/// Base class for LoadSDNode and StoreSDNode
2190class LSBaseSDNode : public MemSDNode {
2191public:
2192 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2193 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2194 MachineMemOperand *MMO)
2195 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2196 LSBaseSDNodeBits.AddressingMode = AM;
2197 assert(getAddressingMode() == AM && "Value truncated")((getAddressingMode() == AM && "Value truncated") ? static_cast
<void> (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2197, __PRETTY_FUNCTION__))
;
2198 }
2199
2200 const SDValue &getOffset() const {
2201 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2202 }
2203
2204 /// Return the addressing mode for this load or store:
2205 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2206 ISD::MemIndexedMode getAddressingMode() const {
2207 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2208 }
2209
2210 /// Return true if this is a pre/post inc/dec load/store.
2211 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2212
2213 /// Return true if this is NOT a pre/post inc/dec load/store.
2214 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2215
2216 static bool classof(const SDNode *N) {
2217 return N->getOpcode() == ISD::LOAD ||
2218 N->getOpcode() == ISD::STORE;
2219 }
2220};
2221
2222/// This class is used to represent ISD::LOAD nodes.
2223class LoadSDNode : public LSBaseSDNode {
2224 friend class SelectionDAG;
2225
2226 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2227 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2228 MachineMemOperand *MMO)
2229 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2230 LoadSDNodeBits.ExtTy = ETy;
2231 assert(readMem() && "Load MachineMemOperand is not a load!")((readMem() && "Load MachineMemOperand is not a load!"
) ? static_cast<void> (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2231, __PRETTY_FUNCTION__))
;
2232 assert(!writeMem() && "Load MachineMemOperand is a store!")((!writeMem() && "Load MachineMemOperand is a store!"
) ? static_cast<void> (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2232, __PRETTY_FUNCTION__))
;
2233 }
2234
2235public:
2236 /// Return whether this is a plain node,
2237 /// or one of the varieties of value-extending loads.
2238 ISD::LoadExtType getExtensionType() const {
2239 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2240 }
2241
2242 const SDValue &getBasePtr() const { return getOperand(1); }
2243 const SDValue &getOffset() const { return getOperand(2); }
2244
2245 static bool classof(const SDNode *N) {
2246 return N->getOpcode() == ISD::LOAD;
2247 }
2248};
2249
2250/// This class is used to represent ISD::STORE nodes.
2251class StoreSDNode : public LSBaseSDNode {
2252 friend class SelectionDAG;
2253
2254 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2255 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2256 MachineMemOperand *MMO)
2257 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2258 StoreSDNodeBits.IsTruncating = isTrunc;
2259 assert(!readMem() && "Store MachineMemOperand is a load!")((!readMem() && "Store MachineMemOperand is a load!")
? static_cast<void> (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2259, __PRETTY_FUNCTION__))
;
2260 assert(writeMem() && "Store MachineMemOperand is not a store!")((writeMem() && "Store MachineMemOperand is not a store!"
) ? static_cast<void> (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2260, __PRETTY_FUNCTION__))
;
2261 }
2262
2263public:
2264 /// Return true if the op does a truncation before store.
2265 /// For integers this is the same as doing a TRUNCATE and storing the result.
2266 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2267 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2268 void setTruncatingStore(bool Truncating) {
2269 StoreSDNodeBits.IsTruncating = Truncating;
2270 }
2271
2272 const SDValue &getValue() const { return getOperand(1); }
2273 const SDValue &getBasePtr() const { return getOperand(2); }
2274 const SDValue &getOffset() const { return getOperand(3); }
2275
2276 static bool classof(const SDNode *N) {
2277 return N->getOpcode() == ISD::STORE;
2278 }
2279};
2280
2281/// This base class is used to represent MLOAD and MSTORE nodes
2282class MaskedLoadStoreSDNode : public MemSDNode {
2283public:
2284 friend class SelectionDAG;
2285
2286 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2287 const DebugLoc &dl, SDVTList VTs,
2288 ISD::MemIndexedMode AM, EVT MemVT,
2289 MachineMemOperand *MMO)
2290 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2291 LSBaseSDNodeBits.AddressingMode = AM;
2292 assert(getAddressingMode() == AM && "Value truncated")((getAddressingMode() == AM && "Value truncated") ? static_cast
<void> (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-10~++20200112100611+7fa5290d5bd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2292, __PRETTY_FUNCTION__))
;
2293 }
2294
2295 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2296 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2297 // Mask is a vector of i1 elements
2298 const SDValue &getBasePtr() const {
2299 return getOperand(getOpcode() == ISD::MLOAD ? 1 : 2);
2300 }
2301 const SDValue &getOffset() const {
2302 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2303 }
2304 const SDValue &getMask() const {
2305 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2306 }
2307
2308 /// Return the addressing mode for this load or store:
2309 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2310 ISD::MemIndexedMode getAddressingMode() const {
2311 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2312 }
2313
2314 /// Return true if this is a pre/post inc/dec load/store.
2315 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2316
2317 /// Return true if this is NOT a pre/post inc/dec load/store.