Bug Summary

File:include/llvm/CodeGen/SelectionDAGNodes.h
Warning:line 1159, column 10
Called C++ object pointer is null

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -disable-llvm-verifier -discard-value-names -main-file-name WebAssemblyISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mthread-model posix -mframe-pointer=none -fmath-errno -masm-verbose -mconstructor-aliases -munwind-tables -fuse-init-array -target-cpu x86-64 -dwarf-column-info -debugger-tuning=gdb -ffunction-sections -fdata-sections -resource-dir /usr/lib/llvm-10/lib/clang/10.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/include -I /build/llvm-toolchain-snapshot-10~svn374877/include -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/x86_64-linux-gnu/c++/6.3.0 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/6.3.0/../../../../include/c++/6.3.0/backward -internal-isystem /usr/local/include -internal-isystem /usr/lib/llvm-10/lib/clang/10.0.0/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir /build/llvm-toolchain-snapshot-10~svn374877/build-llvm/lib/Target/WebAssembly -fdebug-prefix-map=/build/llvm-toolchain-snapshot-10~svn374877=. -ferror-limit 19 -fmessage-length 0 -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fobjc-runtime=gcc -fdiagnostics-show-option -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -o /tmp/scan-build-2019-10-15-233810-7101-1 -x c++ /build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "WebAssemblyMachineFunctionInfo.h"
17#include "WebAssemblySubtarget.h"
18#include "WebAssemblyTargetMachine.h"
19#include "llvm/CodeGen/Analysis.h"
20#include "llvm/CodeGen/CallingConvLower.h"
21#include "llvm/CodeGen/MachineInstrBuilder.h"
22#include "llvm/CodeGen/MachineJumpTableInfo.h"
23#include "llvm/CodeGen/MachineModuleInfo.h"
24#include "llvm/CodeGen/MachineRegisterInfo.h"
25#include "llvm/CodeGen/SelectionDAG.h"
26#include "llvm/CodeGen/WasmEHFuncInfo.h"
27#include "llvm/IR/DiagnosticInfo.h"
28#include "llvm/IR/DiagnosticPrinter.h"
29#include "llvm/IR/Function.h"
30#include "llvm/IR/Intrinsics.h"
31#include "llvm/Support/Debug.h"
32#include "llvm/Support/ErrorHandling.h"
33#include "llvm/Support/raw_ostream.h"
34#include "llvm/Target/TargetOptions.h"
35using namespace llvm;
36
37#define DEBUG_TYPE"wasm-lower" "wasm-lower"
38
39WebAssemblyTargetLowering::WebAssemblyTargetLowering(
40 const TargetMachine &TM, const WebAssemblySubtarget &STI)
41 : TargetLowering(TM), Subtarget(&STI) {
42 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
43
44 // Booleans always contain 0 or 1.
45 setBooleanContents(ZeroOrOneBooleanContent);
46 // Except in SIMD vectors
47 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
48 // We don't know the microarchitecture here, so just reduce register pressure.
49 setSchedulingPreference(Sched::RegPressure);
50 // Tell ISel that we have a stack pointer.
51 setStackPointerRegisterToSaveRestore(
52 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
53 // Set up the register classes.
54 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
55 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
56 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
57 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
58 if (Subtarget->hasSIMD128()) {
59 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
60 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
61 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
62 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
63 }
64 if (Subtarget->hasUnimplementedSIMD128()) {
65 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
66 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
67 }
68 // Compute derived properties from the register classes.
69 computeRegisterProperties(Subtarget->getRegisterInfo());
70
71 setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
72 setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
73 setOperationAction(ISD::JumpTable, MVTPtr, Custom);
74 setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
75 setOperationAction(ISD::BRIND, MVT::Other, Custom);
76
77 // Take the default expansion for va_arg, va_copy, and va_end. There is no
78 // default action for va_start, so we do that custom.
79 setOperationAction(ISD::VASTART, MVT::Other, Custom);
80 setOperationAction(ISD::VAARG, MVT::Other, Expand);
81 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
82 setOperationAction(ISD::VAEND, MVT::Other, Expand);
83
84 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
85 // Don't expand the floating-point types to constant pools.
86 setOperationAction(ISD::ConstantFP, T, Legal);
87 // Expand floating-point comparisons.
88 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
89 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
90 setCondCodeAction(CC, T, Expand);
91 // Expand floating-point library function operators.
92 for (auto Op :
93 {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
94 setOperationAction(Op, T, Expand);
95 // Note supported floating-point library function operators that otherwise
96 // default to expand.
97 for (auto Op :
98 {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
99 setOperationAction(Op, T, Legal);
100 // Support minimum and maximum, which otherwise default to expand.
101 setOperationAction(ISD::FMINIMUM, T, Legal);
102 setOperationAction(ISD::FMAXIMUM, T, Legal);
103 // WebAssembly currently has no builtin f16 support.
104 setOperationAction(ISD::FP16_TO_FP, T, Expand);
105 setOperationAction(ISD::FP_TO_FP16, T, Expand);
106 setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
107 setTruncStoreAction(T, MVT::f16, Expand);
108 }
109
110 // Expand unavailable integer operations.
111 for (auto Op :
112 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
113 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
114 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
115 for (auto T : {MVT::i32, MVT::i64})
116 setOperationAction(Op, T, Expand);
117 if (Subtarget->hasSIMD128())
118 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
119 setOperationAction(Op, T, Expand);
120 if (Subtarget->hasUnimplementedSIMD128())
121 setOperationAction(Op, MVT::v2i64, Expand);
122 }
123
124 // SIMD-specific configuration
125 if (Subtarget->hasSIMD128()) {
126 // Support saturating add for i8x16 and i16x8
127 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
128 for (auto T : {MVT::v16i8, MVT::v8i16})
129 setOperationAction(Op, T, Legal);
130
131 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
132 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
133 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
134 if (Subtarget->hasUnimplementedSIMD128())
135 for (auto T : {MVT::v2i64, MVT::v2f64})
136 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
137
138 // We have custom shuffle lowering to expose the shuffle mask
139 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
140 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
141 if (Subtarget->hasUnimplementedSIMD128())
142 for (auto T: {MVT::v2i64, MVT::v2f64})
143 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
144
145 // Custom lowering since wasm shifts must have a scalar shift amount
146 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL}) {
147 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
148 setOperationAction(Op, T, Custom);
149 if (Subtarget->hasUnimplementedSIMD128())
150 setOperationAction(Op, MVT::v2i64, Custom);
151 }
152
153 // Custom lower lane accesses to expand out variable indices
154 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}) {
155 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
156 setOperationAction(Op, T, Custom);
157 if (Subtarget->hasUnimplementedSIMD128())
158 for (auto T : {MVT::v2i64, MVT::v2f64})
159 setOperationAction(Op, T, Custom);
160 }
161
162 // There is no i64x2.mul instruction
163 setOperationAction(ISD::MUL, MVT::v2i64, Expand);
164
165 // There are no vector select instructions
166 for (auto Op : {ISD::VSELECT, ISD::SELECT_CC, ISD::SELECT}) {
167 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32})
168 setOperationAction(Op, T, Expand);
169 if (Subtarget->hasUnimplementedSIMD128())
170 for (auto T : {MVT::v2i64, MVT::v2f64})
171 setOperationAction(Op, T, Expand);
172 }
173
174 // Expand integer operations supported for scalars but not SIMD
175 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP, ISD::SDIV, ISD::UDIV,
176 ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR}) {
177 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
178 setOperationAction(Op, T, Expand);
179 if (Subtarget->hasUnimplementedSIMD128())
180 setOperationAction(Op, MVT::v2i64, Expand);
181 }
182
183 // Expand float operations supported for scalars but not SIMD
184 for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
185 ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
186 ISD::FEXP, ISD::FEXP2, ISD::FRINT}) {
187 setOperationAction(Op, MVT::v4f32, Expand);
188 if (Subtarget->hasUnimplementedSIMD128())
189 setOperationAction(Op, MVT::v2f64, Expand);
190 }
191
192 // Expand additional SIMD ops that V8 hasn't implemented yet
193 if (!Subtarget->hasUnimplementedSIMD128()) {
194 setOperationAction(ISD::FSQRT, MVT::v4f32, Expand);
195 setOperationAction(ISD::FDIV, MVT::v4f32, Expand);
196 }
197 }
198
199 // As a special case, these operators use the type to mean the type to
200 // sign-extend from.
201 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
202 if (!Subtarget->hasSignExt()) {
203 // Sign extends are legal only when extending a vector extract
204 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
205 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
206 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
207 }
208 for (auto T : MVT::integer_fixedlen_vector_valuetypes())
209 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
210
211 // Dynamic stack allocation: use the default expansion.
212 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
213 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
214 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);
215
216 setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
217 setOperationAction(ISD::CopyToReg, MVT::Other, Custom);
218
219 // Expand these forms; we pattern-match the forms that we can handle in isel.
220 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
221 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
222 setOperationAction(Op, T, Expand);
223
224 // We have custom switch handling.
225 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
226
227 // WebAssembly doesn't have:
228 // - Floating-point extending loads.
229 // - Floating-point truncating stores.
230 // - i1 extending loads.
231 // - truncating SIMD stores and most extending loads
232 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
233 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
234 for (auto T : MVT::integer_valuetypes())
235 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
236 setLoadExtAction(Ext, T, MVT::i1, Promote);
237 if (Subtarget->hasSIMD128()) {
238 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
239 MVT::v2f64}) {
240 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
241 if (MVT(T) != MemT) {
242 setTruncStoreAction(T, MemT, Expand);
243 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
244 setLoadExtAction(Ext, T, MemT, Expand);
245 }
246 }
247 }
248 // But some vector extending loads are legal
249 if (Subtarget->hasUnimplementedSIMD128()) {
250 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
251 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
252 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
253 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
254 }
255 }
256 }
257
258 // Don't do anything clever with build_pairs
259 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
260
261 // Trap lowers to wasm unreachable
262 setOperationAction(ISD::TRAP, MVT::Other, Legal);
263
264 // Exception handling intrinsics
265 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
266 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
267
268 setMaxAtomicSizeInBitsSupported(64);
269
270 // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
271 // consistent with the f64 and f128 names.
272 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
273 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
274
275 // Define the emscripten name for return address helper.
276 // TODO: when implementing other WASM backends, make this generic or only do
277 // this on emscripten depending on what they end up doing.
278 setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
279
280 // Always convert switches to br_tables unless there is only one case, which
281 // is equivalent to a simple branch. This reduces code size for wasm, and we
282 // defer possible jump table optimizations to the VM.
283 setMinimumJumpTableEntries(2);
284}
285
286TargetLowering::AtomicExpansionKind
287WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
288 // We have wasm instructions for these
289 switch (AI->getOperation()) {
290 case AtomicRMWInst::Add:
291 case AtomicRMWInst::Sub:
292 case AtomicRMWInst::And:
293 case AtomicRMWInst::Or:
294 case AtomicRMWInst::Xor:
295 case AtomicRMWInst::Xchg:
296 return AtomicExpansionKind::None;
297 default:
298 break;
299 }
300 return AtomicExpansionKind::CmpXChg;
301}
302
303FastISel *WebAssemblyTargetLowering::createFastISel(
304 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
305 return WebAssembly::createFastISel(FuncInfo, LibInfo);
306}
307
308MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
309 EVT VT) const {
310 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
311 if (BitWidth > 1 && BitWidth < 8)
312 BitWidth = 8;
313
314 if (BitWidth > 64) {
315 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
316 // the count to be an i32.
317 BitWidth = 32;
318 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&((BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && "32-bit shift counts ought to be enough for anyone"
) ? static_cast<void> (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 319, __PRETTY_FUNCTION__))
319 "32-bit shift counts ought to be enough for anyone")((BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && "32-bit shift counts ought to be enough for anyone"
) ? static_cast<void> (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 319, __PRETTY_FUNCTION__))
;
320 }
321
322 MVT Result = MVT::getIntegerVT(BitWidth);
323 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&((Result != MVT::INVALID_SIMPLE_VALUE_TYPE && "Unable to represent scalar shift amount type"
) ? static_cast<void> (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 324, __PRETTY_FUNCTION__))
324 "Unable to represent scalar shift amount type")((Result != MVT::INVALID_SIMPLE_VALUE_TYPE && "Unable to represent scalar shift amount type"
) ? static_cast<void> (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 324, __PRETTY_FUNCTION__))
;
325 return Result;
326}
327
328// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
329// undefined result on invalid/overflow, to the WebAssembly opcode, which
330// traps on invalid/overflow.
331static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
332 MachineBasicBlock *BB,
333 const TargetInstrInfo &TII,
334 bool IsUnsigned, bool Int64,
335 bool Float64, unsigned LoweredOpcode) {
336 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
337
338 Register OutReg = MI.getOperand(0).getReg();
339 Register InReg = MI.getOperand(1).getReg();
340
341 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
342 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
343 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
344 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
345 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
346 unsigned Eqz = WebAssembly::EQZ_I32;
347 unsigned And = WebAssembly::AND_I32;
348 int64_t Limit = Int64 ? INT64_MIN(-9223372036854775807L -1) : INT32_MIN(-2147483647-1);
349 int64_t Substitute = IsUnsigned ? 0 : Limit;
350 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
351 auto &Context = BB->getParent()->getFunction().getContext();
352 Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);
353
354 const BasicBlock *LLVMBB = BB->getBasicBlock();
355 MachineFunction *F = BB->getParent();
356 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
357 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
358 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
359
360 MachineFunction::iterator It = ++BB->getIterator();
361 F->insert(It, FalseMBB);
362 F->insert(It, TrueMBB);
363 F->insert(It, DoneMBB);
364
365 // Transfer the remainder of BB and its successor edges to DoneMBB.
366 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
367 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
368
369 BB->addSuccessor(TrueMBB);
370 BB->addSuccessor(FalseMBB);
371 TrueMBB->addSuccessor(DoneMBB);
372 FalseMBB->addSuccessor(DoneMBB);
373
374 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
375 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
376 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
377 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
378 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
379 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
380 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
381
382 MI.eraseFromParent();
383 // For signed numbers, we can do a single comparison to determine whether
384 // fabs(x) is within range.
385 if (IsUnsigned) {
386 Tmp0 = InReg;
387 } else {
388 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
389 }
390 BuildMI(BB, DL, TII.get(FConst), Tmp1)
391 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
392 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
393
394 // For unsigned numbers, we have to do a separate comparison with zero.
395 if (IsUnsigned) {
396 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
397 Register SecondCmpReg =
398 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
399 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
400 BuildMI(BB, DL, TII.get(FConst), Tmp1)
401 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
402 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
403 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
404 CmpReg = AndReg;
405 }
406
407 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
408
409 // Create the CFG diamond to select between doing the conversion or using
410 // the substitute value.
411 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
412 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
413 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
414 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
415 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
416 .addReg(FalseReg)
417 .addMBB(FalseMBB)
418 .addReg(TrueReg)
419 .addMBB(TrueMBB);
420
421 return DoneMBB;
422}
423
424MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
425 MachineInstr &MI, MachineBasicBlock *BB) const {
426 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
427 DebugLoc DL = MI.getDebugLoc();
428
429 switch (MI.getOpcode()) {
430 default:
431 llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 431)
;
432 case WebAssembly::FP_TO_SINT_I32_F32:
433 return LowerFPToInt(MI, DL, BB, TII, false, false, false,
434 WebAssembly::I32_TRUNC_S_F32);
435 case WebAssembly::FP_TO_UINT_I32_F32:
436 return LowerFPToInt(MI, DL, BB, TII, true, false, false,
437 WebAssembly::I32_TRUNC_U_F32);
438 case WebAssembly::FP_TO_SINT_I64_F32:
439 return LowerFPToInt(MI, DL, BB, TII, false, true, false,
440 WebAssembly::I64_TRUNC_S_F32);
441 case WebAssembly::FP_TO_UINT_I64_F32:
442 return LowerFPToInt(MI, DL, BB, TII, true, true, false,
443 WebAssembly::I64_TRUNC_U_F32);
444 case WebAssembly::FP_TO_SINT_I32_F64:
445 return LowerFPToInt(MI, DL, BB, TII, false, false, true,
446 WebAssembly::I32_TRUNC_S_F64);
447 case WebAssembly::FP_TO_UINT_I32_F64:
448 return LowerFPToInt(MI, DL, BB, TII, true, false, true,
449 WebAssembly::I32_TRUNC_U_F64);
450 case WebAssembly::FP_TO_SINT_I64_F64:
451 return LowerFPToInt(MI, DL, BB, TII, false, true, true,
452 WebAssembly::I64_TRUNC_S_F64);
453 case WebAssembly::FP_TO_UINT_I64_F64:
454 return LowerFPToInt(MI, DL, BB, TII, true, true, true,
455 WebAssembly::I64_TRUNC_U_F64);
456 llvm_unreachable("Unexpected instruction to emit with custom inserter")::llvm::llvm_unreachable_internal("Unexpected instruction to emit with custom inserter"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 456)
;
457 }
458}
459
460const char *
461WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
462 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
463 case WebAssemblyISD::FIRST_NUMBER:
464 break;
465#define HANDLE_NODETYPE(NODE) \
466 case WebAssemblyISD::NODE: \
467 return "WebAssemblyISD::" #NODE;
468#include "WebAssemblyISD.def"
469#undef HANDLE_NODETYPE
470 }
471 return nullptr;
472}
473
474std::pair<unsigned, const TargetRegisterClass *>
475WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
476 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
477 // First, see if this is a constraint that directly corresponds to a
478 // WebAssembly register class.
479 if (Constraint.size() == 1) {
480 switch (Constraint[0]) {
481 case 'r':
482 assert(VT != MVT::iPTR && "Pointer MVT not expected here")((VT != MVT::iPTR && "Pointer MVT not expected here")
? static_cast<void> (0) : __assert_fail ("VT != MVT::iPTR && \"Pointer MVT not expected here\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 482, __PRETTY_FUNCTION__))
;
483 if (Subtarget->hasSIMD128() && VT.isVector()) {
484 if (VT.getSizeInBits() == 128)
485 return std::make_pair(0U, &WebAssembly::V128RegClass);
486 }
487 if (VT.isInteger() && !VT.isVector()) {
488 if (VT.getSizeInBits() <= 32)
489 return std::make_pair(0U, &WebAssembly::I32RegClass);
490 if (VT.getSizeInBits() <= 64)
491 return std::make_pair(0U, &WebAssembly::I64RegClass);
492 }
493 break;
494 default:
495 break;
496 }
497 }
498
499 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
500}
501
502bool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const {
503 // Assume ctz is a relatively cheap operation.
504 return true;
505}
506
507bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const {
508 // Assume clz is a relatively cheap operation.
509 return true;
510}
511
512bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
513 const AddrMode &AM,
514 Type *Ty, unsigned AS,
515 Instruction *I) const {
516 // WebAssembly offsets are added as unsigned without wrapping. The
517 // isLegalAddressingMode gives us no way to determine if wrapping could be
518 // happening, so we approximate this by accepting only non-negative offsets.
519 if (AM.BaseOffs < 0)
520 return false;
521
522 // WebAssembly has no scale register operands.
523 if (AM.Scale != 0)
524 return false;
525
526 // Everything else is legal.
527 return true;
528}
529
530bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
531 EVT /*VT*/, unsigned /*AddrSpace*/, unsigned /*Align*/,
532 MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
533 // WebAssembly supports unaligned accesses, though it should be declared
534 // with the p2align attribute on loads and stores which do so, and there
535 // may be a performance impact. We tell LLVM they're "fast" because
536 // for the kinds of things that LLVM uses this for (merging adjacent stores
537 // of constants, etc.), WebAssembly implementations will either want the
538 // unaligned access or they'll split anyway.
539 if (Fast)
540 *Fast = true;
541 return true;
542}
543
544bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
545 AttributeList Attr) const {
546 // The current thinking is that wasm engines will perform this optimization,
547 // so we can save on code size.
548 return true;
549}
550
551bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
552 if (!Subtarget->hasUnimplementedSIMD128())
553 return false;
554 MVT ExtT = ExtVal.getSimpleValueType();
555 MVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getSimpleValueType(0);
556 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
557 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
558 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
559}
560
561EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
562 LLVMContext &C,
563 EVT VT) const {
564 if (VT.isVector())
565 return VT.changeVectorElementTypeToInteger();
566
567 return TargetLowering::getSetCCResultType(DL, C, VT);
568}
569
570bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
571 const CallInst &I,
572 MachineFunction &MF,
573 unsigned Intrinsic) const {
574 switch (Intrinsic) {
575 case Intrinsic::wasm_atomic_notify:
576 Info.opc = ISD::INTRINSIC_W_CHAIN;
577 Info.memVT = MVT::i32;
578 Info.ptrVal = I.getArgOperand(0);
579 Info.offset = 0;
580 Info.align = Align(4);
581 // atomic.notify instruction does not really load the memory specified with
582 // this argument, but MachineMemOperand should either be load or store, so
583 // we set this to a load.
584 // FIXME Volatile isn't really correct, but currently all LLVM atomic
585 // instructions are treated as volatiles in the backend, so we should be
586 // consistent. The same applies for wasm_atomic_wait intrinsics too.
587 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
588 return true;
589 case Intrinsic::wasm_atomic_wait_i32:
590 Info.opc = ISD::INTRINSIC_W_CHAIN;
591 Info.memVT = MVT::i32;
592 Info.ptrVal = I.getArgOperand(0);
593 Info.offset = 0;
594 Info.align = Align(4);
595 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
596 return true;
597 case Intrinsic::wasm_atomic_wait_i64:
598 Info.opc = ISD::INTRINSIC_W_CHAIN;
599 Info.memVT = MVT::i64;
600 Info.ptrVal = I.getArgOperand(0);
601 Info.offset = 0;
602 Info.align = Align(8);
603 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
604 return true;
605 default:
606 return false;
607 }
608}
609
610//===----------------------------------------------------------------------===//
611// WebAssembly Lowering private implementation.
612//===----------------------------------------------------------------------===//
613
614//===----------------------------------------------------------------------===//
615// Lowering Code
616//===----------------------------------------------------------------------===//
617
618static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
619 MachineFunction &MF = DAG.getMachineFunction();
620 DAG.getContext()->diagnose(
621 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
622}
623
624// Test whether the given calling convention is supported.
625static bool callingConvSupported(CallingConv::ID CallConv) {
626 // We currently support the language-independent target-independent
627 // conventions. We don't yet have a way to annotate calls with properties like
628 // "cold", and we don't have any call-clobbered registers, so these are mostly
629 // all handled the same.
630 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
631 CallConv == CallingConv::Cold ||
632 CallConv == CallingConv::PreserveMost ||
633 CallConv == CallingConv::PreserveAll ||
634 CallConv == CallingConv::CXX_FAST_TLS ||
635 CallConv == CallingConv::WASM_EmscriptenInvoke;
636}
637
638SDValue
639WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
640 SmallVectorImpl<SDValue> &InVals) const {
641 SelectionDAG &DAG = CLI.DAG;
642 SDLoc DL = CLI.DL;
643 SDValue Chain = CLI.Chain;
644 SDValue Callee = CLI.Callee;
645 MachineFunction &MF = DAG.getMachineFunction();
646 auto Layout = MF.getDataLayout();
647
648 CallingConv::ID CallConv = CLI.CallConv;
649 if (!callingConvSupported(CallConv))
650 fail(DL, DAG,
651 "WebAssembly doesn't support language-specific or target-specific "
652 "calling conventions yet");
653 if (CLI.IsPatchPoint)
654 fail(DL, DAG, "WebAssembly doesn't support patch point yet");
655
656 if (CLI.IsTailCall) {
657 bool MustTail = CLI.CS && CLI.CS.isMustTailCall();
658 if (Subtarget->hasTailCall() && !CLI.IsVarArg) {
659 // Do not tail call unless caller and callee return types match
660 const Function &F = MF.getFunction();
661 const TargetMachine &TM = getTargetMachine();
662 Type *RetTy = F.getReturnType();
663 SmallVector<MVT, 4> CallerRetTys;
664 SmallVector<MVT, 4> CalleeRetTys;
665 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
666 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
667 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
668 std::equal(CallerRetTys.begin(), CallerRetTys.end(),
669 CalleeRetTys.begin());
670 if (!TypesMatch) {
671 // musttail in this case would be an LLVM IR validation failure
672 assert(!MustTail)((!MustTail) ? static_cast<void> (0) : __assert_fail ("!MustTail"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 672, __PRETTY_FUNCTION__))
;
673 CLI.IsTailCall = false;
674 }
675 } else {
676 CLI.IsTailCall = false;
677 if (MustTail) {
678 if (CLI.IsVarArg) {
679 // The return would pop the argument buffer
680 fail(DL, DAG, "WebAssembly does not support varargs tail calls");
681 } else {
682 fail(DL, DAG, "WebAssembly 'tail-call' feature not enabled");
683 }
684 }
685 }
686 }
687
688 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
689 if (Ins.size() > 1)
690 fail(DL, DAG, "WebAssembly doesn't support more than 1 returned value yet");
691
692 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
693 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
694
695 // The generic code may have added an sret argument. If we're lowering an
696 // invoke function, the ABI requires that the function pointer be the first
697 // argument, so we may have to swap the arguments.
698 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
699 Outs[0].Flags.isSRet()) {
700 std::swap(Outs[0], Outs[1]);
701 std::swap(OutVals[0], OutVals[1]);
702 }
703
704 unsigned NumFixedArgs = 0;
705 for (unsigned I = 0; I < Outs.size(); ++I) {
706 const ISD::OutputArg &Out = Outs[I];
707 SDValue &OutVal = OutVals[I];
708 if (Out.Flags.isNest())
709 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
710 if (Out.Flags.isInAlloca())
711 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
712 if (Out.Flags.isInConsecutiveRegs())
713 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
714 if (Out.Flags.isInConsecutiveRegsLast())
715 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
716 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
717 auto &MFI = MF.getFrameInfo();
718 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
719 Out.Flags.getByValAlign(),
720 /*isSS=*/false);
721 SDValue SizeNode =
722 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
723 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
724 Chain = DAG.getMemcpy(
725 Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getByValAlign(),
726 /*isVolatile*/ false, /*AlwaysInline=*/false,
727 /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
728 OutVal = FINode;
729 }
730 // Count the number of fixed args *after* legalization.
731 NumFixedArgs += Out.IsFixed;
732 }
733
734 bool IsVarArg = CLI.IsVarArg;
735 auto PtrVT = getPointerTy(Layout);
736
737 // Analyze operands of the call, assigning locations to each operand.
738 SmallVector<CCValAssign, 16> ArgLocs;
739 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
740
741 if (IsVarArg) {
742 // Outgoing non-fixed arguments are placed in a buffer. First
743 // compute their offsets and the total amount of buffer space needed.
744 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
745 const ISD::OutputArg &Out = Outs[I];
746 SDValue &Arg = OutVals[I];
747 EVT VT = Arg.getValueType();
748 assert(VT != MVT::iPTR && "Legalized args should be concrete")((VT != MVT::iPTR && "Legalized args should be concrete"
) ? static_cast<void> (0) : __assert_fail ("VT != MVT::iPTR && \"Legalized args should be concrete\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 748, __PRETTY_FUNCTION__))
;
749 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
750 unsigned Align = std::max(Out.Flags.getOrigAlign(),
751 Layout.getABITypeAlignment(Ty));
752 unsigned Offset = CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty),
753 Align);
754 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
755 Offset, VT.getSimpleVT(),
756 CCValAssign::Full));
757 }
758 }
759
760 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
761
762 SDValue FINode;
763 if (IsVarArg && NumBytes) {
764 // For non-fixed arguments, next emit stores to store the argument values
765 // to the stack buffer at the offsets computed above.
766 int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
767 Layout.getStackAlignment(),
768 /*isSS=*/false);
769 unsigned ValNo = 0;
770 SmallVector<SDValue, 8> Chains;
771 for (SDValue Arg :
772 make_range(OutVals.begin() + NumFixedArgs, OutVals.end())) {
773 assert(ArgLocs[ValNo].getValNo() == ValNo &&((ArgLocs[ValNo].getValNo() == ValNo && "ArgLocs should remain in order and only hold varargs args"
) ? static_cast<void> (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 774, __PRETTY_FUNCTION__))
774 "ArgLocs should remain in order and only hold varargs args")((ArgLocs[ValNo].getValNo() == ValNo && "ArgLocs should remain in order and only hold varargs args"
) ? static_cast<void> (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 774, __PRETTY_FUNCTION__))
;
775 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
776 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
777 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
778 DAG.getConstant(Offset, DL, PtrVT));
779 Chains.push_back(
780 DAG.getStore(Chain, DL, Arg, Add,
781 MachinePointerInfo::getFixedStack(MF, FI, Offset), 0));
782 }
783 if (!Chains.empty())
784 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
785 } else if (IsVarArg) {
786 FINode = DAG.getIntPtrConstant(0, DL);
787 }
788
789 if (Callee->getOpcode() == ISD::GlobalAddress) {
790 // If the callee is a GlobalAddress node (quite common, every direct call
791 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
792 // doesn't at MO_GOT which is not needed for direct calls.
793 GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee);
794 Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
795 getPointerTy(DAG.getDataLayout()),
796 GA->getOffset());
797 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
798 getPointerTy(DAG.getDataLayout()), Callee);
799 }
800
801 // Compute the operands for the CALLn node.
802 SmallVector<SDValue, 16> Ops;
803 Ops.push_back(Chain);
804 Ops.push_back(Callee);
805
806 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
807 // isn't reliable.
808 Ops.append(OutVals.begin(),
809 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
810 // Add a pointer to the vararg buffer.
811 if (IsVarArg)
812 Ops.push_back(FINode);
813
814 SmallVector<EVT, 8> InTys;
815 for (const auto &In : Ins) {
816 assert(!In.Flags.isByVal() && "byval is not valid for return values")((!In.Flags.isByVal() && "byval is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!In.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 816, __PRETTY_FUNCTION__))
;
817 assert(!In.Flags.isNest() && "nest is not valid for return values")((!In.Flags.isNest() && "nest is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!In.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 817, __PRETTY_FUNCTION__))
;
818 if (In.Flags.isInAlloca())
819 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
820 if (In.Flags.isInConsecutiveRegs())
821 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
822 if (In.Flags.isInConsecutiveRegsLast())
823 fail(DL, DAG,
824 "WebAssembly hasn't implemented cons regs last return values");
825 // Ignore In.getOrigAlign() because all our arguments are passed in
826 // registers.
827 InTys.push_back(In.VT);
828 }
829
830 if (CLI.IsTailCall) {
831 // ret_calls do not return values to the current frame
832 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
833 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
834 }
835
836 InTys.push_back(MVT::Other);
837 SDVTList InTyList = DAG.getVTList(InTys);
838 SDValue Res =
839 DAG.getNode(Ins.empty() ? WebAssemblyISD::CALL0 : WebAssemblyISD::CALL1,
840 DL, InTyList, Ops);
841 if (Ins.empty()) {
842 Chain = Res;
843 } else {
844 InVals.push_back(Res);
845 Chain = Res.getValue(1);
846 }
847
848 return Chain;
849}
850
851bool WebAssemblyTargetLowering::CanLowerReturn(
852 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
853 const SmallVectorImpl<ISD::OutputArg> &Outs,
854 LLVMContext & /*Context*/) const {
855 // WebAssembly can only handle returning tuples with multivalue enabled
856 return Subtarget->hasMultivalue() || Outs.size() <= 1;
857}
858
859SDValue WebAssemblyTargetLowering::LowerReturn(
860 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
861 const SmallVectorImpl<ISD::OutputArg> &Outs,
862 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
863 SelectionDAG &DAG) const {
864 assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&(((Subtarget->hasMultivalue() || Outs.size() <= 1) &&
"MVP WebAssembly can only return up to one value") ? static_cast
<void> (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 865, __PRETTY_FUNCTION__))
865 "MVP WebAssembly can only return up to one value")(((Subtarget->hasMultivalue() || Outs.size() <= 1) &&
"MVP WebAssembly can only return up to one value") ? static_cast
<void> (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 865, __PRETTY_FUNCTION__))
;
866 if (!callingConvSupported(CallConv))
867 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
868
869 SmallVector<SDValue, 4> RetOps(1, Chain);
870 RetOps.append(OutVals.begin(), OutVals.end());
871 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
872
873 // Record the number and types of the return values.
874 for (const ISD::OutputArg &Out : Outs) {
875 assert(!Out.Flags.isByVal() && "byval is not valid for return values")((!Out.Flags.isByVal() && "byval is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!Out.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 875, __PRETTY_FUNCTION__))
;
876 assert(!Out.Flags.isNest() && "nest is not valid for return values")((!Out.Flags.isNest() && "nest is not valid for return values"
) ? static_cast<void> (0) : __assert_fail ("!Out.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 876, __PRETTY_FUNCTION__))
;
877 assert(Out.IsFixed && "non-fixed return value is not valid")((Out.IsFixed && "non-fixed return value is not valid"
) ? static_cast<void> (0) : __assert_fail ("Out.IsFixed && \"non-fixed return value is not valid\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 877, __PRETTY_FUNCTION__))
;
878 if (Out.Flags.isInAlloca())
879 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
880 if (Out.Flags.isInConsecutiveRegs())
881 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
882 if (Out.Flags.isInConsecutiveRegsLast())
883 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
884 }
885
886 return Chain;
887}
888
889SDValue WebAssemblyTargetLowering::LowerFormalArguments(
890 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
891 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
892 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
893 if (!callingConvSupported(CallConv))
894 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
895
896 MachineFunction &MF = DAG.getMachineFunction();
897 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
898
899 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
900 // of the incoming values before they're represented by virtual registers.
901 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
902
903 for (const ISD::InputArg &In : Ins) {
904 if (In.Flags.isInAlloca())
905 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
906 if (In.Flags.isNest())
907 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
908 if (In.Flags.isInConsecutiveRegs())
909 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
910 if (In.Flags.isInConsecutiveRegsLast())
911 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
912 // Ignore In.getOrigAlign() because all our arguments are passed in
913 // registers.
914 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
915 DAG.getTargetConstant(InVals.size(),
916 DL, MVT::i32))
917 : DAG.getUNDEF(In.VT));
918
919 // Record the number and types of arguments.
920 MFI->addParam(In.VT);
921 }
922
923 // Varargs are copied into a buffer allocated by the caller, and a pointer to
924 // the buffer is passed as an argument.
925 if (IsVarArg) {
926 MVT PtrVT = getPointerTy(MF.getDataLayout());
927 Register VarargVreg =
928 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
929 MFI->setVarargBufferVreg(VarargVreg);
930 Chain = DAG.getCopyToReg(
931 Chain, DL, VarargVreg,
932 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
933 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
934 MFI->addParam(PtrVT);
935 }
936
937 // Record the number and types of arguments and results.
938 SmallVector<MVT, 4> Params;
939 SmallVector<MVT, 4> Results;
940 computeSignatureVTs(MF.getFunction().getFunctionType(), MF.getFunction(),
941 DAG.getTarget(), Params, Results);
942 for (MVT VT : Results)
943 MFI->addResult(VT);
944 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
945 // the param logic here with ComputeSignatureVTs
946 assert(MFI->getParams().size() == Params.size() &&((MFI->getParams().size() == Params.size() && std::
equal(MFI->getParams().begin(), MFI->getParams().end(),
Params.begin())) ? static_cast<void> (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 948, __PRETTY_FUNCTION__))
947 std::equal(MFI->getParams().begin(), MFI->getParams().end(),((MFI->getParams().size() == Params.size() && std::
equal(MFI->getParams().begin(), MFI->getParams().end(),
Params.begin())) ? static_cast<void> (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 948, __PRETTY_FUNCTION__))
948 Params.begin()))((MFI->getParams().size() == Params.size() && std::
equal(MFI->getParams().begin(), MFI->getParams().end(),
Params.begin())) ? static_cast<void> (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 948, __PRETTY_FUNCTION__))
;
949
950 return Chain;
951}
952
953void WebAssemblyTargetLowering::ReplaceNodeResults(
954 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
955 switch (N->getOpcode()) {
956 case ISD::SIGN_EXTEND_INREG:
957 // Do not add any results, signifying that N should not be custom lowered
958 // after all. This happens because simd128 turns on custom lowering for
959 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
960 // illegal type.
961 break;
962 default:
963 llvm_unreachable(::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 964)
964 "ReplaceNodeResults not implemented for this op for WebAssembly!")::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 964)
;
965 }
966}
967
968//===----------------------------------------------------------------------===//
969// Custom lowering hooks.
970//===----------------------------------------------------------------------===//
971
972SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
973 SelectionDAG &DAG) const {
974 SDLoc DL(Op);
975 switch (Op.getOpcode()) {
1
Control jumps to 'case BUILD_VECTOR:' at line 1010
976 default:
977 llvm_unreachable("unimplemented operation lowering")::llvm::llvm_unreachable_internal("unimplemented operation lowering"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 977)
;
978 return SDValue();
979 case ISD::FrameIndex:
980 return LowerFrameIndex(Op, DAG);
981 case ISD::GlobalAddress:
982 return LowerGlobalAddress(Op, DAG);
983 case ISD::ExternalSymbol:
984 return LowerExternalSymbol(Op, DAG);
985 case ISD::JumpTable:
986 return LowerJumpTable(Op, DAG);
987 case ISD::BR_JT:
988 return LowerBR_JT(Op, DAG);
989 case ISD::VASTART:
990 return LowerVASTART(Op, DAG);
991 case ISD::BlockAddress:
992 case ISD::BRIND:
993 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
994 return SDValue();
995 case ISD::RETURNADDR:
996 return LowerRETURNADDR(Op, DAG);
997 case ISD::FRAMEADDR:
998 return LowerFRAMEADDR(Op, DAG);
999 case ISD::CopyToReg:
1000 return LowerCopyToReg(Op, DAG);
1001 case ISD::EXTRACT_VECTOR_ELT:
1002 case ISD::INSERT_VECTOR_ELT:
1003 return LowerAccessVectorElement(Op, DAG);
1004 case ISD::INTRINSIC_VOID:
1005 case ISD::INTRINSIC_WO_CHAIN:
1006 case ISD::INTRINSIC_W_CHAIN:
1007 return LowerIntrinsic(Op, DAG);
1008 case ISD::SIGN_EXTEND_INREG:
1009 return LowerSIGN_EXTEND_INREG(Op, DAG);
1010 case ISD::BUILD_VECTOR:
1011 return LowerBUILD_VECTOR(Op, DAG);
2
Calling 'WebAssemblyTargetLowering::LowerBUILD_VECTOR'
1012 case ISD::VECTOR_SHUFFLE:
1013 return LowerVECTOR_SHUFFLE(Op, DAG);
1014 case ISD::SHL:
1015 case ISD::SRA:
1016 case ISD::SRL:
1017 return LowerShift(Op, DAG);
1018 }
1019}
1020
1021SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1022 SelectionDAG &DAG) const {
1023 SDValue Src = Op.getOperand(2);
1024 if (isa<FrameIndexSDNode>(Src.getNode())) {
1025 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1026 // the FI to some LEA-like instruction, but since we don't have that, we
1027 // need to insert some kind of instruction that can take an FI operand and
1028 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1029 // local.copy between Op and its FI operand.
1030 SDValue Chain = Op.getOperand(0);
1031 SDLoc DL(Op);
1032 unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
1033 EVT VT = Src.getValueType();
1034 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1035 : WebAssembly::COPY_I64,
1036 DL, VT, Src),
1037 0);
1038 return Op.getNode()->getNumValues() == 1
1039 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
1040 : DAG.getCopyToReg(Chain, DL, Reg, Copy,
1041 Op.getNumOperands() == 4 ? Op.getOperand(3)
1042 : SDValue());
1043 }
1044 return SDValue();
1045}
1046
1047SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1048 SelectionDAG &DAG) const {
1049 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
1050 return DAG.getTargetFrameIndex(FI, Op.getValueType());
1051}
1052
1053SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1054 SelectionDAG &DAG) const {
1055 SDLoc DL(Op);
1056
1057 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1058 fail(DL, DAG,
1059 "Non-Emscripten WebAssembly hasn't implemented "
1060 "__builtin_return_address");
1061 return SDValue();
1062 }
1063
1064 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1065 return SDValue();
1066
1067 unsigned Depth = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1068 MakeLibCallOptions CallOptions;
1069 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
1070 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
1071 .first;
1072}
1073
1074SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1075 SelectionDAG &DAG) const {
1076 // Non-zero depths are not supported by WebAssembly currently. Use the
1077 // legalizer's default expansion, which is to return 0 (what this function is
1078 // documented to do).
1079 if (Op.getConstantOperandVal(0) > 0)
1080 return SDValue();
1081
1082 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
1083 EVT VT = Op.getValueType();
1084 Register FP =
1085 Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
1086 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1087}
1088
1089SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1090 SelectionDAG &DAG) const {
1091 SDLoc DL(Op);
1092 const auto *GA = cast<GlobalAddressSDNode>(Op);
1093 EVT VT = Op.getValueType();
1094 assert(GA->getTargetFlags() == 0 &&((GA->getTargetFlags() == 0 && "Unexpected target flags on generic GlobalAddressSDNode"
) ? static_cast<void> (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1095, __PRETTY_FUNCTION__))
1095 "Unexpected target flags on generic GlobalAddressSDNode")((GA->getTargetFlags() == 0 && "Unexpected target flags on generic GlobalAddressSDNode"
) ? static_cast<void> (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1095, __PRETTY_FUNCTION__))
;
1096 if (GA->getAddressSpace() != 0)
1097 fail(DL, DAG, "WebAssembly only expects the 0 address space");
1098
1099 unsigned OperandFlags = 0;
1100 if (isPositionIndependent()) {
1101 const GlobalValue *GV = GA->getGlobal();
1102 if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
1103 MachineFunction &MF = DAG.getMachineFunction();
1104 MVT PtrVT = getPointerTy(MF.getDataLayout());
1105 const char *BaseName;
1106 if (GV->getValueType()->isFunctionTy()) {
1107 BaseName = MF.createExternalSymbolName("__table_base");
1108 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
1109 }
1110 else {
1111 BaseName = MF.createExternalSymbolName("__memory_base");
1112 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
1113 }
1114 SDValue BaseAddr =
1115 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1116 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1117
1118 SDValue SymAddr = DAG.getNode(
1119 WebAssemblyISD::WrapperPIC, DL, VT,
1120 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
1121 OperandFlags));
1122
1123 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
1124 } else {
1125 OperandFlags = WebAssemblyII::MO_GOT;
1126 }
1127 }
1128
1129 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1130 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1131 GA->getOffset(), OperandFlags));
1132}
1133
1134SDValue
1135WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
1136 SelectionDAG &DAG) const {
1137 SDLoc DL(Op);
1138 const auto *ES = cast<ExternalSymbolSDNode>(Op);
1139 EVT VT = Op.getValueType();
1140 assert(ES->getTargetFlags() == 0 &&((ES->getTargetFlags() == 0 && "Unexpected target flags on generic ExternalSymbolSDNode"
) ? static_cast<void> (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1141, __PRETTY_FUNCTION__))
1141 "Unexpected target flags on generic ExternalSymbolSDNode")((ES->getTargetFlags() == 0 && "Unexpected target flags on generic ExternalSymbolSDNode"
) ? static_cast<void> (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1141, __PRETTY_FUNCTION__))
;
1142 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1143 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1144}
1145
1146SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1147 SelectionDAG &DAG) const {
1148 // There's no need for a Wrapper node because we always incorporate a jump
1149 // table operand into a BR_TABLE instruction, rather than ever
1150 // materializing it in a register.
1151 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
1152 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
1153 JT->getTargetFlags());
1154}
1155
1156SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1157 SelectionDAG &DAG) const {
1158 SDLoc DL(Op);
1159 SDValue Chain = Op.getOperand(0);
1160 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
1161 SDValue Index = Op.getOperand(2);
1162 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags")((JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags"
) ? static_cast<void> (0) : __assert_fail ("JT->getTargetFlags() == 0 && \"WebAssembly doesn't set target flags\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1162, __PRETTY_FUNCTION__))
;
1163
1164 SmallVector<SDValue, 8> Ops;
1165 Ops.push_back(Chain);
1166 Ops.push_back(Index);
1167
1168 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
1169 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
1170
1171 // Add an operand for each case.
1172 for (auto MBB : MBBs)
1173 Ops.push_back(DAG.getBasicBlock(MBB));
1174
1175 // TODO: For now, we just pick something arbitrary for a default case for now.
1176 // We really want to sniff out the guard and put in the real default case (and
1177 // delete the guard).
1178 Ops.push_back(DAG.getBasicBlock(MBBs[0]));
1179
1180 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1181}
1182
1183SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1184 SelectionDAG &DAG) const {
1185 SDLoc DL(Op);
1186 EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());
1187
1188 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
1189 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
1190
1191 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
1192 MFI->getVarargBufferVreg(), PtrVT);
1193 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
1194 MachinePointerInfo(SV), 0);
1195}
1196
1197SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1198 SelectionDAG &DAG) const {
1199 MachineFunction &MF = DAG.getMachineFunction();
1200 unsigned IntNo;
1201 switch (Op.getOpcode()) {
1202 case ISD::INTRINSIC_VOID:
1203 case ISD::INTRINSIC_W_CHAIN:
1204 IntNo = cast<ConstantSDNode>(Op.getOperand(1))->getZExtValue();
1205 break;
1206 case ISD::INTRINSIC_WO_CHAIN:
1207 IntNo = cast<ConstantSDNode>(Op.getOperand(0))->getZExtValue();
1208 break;
1209 default:
1210 llvm_unreachable("Invalid intrinsic")::llvm::llvm_unreachable_internal("Invalid intrinsic", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1210)
;
1211 }
1212 SDLoc DL(Op);
1213
1214 switch (IntNo) {
1215 default:
1216 return SDValue(); // Don't custom lower most intrinsics.
1217
1218 case Intrinsic::wasm_lsda: {
1219 EVT VT = Op.getValueType();
1220 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1221 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
1222 auto &Context = MF.getMMI().getContext();
1223 MCSymbol *S = Context.getOrCreateSymbol(Twine("GCC_except_table") +
1224 Twine(MF.getFunctionNumber()));
1225 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1226 DAG.getMCSymbol(S, PtrVT));
1227 }
1228
1229 case Intrinsic::wasm_throw: {
1230 // We only support C++ exceptions for now
1231 int Tag = cast<ConstantSDNode>(Op.getOperand(2).getNode())->getZExtValue();
1232 if (Tag != CPP_EXCEPTION)
1233 llvm_unreachable("Invalid tag!")::llvm::llvm_unreachable_internal("Invalid tag!", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1233)
;
1234 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
1235 MVT PtrVT = TLI.getPointerTy(DAG.getDataLayout());
1236 const char *SymName = MF.createExternalSymbolName("__cpp_exception");
1237 SDValue SymNode = DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1238 DAG.getTargetExternalSymbol(SymName, PtrVT));
1239 return DAG.getNode(WebAssemblyISD::THROW, DL,
1240 MVT::Other, // outchain type
1241 {
1242 Op.getOperand(0), // inchain
1243 SymNode, // exception symbol
1244 Op.getOperand(3) // thrown value
1245 });
1246 }
1247 }
1248}
1249
1250SDValue
1251WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1252 SelectionDAG &DAG) const {
1253 SDLoc DL(Op);
1254 // If sign extension operations are disabled, allow sext_inreg only if operand
1255 // is a vector extract. SIMD does not depend on sign extension operations, but
1256 // allowing sext_inreg in this context lets us have simple patterns to select
1257 // extract_lane_s instructions. Expanding sext_inreg everywhere would be
1258 // simpler in this file, but would necessitate large and brittle patterns to
1259 // undo the expansion and select extract_lane_s instructions.
1260 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128())((!Subtarget->hasSignExt() && Subtarget->hasSIMD128
()) ? static_cast<void> (0) : __assert_fail ("!Subtarget->hasSignExt() && Subtarget->hasSIMD128()"
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1260, __PRETTY_FUNCTION__))
;
1261 if (Op.getOperand(0).getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
1262 const SDValue &Extract = Op.getOperand(0);
1263 MVT VecT = Extract.getOperand(0).getSimpleValueType();
1264 MVT ExtractedLaneT = static_cast<VTSDNode *>(Op.getOperand(1).getNode())
1265 ->getVT()
1266 .getSimpleVT();
1267 MVT ExtractedVecT =
1268 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
1269 if (ExtractedVecT == VecT)
1270 return Op;
1271 // Bitcast vector to appropriate type to ensure ISel pattern coverage
1272 const SDValue &Index = Extract.getOperand(1);
1273 unsigned IndexVal =
1274 static_cast<ConstantSDNode *>(Index.getNode())->getZExtValue();
1275 unsigned Scale =
1276 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
1277 assert(Scale > 1)((Scale > 1) ? static_cast<void> (0) : __assert_fail
("Scale > 1", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1277, __PRETTY_FUNCTION__))
;
1278 SDValue NewIndex =
1279 DAG.getConstant(IndexVal * Scale, DL, Index.getValueType());
1280 SDValue NewExtract = DAG.getNode(
1281 ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
1282 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
1283 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(),
1284 NewExtract, Op.getOperand(1));
1285 }
1286 // Otherwise expand
1287 return SDValue();
1288}
1289
1290SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1291 SelectionDAG &DAG) const {
1292 SDLoc DL(Op);
1293 const EVT VecT = Op.getValueType();
1294 const EVT LaneT = Op.getOperand(0).getValueType();
1295 const size_t Lanes = Op.getNumOperands();
1296 bool CanSwizzle = Subtarget->hasUnimplementedSIMD128() && VecT == MVT::v16i8;
1297
1298 // BUILD_VECTORs are lowered to the instruction that initializes the highest
1299 // possible number of lanes at once followed by a sequence of replace_lane
1300 // instructions to individually initialize any remaining lanes.
1301
1302 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
1303 // swizzled lanes should be given greater weight.
1304
1305 // TODO: Investigate building vectors by shuffling together vectors built by
1306 // separately specialized means.
1307
1308 auto IsConstant = [](const SDValue &V) {
1309 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
1310 };
1311
1312 // Returns the source vector and index vector pair if they exist. Checks for:
1313 // (extract_vector_elt
1314 // $src,
1315 // (sign_extend_inreg (extract_vector_elt $indices, $i))
1316 // )
1317 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
1318 auto Bail = std::make_pair(SDValue(), SDValue());
1319 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1320 return Bail;
1321 const SDValue &SwizzleSrc = Lane->getOperand(0);
1322 const SDValue &IndexExt = Lane->getOperand(1);
1323 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
1324 return Bail;
1325 const SDValue &Index = IndexExt->getOperand(0);
1326 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1327 return Bail;
1328 const SDValue &SwizzleIndices = Index->getOperand(0);
1329 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
1330 SwizzleIndices.getValueType() != MVT::v16i8 ||
1331 Index->getOperand(1)->getOpcode() != ISD::Constant ||
1332 Index->getConstantOperandVal(1) != I)
1333 return Bail;
1334 return std::make_pair(SwizzleSrc, SwizzleIndices);
1335 };
1336
1337 using ValueEntry = std::pair<SDValue, size_t>;
1338 SmallVector<ValueEntry, 16> SplatValueCounts;
1339
1340 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
1341 SmallVector<SwizzleEntry, 16> SwizzleCounts;
1342
1343 auto AddCount = [](auto &Counts, const auto &Val) {
1344 auto CountIt = std::find_if(Counts.begin(), Counts.end(),
1345 [&Val](auto E) { return E.first == Val; });
1346 if (CountIt == Counts.end()) {
1347 Counts.emplace_back(Val, 1);
1348 } else {
1349 CountIt->second++;
1350 }
1351 };
1352
1353 auto GetMostCommon = [](auto &Counts) {
1354 auto CommonIt =
1355 std::max_element(Counts.begin(), Counts.end(),
1356 [](auto A, auto B) { return A.second < B.second; });
1357 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector")((CommonIt != Counts.end() && "Unexpected all-undef build_vector"
) ? static_cast<void> (0) : __assert_fail ("CommonIt != Counts.end() && \"Unexpected all-undef build_vector\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1357, __PRETTY_FUNCTION__))
;
1358 return *CommonIt;
1359 };
1360
1361 size_t NumConstantLanes = 0;
1362
1363 // Count eligible lanes for each type of vector creation op
1364 for (size_t I = 0; I
2.1
'I' is < 'Lanes'
2.1
'I' is < 'Lanes'
2.1
'I' is < 'Lanes'
< Lanes
; ++I) {
3
Loop condition is true. Entering loop body
6
Assuming 'I' is >= 'Lanes'
7
Loop condition is false. Execution continues on line 1380
1365 const SDValue &Lane = Op->getOperand(I);
1366 if (Lane.isUndef())
4
Taking true branch
1367 continue;
5
Execution continues on line 1364
1368
1369 AddCount(SplatValueCounts, Lane);
1370
1371 if (IsConstant(Lane)) {
1372 NumConstantLanes++;
1373 } else if (CanSwizzle) {
1374 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
1375 if (SwizzleSrcs.first)
1376 AddCount(SwizzleCounts, SwizzleSrcs);
1377 }
1378 }
1379
1380 SDValue SplatValue;
1381 size_t NumSplatLanes;
1382 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
1383
1384 SDValue SwizzleSrc;
1385 SDValue SwizzleIndices;
1386 size_t NumSwizzleLanes = 0;
1387 if (SwizzleCounts.size())
8
Assuming the condition is false
9
Taking false branch
1388 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
1389 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
1390
1391 // Predicate returning true if the lane is properly initialized by the
1392 // original instruction
1393 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
1394 SDValue Result;
1395 if (Subtarget->hasUnimplementedSIMD128()) {
10
Taking true branch
1396 // Prefer swizzles over vector consts over splats
1397 if (NumSwizzleLanes >= NumSplatLanes &&
11
Assuming 'NumSwizzleLanes' is < 'NumSplatLanes'
1398 NumSwizzleLanes >= NumConstantLanes) {
1399 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
1400 SwizzleIndices);
1401 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
1402 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
1403 return Swizzled == GetSwizzleSrcs(I, Lane);
1404 };
1405 } else if (NumConstantLanes
11.1
'NumConstantLanes' is < 'NumSplatLanes'
11.1
'NumConstantLanes' is < 'NumSplatLanes'
11.1
'NumConstantLanes' is < 'NumSplatLanes'
>= NumSplatLanes) {
12
Taking false branch
1406 SmallVector<SDValue, 16> ConstLanes;
1407 for (const SDValue &Lane : Op->op_values()) {
1408 if (IsConstant(Lane)) {
1409 ConstLanes.push_back(Lane);
1410 } else if (LaneT.isFloatingPoint()) {
1411 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
1412 } else {
1413 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
1414 }
1415 }
1416 Result = DAG.getBuildVector(VecT, DL, ConstLanes);
1417 IsLaneConstructed = [&](size_t _, const SDValue &Lane) {
1418 return IsConstant(Lane);
1419 };
1420 }
1421 }
1422 if (!Result) {
13
Taking true branch
1423 // Use a splat, but possibly a load_splat
1424 LoadSDNode *SplattedLoad;
1425 if (Subtarget->hasUnimplementedSIMD128() &&
15
Assuming pointer value is null
16
Taking false branch
1426 (SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
14
Assuming 'SplattedLoad' is null
1427 SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
1428 Result = DAG.getNode(WebAssemblyISD::LOAD_SPLAT, DL, VecT, SplatValue);
1429 } else {
1430 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
17
Value assigned to 'Op.Node'
18
Calling 'SelectionDAG::getSplatBuildVector'
1431 }
1432 IsLaneConstructed = [&](size_t _, const SDValue &Lane) {
1433 return Lane == SplatValue;
1434 };
1435 }
1436
1437 // Add replace_lane instructions for any unhandled values
1438 for (size_t I = 0; I < Lanes; ++I) {
1439 const SDValue &Lane = Op->getOperand(I);
1440 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
1441 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
1442 DAG.getConstant(I, DL, MVT::i32));
1443 }
1444
1445 return Result;
1446}
1447
1448SDValue
1449WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
1450 SelectionDAG &DAG) const {
1451 SDLoc DL(Op);
1452 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
1453 MVT VecType = Op.getOperand(0).getSimpleValueType();
1454 assert(VecType.is128BitVector() && "Unexpected shuffle vector type")((VecType.is128BitVector() && "Unexpected shuffle vector type"
) ? static_cast<void> (0) : __assert_fail ("VecType.is128BitVector() && \"Unexpected shuffle vector type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1454, __PRETTY_FUNCTION__))
;
1455 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
1456
1457 // Space for two vector args and sixteen mask indices
1458 SDValue Ops[18];
1459 size_t OpIdx = 0;
1460 Ops[OpIdx++] = Op.getOperand(0);
1461 Ops[OpIdx++] = Op.getOperand(1);
1462
1463 // Expand mask indices to byte indices and materialize them as operands
1464 for (int M : Mask) {
1465 for (size_t J = 0; J < LaneBytes; ++J) {
1466 // Lower undefs (represented by -1 in mask) to zero
1467 uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J;
1468 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
1469 }
1470 }
1471
1472 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
1473}
1474
1475SDValue
1476WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
1477 SelectionDAG &DAG) const {
1478 // Allow constant lane indices, expand variable lane indices
1479 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
1480 if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
1481 return Op;
1482 else
1483 // Perform default expansion
1484 return SDValue();
1485}
1486
1487static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
1488 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
1489 // 32-bit and 64-bit unrolled shifts will have proper semantics
1490 if (LaneT.bitsGE(MVT::i32))
1491 return DAG.UnrollVectorOp(Op.getNode());
1492 // Otherwise mask the shift value to get proper semantics from 32-bit shift
1493 SDLoc DL(Op);
1494 SDValue ShiftVal = Op.getOperand(1);
1495 uint64_t MaskVal = LaneT.getSizeInBits() - 1;
1496 SDValue MaskedShiftVal = DAG.getNode(
1497 ISD::AND, // mask opcode
1498 DL, ShiftVal.getValueType(), // masked value type
1499 ShiftVal, // original shift value operand
1500 DAG.getConstant(MaskVal, DL, ShiftVal.getValueType()) // mask operand
1501 );
1502
1503 return DAG.UnrollVectorOp(
1504 DAG.getNode(Op.getOpcode(), // original shift opcode
1505 DL, Op.getValueType(), // original return type
1506 Op.getOperand(0), // original vector operand,
1507 MaskedShiftVal // new masked shift value operand
1508 )
1509 .getNode());
1510}
1511
1512SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
1513 SelectionDAG &DAG) const {
1514 SDLoc DL(Op);
1515
1516 // Only manually lower vector shifts
1517 assert(Op.getSimpleValueType().isVector())((Op.getSimpleValueType().isVector()) ? static_cast<void>
(0) : __assert_fail ("Op.getSimpleValueType().isVector()", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1517, __PRETTY_FUNCTION__))
;
1518
1519 // Unroll non-splat vector shifts
1520 BuildVectorSDNode *ShiftVec;
1521 SDValue SplatVal;
1522 if (!(ShiftVec = dyn_cast<BuildVectorSDNode>(Op.getOperand(1).getNode())) ||
1523 !(SplatVal = ShiftVec->getSplatValue()))
1524 return unrollVectorShift(Op, DAG);
1525
1526 // All splats except i64x2 const splats are handled by patterns
1527 auto *SplatConst = dyn_cast<ConstantSDNode>(SplatVal);
1528 if (!SplatConst || Op.getSimpleValueType() != MVT::v2i64)
1529 return Op;
1530
1531 // i64x2 const splats are custom lowered to avoid unnecessary wraps
1532 unsigned Opcode;
1533 switch (Op.getOpcode()) {
1534 case ISD::SHL:
1535 Opcode = WebAssemblyISD::VEC_SHL;
1536 break;
1537 case ISD::SRA:
1538 Opcode = WebAssemblyISD::VEC_SHR_S;
1539 break;
1540 case ISD::SRL:
1541 Opcode = WebAssemblyISD::VEC_SHR_U;
1542 break;
1543 default:
1544 llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-10~svn374877/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1544)
;
1545 }
1546 APInt Shift = SplatConst->getAPIntValue().zextOrTrunc(32);
1547 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0),
1548 DAG.getConstant(Shift, DL, MVT::i32));
1549}
1550
1551//===----------------------------------------------------------------------===//
1552// WebAssembly Optimization Hooks
1553//===----------------------------------------------------------------------===//

/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h

1//===- llvm/CodeGen/SelectionDAG.h - InstSelection DAG ----------*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SelectionDAG class, and transitively defines the
10// SDNode class and subclasses.
11//
12//===----------------------------------------------------------------------===//
13
14#ifndef LLVM_CODEGEN_SELECTIONDAG_H
15#define LLVM_CODEGEN_SELECTIONDAG_H
16
17#include "llvm/ADT/APFloat.h"
18#include "llvm/ADT/APInt.h"
19#include "llvm/ADT/ArrayRef.h"
20#include "llvm/ADT/DenseMap.h"
21#include "llvm/ADT/DenseSet.h"
22#include "llvm/ADT/FoldingSet.h"
23#include "llvm/ADT/SetVector.h"
24#include "llvm/ADT/SmallVector.h"
25#include "llvm/ADT/StringMap.h"
26#include "llvm/ADT/ilist.h"
27#include "llvm/ADT/iterator.h"
28#include "llvm/ADT/iterator_range.h"
29#include "llvm/Analysis/AliasAnalysis.h"
30#include "llvm/Analysis/LegacyDivergenceAnalysis.h"
31#include "llvm/CodeGen/DAGCombine.h"
32#include "llvm/CodeGen/FunctionLoweringInfo.h"
33#include "llvm/CodeGen/ISDOpcodes.h"
34#include "llvm/CodeGen/MachineFunction.h"
35#include "llvm/CodeGen/MachineMemOperand.h"
36#include "llvm/CodeGen/SelectionDAGNodes.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/IR/DebugLoc.h"
39#include "llvm/IR/Instructions.h"
40#include "llvm/IR/Metadata.h"
41#include "llvm/Support/Allocator.h"
42#include "llvm/Support/ArrayRecycler.h"
43#include "llvm/Support/AtomicOrdering.h"
44#include "llvm/Support/Casting.h"
45#include "llvm/Support/CodeGen.h"
46#include "llvm/Support/ErrorHandling.h"
47#include "llvm/Support/MachineValueType.h"
48#include "llvm/Support/RecyclingAllocator.h"
49#include <algorithm>
50#include <cassert>
51#include <cstdint>
52#include <functional>
53#include <map>
54#include <string>
55#include <tuple>
56#include <utility>
57#include <vector>
58
59namespace llvm {
60
61class BlockAddress;
62class Constant;
63class ConstantFP;
64class ConstantInt;
65class DataLayout;
66struct fltSemantics;
67class GlobalValue;
68struct KnownBits;
69class LLVMContext;
70class MachineBasicBlock;
71class MachineConstantPoolValue;
72class MCSymbol;
73class OptimizationRemarkEmitter;
74class SDDbgValue;
75class SDDbgLabel;
76class SelectionDAG;
77class SelectionDAGTargetInfo;
78class TargetLibraryInfo;
79class TargetLowering;
80class TargetMachine;
81class TargetSubtargetInfo;
82class Value;
83
84class SDVTListNode : public FoldingSetNode {
85 friend struct FoldingSetTrait<SDVTListNode>;
86
87 /// A reference to an Interned FoldingSetNodeID for this node.
88 /// The Allocator in SelectionDAG holds the data.
89 /// SDVTList contains all types which are frequently accessed in SelectionDAG.
90 /// The size of this list is not expected to be big so it won't introduce
91 /// a memory penalty.
92 FoldingSetNodeIDRef FastID;
93 const EVT *VTs;
94 unsigned int NumVTs;
95 /// The hash value for SDVTList is fixed, so cache it to avoid
96 /// hash calculation.
97 unsigned HashValue;
98
99public:
100 SDVTListNode(const FoldingSetNodeIDRef ID, const EVT *VT, unsigned int Num) :
101 FastID(ID), VTs(VT), NumVTs(Num) {
102 HashValue = ID.ComputeHash();
103 }
104
105 SDVTList getSDVTList() {
106 SDVTList result = {VTs, NumVTs};
107 return result;
108 }
109};
110
111/// Specialize FoldingSetTrait for SDVTListNode
112/// to avoid computing temp FoldingSetNodeID and hash value.
113template<> struct FoldingSetTrait<SDVTListNode> : DefaultFoldingSetTrait<SDVTListNode> {
114 static void Profile(const SDVTListNode &X, FoldingSetNodeID& ID) {
115 ID = X.FastID;
116 }
117
118 static bool Equals(const SDVTListNode &X, const FoldingSetNodeID &ID,
119 unsigned IDHash, FoldingSetNodeID &TempID) {
120 if (X.HashValue != IDHash)
121 return false;
122 return ID == X.FastID;
123 }
124
125 static unsigned ComputeHash(const SDVTListNode &X, FoldingSetNodeID &TempID) {
126 return X.HashValue;
127 }
128};
129
130template <> struct ilist_alloc_traits<SDNode> {
131 static void deleteNode(SDNode *) {
132 llvm_unreachable("ilist_traits<SDNode> shouldn't see a deleteNode call!")::llvm::llvm_unreachable_internal("ilist_traits<SDNode> shouldn't see a deleteNode call!"
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 132)
;
133 }
134};
135
136/// Keeps track of dbg_value information through SDISel. We do
137/// not build SDNodes for these so as not to perturb the generated code;
138/// instead the info is kept off to the side in this structure. Each SDNode may
139/// have one or more associated dbg_value entries. This information is kept in
140/// DbgValMap.
141/// Byval parameters are handled separately because they don't use alloca's,
142/// which busts the normal mechanism. There is good reason for handling all
143/// parameters separately: they may not have code generated for them, they
144/// should always go at the beginning of the function regardless of other code
145/// motion, and debug info for them is potentially useful even if the parameter
146/// is unused. Right now only byval parameters are handled separately.
147class SDDbgInfo {
148 BumpPtrAllocator Alloc;
149 SmallVector<SDDbgValue*, 32> DbgValues;
150 SmallVector<SDDbgValue*, 32> ByvalParmDbgValues;
151 SmallVector<SDDbgLabel*, 4> DbgLabels;
152 using DbgValMapType = DenseMap<const SDNode *, SmallVector<SDDbgValue *, 2>>;
153 DbgValMapType DbgValMap;
154
155public:
156 SDDbgInfo() = default;
157 SDDbgInfo(const SDDbgInfo &) = delete;
158 SDDbgInfo &operator=(const SDDbgInfo &) = delete;
159
160 void add(SDDbgValue *V, const SDNode *Node, bool isParameter) {
161 if (isParameter) {
162 ByvalParmDbgValues.push_back(V);
163 } else DbgValues.push_back(V);
164 if (Node)
165 DbgValMap[Node].push_back(V);
166 }
167
168 void add(SDDbgLabel *L) {
169 DbgLabels.push_back(L);
170 }
171
172 /// Invalidate all DbgValues attached to the node and remove
173 /// it from the Node-to-DbgValues map.
174 void erase(const SDNode *Node);
175
176 void clear() {
177 DbgValMap.clear();
178 DbgValues.clear();
179 ByvalParmDbgValues.clear();
180 DbgLabels.clear();
181 Alloc.Reset();
182 }
183
184 BumpPtrAllocator &getAlloc() { return Alloc; }
185
186 bool empty() const {
187 return DbgValues.empty() && ByvalParmDbgValues.empty() && DbgLabels.empty();
188 }
189
190 ArrayRef<SDDbgValue*> getSDDbgValues(const SDNode *Node) const {
191 auto I = DbgValMap.find(Node);
192 if (I != DbgValMap.end())
193 return I->second;
194 return ArrayRef<SDDbgValue*>();
195 }
196
197 using DbgIterator = SmallVectorImpl<SDDbgValue*>::iterator;
198 using DbgLabelIterator = SmallVectorImpl<SDDbgLabel*>::iterator;
199
200 DbgIterator DbgBegin() { return DbgValues.begin(); }
201 DbgIterator DbgEnd() { return DbgValues.end(); }
202 DbgIterator ByvalParmDbgBegin() { return ByvalParmDbgValues.begin(); }
203 DbgIterator ByvalParmDbgEnd() { return ByvalParmDbgValues.end(); }
204 DbgLabelIterator DbgLabelBegin() { return DbgLabels.begin(); }
205 DbgLabelIterator DbgLabelEnd() { return DbgLabels.end(); }
206};
207
208void checkForCycles(const SelectionDAG *DAG, bool force = false);
209
210/// This is used to represent a portion of an LLVM function in a low-level
211/// Data Dependence DAG representation suitable for instruction selection.
212/// This DAG is constructed as the first step of instruction selection in order
213/// to allow implementation of machine specific optimizations
214/// and code simplifications.
215///
216/// The representation used by the SelectionDAG is a target-independent
217/// representation, which has some similarities to the GCC RTL representation,
218/// but is significantly more simple, powerful, and is a graph form instead of a
219/// linear form.
220///
221class SelectionDAG {
222 const TargetMachine &TM;
223 const SelectionDAGTargetInfo *TSI = nullptr;
224 const TargetLowering *TLI = nullptr;
225 const TargetLibraryInfo *LibInfo = nullptr;
226 MachineFunction *MF;
227 Pass *SDAGISelPass = nullptr;
228 LLVMContext *Context;
229 CodeGenOpt::Level OptLevel;
230
231 LegacyDivergenceAnalysis * DA = nullptr;
232 FunctionLoweringInfo * FLI = nullptr;
233
234 /// The function-level optimization remark emitter. Used to emit remarks
235 /// whenever manipulating the DAG.
236 OptimizationRemarkEmitter *ORE;
237
238 /// The starting token.
239 SDNode EntryNode;
240
241 /// The root of the entire DAG.
242 SDValue Root;
243
244 /// A linked list of nodes in the current DAG.
245 ilist<SDNode> AllNodes;
246
247 /// The AllocatorType for allocating SDNodes. We use
248 /// pool allocation with recycling.
249 using NodeAllocatorType = RecyclingAllocator<BumpPtrAllocator, SDNode,
250 sizeof(LargestSDNode),
251 alignof(MostAlignedSDNode)>;
252
253 /// Pool allocation for nodes.
254 NodeAllocatorType NodeAllocator;
255
256 /// This structure is used to memoize nodes, automatically performing
257 /// CSE with existing nodes when a duplicate is requested.
258 FoldingSet<SDNode> CSEMap;
259
260 /// Pool allocation for machine-opcode SDNode operands.
261 BumpPtrAllocator OperandAllocator;
262 ArrayRecycler<SDUse> OperandRecycler;
263
264 /// Pool allocation for misc. objects that are created once per SelectionDAG.
265 BumpPtrAllocator Allocator;
266
267 /// Tracks dbg_value and dbg_label information through SDISel.
268 SDDbgInfo *DbgInfo;
269
270 using CallSiteInfo = MachineFunction::CallSiteInfo;
271 using CallSiteInfoImpl = MachineFunction::CallSiteInfoImpl;
272
273 struct CallSiteDbgInfo {
274 CallSiteInfo CSInfo;
275 MDNode *HeapAllocSite = nullptr;
276 };
277
278 DenseMap<const SDNode *, CallSiteDbgInfo> SDCallSiteDbgInfo;
279
280 uint16_t NextPersistentId = 0;
281
282public:
283 /// Clients of various APIs that cause global effects on
284 /// the DAG can optionally implement this interface. This allows the clients
285 /// to handle the various sorts of updates that happen.
286 ///
287 /// A DAGUpdateListener automatically registers itself with DAG when it is
288 /// constructed, and removes itself when destroyed in RAII fashion.
289 struct DAGUpdateListener {
290 DAGUpdateListener *const Next;
291 SelectionDAG &DAG;
292
293 explicit DAGUpdateListener(SelectionDAG &D)
294 : Next(D.UpdateListeners), DAG(D) {
295 DAG.UpdateListeners = this;
296 }
297
298 virtual ~DAGUpdateListener() {
299 assert(DAG.UpdateListeners == this &&((DAG.UpdateListeners == this && "DAGUpdateListeners must be destroyed in LIFO order"
) ? static_cast<void> (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 300, __PRETTY_FUNCTION__))
300 "DAGUpdateListeners must be destroyed in LIFO order")((DAG.UpdateListeners == this && "DAGUpdateListeners must be destroyed in LIFO order"
) ? static_cast<void> (0) : __assert_fail ("DAG.UpdateListeners == this && \"DAGUpdateListeners must be destroyed in LIFO order\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 300, __PRETTY_FUNCTION__))
;
301 DAG.UpdateListeners = Next;
302 }
303
304 /// The node N that was deleted and, if E is not null, an
305 /// equivalent node E that replaced it.
306 virtual void NodeDeleted(SDNode *N, SDNode *E);
307
308 /// The node N that was updated.
309 virtual void NodeUpdated(SDNode *N);
310
311 /// The node N that was inserted.
312 virtual void NodeInserted(SDNode *N);
313 };
314
315 struct DAGNodeDeletedListener : public DAGUpdateListener {
316 std::function<void(SDNode *, SDNode *)> Callback;
317
318 DAGNodeDeletedListener(SelectionDAG &DAG,
319 std::function<void(SDNode *, SDNode *)> Callback)
320 : DAGUpdateListener(DAG), Callback(std::move(Callback)) {}
321
322 void NodeDeleted(SDNode *N, SDNode *E) override { Callback(N, E); }
323
324 private:
325 virtual void anchor();
326 };
327
328 /// When true, additional steps are taken to
329 /// ensure that getConstant() and similar functions return DAG nodes that
330 /// have legal types. This is important after type legalization since
331 /// any illegally typed nodes generated after this point will not experience
332 /// type legalization.
333 bool NewNodesMustHaveLegalTypes = false;
334
335private:
336 /// DAGUpdateListener is a friend so it can manipulate the listener stack.
337 friend struct DAGUpdateListener;
338
339 /// Linked list of registered DAGUpdateListener instances.
340 /// This stack is maintained by DAGUpdateListener RAII.
341 DAGUpdateListener *UpdateListeners = nullptr;
342
343 /// Implementation of setSubgraphColor.
344 /// Return whether we had to truncate the search.
345 bool setSubgraphColorHelper(SDNode *N, const char *Color,
346 DenseSet<SDNode *> &visited,
347 int level, bool &printed);
348
349 template <typename SDNodeT, typename... ArgTypes>
350 SDNodeT *newSDNode(ArgTypes &&... Args) {
351 return new (NodeAllocator.template Allocate<SDNodeT>())
352 SDNodeT(std::forward<ArgTypes>(Args)...);
353 }
354
355 /// Build a synthetic SDNodeT with the given args and extract its subclass
356 /// data as an integer (e.g. for use in a folding set).
357 ///
358 /// The args to this function are the same as the args to SDNodeT's
359 /// constructor, except the second arg (assumed to be a const DebugLoc&) is
360 /// omitted.
361 template <typename SDNodeT, typename... ArgTypes>
362 static uint16_t getSyntheticNodeSubclassData(unsigned IROrder,
363 ArgTypes &&... Args) {
364 // The compiler can reduce this expression to a constant iff we pass an
365 // empty DebugLoc. Thankfully, the debug location doesn't have any bearing
366 // on the subclass data.
367 return SDNodeT(IROrder, DebugLoc(), std::forward<ArgTypes>(Args)...)
368 .getRawSubclassData();
369 }
370
371 template <typename SDNodeTy>
372 static uint16_t getSyntheticNodeSubclassData(unsigned Opc, unsigned Order,
373 SDVTList VTs, EVT MemoryVT,
374 MachineMemOperand *MMO) {
375 return SDNodeTy(Opc, Order, DebugLoc(), VTs, MemoryVT, MMO)
376 .getRawSubclassData();
377 }
378
379 void createOperands(SDNode *Node, ArrayRef<SDValue> Vals);
380
381 void removeOperands(SDNode *Node) {
382 if (!Node->OperandList)
383 return;
384 OperandRecycler.deallocate(
385 ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands),
386 Node->OperandList);
387 Node->NumOperands = 0;
388 Node->OperandList = nullptr;
389 }
390 void CreateTopologicalOrder(std::vector<SDNode*>& Order);
391
392public:
393 // Maximum depth for recursive analysis such as computeKnownBits, etc.
394 static constexpr unsigned MaxRecursionDepth = 6;
395
396 explicit SelectionDAG(const TargetMachine &TM, CodeGenOpt::Level);
397 SelectionDAG(const SelectionDAG &) = delete;
398 SelectionDAG &operator=(const SelectionDAG &) = delete;
399 ~SelectionDAG();
400
401 /// Prepare this SelectionDAG to process code in the given MachineFunction.
402 void init(MachineFunction &NewMF, OptimizationRemarkEmitter &NewORE,
403 Pass *PassPtr, const TargetLibraryInfo *LibraryInfo,
404 LegacyDivergenceAnalysis * Divergence);
405
406 void setFunctionLoweringInfo(FunctionLoweringInfo * FuncInfo) {
407 FLI = FuncInfo;
408 }
409
410 /// Clear state and free memory necessary to make this
411 /// SelectionDAG ready to process a new block.
412 void clear();
413
414 MachineFunction &getMachineFunction() const { return *MF; }
415 const Pass *getPass() const { return SDAGISelPass; }
416
417 const DataLayout &getDataLayout() const { return MF->getDataLayout(); }
418 const TargetMachine &getTarget() const { return TM; }
419 const TargetSubtargetInfo &getSubtarget() const { return MF->getSubtarget(); }
420 const TargetLowering &getTargetLoweringInfo() const { return *TLI; }
421 const TargetLibraryInfo &getLibInfo() const { return *LibInfo; }
422 const SelectionDAGTargetInfo &getSelectionDAGInfo() const { return *TSI; }
423 const LegacyDivergenceAnalysis *getDivergenceAnalysis() const { return DA; }
424 LLVMContext *getContext() const {return Context; }
425 OptimizationRemarkEmitter &getORE() const { return *ORE; }
426
427 /// Pop up a GraphViz/gv window with the DAG rendered using 'dot'.
428 void viewGraph(const std::string &Title);
429 void viewGraph();
430
431#ifndef NDEBUG
432 std::map<const SDNode *, std::string> NodeGraphAttrs;
433#endif
434
435 /// Clear all previously defined node graph attributes.
436 /// Intended to be used from a debugging tool (eg. gdb).
437 void clearGraphAttrs();
438
439 /// Set graph attributes for a node. (eg. "color=red".)
440 void setGraphAttrs(const SDNode *N, const char *Attrs);
441
442 /// Get graph attributes for a node. (eg. "color=red".)
443 /// Used from getNodeAttributes.
444 const std::string getGraphAttrs(const SDNode *N) const;
445
446 /// Convenience for setting node color attribute.
447 void setGraphColor(const SDNode *N, const char *Color);
448
449 /// Convenience for setting subgraph color attribute.
450 void setSubgraphColor(SDNode *N, const char *Color);
451
452 using allnodes_const_iterator = ilist<SDNode>::const_iterator;
453
454 allnodes_const_iterator allnodes_begin() const { return AllNodes.begin(); }
455 allnodes_const_iterator allnodes_end() const { return AllNodes.end(); }
456
457 using allnodes_iterator = ilist<SDNode>::iterator;
458
459 allnodes_iterator allnodes_begin() { return AllNodes.begin(); }
460 allnodes_iterator allnodes_end() { return AllNodes.end(); }
461
462 ilist<SDNode>::size_type allnodes_size() const {
463 return AllNodes.size();
464 }
465
466 iterator_range<allnodes_iterator> allnodes() {
467 return make_range(allnodes_begin(), allnodes_end());
468 }
469 iterator_range<allnodes_const_iterator> allnodes() const {
470 return make_range(allnodes_begin(), allnodes_end());
471 }
472
473 /// Return the root tag of the SelectionDAG.
474 const SDValue &getRoot() const { return Root; }
475
476 /// Return the token chain corresponding to the entry of the function.
477 SDValue getEntryNode() const {
478 return SDValue(const_cast<SDNode *>(&EntryNode), 0);
479 }
480
481 /// Set the current root tag of the SelectionDAG.
482 ///
483 const SDValue &setRoot(SDValue N) {
484 assert((!N.getNode() || N.getValueType() == MVT::Other) &&(((!N.getNode() || N.getValueType() == MVT::Other) &&
"DAG root value is not a chain!") ? static_cast<void> (
0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 485, __PRETTY_FUNCTION__))
485 "DAG root value is not a chain!")(((!N.getNode() || N.getValueType() == MVT::Other) &&
"DAG root value is not a chain!") ? static_cast<void> (
0) : __assert_fail ("(!N.getNode() || N.getValueType() == MVT::Other) && \"DAG root value is not a chain!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 485, __PRETTY_FUNCTION__))
;
486 if (N.getNode())
487 checkForCycles(N.getNode(), this);
488 Root = N;
489 if (N.getNode())
490 checkForCycles(this);
491 return Root;
492 }
493
494#ifndef NDEBUG
495 void VerifyDAGDiverence();
496#endif
497
498 /// This iterates over the nodes in the SelectionDAG, folding
499 /// certain types of nodes together, or eliminating superfluous nodes. The
500 /// Level argument controls whether Combine is allowed to produce nodes and
501 /// types that are illegal on the target.
502 void Combine(CombineLevel Level, AliasAnalysis *AA,
503 CodeGenOpt::Level OptLevel);
504
505 /// This transforms the SelectionDAG into a SelectionDAG that
506 /// only uses types natively supported by the target.
507 /// Returns "true" if it made any changes.
508 ///
509 /// Note that this is an involved process that may invalidate pointers into
510 /// the graph.
511 bool LegalizeTypes();
512
513 /// This transforms the SelectionDAG into a SelectionDAG that is
514 /// compatible with the target instruction selector, as indicated by the
515 /// TargetLowering object.
516 ///
517 /// Note that this is an involved process that may invalidate pointers into
518 /// the graph.
519 void Legalize();
520
521 /// Transforms a SelectionDAG node and any operands to it into a node
522 /// that is compatible with the target instruction selector, as indicated by
523 /// the TargetLowering object.
524 ///
525 /// \returns true if \c N is a valid, legal node after calling this.
526 ///
527 /// This essentially runs a single recursive walk of the \c Legalize process
528 /// over the given node (and its operands). This can be used to incrementally
529 /// legalize the DAG. All of the nodes which are directly replaced,
530 /// potentially including N, are added to the output parameter \c
531 /// UpdatedNodes so that the delta to the DAG can be understood by the
532 /// caller.
533 ///
534 /// When this returns false, N has been legalized in a way that make the
535 /// pointer passed in no longer valid. It may have even been deleted from the
536 /// DAG, and so it shouldn't be used further. When this returns true, the
537 /// N passed in is a legal node, and can be immediately processed as such.
538 /// This may still have done some work on the DAG, and will still populate
539 /// UpdatedNodes with any new nodes replacing those originally in the DAG.
540 bool LegalizeOp(SDNode *N, SmallSetVector<SDNode *, 16> &UpdatedNodes);
541
542 /// This transforms the SelectionDAG into a SelectionDAG
543 /// that only uses vector math operations supported by the target. This is
544 /// necessary as a separate step from Legalize because unrolling a vector
545 /// operation can introduce illegal types, which requires running
546 /// LegalizeTypes again.
547 ///
548 /// This returns true if it made any changes; in that case, LegalizeTypes
549 /// is called again before Legalize.
550 ///
551 /// Note that this is an involved process that may invalidate pointers into
552 /// the graph.
553 bool LegalizeVectors();
554
555 /// This method deletes all unreachable nodes in the SelectionDAG.
556 void RemoveDeadNodes();
557
558 /// Remove the specified node from the system. This node must
559 /// have no referrers.
560 void DeleteNode(SDNode *N);
561
562 /// Return an SDVTList that represents the list of values specified.
563 SDVTList getVTList(EVT VT);
564 SDVTList getVTList(EVT VT1, EVT VT2);
565 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3);
566 SDVTList getVTList(EVT VT1, EVT VT2, EVT VT3, EVT VT4);
567 SDVTList getVTList(ArrayRef<EVT> VTs);
568
569 //===--------------------------------------------------------------------===//
570 // Node creation methods.
571
572 /// Create a ConstantSDNode wrapping a constant value.
573 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
574 ///
575 /// If only legal types can be produced, this does the necessary
576 /// transformations (e.g., if the vector element type is illegal).
577 /// @{
578 SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT,
579 bool isTarget = false, bool isOpaque = false);
580 SDValue getConstant(const APInt &Val, const SDLoc &DL, EVT VT,
581 bool isTarget = false, bool isOpaque = false);
582
583 SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget = false,
584 bool IsOpaque = false) {
585 return getConstant(APInt::getAllOnesValue(VT.getScalarSizeInBits()), DL,
586 VT, IsTarget, IsOpaque);
587 }
588
589 SDValue getConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
590 bool isTarget = false, bool isOpaque = false);
591 SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL,
592 bool isTarget = false);
593 SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL,
594 bool LegalTypes = true);
595
596 SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT,
597 bool isOpaque = false) {
598 return getConstant(Val, DL, VT, true, isOpaque);
599 }
600 SDValue getTargetConstant(const APInt &Val, const SDLoc &DL, EVT VT,
601 bool isOpaque = false) {
602 return getConstant(Val, DL, VT, true, isOpaque);
603 }
604 SDValue getTargetConstant(const ConstantInt &Val, const SDLoc &DL, EVT VT,
605 bool isOpaque = false) {
606 return getConstant(Val, DL, VT, true, isOpaque);
607 }
608
609 /// Create a true or false constant of type \p VT using the target's
610 /// BooleanContent for type \p OpVT.
611 SDValue getBoolConstant(bool V, const SDLoc &DL, EVT VT, EVT OpVT);
612 /// @}
613
614 /// Create a ConstantFPSDNode wrapping a constant value.
615 /// If VT is a vector type, the constant is splatted into a BUILD_VECTOR.
616 ///
617 /// If only legal types can be produced, this does the necessary
618 /// transformations (e.g., if the vector element type is illegal).
619 /// The forms that take a double should only be used for simple constants
620 /// that can be exactly represented in VT. No checks are made.
621 /// @{
622 SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT,
623 bool isTarget = false);
624 SDValue getConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT,
625 bool isTarget = false);
626 SDValue getConstantFP(const ConstantFP &V, const SDLoc &DL, EVT VT,
627 bool isTarget = false);
628 SDValue getTargetConstantFP(double Val, const SDLoc &DL, EVT VT) {
629 return getConstantFP(Val, DL, VT, true);
630 }
631 SDValue getTargetConstantFP(const APFloat &Val, const SDLoc &DL, EVT VT) {
632 return getConstantFP(Val, DL, VT, true);
633 }
634 SDValue getTargetConstantFP(const ConstantFP &Val, const SDLoc &DL, EVT VT) {
635 return getConstantFP(Val, DL, VT, true);
636 }
637 /// @}
638
639 SDValue getGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
640 int64_t offset = 0, bool isTargetGA = false,
641 unsigned TargetFlags = 0);
642 SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT,
643 int64_t offset = 0, unsigned TargetFlags = 0) {
644 return getGlobalAddress(GV, DL, VT, offset, true, TargetFlags);
645 }
646 SDValue getFrameIndex(int FI, EVT VT, bool isTarget = false);
647 SDValue getTargetFrameIndex(int FI, EVT VT) {
648 return getFrameIndex(FI, VT, true);
649 }
650 SDValue getJumpTable(int JTI, EVT VT, bool isTarget = false,
651 unsigned TargetFlags = 0);
652 SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags = 0) {
653 return getJumpTable(JTI, VT, true, TargetFlags);
654 }
655 SDValue getConstantPool(const Constant *C, EVT VT, unsigned Align = 0,
656 int Offs = 0, bool isT = false,
657 unsigned TargetFlags = 0);
658 SDValue getTargetConstantPool(const Constant *C, EVT VT, unsigned Align = 0,
659 int Offset = 0, unsigned TargetFlags = 0) {
660 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
661 }
662 SDValue getConstantPool(MachineConstantPoolValue *C, EVT VT,
663 unsigned Align = 0, int Offs = 0, bool isT=false,
664 unsigned TargetFlags = 0);
665 SDValue getTargetConstantPool(MachineConstantPoolValue *C, EVT VT,
666 unsigned Align = 0, int Offset = 0,
667 unsigned TargetFlags = 0) {
668 return getConstantPool(C, VT, Align, Offset, true, TargetFlags);
669 }
670 SDValue getTargetIndex(int Index, EVT VT, int64_t Offset = 0,
671 unsigned TargetFlags = 0);
672 // When generating a branch to a BB, we don't in general know enough
673 // to provide debug info for the BB at that time, so keep this one around.
674 SDValue getBasicBlock(MachineBasicBlock *MBB);
675 SDValue getBasicBlock(MachineBasicBlock *MBB, SDLoc dl);
676 SDValue getExternalSymbol(const char *Sym, EVT VT);
677 SDValue getExternalSymbol(const char *Sym, const SDLoc &dl, EVT VT);
678 SDValue getTargetExternalSymbol(const char *Sym, EVT VT,
679 unsigned TargetFlags = 0);
680 SDValue getMCSymbol(MCSymbol *Sym, EVT VT);
681
682 SDValue getValueType(EVT);
683 SDValue getRegister(unsigned Reg, EVT VT);
684 SDValue getRegisterMask(const uint32_t *RegMask);
685 SDValue getEHLabel(const SDLoc &dl, SDValue Root, MCSymbol *Label);
686 SDValue getLabelNode(unsigned Opcode, const SDLoc &dl, SDValue Root,
687 MCSymbol *Label);
688 SDValue getBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset = 0,
689 bool isTarget = false, unsigned TargetFlags = 0);
690 SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT,
691 int64_t Offset = 0, unsigned TargetFlags = 0) {
692 return getBlockAddress(BA, VT, Offset, true, TargetFlags);
693 }
694
695 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg,
696 SDValue N) {
697 return getNode(ISD::CopyToReg, dl, MVT::Other, Chain,
698 getRegister(Reg, N.getValueType()), N);
699 }
700
701 // This version of the getCopyToReg method takes an extra operand, which
702 // indicates that there is potentially an incoming glue value (if Glue is not
703 // null) and that there should be a glue result.
704 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, unsigned Reg, SDValue N,
705 SDValue Glue) {
706 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
707 SDValue Ops[] = { Chain, getRegister(Reg, N.getValueType()), N, Glue };
708 return getNode(ISD::CopyToReg, dl, VTs,
709 makeArrayRef(Ops, Glue.getNode() ? 4 : 3));
710 }
711
712 // Similar to last getCopyToReg() except parameter Reg is a SDValue
713 SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, SDValue Reg, SDValue N,
714 SDValue Glue) {
715 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
716 SDValue Ops[] = { Chain, Reg, N, Glue };
717 return getNode(ISD::CopyToReg, dl, VTs,
718 makeArrayRef(Ops, Glue.getNode() ? 4 : 3));
719 }
720
721 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT) {
722 SDVTList VTs = getVTList(VT, MVT::Other);
723 SDValue Ops[] = { Chain, getRegister(Reg, VT) };
724 return getNode(ISD::CopyFromReg, dl, VTs, Ops);
725 }
726
727 // This version of the getCopyFromReg method takes an extra operand, which
728 // indicates that there is potentially an incoming glue value (if Glue is not
729 // null) and that there should be a glue result.
730 SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, unsigned Reg, EVT VT,
731 SDValue Glue) {
732 SDVTList VTs = getVTList(VT, MVT::Other, MVT::Glue);
733 SDValue Ops[] = { Chain, getRegister(Reg, VT), Glue };
734 return getNode(ISD::CopyFromReg, dl, VTs,
735 makeArrayRef(Ops, Glue.getNode() ? 3 : 2));
736 }
737
738 SDValue getCondCode(ISD::CondCode Cond);
739
740 /// Return an ISD::VECTOR_SHUFFLE node. The number of elements in VT,
741 /// which must be a vector type, must match the number of mask elements
742 /// NumElts. An integer mask element equal to -1 is treated as undefined.
743 SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2,
744 ArrayRef<int> Mask);
745
746 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
747 /// which must be a vector type, must match the number of operands in Ops.
748 /// The operands must have the same type as (or, for integers, a type wider
749 /// than) VT's element type.
750 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDValue> Ops) {
751 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
752 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
753 }
754
755 /// Return an ISD::BUILD_VECTOR node. The number of elements in VT,
756 /// which must be a vector type, must match the number of operands in Ops.
757 /// The operands must have the same type as (or, for integers, a type wider
758 /// than) VT's element type.
759 SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef<SDUse> Ops) {
760 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
761 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
762 }
763
764 /// Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all
765 /// elements. VT must be a vector type. Op's type must be the same as (or,
766 /// for integers, a type wider than) VT's element type.
767 SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op) {
768 // VerifySDNode (via InsertNode) checks BUILD_VECTOR later.
769 if (Op.getOpcode() == ISD::UNDEF) {
19
Calling 'SDValue::getOpcode'
770 assert((VT.getVectorElementType() == Op.getValueType() ||(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 774, __PRETTY_FUNCTION__))
771 (VT.isInteger() &&(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 774, __PRETTY_FUNCTION__))
772 VT.getVectorElementType().bitsLE(Op.getValueType()))) &&(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 774, __PRETTY_FUNCTION__))
773 "A splatted value must have a width equal or (for integers) "(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 774, __PRETTY_FUNCTION__))
774 "greater than the vector element type!")(((VT.getVectorElementType() == Op.getValueType() || (VT.isInteger
() && VT.getVectorElementType().bitsLE(Op.getValueType
()))) && "A splatted value must have a width equal or (for integers) "
"greater than the vector element type!") ? static_cast<void
> (0) : __assert_fail ("(VT.getVectorElementType() == Op.getValueType() || (VT.isInteger() && VT.getVectorElementType().bitsLE(Op.getValueType()))) && \"A splatted value must have a width equal or (for integers) \" \"greater than the vector element type!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 774, __PRETTY_FUNCTION__))
;
775 return getNode(ISD::UNDEF, SDLoc(), VT);
776 }
777
778 SmallVector<SDValue, 16> Ops(VT.getVectorNumElements(), Op);
779 return getNode(ISD::BUILD_VECTOR, DL, VT, Ops);
780 }
781
782 /// Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to
783 /// the shuffle node in input but with swapped operands.
784 ///
785 /// Example: shuffle A, B, <0,5,2,7> -> shuffle B, A, <4,1,6,3>
786 SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV);
787
788 /// Convert Op, which must be of float type, to the
789 /// float type VT, by either extending or rounding (by truncation).
790 SDValue getFPExtendOrRound(SDValue Op, const SDLoc &DL, EVT VT);
791
792 /// Convert Op, which must be of integer type, to the
793 /// integer type VT, by either any-extending or truncating it.
794 SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
795
796 /// Convert Op, which must be of integer type, to the
797 /// integer type VT, by either sign-extending or truncating it.
798 SDValue getSExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
799
800 /// Convert Op, which must be of integer type, to the
801 /// integer type VT, by either zero-extending or truncating it.
802 SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
803
804 /// Return the expression required to zero extend the Op
805 /// value assuming it was the smaller SrcTy value.
806 SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
807
808 /// Convert Op, which must be of integer type, to the integer type VT, by
809 /// either truncating it or performing either zero or sign extension as
810 /// appropriate extension for the pointer's semantics.
811 SDValue getPtrExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT);
812
813 /// Return the expression required to extend the Op as a pointer value
814 /// assuming it was the smaller SrcTy value. This may be either a zero extend
815 /// or a sign extend.
816 SDValue getPtrExtendInReg(SDValue Op, const SDLoc &DL, EVT VT);
817
818 /// Convert Op, which must be of integer type, to the integer type VT,
819 /// by using an extension appropriate for the target's
820 /// BooleanContent for type OpVT or truncating it.
821 SDValue getBoolExtOrTrunc(SDValue Op, const SDLoc &SL, EVT VT, EVT OpVT);
822
823 /// Create a bitwise NOT operation as (XOR Val, -1).
824 SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT);
825
826 /// Create a logical NOT operation as (XOR Val, BooleanOne).
827 SDValue getLogicalNOT(const SDLoc &DL, SDValue Val, EVT VT);
828
829 /// Create an add instruction with appropriate flags when used for
830 /// addressing some offset of an object. i.e. if a load is split into multiple
831 /// components, create an add nuw from the base pointer to the offset.
832 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, int64_t Offset) {
833 EVT VT = Op.getValueType();
834 return getObjectPtrOffset(SL, Op, getConstant(Offset, SL, VT));
835 }
836
837 SDValue getObjectPtrOffset(const SDLoc &SL, SDValue Op, SDValue Offset) {
838 EVT VT = Op.getValueType();
839
840 // The object itself can't wrap around the address space, so it shouldn't be
841 // possible for the adds of the offsets to the split parts to overflow.
842 SDNodeFlags Flags;
843 Flags.setNoUnsignedWrap(true);
844 return getNode(ISD::ADD, SL, VT, Op, Offset, Flags);
845 }
846
847 /// Return a new CALLSEQ_START node, that starts new call frame, in which
848 /// InSize bytes are set up inside CALLSEQ_START..CALLSEQ_END sequence and
849 /// OutSize specifies part of the frame set up prior to the sequence.
850 SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize,
851 const SDLoc &DL) {
852 SDVTList VTs = getVTList(MVT::Other, MVT::Glue);
853 SDValue Ops[] = { Chain,
854 getIntPtrConstant(InSize, DL, true),
855 getIntPtrConstant(OutSize, DL, true) };
856 return getNode(ISD::CALLSEQ_START, DL, VTs, Ops);
857 }
858
859 /// Return a new CALLSEQ_END node, which always must have a
860 /// glue result (to ensure it's not CSE'd).
861 /// CALLSEQ_END does not have a useful SDLoc.
862 SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2,
863 SDValue InGlue, const SDLoc &DL) {
864 SDVTList NodeTys = getVTList(MVT::Other, MVT::Glue);
865 SmallVector<SDValue, 4> Ops;
866 Ops.push_back(Chain);
867 Ops.push_back(Op1);
868 Ops.push_back(Op2);
869 if (InGlue.getNode())
870 Ops.push_back(InGlue);
871 return getNode(ISD::CALLSEQ_END, DL, NodeTys, Ops);
872 }
873
874 /// Return true if the result of this operation is always undefined.
875 bool isUndef(unsigned Opcode, ArrayRef<SDValue> Ops);
876
877 /// Return an UNDEF node. UNDEF does not have a useful SDLoc.
878 SDValue getUNDEF(EVT VT) {
879 return getNode(ISD::UNDEF, SDLoc(), VT);
880 }
881
882 /// Return a GLOBAL_OFFSET_TABLE node. This does not have a useful SDLoc.
883 SDValue getGLOBAL_OFFSET_TABLE(EVT VT) {
884 return getNode(ISD::GLOBAL_OFFSET_TABLE, SDLoc(), VT);
885 }
886
887 /// Gets or creates the specified node.
888 ///
889 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
890 ArrayRef<SDUse> Ops);
891 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
892 ArrayRef<SDValue> Ops, const SDNodeFlags Flags = SDNodeFlags());
893 SDValue getNode(unsigned Opcode, const SDLoc &DL, ArrayRef<EVT> ResultTys,
894 ArrayRef<SDValue> Ops);
895 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList,
896 ArrayRef<SDValue> Ops);
897
898 // Specialize based on number of operands.
899 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT);
900 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue Operand,
901 const SDNodeFlags Flags = SDNodeFlags());
902 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
903 SDValue N2, const SDNodeFlags Flags = SDNodeFlags());
904 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
905 SDValue N2, SDValue N3,
906 const SDNodeFlags Flags = SDNodeFlags());
907 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
908 SDValue N2, SDValue N3, SDValue N4);
909 SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, SDValue N1,
910 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
911
912 // Specialize again based on number of operands for nodes with a VTList
913 // rather than a single VT.
914 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList);
915 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N);
916 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
917 SDValue N2);
918 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
919 SDValue N2, SDValue N3);
920 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
921 SDValue N2, SDValue N3, SDValue N4);
922 SDValue getNode(unsigned Opcode, const SDLoc &DL, SDVTList VTList, SDValue N1,
923 SDValue N2, SDValue N3, SDValue N4, SDValue N5);
924
925 /// Compute a TokenFactor to force all the incoming stack arguments to be
926 /// loaded from the stack. This is used in tail call lowering to protect
927 /// stack arguments from being clobbered.
928 SDValue getStackArgumentTokenFactor(SDValue Chain);
929
930 SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
931 SDValue Size, unsigned Align, bool isVol, bool AlwaysInline,
932 bool isTailCall, MachinePointerInfo DstPtrInfo,
933 MachinePointerInfo SrcPtrInfo);
934
935 SDValue getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
936 SDValue Size, unsigned Align, bool isVol, bool isTailCall,
937 MachinePointerInfo DstPtrInfo,
938 MachinePointerInfo SrcPtrInfo);
939
940 SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src,
941 SDValue Size, unsigned Align, bool isVol, bool isTailCall,
942 MachinePointerInfo DstPtrInfo);
943
944 SDValue getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst,
945 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
946 SDValue Size, Type *SizeTy, unsigned ElemSz,
947 bool isTailCall, MachinePointerInfo DstPtrInfo,
948 MachinePointerInfo SrcPtrInfo);
949
950 SDValue getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst,
951 unsigned DstAlign, SDValue Src, unsigned SrcAlign,
952 SDValue Size, Type *SizeTy, unsigned ElemSz,
953 bool isTailCall, MachinePointerInfo DstPtrInfo,
954 MachinePointerInfo SrcPtrInfo);
955
956 SDValue getAtomicMemset(SDValue Chain, const SDLoc &dl, SDValue Dst,
957 unsigned DstAlign, SDValue Value, SDValue Size,
958 Type *SizeTy, unsigned ElemSz, bool isTailCall,
959 MachinePointerInfo DstPtrInfo);
960
961 /// Helper function to make it easier to build SetCC's if you just have an
962 /// ISD::CondCode instead of an SDValue.
963 SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS,
964 ISD::CondCode Cond) {
965 assert(LHS.getValueType().isVector() == RHS.getValueType().isVector() &&((LHS.getValueType().isVector() == RHS.getValueType().isVector
() && "Cannot compare scalars to vectors") ? static_cast
<void> (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 966, __PRETTY_FUNCTION__))
966 "Cannot compare scalars to vectors")((LHS.getValueType().isVector() == RHS.getValueType().isVector
() && "Cannot compare scalars to vectors") ? static_cast
<void> (0) : __assert_fail ("LHS.getValueType().isVector() == RHS.getValueType().isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 966, __PRETTY_FUNCTION__))
;
967 assert(LHS.getValueType().isVector() == VT.isVector() &&((LHS.getValueType().isVector() == VT.isVector() && "Cannot compare scalars to vectors"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 968, __PRETTY_FUNCTION__))
968 "Cannot compare scalars to vectors")((LHS.getValueType().isVector() == VT.isVector() && "Cannot compare scalars to vectors"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType().isVector() == VT.isVector() && \"Cannot compare scalars to vectors\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 968, __PRETTY_FUNCTION__))
;
969 assert(Cond != ISD::SETCC_INVALID &&((Cond != ISD::SETCC_INVALID && "Cannot create a setCC of an invalid node."
) ? static_cast<void> (0) : __assert_fail ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 970, __PRETTY_FUNCTION__))
970 "Cannot create a setCC of an invalid node.")((Cond != ISD::SETCC_INVALID && "Cannot create a setCC of an invalid node."
) ? static_cast<void> (0) : __assert_fail ("Cond != ISD::SETCC_INVALID && \"Cannot create a setCC of an invalid node.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 970, __PRETTY_FUNCTION__))
;
971 return getNode(ISD::SETCC, DL, VT, LHS, RHS, getCondCode(Cond));
972 }
973
974 /// Helper function to make it easier to build Select's if you just have
975 /// operands and don't want to check for vector.
976 SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
977 SDValue RHS) {
978 assert(LHS.getValueType() == RHS.getValueType() &&((LHS.getValueType() == RHS.getValueType() && "Cannot use select on differing types"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 979, __PRETTY_FUNCTION__))
979 "Cannot use select on differing types")((LHS.getValueType() == RHS.getValueType() && "Cannot use select on differing types"
) ? static_cast<void> (0) : __assert_fail ("LHS.getValueType() == RHS.getValueType() && \"Cannot use select on differing types\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 979, __PRETTY_FUNCTION__))
;
980 assert(VT.isVector() == LHS.getValueType().isVector() &&((VT.isVector() == LHS.getValueType().isVector() && "Cannot mix vectors and scalars"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 981, __PRETTY_FUNCTION__))
981 "Cannot mix vectors and scalars")((VT.isVector() == LHS.getValueType().isVector() && "Cannot mix vectors and scalars"
) ? static_cast<void> (0) : __assert_fail ("VT.isVector() == LHS.getValueType().isVector() && \"Cannot mix vectors and scalars\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 981, __PRETTY_FUNCTION__))
;
982 auto Opcode = Cond.getValueType().isVector() ? ISD::VSELECT : ISD::SELECT;
983 return getNode(Opcode, DL, VT, Cond, LHS, RHS);
984 }
985
986 /// Helper function to make it easier to build SelectCC's if you just have an
987 /// ISD::CondCode instead of an SDValue.
988 SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
989 SDValue False, ISD::CondCode Cond) {
990 return getNode(ISD::SELECT_CC, DL, True.getValueType(), LHS, RHS, True,
991 False, getCondCode(Cond));
992 }
993
994 /// Try to simplify a select/vselect into 1 of its operands or a constant.
995 SDValue simplifySelect(SDValue Cond, SDValue TVal, SDValue FVal);
996
997 /// Try to simplify a shift into 1 of its operands or a constant.
998 SDValue simplifyShift(SDValue X, SDValue Y);
999
1000 /// Try to simplify a floating-point binary operation into 1 of its operands
1001 /// or a constant.
1002 SDValue simplifyFPBinop(unsigned Opcode, SDValue X, SDValue Y);
1003
1004 /// VAArg produces a result and token chain, and takes a pointer
1005 /// and a source value as input.
1006 SDValue getVAArg(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1007 SDValue SV, unsigned Align);
1008
1009 /// Gets a node for an atomic cmpxchg op. There are two
1010 /// valid Opcodes. ISD::ATOMIC_CMO_SWAP produces the value loaded and a
1011 /// chain result. ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS produces the value loaded,
1012 /// a success flag (initially i1), and a chain.
1013 SDValue getAtomicCmpSwap(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1014 SDVTList VTs, SDValue Chain, SDValue Ptr,
1015 SDValue Cmp, SDValue Swp, MachineMemOperand *MMO);
1016
1017 /// Gets a node for an atomic op, produces result (if relevant)
1018 /// and chain and takes 2 operands.
1019 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, SDValue Chain,
1020 SDValue Ptr, SDValue Val, MachineMemOperand *MMO);
1021
1022 /// Gets a node for an atomic op, produces result and chain and
1023 /// takes 1 operand.
1024 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT, EVT VT,
1025 SDValue Chain, SDValue Ptr, MachineMemOperand *MMO);
1026
1027 /// Gets a node for an atomic op, produces result and chain and takes N
1028 /// operands.
1029 SDValue getAtomic(unsigned Opcode, const SDLoc &dl, EVT MemVT,
1030 SDVTList VTList, ArrayRef<SDValue> Ops,
1031 MachineMemOperand *MMO);
1032
1033 /// Creates a MemIntrinsicNode that may produce a
1034 /// result and takes a list of operands. Opcode may be INTRINSIC_VOID,
1035 /// INTRINSIC_W_CHAIN, or a target-specific opcode with a value not
1036 /// less than FIRST_TARGET_MEMORY_OPCODE.
1037 SDValue getMemIntrinsicNode(
1038 unsigned Opcode, const SDLoc &dl, SDVTList VTList,
1039 ArrayRef<SDValue> Ops, EVT MemVT,
1040 MachinePointerInfo PtrInfo,
1041 unsigned Align = 0,
1042 MachineMemOperand::Flags Flags
1043 = MachineMemOperand::MOLoad | MachineMemOperand::MOStore,
1044 uint64_t Size = 0,
1045 const AAMDNodes &AAInfo = AAMDNodes());
1046
1047 SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList,
1048 ArrayRef<SDValue> Ops, EVT MemVT,
1049 MachineMemOperand *MMO);
1050
1051 /// Creates a LifetimeSDNode that starts (`IsStart==true`) or ends
1052 /// (`IsStart==false`) the lifetime of the portion of `FrameIndex` between
1053 /// offsets `Offset` and `Offset + Size`.
1054 SDValue getLifetimeNode(bool IsStart, const SDLoc &dl, SDValue Chain,
1055 int FrameIndex, int64_t Size, int64_t Offset = -1);
1056
1057 /// Create a MERGE_VALUES node from the given operands.
1058 SDValue getMergeValues(ArrayRef<SDValue> Ops, const SDLoc &dl);
1059
1060 /// Loads are not normal binary operators: their result type is not
1061 /// determined by their operands, and they produce a value AND a token chain.
1062 ///
1063 /// This function will set the MOLoad flag on MMOFlags, but you can set it if
1064 /// you want. The MOStore flag must not be set.
1065 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1066 MachinePointerInfo PtrInfo, unsigned Alignment = 0,
1067 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1068 const AAMDNodes &AAInfo = AAMDNodes(),
1069 const MDNode *Ranges = nullptr);
1070 SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1071 MachineMemOperand *MMO);
1072 SDValue
1073 getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain,
1074 SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT,
1075 unsigned Alignment = 0,
1076 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1077 const AAMDNodes &AAInfo = AAMDNodes());
1078 SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT,
1079 SDValue Chain, SDValue Ptr, EVT MemVT,
1080 MachineMemOperand *MMO);
1081 SDValue getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, SDValue Base,
1082 SDValue Offset, ISD::MemIndexedMode AM);
1083 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1084 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1085 MachinePointerInfo PtrInfo, EVT MemVT, unsigned Alignment = 0,
1086 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1087 const AAMDNodes &AAInfo = AAMDNodes(),
1088 const MDNode *Ranges = nullptr);
1089 SDValue getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, EVT VT,
1090 const SDLoc &dl, SDValue Chain, SDValue Ptr, SDValue Offset,
1091 EVT MemVT, MachineMemOperand *MMO);
1092
1093 /// Helper function to build ISD::STORE nodes.
1094 ///
1095 /// This function will set the MOStore flag on MMOFlags, but you can set it if
1096 /// you want. The MOLoad and MOInvariant flags must not be set.
1097 SDValue
1098 getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1099 MachinePointerInfo PtrInfo, unsigned Alignment = 0,
1100 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1101 const AAMDNodes &AAInfo = AAMDNodes());
1102 SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1103 MachineMemOperand *MMO);
1104 SDValue
1105 getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr,
1106 MachinePointerInfo PtrInfo, EVT SVT, unsigned Alignment = 0,
1107 MachineMemOperand::Flags MMOFlags = MachineMemOperand::MONone,
1108 const AAMDNodes &AAInfo = AAMDNodes());
1109 SDValue getTruncStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1110 SDValue Ptr, EVT SVT, MachineMemOperand *MMO);
1111 SDValue getIndexedStore(SDValue OrigStore, const SDLoc &dl, SDValue Base,
1112 SDValue Offset, ISD::MemIndexedMode AM);
1113
1114 /// Returns sum of the base pointer and offset.
1115 SDValue getMemBasePlusOffset(SDValue Base, unsigned Offset, const SDLoc &DL);
1116
1117 SDValue getMaskedLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr,
1118 SDValue Mask, SDValue Src0, EVT MemVT,
1119 MachineMemOperand *MMO, ISD::LoadExtType,
1120 bool IsExpanding = false);
1121 SDValue getMaskedStore(SDValue Chain, const SDLoc &dl, SDValue Val,
1122 SDValue Ptr, SDValue Mask, EVT MemVT,
1123 MachineMemOperand *MMO, bool IsTruncating = false,
1124 bool IsCompressing = false);
1125 SDValue getMaskedGather(SDVTList VTs, EVT VT, const SDLoc &dl,
1126 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1127 ISD::MemIndexType IndexType);
1128 SDValue getMaskedScatter(SDVTList VTs, EVT VT, const SDLoc &dl,
1129 ArrayRef<SDValue> Ops, MachineMemOperand *MMO,
1130 ISD::MemIndexType IndexType);
1131
1132 /// Return (create a new or find existing) a target-specific node.
1133 /// TargetMemSDNode should be derived class from MemSDNode.
1134 template <class TargetMemSDNode>
1135 SDValue getTargetMemSDNode(SDVTList VTs, ArrayRef<SDValue> Ops,
1136 const SDLoc &dl, EVT MemVT,
1137 MachineMemOperand *MMO);
1138
1139 /// Construct a node to track a Value* through the backend.
1140 SDValue getSrcValue(const Value *v);
1141
1142 /// Return an MDNodeSDNode which holds an MDNode.
1143 SDValue getMDNode(const MDNode *MD);
1144
1145 /// Return a bitcast using the SDLoc of the value operand, and casting to the
1146 /// provided type. Use getNode to set a custom SDLoc.
1147 SDValue getBitcast(EVT VT, SDValue V);
1148
1149 /// Return an AddrSpaceCastSDNode.
1150 SDValue getAddrSpaceCast(const SDLoc &dl, EVT VT, SDValue Ptr, unsigned SrcAS,
1151 unsigned DestAS);
1152
1153 /// Return the specified value casted to
1154 /// the target's desired shift amount type.
1155 SDValue getShiftAmountOperand(EVT LHSTy, SDValue Op);
1156
1157 /// Expand the specified \c ISD::VAARG node as the Legalize pass would.
1158 SDValue expandVAArg(SDNode *Node);
1159
1160 /// Expand the specified \c ISD::VACOPY node as the Legalize pass would.
1161 SDValue expandVACopy(SDNode *Node);
1162
1163 /// Returs an GlobalAddress of the function from the current module with
1164 /// name matching the given ExternalSymbol. Additionally can provide the
1165 /// matched function.
1166 /// Panics the function doesn't exists.
1167 SDValue getSymbolFunctionGlobalAddress(SDValue Op,
1168 Function **TargetFunction = nullptr);
1169
1170 /// *Mutate* the specified node in-place to have the
1171 /// specified operands. If the resultant node already exists in the DAG,
1172 /// this does not modify the specified node, instead it returns the node that
1173 /// already exists. If the resultant node does not exist in the DAG, the
1174 /// input node is returned. As a degenerate case, if you specify the same
1175 /// input operands as the node already has, the input node is returned.
1176 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op);
1177 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2);
1178 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1179 SDValue Op3);
1180 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1181 SDValue Op3, SDValue Op4);
1182 SDNode *UpdateNodeOperands(SDNode *N, SDValue Op1, SDValue Op2,
1183 SDValue Op3, SDValue Op4, SDValue Op5);
1184 SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops);
1185
1186 /// Creates a new TokenFactor containing \p Vals. If \p Vals contains 64k
1187 /// values or more, move values into new TokenFactors in 64k-1 blocks, until
1188 /// the final TokenFactor has less than 64k operands.
1189 SDValue getTokenFactor(const SDLoc &DL, SmallVectorImpl<SDValue> &Vals);
1190
1191 /// *Mutate* the specified machine node's memory references to the provided
1192 /// list.
1193 void setNodeMemRefs(MachineSDNode *N,
1194 ArrayRef<MachineMemOperand *> NewMemRefs);
1195
1196 // Propagates the change in divergence to users
1197 void updateDivergence(SDNode * N);
1198
1199 /// These are used for target selectors to *mutate* the
1200 /// specified node to have the specified return type, Target opcode, and
1201 /// operands. Note that target opcodes are stored as
1202 /// ~TargetOpcode in the node opcode field. The resultant node is returned.
1203 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT);
1204 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT, SDValue Op1);
1205 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1206 SDValue Op1, SDValue Op2);
1207 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1208 SDValue Op1, SDValue Op2, SDValue Op3);
1209 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT,
1210 ArrayRef<SDValue> Ops);
1211 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1, EVT VT2);
1212 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1213 EVT VT2, ArrayRef<SDValue> Ops);
1214 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1215 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1216 SDNode *SelectNodeTo(SDNode *N, unsigned TargetOpc, EVT VT1,
1217 EVT VT2, SDValue Op1);
1218 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, EVT VT1,
1219 EVT VT2, SDValue Op1, SDValue Op2);
1220 SDNode *SelectNodeTo(SDNode *N, unsigned MachineOpc, SDVTList VTs,
1221 ArrayRef<SDValue> Ops);
1222
1223 /// This *mutates* the specified node to have the specified
1224 /// return type, opcode, and operands.
1225 SDNode *MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs,
1226 ArrayRef<SDValue> Ops);
1227
1228 /// Mutate the specified strict FP node to its non-strict equivalent,
1229 /// unlinking the node from its chain and dropping the metadata arguments.
1230 /// The node must be a strict FP node.
1231 SDNode *mutateStrictFPToFP(SDNode *Node);
1232
1233 /// These are used for target selectors to create a new node
1234 /// with specified return type(s), MachineInstr opcode, and operands.
1235 ///
1236 /// Note that getMachineNode returns the resultant node. If there is already
1237 /// a node of the specified opcode and operands, it returns that node instead
1238 /// of the current one.
1239 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT);
1240 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1241 SDValue Op1);
1242 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1243 SDValue Op1, SDValue Op2);
1244 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1245 SDValue Op1, SDValue Op2, SDValue Op3);
1246 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT,
1247 ArrayRef<SDValue> Ops);
1248 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1249 EVT VT2, SDValue Op1, SDValue Op2);
1250 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1251 EVT VT2, SDValue Op1, SDValue Op2, SDValue Op3);
1252 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1253 EVT VT2, ArrayRef<SDValue> Ops);
1254 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1255 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2);
1256 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1257 EVT VT2, EVT VT3, SDValue Op1, SDValue Op2,
1258 SDValue Op3);
1259 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT1,
1260 EVT VT2, EVT VT3, ArrayRef<SDValue> Ops);
1261 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl,
1262 ArrayRef<EVT> ResultTys, ArrayRef<SDValue> Ops);
1263 MachineSDNode *getMachineNode(unsigned Opcode, const SDLoc &dl, SDVTList VTs,
1264 ArrayRef<SDValue> Ops);
1265
1266 /// A convenience function for creating TargetInstrInfo::EXTRACT_SUBREG nodes.
1267 SDValue getTargetExtractSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1268 SDValue Operand);
1269
1270 /// A convenience function for creating TargetInstrInfo::INSERT_SUBREG nodes.
1271 SDValue getTargetInsertSubreg(int SRIdx, const SDLoc &DL, EVT VT,
1272 SDValue Operand, SDValue Subreg);
1273
1274 /// Get the specified node if it's already available, or else return NULL.
1275 SDNode *getNodeIfExists(unsigned Opcode, SDVTList VTList, ArrayRef<SDValue> Ops,
1276 const SDNodeFlags Flags = SDNodeFlags());
1277
1278 /// Creates a SDDbgValue node.
1279 SDDbgValue *getDbgValue(DIVariable *Var, DIExpression *Expr, SDNode *N,
1280 unsigned R, bool IsIndirect, const DebugLoc &DL,
1281 unsigned O);
1282
1283 /// Creates a constant SDDbgValue node.
1284 SDDbgValue *getConstantDbgValue(DIVariable *Var, DIExpression *Expr,
1285 const Value *C, const DebugLoc &DL,
1286 unsigned O);
1287
1288 /// Creates a FrameIndex SDDbgValue node.
1289 SDDbgValue *getFrameIndexDbgValue(DIVariable *Var, DIExpression *Expr,
1290 unsigned FI, bool IsIndirect,
1291 const DebugLoc &DL, unsigned O);
1292
1293 /// Creates a VReg SDDbgValue node.
1294 SDDbgValue *getVRegDbgValue(DIVariable *Var, DIExpression *Expr,
1295 unsigned VReg, bool IsIndirect,
1296 const DebugLoc &DL, unsigned O);
1297
1298 /// Creates a SDDbgLabel node.
1299 SDDbgLabel *getDbgLabel(DILabel *Label, const DebugLoc &DL, unsigned O);
1300
1301 /// Transfer debug values from one node to another, while optionally
1302 /// generating fragment expressions for split-up values. If \p InvalidateDbg
1303 /// is set, debug values are invalidated after they are transferred.
1304 void transferDbgValues(SDValue From, SDValue To, unsigned OffsetInBits = 0,
1305 unsigned SizeInBits = 0, bool InvalidateDbg = true);
1306
1307 /// Remove the specified node from the system. If any of its
1308 /// operands then becomes dead, remove them as well. Inform UpdateListener
1309 /// for each node deleted.
1310 void RemoveDeadNode(SDNode *N);
1311
1312 /// This method deletes the unreachable nodes in the
1313 /// given list, and any nodes that become unreachable as a result.
1314 void RemoveDeadNodes(SmallVectorImpl<SDNode *> &DeadNodes);
1315
1316 /// Modify anything using 'From' to use 'To' instead.
1317 /// This can cause recursive merging of nodes in the DAG. Use the first
1318 /// version if 'From' is known to have a single result, use the second
1319 /// if you have two nodes with identical results (or if 'To' has a superset
1320 /// of the results of 'From'), use the third otherwise.
1321 ///
1322 /// These methods all take an optional UpdateListener, which (if not null) is
1323 /// informed about nodes that are deleted and modified due to recursive
1324 /// changes in the dag.
1325 ///
1326 /// These functions only replace all existing uses. It's possible that as
1327 /// these replacements are being performed, CSE may cause the From node
1328 /// to be given new uses. These new uses of From are left in place, and
1329 /// not automatically transferred to To.
1330 ///
1331 void ReplaceAllUsesWith(SDValue From, SDValue To);
1332 void ReplaceAllUsesWith(SDNode *From, SDNode *To);
1333 void ReplaceAllUsesWith(SDNode *From, const SDValue *To);
1334
1335 /// Replace any uses of From with To, leaving
1336 /// uses of other values produced by From.getNode() alone.
1337 void ReplaceAllUsesOfValueWith(SDValue From, SDValue To);
1338
1339 /// Like ReplaceAllUsesOfValueWith, but for multiple values at once.
1340 /// This correctly handles the case where
1341 /// there is an overlap between the From values and the To values.
1342 void ReplaceAllUsesOfValuesWith(const SDValue *From, const SDValue *To,
1343 unsigned Num);
1344
1345 /// If an existing load has uses of its chain, create a token factor node with
1346 /// that chain and the new memory node's chain and update users of the old
1347 /// chain to the token factor. This ensures that the new memory node will have
1348 /// the same relative memory dependency position as the old load. Returns the
1349 /// new merged load chain.
1350 SDValue makeEquivalentMemoryOrdering(LoadSDNode *Old, SDValue New);
1351
1352 /// Topological-sort the AllNodes list and a
1353 /// assign a unique node id for each node in the DAG based on their
1354 /// topological order. Returns the number of nodes.
1355 unsigned AssignTopologicalOrder();
1356
1357 /// Move node N in the AllNodes list to be immediately
1358 /// before the given iterator Position. This may be used to update the
1359 /// topological ordering when the list of nodes is modified.
1360 void RepositionNode(allnodes_iterator Position, SDNode *N) {
1361 AllNodes.insert(Position, AllNodes.remove(N));
1362 }
1363
1364 /// Returns an APFloat semantics tag appropriate for the given type. If VT is
1365 /// a vector type, the element semantics are returned.
1366 static const fltSemantics &EVTToAPFloatSemantics(EVT VT) {
1367 switch (VT.getScalarType().getSimpleVT().SimpleTy) {
1368 default: llvm_unreachable("Unknown FP format")::llvm::llvm_unreachable_internal("Unknown FP format", "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAG.h"
, 1368)
;
1369 case MVT::f16: return APFloat::IEEEhalf();
1370 case MVT::f32: return APFloat::IEEEsingle();
1371 case MVT::f64: return APFloat::IEEEdouble();
1372 case MVT::f80: return APFloat::x87DoubleExtended();
1373 case MVT::f128: return APFloat::IEEEquad();
1374 case MVT::ppcf128: return APFloat::PPCDoubleDouble();
1375 }
1376 }
1377
1378 /// Add a dbg_value SDNode. If SD is non-null that means the
1379 /// value is produced by SD.
1380 void AddDbgValue(SDDbgValue *DB, SDNode *SD, bool isParameter);
1381
1382 /// Add a dbg_label SDNode.
1383 void AddDbgLabel(SDDbgLabel *DB);
1384
1385 /// Get the debug values which reference the given SDNode.
1386 ArrayRef<SDDbgValue*> GetDbgValues(const SDNode* SD) const {
1387 return DbgInfo->getSDDbgValues(SD);
1388 }
1389
1390public:
1391 /// Return true if there are any SDDbgValue nodes associated
1392 /// with this SelectionDAG.
1393 bool hasDebugValues() const { return !DbgInfo->empty(); }
1394
1395 SDDbgInfo::DbgIterator DbgBegin() const { return DbgInfo->DbgBegin(); }
1396 SDDbgInfo::DbgIterator DbgEnd() const { return DbgInfo->DbgEnd(); }
1397
1398 SDDbgInfo::DbgIterator ByvalParmDbgBegin() const {
1399 return DbgInfo->ByvalParmDbgBegin();
1400 }
1401 SDDbgInfo::DbgIterator ByvalParmDbgEnd() const {
1402 return DbgInfo->ByvalParmDbgEnd();
1403 }
1404
1405 SDDbgInfo::DbgLabelIterator DbgLabelBegin() const {
1406 return DbgInfo->DbgLabelBegin();
1407 }
1408 SDDbgInfo::DbgLabelIterator DbgLabelEnd() const {
1409 return DbgInfo->DbgLabelEnd();
1410 }
1411
1412 /// To be invoked on an SDNode that is slated to be erased. This
1413 /// function mirrors \c llvm::salvageDebugInfo.
1414 void salvageDebugInfo(SDNode &N);
1415
1416 void dump() const;
1417
1418 /// Create a stack temporary, suitable for holding the specified value type.
1419 /// If minAlign is specified, the slot size will have at least that alignment.
1420 SDValue CreateStackTemporary(EVT VT, unsigned minAlign = 1);
1421
1422 /// Create a stack temporary suitable for holding either of the specified
1423 /// value types.
1424 SDValue CreateStackTemporary(EVT VT1, EVT VT2);
1425
1426 SDValue FoldSymbolOffset(unsigned Opcode, EVT VT,
1427 const GlobalAddressSDNode *GA,
1428 const SDNode *N2);
1429
1430 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1431 SDNode *N1, SDNode *N2);
1432
1433 SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1434 const ConstantSDNode *C1,
1435 const ConstantSDNode *C2);
1436
1437 SDValue FoldConstantVectorArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT,
1438 ArrayRef<SDValue> Ops,
1439 const SDNodeFlags Flags = SDNodeFlags());
1440
1441 /// Fold floating-point operations with 2 operands when both operands are
1442 /// constants and/or undefined.
1443 SDValue foldConstantFPMath(unsigned Opcode, const SDLoc &DL, EVT VT,
1444 SDValue N1, SDValue N2);
1445
1446 /// Constant fold a setcc to true or false.
1447 SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond,
1448 const SDLoc &dl);
1449
1450 /// See if the specified operand can be simplified with the knowledge that
1451 /// only the bits specified by DemandedBits are used. If so, return the
1452 /// simpler operand, otherwise return a null SDValue.
1453 ///
1454 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
1455 /// simplify nodes with multiple uses more aggressively.)
1456 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits);
1457
1458 /// See if the specified operand can be simplified with the knowledge that
1459 /// only the bits specified by DemandedBits are used in the elements specified
1460 /// by DemandedElts. If so, return the simpler operand, otherwise return a
1461 /// null SDValue.
1462 ///
1463 /// (This exists alongside SimplifyDemandedBits because GetDemandedBits can
1464 /// simplify nodes with multiple uses more aggressively.)
1465 SDValue GetDemandedBits(SDValue V, const APInt &DemandedBits,
1466 const APInt &DemandedElts);
1467
1468 /// Return true if the sign bit of Op is known to be zero.
1469 /// We use this predicate to simplify operations downstream.
1470 bool SignBitIsZero(SDValue Op, unsigned Depth = 0) const;
1471
1472 /// Return true if 'Op & Mask' is known to be zero. We
1473 /// use this predicate to simplify operations downstream. Op and Mask are
1474 /// known to be the same type.
1475 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1476 unsigned Depth = 0) const;
1477
1478 /// Return true if 'Op & Mask' is known to be zero in DemandedElts. We
1479 /// use this predicate to simplify operations downstream. Op and Mask are
1480 /// known to be the same type.
1481 bool MaskedValueIsZero(SDValue Op, const APInt &Mask,
1482 const APInt &DemandedElts, unsigned Depth = 0) const;
1483
1484 /// Return true if '(Op & Mask) == Mask'.
1485 /// Op and Mask are known to be the same type.
1486 bool MaskedValueIsAllOnes(SDValue Op, const APInt &Mask,
1487 unsigned Depth = 0) const;
1488
1489 /// Determine which bits of Op are known to be either zero or one and return
1490 /// them in Known. For vectors, the known bits are those that are shared by
1491 /// every vector element.
1492 /// Targets can implement the computeKnownBitsForTargetNode method in the
1493 /// TargetLowering class to allow target nodes to be understood.
1494 KnownBits computeKnownBits(SDValue Op, unsigned Depth = 0) const;
1495
1496 /// Determine which bits of Op are known to be either zero or one and return
1497 /// them in Known. The DemandedElts argument allows us to only collect the
1498 /// known bits that are shared by the requested vector elements.
1499 /// Targets can implement the computeKnownBitsForTargetNode method in the
1500 /// TargetLowering class to allow target nodes to be understood.
1501 KnownBits computeKnownBits(SDValue Op, const APInt &DemandedElts,
1502 unsigned Depth = 0) const;
1503
1504 /// Used to represent the possible overflow behavior of an operation.
1505 /// Never: the operation cannot overflow.
1506 /// Always: the operation will always overflow.
1507 /// Sometime: the operation may or may not overflow.
1508 enum OverflowKind {
1509 OFK_Never,
1510 OFK_Sometime,
1511 OFK_Always,
1512 };
1513
1514 /// Determine if the result of the addition of 2 node can overflow.
1515 OverflowKind computeOverflowKind(SDValue N0, SDValue N1) const;
1516
1517 /// Test if the given value is known to have exactly one bit set. This differs
1518 /// from computeKnownBits in that it doesn't necessarily determine which bit
1519 /// is set.
1520 bool isKnownToBeAPowerOfTwo(SDValue Val) const;
1521
1522 /// Return the number of times the sign bit of the register is replicated into
1523 /// the other bits. We know that at least 1 bit is always equal to the sign
1524 /// bit (itself), but other cases can give us information. For example,
1525 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
1526 /// to each other, so we return 3. Targets can implement the
1527 /// ComputeNumSignBitsForTarget method in the TargetLowering class to allow
1528 /// target nodes to be understood.
1529 unsigned ComputeNumSignBits(SDValue Op, unsigned Depth = 0) const;
1530
1531 /// Return the number of times the sign bit of the register is replicated into
1532 /// the other bits. We know that at least 1 bit is always equal to the sign
1533 /// bit (itself), but other cases can give us information. For example,
1534 /// immediately after an "SRA X, 2", we know that the top 3 bits are all equal
1535 /// to each other, so we return 3. The DemandedElts argument allows
1536 /// us to only collect the minimum sign bits of the requested vector elements.
1537 /// Targets can implement the ComputeNumSignBitsForTarget method in the
1538 /// TargetLowering class to allow target nodes to be understood.
1539 unsigned ComputeNumSignBits(SDValue Op, const APInt &DemandedElts,
1540 unsigned Depth = 0) const;
1541
1542 /// Return true if the specified operand is an ISD::ADD with a ConstantSDNode
1543 /// on the right-hand side, or if it is an ISD::OR with a ConstantSDNode that
1544 /// is guaranteed to have the same semantics as an ADD. This handles the
1545 /// equivalence:
1546 /// X|Cst == X+Cst iff X&Cst = 0.
1547 bool isBaseWithConstantOffset(SDValue Op) const;
1548
1549 /// Test whether the given SDValue is known to never be NaN. If \p SNaN is
1550 /// true, returns if \p Op is known to never be a signaling NaN (it may still
1551 /// be a qNaN).
1552 bool isKnownNeverNaN(SDValue Op, bool SNaN = false, unsigned Depth = 0) const;
1553
1554 /// \returns true if \p Op is known to never be a signaling NaN.
1555 bool isKnownNeverSNaN(SDValue Op, unsigned Depth = 0) const {
1556 return isKnownNeverNaN(Op, true, Depth);
1557 }
1558
1559 /// Test whether the given floating point SDValue is known to never be
1560 /// positive or negative zero.
1561 bool isKnownNeverZeroFloat(SDValue Op) const;
1562
1563 /// Test whether the given SDValue is known to contain non-zero value(s).
1564 bool isKnownNeverZero(SDValue Op) const;
1565
1566 /// Test whether two SDValues are known to compare equal. This
1567 /// is true if they are the same value, or if one is negative zero and the
1568 /// other positive zero.
1569 bool isEqualTo(SDValue A, SDValue B) const;
1570
1571 /// Return true if A and B have no common bits set. As an example, this can
1572 /// allow an 'add' to be transformed into an 'or'.
1573 bool haveNoCommonBitsSet(SDValue A, SDValue B) const;
1574
1575 /// Test whether \p V has a splatted value for all the demanded elements.
1576 ///
1577 /// On success \p UndefElts will indicate the elements that have UNDEF
1578 /// values instead of the splat value, this is only guaranteed to be correct
1579 /// for \p DemandedElts.
1580 ///
1581 /// NOTE: The function will return true for a demanded splat of UNDEF values.
1582 bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts);
1583
1584 /// Test whether \p V has a splatted value.
1585 bool isSplatValue(SDValue V, bool AllowUndefs = false);
1586
1587 /// If V is a splatted value, return the source vector and its splat index.
1588 SDValue getSplatSourceVector(SDValue V, int &SplatIndex);
1589
1590 /// If V is a splat vector, return its scalar source operand by extracting
1591 /// that element from the source vector.
1592 SDValue getSplatValue(SDValue V);
1593
1594 /// Match a binop + shuffle pyramid that represents a horizontal reduction
1595 /// over the elements of a vector starting from the EXTRACT_VECTOR_ELT node /p
1596 /// Extract. The reduction must use one of the opcodes listed in /p
1597 /// CandidateBinOps and on success /p BinOp will contain the matching opcode.
1598 /// Returns the vector that is being reduced on, or SDValue() if a reduction
1599 /// was not matched. If \p AllowPartials is set then in the case of a
1600 /// reduction pattern that only matches the first few stages, the extracted
1601 /// subvector of the start of the reduction is returned.
1602 SDValue matchBinOpReduction(SDNode *Extract, ISD::NodeType &BinOp,
1603 ArrayRef<ISD::NodeType> CandidateBinOps,
1604 bool AllowPartials = false);
1605
1606 /// Utility function used by legalize and lowering to
1607 /// "unroll" a vector operation by splitting out the scalars and operating
1608 /// on each element individually. If the ResNE is 0, fully unroll the vector
1609 /// op. If ResNE is less than the width of the vector op, unroll up to ResNE.
1610 /// If the ResNE is greater than the width of the vector op, unroll the
1611 /// vector op and fill the end of the resulting vector with UNDEFS.
1612 SDValue UnrollVectorOp(SDNode *N, unsigned ResNE = 0);
1613
1614 /// Like UnrollVectorOp(), but for the [US](ADD|SUB|MUL)O family of opcodes.
1615 /// This is a separate function because those opcodes have two results.
1616 std::pair<SDValue, SDValue> UnrollVectorOverflowOp(SDNode *N,
1617 unsigned ResNE = 0);
1618
1619 /// Return true if loads are next to each other and can be
1620 /// merged. Check that both are nonvolatile and if LD is loading
1621 /// 'Bytes' bytes from a location that is 'Dist' units away from the
1622 /// location that the 'Base' load is loading from.
1623 bool areNonVolatileConsecutiveLoads(LoadSDNode *LD, LoadSDNode *Base,
1624 unsigned Bytes, int Dist) const;
1625
1626 /// Infer alignment of a load / store address. Return 0 if
1627 /// it cannot be inferred.
1628 unsigned InferPtrAlignment(SDValue Ptr) const;
1629
1630 /// Compute the VTs needed for the low/hi parts of a type
1631 /// which is split (or expanded) into two not necessarily identical pieces.
1632 std::pair<EVT, EVT> GetSplitDestVTs(const EVT &VT) const;
1633
1634 /// Split the vector with EXTRACT_SUBVECTOR using the provides
1635 /// VTs and return the low/high part.
1636 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL,
1637 const EVT &LoVT, const EVT &HiVT);
1638
1639 /// Split the vector with EXTRACT_SUBVECTOR and return the low/high part.
1640 std::pair<SDValue, SDValue> SplitVector(const SDValue &N, const SDLoc &DL) {
1641 EVT LoVT, HiVT;
1642 std::tie(LoVT, HiVT) = GetSplitDestVTs(N.getValueType());
1643 return SplitVector(N, DL, LoVT, HiVT);
1644 }
1645
1646 /// Split the node's operand with EXTRACT_SUBVECTOR and
1647 /// return the low/high part.
1648 std::pair<SDValue, SDValue> SplitVectorOperand(const SDNode *N, unsigned OpNo)
1649 {
1650 return SplitVector(N->getOperand(OpNo), SDLoc(N));
1651 }
1652
1653 /// Widen the vector up to the next power of two using INSERT_SUBVECTOR.
1654 SDValue WidenVector(const SDValue &N, const SDLoc &DL);
1655
1656 /// Append the extracted elements from Start to Count out of the vector Op
1657 /// in Args. If Count is 0, all of the elements will be extracted.
1658 void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args,
1659 unsigned Start = 0, unsigned Count = 0);
1660
1661 /// Compute the default alignment value for the given type.
1662 unsigned getEVTAlignment(EVT MemoryVT) const;
1663
1664 /// Test whether the given value is a constant int or similar node.
1665 SDNode *isConstantIntBuildVectorOrConstantInt(SDValue N);
1666
1667 /// Test whether the given value is a constant FP or similar node.
1668 SDNode *isConstantFPBuildVectorOrConstantFP(SDValue N);
1669
1670 /// \returns true if \p N is any kind of constant or build_vector of
1671 /// constants, int or float. If a vector, it may not necessarily be a splat.
1672 inline bool isConstantValueOfAnyType(SDValue N) {
1673 return isConstantIntBuildVectorOrConstantInt(N) ||
1674 isConstantFPBuildVectorOrConstantFP(N);
1675 }
1676
1677 void addCallSiteInfo(const SDNode *CallNode, CallSiteInfoImpl &&CallInfo) {
1678 SDCallSiteDbgInfo[CallNode].CSInfo = std::move(CallInfo);
1679 }
1680
1681 CallSiteInfo getSDCallSiteInfo(const SDNode *CallNode) {
1682 auto I = SDCallSiteDbgInfo.find(CallNode);
1683 if (I != SDCallSiteDbgInfo.end())
1684 return std::move(I->second).CSInfo;
1685 return CallSiteInfo();
1686 }
1687
1688 void addHeapAllocSite(const SDNode *Node, MDNode *MD) {
1689 SDCallSiteDbgInfo[Node].HeapAllocSite = MD;
1690 }
1691
1692 /// Return the HeapAllocSite type associated with the SDNode, if it exists.
1693 MDNode *getHeapAllocSite(const SDNode *Node) {
1694 auto It = SDCallSiteDbgInfo.find(Node);
1695 if (It == SDCallSiteDbgInfo.end())
1696 return nullptr;
1697 return It->second.HeapAllocSite;
1698 }
1699
1700private:
1701 void InsertNode(SDNode *N);
1702 bool RemoveNodeFromCSEMaps(SDNode *N);
1703 void AddModifiedNodeToCSEMaps(SDNode *N);
1704 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op, void *&InsertPos);
1705 SDNode *FindModifiedNodeSlot(SDNode *N, SDValue Op1, SDValue Op2,
1706 void *&InsertPos);
1707 SDNode *FindModifiedNodeSlot(SDNode *N, ArrayRef<SDValue> Ops,
1708 void *&InsertPos);
1709 SDNode *UpdateSDLocOnMergeSDNode(SDNode *N, const SDLoc &loc);
1710
1711 void DeleteNodeNotInCSEMaps(SDNode *N);
1712 void DeallocateNode(SDNode *N);
1713
1714 void allnodes_clear();
1715
1716 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
1717 /// not, return the insertion token that will make insertion faster. This
1718 /// overload is for nodes other than Constant or ConstantFP, use the other one
1719 /// for those.
1720 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, void *&InsertPos);
1721
1722 /// Look up the node specified by ID in CSEMap. If it exists, return it. If
1723 /// not, return the insertion token that will make insertion faster. Performs
1724 /// additional processing for constant nodes.
1725 SDNode *FindNodeOrInsertPos(const FoldingSetNodeID &ID, const SDLoc &DL,
1726 void *&InsertPos);
1727
1728 /// List of non-single value types.
1729 FoldingSet<SDVTListNode> VTListMap;
1730
1731 /// Maps to auto-CSE operations.
1732 std::vector<CondCodeSDNode*> CondCodeNodes;
1733
1734 std::vector<SDNode*> ValueTypeNodes;
1735 std::map<EVT, SDNode*, EVT::compareRawBits> ExtendedValueTypeNodes;
1736 StringMap<SDNode*> ExternalSymbols;
1737
1738 std::map<std::pair<std::string, unsigned>, SDNode *> TargetExternalSymbols;
1739 DenseMap<MCSymbol *, SDNode *> MCSymbols;
1740};
1741
1742template <> struct GraphTraits<SelectionDAG*> : public GraphTraits<SDNode*> {
1743 using nodes_iterator = pointer_iterator<SelectionDAG::allnodes_iterator>;
1744
1745 static nodes_iterator nodes_begin(SelectionDAG *G) {
1746 return nodes_iterator(G->allnodes_begin());
1747 }
1748
1749 static nodes_iterator nodes_end(SelectionDAG *G) {
1750 return nodes_iterator(G->allnodes_end());
1751 }
1752};
1753
1754template <class TargetMemSDNode>
1755SDValue SelectionDAG::getTargetMemSDNode(SDVTList VTs,
1756 ArrayRef<SDValue> Ops,
1757 const SDLoc &dl, EVT MemVT,
1758 MachineMemOperand *MMO) {
1759 /// Compose node ID and try to find an existing node.
1760 FoldingSetNodeID ID;
1761 unsigned Opcode =
1762 TargetMemSDNode(dl.getIROrder(), DebugLoc(), VTs, MemVT, MMO).getOpcode();
1763 ID.AddInteger(Opcode);
1764 ID.AddPointer(VTs.VTs);
1765 for (auto& Op : Ops) {
1766 ID.AddPointer(Op.getNode());
1767 ID.AddInteger(Op.getResNo());
1768 }
1769 ID.AddInteger(MemVT.getRawBits());
1770 ID.AddInteger(MMO->getPointerInfo().getAddrSpace());
1771 ID.AddInteger(getSyntheticNodeSubclassData<TargetMemSDNode>(
1772 dl.getIROrder(), VTs, MemVT, MMO));
1773
1774 void *IP = nullptr;
1775 if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) {
1776 cast<TargetMemSDNode>(E)->refineAlignment(MMO);
1777 return SDValue(E, 0);
1778 }
1779
1780 /// Existing node was not found. Create a new one.
1781 auto *N = newSDNode<TargetMemSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs,
1782 MemVT, MMO);
1783 createOperands(N, Ops);
1784 CSEMap.InsertNode(N, IP);
1785 InsertNode(N);
1786 return SDValue(N, 0);
1787}
1788
1789} // end namespace llvm
1790
1791#endif // LLVM_CODEGEN_SELECTIONDAG_H

/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/ValueTypes.h"
34#include "llvm/IR/Constants.h"
35#include "llvm/IR/DebugLoc.h"
36#include "llvm/IR/Instruction.h"
37#include "llvm/IR/Instructions.h"
38#include "llvm/IR/Metadata.h"
39#include "llvm/IR/Operator.h"
40#include "llvm/Support/AlignOf.h"
41#include "llvm/Support/AtomicOrdering.h"
42#include "llvm/Support/Casting.h"
43#include "llvm/Support/ErrorHandling.h"
44#include "llvm/Support/MachineValueType.h"
45#include <algorithm>
46#include <cassert>
47#include <climits>
48#include <cstddef>
49#include <cstdint>
50#include <cstring>
51#include <iterator>
52#include <string>
53#include <tuple>
54
55namespace llvm {
56
57class APInt;
58class Constant;
59template <typename T> struct DenseMapInfo;
60class GlobalValue;
61class MachineBasicBlock;
62class MachineConstantPoolValue;
63class MCSymbol;
64class raw_ostream;
65class SDNode;
66class SelectionDAG;
67class Type;
68class Value;
69
70void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
71 bool force = false);
72
73/// This represents a list of ValueType's that has been intern'd by
74/// a SelectionDAG. Instances of this simple value class are returned by
75/// SelectionDAG::getVTList(...).
76///
77struct SDVTList {
78 const EVT *VTs;
79 unsigned int NumVTs;
80};
81
82namespace ISD {
83
84 /// Node predicates
85
86 /// If N is a BUILD_VECTOR node whose elements are all the same constant or
87 /// undefined, return true and return the constant value in \p SplatValue.
88 bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
89
90 /// Return true if the specified node is a BUILD_VECTOR where all of the
91 /// elements are ~0 or undef.
92 bool isBuildVectorAllOnes(const SDNode *N);
93
94 /// Return true if the specified node is a BUILD_VECTOR where all of the
95 /// elements are 0 or undef.
96 bool isBuildVectorAllZeros(const SDNode *N);
97
98 /// Return true if the specified node is a BUILD_VECTOR node of all
99 /// ConstantSDNode or undef.
100 bool isBuildVectorOfConstantSDNodes(const SDNode *N);
101
102 /// Return true if the specified node is a BUILD_VECTOR node of all
103 /// ConstantFPSDNode or undef.
104 bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
105
106 /// Return true if the node has at least one operand and all operands of the
107 /// specified node are ISD::UNDEF.
108 bool allOperandsUndef(const SDNode *N);
109
110} // end namespace ISD
111
112//===----------------------------------------------------------------------===//
113/// Unlike LLVM values, Selection DAG nodes may return multiple
114/// values as the result of a computation. Many nodes return multiple values,
115/// from loads (which define a token and a return value) to ADDC (which returns
116/// a result and a carry value), to calls (which may return an arbitrary number
117/// of values).
118///
119/// As such, each use of a SelectionDAG computation must indicate the node that
120/// computes it as well as which return value to use from that node. This pair
121/// of information is represented with the SDValue value type.
122///
123class SDValue {
124 friend struct DenseMapInfo<SDValue>;
125
126 SDNode *Node = nullptr; // The node defining the value we are using.
127 unsigned ResNo = 0; // Which return value of the node we are using.
128
129public:
130 SDValue() = default;
131 SDValue(SDNode *node, unsigned resno);
132
133 /// get the index which selects a specific result in the SDNode
134 unsigned getResNo() const { return ResNo; }
135
136 /// get the SDNode which holds the desired result
137 SDNode *getNode() const { return Node; }
138
139 /// set the SDNode
140 void setNode(SDNode *N) { Node = N; }
141
142 inline SDNode *operator->() const { return Node; }
143
144 bool operator==(const SDValue &O) const {
145 return Node == O.Node && ResNo == O.ResNo;
146 }
147 bool operator!=(const SDValue &O) const {
148 return !operator==(O);
149 }
150 bool operator<(const SDValue &O) const {
151 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
152 }
153 explicit operator bool() const {
154 return Node != nullptr;
155 }
156
157 SDValue getValue(unsigned R) const {
158 return SDValue(Node, R);
159 }
160
161 /// Return true if this node is an operand of N.
162 bool isOperandOf(const SDNode *N) const;
163
164 /// Return the ValueType of the referenced return value.
165 inline EVT getValueType() const;
166
167 /// Return the simple ValueType of the referenced return value.
168 MVT getSimpleValueType() const {
169 return getValueType().getSimpleVT();
170 }
171
172 /// Returns the size of the value in bits.
173 unsigned getValueSizeInBits() const {
174 return getValueType().getSizeInBits();
175 }
176
177 unsigned getScalarValueSizeInBits() const {
178 return getValueType().getScalarType().getSizeInBits();
179 }
180
181 // Forwarding methods - These forward to the corresponding methods in SDNode.
182 inline unsigned getOpcode() const;
183 inline unsigned getNumOperands() const;
184 inline const SDValue &getOperand(unsigned i) const;
185 inline uint64_t getConstantOperandVal(unsigned i) const;
186 inline const APInt &getConstantOperandAPInt(unsigned i) const;
187 inline bool isTargetMemoryOpcode() const;
188 inline bool isTargetOpcode() const;
189 inline bool isMachineOpcode() const;
190 inline bool isUndef() const;
191 inline unsigned getMachineOpcode() const;
192 inline const DebugLoc &getDebugLoc() const;
193 inline void dump() const;
194 inline void dump(const SelectionDAG *G) const;
195 inline void dumpr() const;
196 inline void dumpr(const SelectionDAG *G) const;
197
198 /// Return true if this operand (which must be a chain) reaches the
199 /// specified operand without crossing any side-effecting instructions.
200 /// In practice, this looks through token factors and non-volatile loads.
201 /// In order to remain efficient, this only
202 /// looks a couple of nodes in, it does not do an exhaustive search.
203 bool reachesChainWithoutSideEffects(SDValue Dest,
204 unsigned Depth = 2) const;
205
206 /// Return true if there are no nodes using value ResNo of Node.
207 inline bool use_empty() const;
208
209 /// Return true if there is exactly one node using value ResNo of Node.
210 inline bool hasOneUse() const;
211};
212
213template<> struct DenseMapInfo<SDValue> {
214 static inline SDValue getEmptyKey() {
215 SDValue V;
216 V.ResNo = -1U;
217 return V;
218 }
219
220 static inline SDValue getTombstoneKey() {
221 SDValue V;
222 V.ResNo = -2U;
223 return V;
224 }
225
226 static unsigned getHashValue(const SDValue &Val) {
227 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
228 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
229 }
230
231 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
232 return LHS == RHS;
233 }
234};
235
236/// Allow casting operators to work directly on
237/// SDValues as if they were SDNode*'s.
238template<> struct simplify_type<SDValue> {
239 using SimpleType = SDNode *;
240
241 static SimpleType getSimplifiedValue(SDValue &Val) {
242 return Val.getNode();
243 }
244};
245template<> struct simplify_type<const SDValue> {
246 using SimpleType = /*const*/ SDNode *;
247
248 static SimpleType getSimplifiedValue(const SDValue &Val) {
249 return Val.getNode();
250 }
251};
252
253/// Represents a use of a SDNode. This class holds an SDValue,
254/// which records the SDNode being used and the result number, a
255/// pointer to the SDNode using the value, and Next and Prev pointers,
256/// which link together all the uses of an SDNode.
257///
258class SDUse {
259 /// Val - The value being used.
260 SDValue Val;
261 /// User - The user of this value.
262 SDNode *User = nullptr;
263 /// Prev, Next - Pointers to the uses list of the SDNode referred by
264 /// this operand.
265 SDUse **Prev = nullptr;
266 SDUse *Next = nullptr;
267
268public:
269 SDUse() = default;
270 SDUse(const SDUse &U) = delete;
271 SDUse &operator=(const SDUse &) = delete;
272
273 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
274 operator const SDValue&() const { return Val; }
275
276 /// If implicit conversion to SDValue doesn't work, the get() method returns
277 /// the SDValue.
278 const SDValue &get() const { return Val; }
279
280 /// This returns the SDNode that contains this Use.
281 SDNode *getUser() { return User; }
282
283 /// Get the next SDUse in the use list.
284 SDUse *getNext() const { return Next; }
285
286 /// Convenience function for get().getNode().
287 SDNode *getNode() const { return Val.getNode(); }
288 /// Convenience function for get().getResNo().
289 unsigned getResNo() const { return Val.getResNo(); }
290 /// Convenience function for get().getValueType().
291 EVT getValueType() const { return Val.getValueType(); }
292
293 /// Convenience function for get().operator==
294 bool operator==(const SDValue &V) const {
295 return Val == V;
296 }
297
298 /// Convenience function for get().operator!=
299 bool operator!=(const SDValue &V) const {
300 return Val != V;
301 }
302
303 /// Convenience function for get().operator<
304 bool operator<(const SDValue &V) const {
305 return Val < V;
306 }
307
308private:
309 friend class SelectionDAG;
310 friend class SDNode;
311 // TODO: unfriend HandleSDNode once we fix its operand handling.
312 friend class HandleSDNode;
313
314 void setUser(SDNode *p) { User = p; }
315
316 /// Remove this use from its existing use list, assign it the
317 /// given value, and add it to the new value's node's use list.
318 inline void set(const SDValue &V);
319 /// Like set, but only supports initializing a newly-allocated
320 /// SDUse with a non-null value.
321 inline void setInitial(const SDValue &V);
322 /// Like set, but only sets the Node portion of the value,
323 /// leaving the ResNo portion unmodified.
324 inline void setNode(SDNode *N);
325
326 void addToList(SDUse **List) {
327 Next = *List;
328 if (Next) Next->Prev = &Next;
329 Prev = List;
330 *List = this;
331 }
332
333 void removeFromList() {
334 *Prev = Next;
335 if (Next) Next->Prev = Prev;
336 }
337};
338
339/// simplify_type specializations - Allow casting operators to work directly on
340/// SDValues as if they were SDNode*'s.
341template<> struct simplify_type<SDUse> {
342 using SimpleType = SDNode *;
343
344 static SimpleType getSimplifiedValue(SDUse &Val) {
345 return Val.getNode();
346 }
347};
348
349/// These are IR-level optimization flags that may be propagated to SDNodes.
350/// TODO: This data structure should be shared by the IR optimizer and the
351/// the backend.
352struct SDNodeFlags {
353private:
354 // This bit is used to determine if the flags are in a defined state.
355 // Flag bits can only be masked out during intersection if the masking flags
356 // are defined.
357 bool AnyDefined : 1;
358
359 bool NoUnsignedWrap : 1;
360 bool NoSignedWrap : 1;
361 bool Exact : 1;
362 bool NoNaNs : 1;
363 bool NoInfs : 1;
364 bool NoSignedZeros : 1;
365 bool AllowReciprocal : 1;
366 bool VectorReduction : 1;
367 bool AllowContract : 1;
368 bool ApproximateFuncs : 1;
369 bool AllowReassociation : 1;
370
371 // We assume instructions do not raise floating-point exceptions by default,
372 // and only those marked explicitly may do so. We could choose to represent
373 // this via a positive "FPExcept" flags like on the MI level, but having a
374 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
375 // intersection logic more straightforward.
376 bool NoFPExcept : 1;
377
378public:
379 /// Default constructor turns off all optimization flags.
380 SDNodeFlags()
381 : AnyDefined(false), NoUnsignedWrap(false), NoSignedWrap(false),
382 Exact(false), NoNaNs(false), NoInfs(false),
383 NoSignedZeros(false), AllowReciprocal(false), VectorReduction(false),
384 AllowContract(false), ApproximateFuncs(false),
385 AllowReassociation(false), NoFPExcept(true) {}
386
387 /// Propagate the fast-math-flags from an IR FPMathOperator.
388 void copyFMF(const FPMathOperator &FPMO) {
389 setNoNaNs(FPMO.hasNoNaNs());
390 setNoInfs(FPMO.hasNoInfs());
391 setNoSignedZeros(FPMO.hasNoSignedZeros());
392 setAllowReciprocal(FPMO.hasAllowReciprocal());
393 setAllowContract(FPMO.hasAllowContract());
394 setApproximateFuncs(FPMO.hasApproxFunc());
395 setAllowReassociation(FPMO.hasAllowReassoc());
396 }
397
398 /// Sets the state of the flags to the defined state.
399 void setDefined() { AnyDefined = true; }
400 /// Returns true if the flags are in a defined state.
401 bool isDefined() const { return AnyDefined; }
402
403 // These are mutators for each flag.
404 void setNoUnsignedWrap(bool b) {
405 setDefined();
406 NoUnsignedWrap = b;
407 }
408 void setNoSignedWrap(bool b) {
409 setDefined();
410 NoSignedWrap = b;
411 }
412 void setExact(bool b) {
413 setDefined();
414 Exact = b;
415 }
416 void setNoNaNs(bool b) {
417 setDefined();
418 NoNaNs = b;
419 }
420 void setNoInfs(bool b) {
421 setDefined();
422 NoInfs = b;
423 }
424 void setNoSignedZeros(bool b) {
425 setDefined();
426 NoSignedZeros = b;
427 }
428 void setAllowReciprocal(bool b) {
429 setDefined();
430 AllowReciprocal = b;
431 }
432 void setVectorReduction(bool b) {
433 setDefined();
434 VectorReduction = b;
435 }
436 void setAllowContract(bool b) {
437 setDefined();
438 AllowContract = b;
439 }
440 void setApproximateFuncs(bool b) {
441 setDefined();
442 ApproximateFuncs = b;
443 }
444 void setAllowReassociation(bool b) {
445 setDefined();
446 AllowReassociation = b;
447 }
448 void setFPExcept(bool b) {
449 setDefined();
450 NoFPExcept = !b;
451 }
452
453 // These are accessors for each flag.
454 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
455 bool hasNoSignedWrap() const { return NoSignedWrap; }
456 bool hasExact() const { return Exact; }
457 bool hasNoNaNs() const { return NoNaNs; }
458 bool hasNoInfs() const { return NoInfs; }
459 bool hasNoSignedZeros() const { return NoSignedZeros; }
460 bool hasAllowReciprocal() const { return AllowReciprocal; }
461 bool hasVectorReduction() const { return VectorReduction; }
462 bool hasAllowContract() const { return AllowContract; }
463 bool hasApproximateFuncs() const { return ApproximateFuncs; }
464 bool hasAllowReassociation() const { return AllowReassociation; }
465 bool hasFPExcept() const { return !NoFPExcept; }
466
467 bool isFast() const {
468 return NoSignedZeros && AllowReciprocal && NoNaNs && NoInfs && NoFPExcept &&
469 AllowContract && ApproximateFuncs && AllowReassociation;
470 }
471
472 /// Clear any flags in this flag set that aren't also set in Flags.
473 /// If the given Flags are undefined then don't do anything.
474 void intersectWith(const SDNodeFlags Flags) {
475 if (!Flags.isDefined())
476 return;
477 NoUnsignedWrap &= Flags.NoUnsignedWrap;
478 NoSignedWrap &= Flags.NoSignedWrap;
479 Exact &= Flags.Exact;
480 NoNaNs &= Flags.NoNaNs;
481 NoInfs &= Flags.NoInfs;
482 NoSignedZeros &= Flags.NoSignedZeros;
483 AllowReciprocal &= Flags.AllowReciprocal;
484 VectorReduction &= Flags.VectorReduction;
485 AllowContract &= Flags.AllowContract;
486 ApproximateFuncs &= Flags.ApproximateFuncs;
487 AllowReassociation &= Flags.AllowReassociation;
488 NoFPExcept &= Flags.NoFPExcept;
489 }
490};
491
492/// Represents one node in the SelectionDAG.
493///
494class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
495private:
496 /// The operation that this node performs.
497 int16_t NodeType;
498
499protected:
500 // We define a set of mini-helper classes to help us interpret the bits in our
501 // SubclassData. These are designed to fit within a uint16_t so they pack
502 // with NodeType.
503
504#if defined(_AIX) && (!defined(__GNUC__4) || defined(__ibmxl__))
505// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
506// and give the `pack` pragma push semantics.
507#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
508#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
509#else
510#define BEGIN_TWO_BYTE_PACK()
511#define END_TWO_BYTE_PACK()
512#endif
513
514BEGIN_TWO_BYTE_PACK()
515 class SDNodeBitfields {
516 friend class SDNode;
517 friend class MemIntrinsicSDNode;
518 friend class MemSDNode;
519 friend class SelectionDAG;
520
521 uint16_t HasDebugValue : 1;
522 uint16_t IsMemIntrinsic : 1;
523 uint16_t IsDivergent : 1;
524 };
525 enum { NumSDNodeBits = 3 };
526
527 class ConstantSDNodeBitfields {
528 friend class ConstantSDNode;
529
530 uint16_t : NumSDNodeBits;
531
532 uint16_t IsOpaque : 1;
533 };
534
535 class MemSDNodeBitfields {
536 friend class MemSDNode;
537 friend class MemIntrinsicSDNode;
538 friend class AtomicSDNode;
539
540 uint16_t : NumSDNodeBits;
541
542 uint16_t IsVolatile : 1;
543 uint16_t IsNonTemporal : 1;
544 uint16_t IsDereferenceable : 1;
545 uint16_t IsInvariant : 1;
546 };
547 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
548
549 class LSBaseSDNodeBitfields {
550 friend class LSBaseSDNode;
551 friend class MaskedGatherScatterSDNode;
552
553 uint16_t : NumMemSDNodeBits;
554
555 // This storage is shared between disparate class hierarchies to hold an
556 // enumeration specific to the class hierarchy in use.
557 // LSBaseSDNode => enum ISD::MemIndexedMode
558 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
559 uint16_t AddressingMode : 3;
560 };
561 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
562
563 class LoadSDNodeBitfields {
564 friend class LoadSDNode;
565 friend class MaskedLoadSDNode;
566
567 uint16_t : NumLSBaseSDNodeBits;
568
569 uint16_t ExtTy : 2; // enum ISD::LoadExtType
570 uint16_t IsExpanding : 1;
571 };
572
573 class StoreSDNodeBitfields {
574 friend class StoreSDNode;
575 friend class MaskedStoreSDNode;
576
577 uint16_t : NumLSBaseSDNodeBits;
578
579 uint16_t IsTruncating : 1;
580 uint16_t IsCompressing : 1;
581 };
582
583 union {
584 char RawSDNodeBits[sizeof(uint16_t)];
585 SDNodeBitfields SDNodeBits;
586 ConstantSDNodeBitfields ConstantSDNodeBits;
587 MemSDNodeBitfields MemSDNodeBits;
588 LSBaseSDNodeBitfields LSBaseSDNodeBits;
589 LoadSDNodeBitfields LoadSDNodeBits;
590 StoreSDNodeBitfields StoreSDNodeBits;
591 };
592END_TWO_BYTE_PACK()
593#undef BEGIN_TWO_BYTE_PACK
594#undef END_TWO_BYTE_PACK
595
596 // RawSDNodeBits must cover the entirety of the union. This means that all of
597 // the union's members must have size <= RawSDNodeBits. We write the RHS as
598 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
599 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
600 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
601 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
602 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
603 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
604 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
605
606private:
607 friend class SelectionDAG;
608 // TODO: unfriend HandleSDNode once we fix its operand handling.
609 friend class HandleSDNode;
610
611 /// Unique id per SDNode in the DAG.
612 int NodeId = -1;
613
614 /// The values that are used by this operation.
615 SDUse *OperandList = nullptr;
616
617 /// The types of the values this node defines. SDNode's may
618 /// define multiple values simultaneously.
619 const EVT *ValueList;
620
621 /// List of uses for this SDNode.
622 SDUse *UseList = nullptr;
623
624 /// The number of entries in the Operand/Value list.
625 unsigned short NumOperands = 0;
626 unsigned short NumValues;
627
628 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
629 // original LLVM instructions.
630 // This is used for turning off scheduling, because we'll forgo
631 // the normal scheduling algorithms and output the instructions according to
632 // this ordering.
633 unsigned IROrder;
634
635 /// Source line information.
636 DebugLoc debugLoc;
637
638 /// Return a pointer to the specified value type.
639 static const EVT *getValueTypeList(EVT VT);
640
641 SDNodeFlags Flags;
642
643public:
644 /// Unique and persistent id per SDNode in the DAG.
645 /// Used for debug printing.
646 uint16_t PersistentId;
647
648 //===--------------------------------------------------------------------===//
649 // Accessors
650 //
651
652 /// Return the SelectionDAG opcode value for this node. For
653 /// pre-isel nodes (those for which isMachineOpcode returns false), these
654 /// are the opcode values in the ISD and <target>ISD namespaces. For
655 /// post-isel opcodes, see getMachineOpcode.
656 unsigned getOpcode() const { return (unsigned short)NodeType; }
657
658 /// Test if this node has a target-specific opcode (in the
659 /// \<target\>ISD namespace).
660 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
661
662 /// Test if this node has a target-specific
663 /// memory-referencing opcode (in the \<target\>ISD namespace and
664 /// greater than FIRST_TARGET_MEMORY_OPCODE).
665 bool isTargetMemoryOpcode() const {
666 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
667 }
668
669 /// Return true if the type of the node type undefined.
670 bool isUndef() const { return NodeType == ISD::UNDEF; }
671
672 /// Test if this node is a memory intrinsic (with valid pointer information).
673 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
674 /// non-memory intrinsics (with chains) that are not really instances of
675 /// MemSDNode. For such nodes, we need some extra state to determine the
676 /// proper classof relationship.
677 bool isMemIntrinsic() const {
678 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
679 NodeType == ISD::INTRINSIC_VOID) &&
680 SDNodeBits.IsMemIntrinsic;
681 }
682
683 /// Test if this node is a strict floating point pseudo-op.
684 bool isStrictFPOpcode() {
685 switch (NodeType) {
686 default:
687 return false;
688 case ISD::STRICT_FADD:
689 case ISD::STRICT_FSUB:
690 case ISD::STRICT_FMUL:
691 case ISD::STRICT_FDIV:
692 case ISD::STRICT_FREM:
693 case ISD::STRICT_FMA:
694 case ISD::STRICT_FSQRT:
695 case ISD::STRICT_FPOW:
696 case ISD::STRICT_FPOWI:
697 case ISD::STRICT_FSIN:
698 case ISD::STRICT_FCOS:
699 case ISD::STRICT_FEXP:
700 case ISD::STRICT_FEXP2:
701 case ISD::STRICT_FLOG:
702 case ISD::STRICT_FLOG10:
703 case ISD::STRICT_FLOG2:
704 case ISD::STRICT_LRINT:
705 case ISD::STRICT_LLRINT:
706 case ISD::STRICT_FRINT:
707 case ISD::STRICT_FNEARBYINT:
708 case ISD::STRICT_FMAXNUM:
709 case ISD::STRICT_FMINNUM:
710 case ISD::STRICT_FCEIL:
711 case ISD::STRICT_FFLOOR:
712 case ISD::STRICT_LROUND:
713 case ISD::STRICT_LLROUND:
714 case ISD::STRICT_FROUND:
715 case ISD::STRICT_FTRUNC:
716 case ISD::STRICT_FP_TO_SINT:
717 case ISD::STRICT_FP_TO_UINT:
718 case ISD::STRICT_FP_ROUND:
719 case ISD::STRICT_FP_EXTEND:
720 return true;
721 }
722 }
723
724 /// Test if this node has a post-isel opcode, directly
725 /// corresponding to a MachineInstr opcode.
726 bool isMachineOpcode() const { return NodeType < 0; }
727
728 /// This may only be called if isMachineOpcode returns
729 /// true. It returns the MachineInstr opcode value that the node's opcode
730 /// corresponds to.
731 unsigned getMachineOpcode() const {
732 assert(isMachineOpcode() && "Not a MachineInstr opcode!")((isMachineOpcode() && "Not a MachineInstr opcode!") ?
static_cast<void> (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 732, __PRETTY_FUNCTION__))
;
733 return ~NodeType;
734 }
735
736 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
737 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
738
739 bool isDivergent() const { return SDNodeBits.IsDivergent; }
740
741 /// Return true if there are no uses of this node.
742 bool use_empty() const { return UseList == nullptr; }
743
744 /// Return true if there is exactly one use of this node.
745 bool hasOneUse() const {
746 return !use_empty() && std::next(use_begin()) == use_end();
747 }
748
749 /// Return the number of uses of this node. This method takes
750 /// time proportional to the number of uses.
751 size_t use_size() const { return std::distance(use_begin(), use_end()); }
752
753 /// Return the unique node id.
754 int getNodeId() const { return NodeId; }
755
756 /// Set unique node id.
757 void setNodeId(int Id) { NodeId = Id; }
758
759 /// Return the node ordering.
760 unsigned getIROrder() const { return IROrder; }
761
762 /// Set the node ordering.
763 void setIROrder(unsigned Order) { IROrder = Order; }
764
765 /// Return the source location info.
766 const DebugLoc &getDebugLoc() const { return debugLoc; }
767
768 /// Set source location info. Try to avoid this, putting
769 /// it in the constructor is preferable.
770 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
771
772 /// This class provides iterator support for SDUse
773 /// operands that use a specific SDNode.
774 class use_iterator
775 : public std::iterator<std::forward_iterator_tag, SDUse, ptrdiff_t> {
776 friend class SDNode;
777
778 SDUse *Op = nullptr;
779
780 explicit use_iterator(SDUse *op) : Op(op) {}
781
782 public:
783 using reference = std::iterator<std::forward_iterator_tag,
784 SDUse, ptrdiff_t>::reference;
785 using pointer = std::iterator<std::forward_iterator_tag,
786 SDUse, ptrdiff_t>::pointer;
787
788 use_iterator() = default;
789 use_iterator(const use_iterator &I) : Op(I.Op) {}
790
791 bool operator==(const use_iterator &x) const {
792 return Op == x.Op;
793 }
794 bool operator!=(const use_iterator &x) const {
795 return !operator==(x);
796 }
797
798 /// Return true if this iterator is at the end of uses list.
799 bool atEnd() const { return Op == nullptr; }
800
801 // Iterator traversal: forward iteration only.
802 use_iterator &operator++() { // Preincrement
803 assert(Op && "Cannot increment end iterator!")((Op && "Cannot increment end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 803, __PRETTY_FUNCTION__))
;
804 Op = Op->getNext();
805 return *this;
806 }
807
808 use_iterator operator++(int) { // Postincrement
809 use_iterator tmp = *this; ++*this; return tmp;
810 }
811
812 /// Retrieve a pointer to the current user node.
813 SDNode *operator*() const {
814 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 814, __PRETTY_FUNCTION__))
;
815 return Op->getUser();
816 }
817
818 SDNode *operator->() const { return operator*(); }
819
820 SDUse &getUse() const { return *Op; }
821
822 /// Retrieve the operand # of this use in its user.
823 unsigned getOperandNo() const {
824 assert(Op && "Cannot dereference end iterator!")((Op && "Cannot dereference end iterator!") ? static_cast
<void> (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 824, __PRETTY_FUNCTION__))
;
825 return (unsigned)(Op - Op->getUser()->OperandList);
826 }
827 };
828
829 /// Provide iteration support to walk over all uses of an SDNode.
830 use_iterator use_begin() const {
831 return use_iterator(UseList);
832 }
833
834 static use_iterator use_end() { return use_iterator(nullptr); }
835
836 inline iterator_range<use_iterator> uses() {
837 return make_range(use_begin(), use_end());
838 }
839 inline iterator_range<use_iterator> uses() const {
840 return make_range(use_begin(), use_end());
841 }
842
843 /// Return true if there are exactly NUSES uses of the indicated value.
844 /// This method ignores uses of other values defined by this operation.
845 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
846
847 /// Return true if there are any use of the indicated value.
848 /// This method ignores uses of other values defined by this operation.
849 bool hasAnyUseOfValue(unsigned Value) const;
850
851 /// Return true if this node is the only use of N.
852 bool isOnlyUserOf(const SDNode *N) const;
853
854 /// Return true if this node is an operand of N.
855 bool isOperandOf(const SDNode *N) const;
856
857 /// Return true if this node is a predecessor of N.
858 /// NOTE: Implemented on top of hasPredecessor and every bit as
859 /// expensive. Use carefully.
860 bool isPredecessorOf(const SDNode *N) const {
861 return N->hasPredecessor(this);
862 }
863
864 /// Return true if N is a predecessor of this node.
865 /// N is either an operand of this node, or can be reached by recursively
866 /// traversing up the operands.
867 /// NOTE: This is an expensive method. Use it carefully.
868 bool hasPredecessor(const SDNode *N) const;
869
870 /// Returns true if N is a predecessor of any node in Worklist. This
871 /// helper keeps Visited and Worklist sets externally to allow unions
872 /// searches to be performed in parallel, caching of results across
873 /// queries and incremental addition to Worklist. Stops early if N is
874 /// found but will resume. Remember to clear Visited and Worklists
875 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
876 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
877 /// topologically ordered (Operands have strictly smaller node id) and search
878 /// can be pruned leveraging this.
879 static bool hasPredecessorHelper(const SDNode *N,
880 SmallPtrSetImpl<const SDNode *> &Visited,
881 SmallVectorImpl<const SDNode *> &Worklist,
882 unsigned int MaxSteps = 0,
883 bool TopologicalPrune = false) {
884 SmallVector<const SDNode *, 8> DeferredNodes;
885 if (Visited.count(N))
886 return true;
887
888 // Node Id's are assigned in three places: As a topological
889 // ordering (> 0), during legalization (results in values set to
890 // 0), new nodes (set to -1). If N has a topolgical id then we
891 // know that all nodes with ids smaller than it cannot be
892 // successors and we need not check them. Filter out all node
893 // that can't be matches. We add them to the worklist before exit
894 // in case of multiple calls. Note that during selection the topological id
895 // may be violated if a node's predecessor is selected before it. We mark
896 // this at selection negating the id of unselected successors and
897 // restricting topological pruning to positive ids.
898
899 int NId = N->getNodeId();
900 // If we Invalidated the Id, reconstruct original NId.
901 if (NId < -1)
902 NId = -(NId + 1);
903
904 bool Found = false;
905 while (!Worklist.empty()) {
906 const SDNode *M = Worklist.pop_back_val();
907 int MId = M->getNodeId();
908 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
909 (MId > 0) && (MId < NId)) {
910 DeferredNodes.push_back(M);
911 continue;
912 }
913 for (const SDValue &OpV : M->op_values()) {
914 SDNode *Op = OpV.getNode();
915 if (Visited.insert(Op).second)
916 Worklist.push_back(Op);
917 if (Op == N)
918 Found = true;
919 }
920 if (Found)
921 break;
922 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
923 break;
924 }
925 // Push deferred nodes back on worklist.
926 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
927 // If we bailed early, conservatively return found.
928 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
929 return true;
930 return Found;
931 }
932
933 /// Return true if all the users of N are contained in Nodes.
934 /// NOTE: Requires at least one match, but doesn't require them all.
935 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
936
937 /// Return the number of values used by this operation.
938 unsigned getNumOperands() const { return NumOperands; }
939
940 /// Return the maximum number of operands that a SDNode can hold.
941 static constexpr size_t getMaxNumOperands() {
942 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
943 }
944
945 /// Helper method returns the integer value of a ConstantSDNode operand.
946 inline uint64_t getConstantOperandVal(unsigned Num) const;
947
948 /// Helper method returns the APInt of a ConstantSDNode operand.
949 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
950
951 const SDValue &getOperand(unsigned Num) const {
952 assert(Num < NumOperands && "Invalid child # of SDNode!")((Num < NumOperands && "Invalid child # of SDNode!"
) ? static_cast<void> (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 952, __PRETTY_FUNCTION__))
;
953 return OperandList[Num];
954 }
955
956 using op_iterator = SDUse *;
957
958 op_iterator op_begin() const { return OperandList; }
959 op_iterator op_end() const { return OperandList+NumOperands; }
960 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
961
962 /// Iterator for directly iterating over the operand SDValue's.
963 struct value_op_iterator
964 : iterator_adaptor_base<value_op_iterator, op_iterator,
965 std::random_access_iterator_tag, SDValue,
966 ptrdiff_t, value_op_iterator *,
967 value_op_iterator *> {
968 explicit value_op_iterator(SDUse *U = nullptr)
969 : iterator_adaptor_base(U) {}
970
971 const SDValue &operator*() const { return I->get(); }
972 };
973
974 iterator_range<value_op_iterator> op_values() const {
975 return make_range(value_op_iterator(op_begin()),
976 value_op_iterator(op_end()));
977 }
978
979 SDVTList getVTList() const {
980 SDVTList X = { ValueList, NumValues };
981 return X;
982 }
983
984 /// If this node has a glue operand, return the node
985 /// to which the glue operand points. Otherwise return NULL.
986 SDNode *getGluedNode() const {
987 if (getNumOperands() != 0 &&
988 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
989 return getOperand(getNumOperands()-1).getNode();
990 return nullptr;
991 }
992
993 /// If this node has a glue value with a user, return
994 /// the user (there is at most one). Otherwise return NULL.
995 SDNode *getGluedUser() const {
996 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
997 if (UI.getUse().get().getValueType() == MVT::Glue)
998 return *UI;
999 return nullptr;
1000 }
1001
1002 const SDNodeFlags getFlags() const { return Flags; }
1003 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
1004 bool isFast() { return Flags.isFast(); }
1005
1006 /// Clear any flags in this node that aren't also set in Flags.
1007 /// If Flags is not in a defined state then this has no effect.
1008 void intersectFlagsWith(const SDNodeFlags Flags);
1009
1010 /// Return the number of values defined/returned by this operator.
1011 unsigned getNumValues() const { return NumValues; }
1012
1013 /// Return the type of a specified result.
1014 EVT getValueType(unsigned ResNo) const {
1015 assert(ResNo < NumValues && "Illegal result number!")((ResNo < NumValues && "Illegal result number!") ?
static_cast<void> (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1015, __PRETTY_FUNCTION__))
;
1016 return ValueList[ResNo];
1017 }
1018
1019 /// Return the type of a specified result as a simple type.
1020 MVT getSimpleValueType(unsigned ResNo) const {
1021 return getValueType(ResNo).getSimpleVT();
1022 }
1023
1024 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
1025 unsigned getValueSizeInBits(unsigned ResNo) const {
1026 return getValueType(ResNo).getSizeInBits();
1027 }
1028
1029 using value_iterator = const EVT *;
1030
1031 value_iterator value_begin() const { return ValueList; }
1032 value_iterator value_end() const { return ValueList+NumValues; }
1033
1034 /// Return the opcode of this operation for printing.
1035 std::string getOperationName(const SelectionDAG *G = nullptr) const;
1036 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
1037 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
1038 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
1039 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1040 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1041
1042 /// Print a SelectionDAG node and all children down to
1043 /// the leaves. The given SelectionDAG allows target-specific nodes
1044 /// to be printed in human-readable form. Unlike printr, this will
1045 /// print the whole DAG, including children that appear multiple
1046 /// times.
1047 ///
1048 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1049
1050 /// Print a SelectionDAG node and children up to
1051 /// depth "depth." The given SelectionDAG allows target-specific
1052 /// nodes to be printed in human-readable form. Unlike printr, this
1053 /// will print children that appear multiple times wherever they are
1054 /// used.
1055 ///
1056 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1057 unsigned depth = 100) const;
1058
1059 /// Dump this node, for debugging.
1060 void dump() const;
1061
1062 /// Dump (recursively) this node and its use-def subgraph.
1063 void dumpr() const;
1064
1065 /// Dump this node, for debugging.
1066 /// The given SelectionDAG allows target-specific nodes to be printed
1067 /// in human-readable form.
1068 void dump(const SelectionDAG *G) const;
1069
1070 /// Dump (recursively) this node and its use-def subgraph.
1071 /// The given SelectionDAG allows target-specific nodes to be printed
1072 /// in human-readable form.
1073 void dumpr(const SelectionDAG *G) const;
1074
1075 /// printrFull to dbgs(). The given SelectionDAG allows
1076 /// target-specific nodes to be printed in human-readable form.
1077 /// Unlike dumpr, this will print the whole DAG, including children
1078 /// that appear multiple times.
1079 void dumprFull(const SelectionDAG *G = nullptr) const;
1080
1081 /// printrWithDepth to dbgs(). The given
1082 /// SelectionDAG allows target-specific nodes to be printed in
1083 /// human-readable form. Unlike dumpr, this will print children
1084 /// that appear multiple times wherever they are used.
1085 ///
1086 void dumprWithDepth(const SelectionDAG *G = nullptr,
1087 unsigned depth = 100) const;
1088
1089 /// Gather unique data for the node.
1090 void Profile(FoldingSetNodeID &ID) const;
1091
1092 /// This method should only be used by the SDUse class.
1093 void addUse(SDUse &U) { U.addToList(&UseList); }
1094
1095protected:
1096 static SDVTList getSDVTList(EVT VT) {
1097 SDVTList Ret = { getValueTypeList(VT), 1 };
1098 return Ret;
1099 }
1100
1101 /// Create an SDNode.
1102 ///
1103 /// SDNodes are created without any operands, and never own the operand
1104 /// storage. To add operands, see SelectionDAG::createOperands.
1105 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1106 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1107 IROrder(Order), debugLoc(std::move(dl)) {
1108 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1109 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")((debugLoc.hasTrivialDestructor() && "Expected trivial destructor"
) ? static_cast<void> (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1109, __PRETTY_FUNCTION__))
;
1110 assert(NumValues == VTs.NumVTs &&((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1111, __PRETTY_FUNCTION__))
1111 "NumValues wasn't wide enough for its operands!")((NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"
) ? static_cast<void> (0) : __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1111, __PRETTY_FUNCTION__))
;
1112 }
1113
1114 /// Release the operands and set this node to have zero operands.
1115 void DropOperands();
1116};
1117
1118/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1119/// into SDNode creation functions.
1120/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1121/// from the original Instruction, and IROrder is the ordinal position of
1122/// the instruction.
1123/// When an SDNode is created after the DAG is being built, both DebugLoc and
1124/// the IROrder are propagated from the original SDNode.
1125/// So SDLoc class provides two constructors besides the default one, one to
1126/// be used by the DAGBuilder, the other to be used by others.
1127class SDLoc {
1128private:
1129 DebugLoc DL;
1130 int IROrder = 0;
1131
1132public:
1133 SDLoc() = default;
1134 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1135 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1136 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1137 assert(Order >= 0 && "bad IROrder")((Order >= 0 && "bad IROrder") ? static_cast<void
> (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1137, __PRETTY_FUNCTION__))
;
1138 if (I)
1139 DL = I->getDebugLoc();
1140 }
1141
1142 unsigned getIROrder() const { return IROrder; }
1143 const DebugLoc &getDebugLoc() const { return DL; }
1144};
1145
1146// Define inline functions from the SDValue class.
1147
1148inline SDValue::SDValue(SDNode *node, unsigned resno)
1149 : Node(node), ResNo(resno) {
1150 // Explicitly check for !ResNo to avoid use-after-free, because there are
1151 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1152 // combines.
1153 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1154, __PRETTY_FUNCTION__))
1154 "Invalid result number for the given node!")(((!Node || !ResNo || ResNo < Node->getNumValues()) &&
"Invalid result number for the given node!") ? static_cast<
void> (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1154, __PRETTY_FUNCTION__))
;
1155 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")((ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? static_cast<void> (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1155, __PRETTY_FUNCTION__))
;
1156}
1157
1158inline unsigned SDValue::getOpcode() const {
1159 return Node->getOpcode();
20
Called C++ object pointer is null
1160}
1161
1162inline EVT SDValue::getValueType() const {
1163 return Node->getValueType(ResNo);
1164}
1165
1166inline unsigned SDValue::getNumOperands() const {
1167 return Node->getNumOperands();
1168}
1169
1170inline const SDValue &SDValue::getOperand(unsigned i) const {
1171 return Node->getOperand(i);
1172}
1173
1174inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1175 return Node->getConstantOperandVal(i);
1176}
1177
1178inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1179 return Node->getConstantOperandAPInt(i);
1180}
1181
1182inline bool SDValue::isTargetOpcode() const {
1183 return Node->isTargetOpcode();
1184}
1185
1186inline bool SDValue::isTargetMemoryOpcode() const {
1187 return Node->isTargetMemoryOpcode();
1188}
1189
1190inline bool SDValue::isMachineOpcode() const {
1191 return Node->isMachineOpcode();
1192}
1193
1194inline unsigned SDValue::getMachineOpcode() const {
1195 return Node->getMachineOpcode();
1196}
1197
1198inline bool SDValue::isUndef() const {
1199 return Node->isUndef();
1200}
1201
1202inline bool SDValue::use_empty() const {
1203 return !Node->hasAnyUseOfValue(ResNo);
1204}
1205
1206inline bool SDValue::hasOneUse() const {
1207 return Node->hasNUsesOfValue(1, ResNo);
1208}
1209
1210inline const DebugLoc &SDValue::getDebugLoc() const {
1211 return Node->getDebugLoc();
1212}
1213
1214inline void SDValue::dump() const {
1215 return Node->dump();
1216}
1217
1218inline void SDValue::dump(const SelectionDAG *G) const {
1219 return Node->dump(G);
1220}
1221
1222inline void SDValue::dumpr() const {
1223 return Node->dumpr();
1224}
1225
1226inline void SDValue::dumpr(const SelectionDAG *G) const {
1227 return Node->dumpr(G);
1228}
1229
1230// Define inline functions from the SDUse class.
1231
1232inline void SDUse::set(const SDValue &V) {
1233 if (Val.getNode()) removeFromList();
1234 Val = V;
1235 if (V.getNode()) V.getNode()->addUse(*this);
1236}
1237
1238inline void SDUse::setInitial(const SDValue &V) {
1239 Val = V;
1240 V.getNode()->addUse(*this);
1241}
1242
1243inline void SDUse::setNode(SDNode *N) {
1244 if (Val.getNode()) removeFromList();
1245 Val.setNode(N);
1246 if (N) N->addUse(*this);
1247}
1248
1249/// This class is used to form a handle around another node that
1250/// is persistent and is updated across invocations of replaceAllUsesWith on its
1251/// operand. This node should be directly created by end-users and not added to
1252/// the AllNodes list.
1253class HandleSDNode : public SDNode {
1254 SDUse Op;
1255
1256public:
1257 explicit HandleSDNode(SDValue X)
1258 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1259 // HandleSDNodes are never inserted into the DAG, so they won't be
1260 // auto-numbered. Use ID 65535 as a sentinel.
1261 PersistentId = 0xffff;
1262
1263 // Manually set up the operand list. This node type is special in that it's
1264 // always stack allocated and SelectionDAG does not manage its operands.
1265 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1266 // be so special.
1267 Op.setUser(this);
1268 Op.setInitial(X);
1269 NumOperands = 1;
1270 OperandList = &Op;
1271 }
1272 ~HandleSDNode();
1273
1274 const SDValue &getValue() const { return Op; }
1275};
1276
1277class AddrSpaceCastSDNode : public SDNode {
1278private:
1279 unsigned SrcAddrSpace;
1280 unsigned DestAddrSpace;
1281
1282public:
1283 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1284 unsigned SrcAS, unsigned DestAS);
1285
1286 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1287 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1288
1289 static bool classof(const SDNode *N) {
1290 return N->getOpcode() == ISD::ADDRSPACECAST;
1291 }
1292};
1293
1294/// This is an abstract virtual class for memory operations.
1295class MemSDNode : public SDNode {
1296private:
1297 // VT of in-memory value.
1298 EVT MemoryVT;
1299
1300protected:
1301 /// Memory reference information.
1302 MachineMemOperand *MMO;
1303
1304public:
1305 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1306 EVT memvt, MachineMemOperand *MMO);
1307
1308 bool readMem() const { return MMO->isLoad(); }
1309 bool writeMem() const { return MMO->isStore(); }
1310
1311 /// Returns alignment and volatility of the memory access
1312 unsigned getOriginalAlignment() const {
1313 return MMO->getBaseAlignment();
1314 }
1315 unsigned getAlignment() const {
1316 return MMO->getAlignment();
1317 }
1318
1319 /// Return the SubclassData value, without HasDebugValue. This contains an
1320 /// encoding of the volatile flag, as well as bits used by subclasses. This
1321 /// function should only be used to compute a FoldingSetNodeID value.
1322 /// The HasDebugValue bit is masked out because CSE map needs to match
1323 /// nodes with debug info with nodes without debug info. Same is about
1324 /// isDivergent bit.
1325 unsigned getRawSubclassData() const {
1326 uint16_t Data;
1327 union {
1328 char RawSDNodeBits[sizeof(uint16_t)];
1329 SDNodeBitfields SDNodeBits;
1330 };
1331 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1332 SDNodeBits.HasDebugValue = 0;
1333 SDNodeBits.IsDivergent = false;
1334 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1335 return Data;
1336 }
1337
1338 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1339 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1340 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1341 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1342
1343 // Returns the offset from the location of the access.
1344 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1345
1346 /// Returns the AA info that describes the dereference.
1347 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1348
1349 /// Returns the Ranges that describes the dereference.
1350 const MDNode *getRanges() const { return MMO->getRanges(); }
1351
1352 /// Returns the synchronization scope ID for this memory operation.
1353 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1354
1355 /// Return the atomic ordering requirements for this memory operation. For
1356 /// cmpxchg atomic operations, return the atomic ordering requirements when
1357 /// store occurs.
1358 AtomicOrdering getOrdering() const { return MMO->getOrdering(); }
1359
1360 /// Return true if the memory operation ordering is Unordered or higher.
1361 bool isAtomic() const { return MMO->isAtomic(); }
1362
1363 /// Returns true if the memory operation doesn't imply any ordering
1364 /// constraints on surrounding memory operations beyond the normal memory
1365 /// aliasing rules.
1366 bool isUnordered() const { return MMO->isUnordered(); }
1367
1368 /// Returns true if the memory operation is neither atomic or volatile.
1369 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1370
1371 /// Return the type of the in-memory value.
1372 EVT getMemoryVT() const { return MemoryVT; }
1373
1374 /// Return a MachineMemOperand object describing the memory
1375 /// reference performed by operation.
1376 MachineMemOperand *getMemOperand() const { return MMO; }
1377
1378 const MachinePointerInfo &getPointerInfo() const {
1379 return MMO->getPointerInfo();
1380 }
1381
1382 /// Return the address space for the associated pointer
1383 unsigned getAddressSpace() const {
1384 return getPointerInfo().getAddrSpace();
1385 }
1386
1387 /// Update this MemSDNode's MachineMemOperand information
1388 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1389 /// This must only be used when the new alignment applies to all users of
1390 /// this MachineMemOperand.
1391 void refineAlignment(const MachineMemOperand *NewMMO) {
1392 MMO->refineAlignment(NewMMO);
1393 }
1394
1395 const SDValue &getChain() const { return getOperand(0); }
1396 const SDValue &getBasePtr() const {
1397 return getOperand(getOpcode() == ISD::STORE ? 2 : 1);
1398 }
1399
1400 // Methods to support isa and dyn_cast
1401 static bool classof(const SDNode *N) {
1402 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1403 // with either an intrinsic or a target opcode.
1404 return N->getOpcode() == ISD::LOAD ||
1405 N->getOpcode() == ISD::STORE ||
1406 N->getOpcode() == ISD::PREFETCH ||
1407 N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1408 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1409 N->getOpcode() == ISD::ATOMIC_SWAP ||
1410 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1411 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1412 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1413 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1414 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1415 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1416 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1417 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1418 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1419 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1420 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1421 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1422 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1423 N->getOpcode() == ISD::ATOMIC_LOAD ||
1424 N->getOpcode() == ISD::ATOMIC_STORE ||
1425 N->getOpcode() == ISD::MLOAD ||
1426 N->getOpcode() == ISD::MSTORE ||
1427 N->getOpcode() == ISD::MGATHER ||
1428 N->getOpcode() == ISD::MSCATTER ||
1429 N->isMemIntrinsic() ||
1430 N->isTargetMemoryOpcode();
1431 }
1432};
1433
1434/// This is an SDNode representing atomic operations.
1435class AtomicSDNode : public MemSDNode {
1436public:
1437 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1438 EVT MemVT, MachineMemOperand *MMO)
1439 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1440 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1441, __PRETTY_FUNCTION__))
1441 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")((((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE
) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? static_cast<void> (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1441, __PRETTY_FUNCTION__))
;
1442 }
1443
1444 const SDValue &getBasePtr() const { return getOperand(1); }
1445 const SDValue &getVal() const { return getOperand(2); }
1446
1447 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1448 /// otherwise.
1449 bool isCompareAndSwap() const {
1450 unsigned Op = getOpcode();
1451 return Op == ISD::ATOMIC_CMP_SWAP ||
1452 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1453 }
1454
1455 /// For cmpxchg atomic operations, return the atomic ordering requirements
1456 /// when store does not occur.
1457 AtomicOrdering getFailureOrdering() const {
1458 assert(isCompareAndSwap() && "Must be cmpxchg operation")((isCompareAndSwap() && "Must be cmpxchg operation") ?
static_cast<void> (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1458, __PRETTY_FUNCTION__))
;
1459 return MMO->getFailureOrdering();
1460 }
1461
1462 // Methods to support isa and dyn_cast
1463 static bool classof(const SDNode *N) {
1464 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1465 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1466 N->getOpcode() == ISD::ATOMIC_SWAP ||
1467 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1468 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1469 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1470 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1471 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1472 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1473 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1474 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1475 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1476 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1477 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1478 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1479 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1480 N->getOpcode() == ISD::ATOMIC_LOAD ||
1481 N->getOpcode() == ISD::ATOMIC_STORE;
1482 }
1483};
1484
1485/// This SDNode is used for target intrinsics that touch
1486/// memory and need an associated MachineMemOperand. Its opcode may be
1487/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1488/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1489class MemIntrinsicSDNode : public MemSDNode {
1490public:
1491 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1492 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1493 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1494 SDNodeBits.IsMemIntrinsic = true;
1495 }
1496
1497 // Methods to support isa and dyn_cast
1498 static bool classof(const SDNode *N) {
1499 // We lower some target intrinsics to their target opcode
1500 // early a node with a target opcode can be of this class
1501 return N->isMemIntrinsic() ||
1502 N->getOpcode() == ISD::PREFETCH ||
1503 N->isTargetMemoryOpcode();
1504 }
1505};
1506
1507/// This SDNode is used to implement the code generator
1508/// support for the llvm IR shufflevector instruction. It combines elements
1509/// from two input vectors into a new input vector, with the selection and
1510/// ordering of elements determined by an array of integers, referred to as
1511/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1512/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1513/// An index of -1 is treated as undef, such that the code generator may put
1514/// any value in the corresponding element of the result.
1515class ShuffleVectorSDNode : public SDNode {
1516 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1517 // is freed when the SelectionDAG object is destroyed.
1518 const int *Mask;
1519
1520protected:
1521 friend class SelectionDAG;
1522
1523 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1524 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1525
1526public:
1527 ArrayRef<int> getMask() const {
1528 EVT VT = getValueType(0);
1529 return makeArrayRef(Mask, VT.getVectorNumElements());
1530 }
1531
1532 int getMaskElt(unsigned Idx) const {
1533 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")((Idx < getValueType(0).getVectorNumElements() && "Idx out of range!"
) ? static_cast<void> (0) : __assert_fail ("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1533, __PRETTY_FUNCTION__))
;
1534 return Mask[Idx];
1535 }
1536
1537 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1538
1539 int getSplatIndex() const {
1540 assert(isSplat() && "Cannot get splat index for non-splat!")((isSplat() && "Cannot get splat index for non-splat!"
) ? static_cast<void> (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1540, __PRETTY_FUNCTION__))
;
1541 EVT VT = getValueType(0);
1542 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1543 if (Mask[i] >= 0)
1544 return Mask[i];
1545
1546 // We can choose any index value here and be correct because all elements
1547 // are undefined. Return 0 for better potential for callers to simplify.
1548 return 0;
1549 }
1550
1551 static bool isSplatMask(const int *Mask, EVT VT);
1552
1553 /// Change values in a shuffle permute mask assuming
1554 /// the two vector operands have swapped position.
1555 static void commuteMask(MutableArrayRef<int> Mask) {
1556 unsigned NumElems = Mask.size();
1557 for (unsigned i = 0; i != NumElems; ++i) {
1558 int idx = Mask[i];
1559 if (idx < 0)
1560 continue;
1561 else if (idx < (int)NumElems)
1562 Mask[i] = idx + NumElems;
1563 else
1564 Mask[i] = idx - NumElems;
1565 }
1566 }
1567
1568 static bool classof(const SDNode *N) {
1569 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1570 }
1571};
1572
1573class ConstantSDNode : public SDNode {
1574 friend class SelectionDAG;
1575
1576 const ConstantInt *Value;
1577
1578 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1579 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1580 getSDVTList(VT)),
1581 Value(val) {
1582 ConstantSDNodeBits.IsOpaque = isOpaque;
1583 }
1584
1585public:
1586 const ConstantInt *getConstantIntValue() const { return Value; }
1587 const APInt &getAPIntValue() const { return Value->getValue(); }
1588 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1589 int64_t getSExtValue() const { return Value->getSExtValue(); }
1590 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1591 return Value->getLimitedValue(Limit);
1592 }
1593
1594 bool isOne() const { return Value->isOne(); }
1595 bool isNullValue() const { return Value->isZero(); }
1596 bool isAllOnesValue() const { return Value->isMinusOne(); }
1597
1598 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1599
1600 static bool classof(const SDNode *N) {
1601 return N->getOpcode() == ISD::Constant ||
1602 N->getOpcode() == ISD::TargetConstant;
1603 }
1604};
1605
1606uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1607 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1608}
1609
1610const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1611 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1612}
1613
1614class ConstantFPSDNode : public SDNode {
1615 friend class SelectionDAG;
1616
1617 const ConstantFP *Value;
1618
1619 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1620 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1621 DebugLoc(), getSDVTList(VT)),
1622 Value(val) {}
1623
1624public:
1625 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1626 const ConstantFP *getConstantFPValue() const { return Value; }
1627
1628 /// Return true if the value is positive or negative zero.
1629 bool isZero() const { return Value->isZero(); }
1630
1631 /// Return true if the value is a NaN.
1632 bool isNaN() const { return Value->isNaN(); }
1633
1634 /// Return true if the value is an infinity
1635 bool isInfinity() const { return Value->isInfinity(); }
1636
1637 /// Return true if the value is negative.
1638 bool isNegative() const { return Value->isNegative(); }
1639
1640 /// We don't rely on operator== working on double values, as
1641 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1642 /// As such, this method can be used to do an exact bit-for-bit comparison of
1643 /// two floating point values.
1644
1645 /// We leave the version with the double argument here because it's just so
1646 /// convenient to write "2.0" and the like. Without this function we'd
1647 /// have to duplicate its logic everywhere it's called.
1648 bool isExactlyValue(double V) const {
1649 return Value->getValueAPF().isExactlyValue(V);
1650 }
1651 bool isExactlyValue(const APFloat& V) const;
1652
1653 static bool isValueValidForType(EVT VT, const APFloat& Val);
1654
1655 static bool classof(const SDNode *N) {
1656 return N->getOpcode() == ISD::ConstantFP ||
1657 N->getOpcode() == ISD::TargetConstantFP;
1658 }
1659};
1660
1661/// Returns true if \p V is a constant integer zero.
1662bool isNullConstant(SDValue V);
1663
1664/// Returns true if \p V is an FP constant with a value of positive zero.
1665bool isNullFPConstant(SDValue V);
1666
1667/// Returns true if \p V is an integer constant with all bits set.
1668bool isAllOnesConstant(SDValue V);
1669
1670/// Returns true if \p V is a constant integer one.
1671bool isOneConstant(SDValue V);
1672
1673/// Return the non-bitcasted source operand of \p V if it exists.
1674/// If \p V is not a bitcasted value, it is returned as-is.
1675SDValue peekThroughBitcasts(SDValue V);
1676
1677/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1678/// If \p V is not a bitcasted one-use value, it is returned as-is.
1679SDValue peekThroughOneUseBitcasts(SDValue V);
1680
1681/// Return the non-extracted vector source operand of \p V if it exists.
1682/// If \p V is not an extracted subvector, it is returned as-is.
1683SDValue peekThroughExtractSubvectors(SDValue V);
1684
1685/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1686/// constant is canonicalized to be operand 1.
1687bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1688
1689/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1690ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1691 bool AllowTruncation = false);
1692
1693/// Returns the SDNode if it is a demanded constant splat BuildVector or
1694/// constant int.
1695ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1696 bool AllowUndefs = false,
1697 bool AllowTruncation = false);
1698
1699/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1700ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1701
1702/// Returns the SDNode if it is a demanded constant splat BuildVector or
1703/// constant float.
1704ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1705 bool AllowUndefs = false);
1706
1707/// Return true if the value is a constant 0 integer or a splatted vector of
1708/// a constant 0 integer (with no undefs by default).
1709/// Build vector implicit truncation is not an issue for null values.
1710bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1711
1712/// Return true if the value is a constant 1 integer or a splatted vector of a
1713/// constant 1 integer (with no undefs).
1714/// Does not permit build vector implicit truncation.
1715bool isOneOrOneSplat(SDValue V);
1716
1717/// Return true if the value is a constant -1 integer or a splatted vector of a
1718/// constant -1 integer (with no undefs).
1719/// Does not permit build vector implicit truncation.
1720bool isAllOnesOrAllOnesSplat(SDValue V);
1721
1722class GlobalAddressSDNode : public SDNode {
1723 friend class SelectionDAG;
1724
1725 const GlobalValue *TheGlobal;
1726 int64_t Offset;
1727 unsigned TargetFlags;
1728
1729 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1730 const GlobalValue *GA, EVT VT, int64_t o,
1731 unsigned TF);
1732
1733public:
1734 const GlobalValue *getGlobal() const { return TheGlobal; }
1735 int64_t getOffset() const { return Offset; }
1736 unsigned getTargetFlags() const { return TargetFlags; }
1737 // Return the address space this GlobalAddress belongs to.
1738 unsigned getAddressSpace() const;
1739
1740 static bool classof(const SDNode *N) {
1741 return N->getOpcode() == ISD::GlobalAddress ||
1742 N->getOpcode() == ISD::TargetGlobalAddress ||
1743 N->getOpcode() == ISD::GlobalTLSAddress ||
1744 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1745 }
1746};
1747
1748class FrameIndexSDNode : public SDNode {
1749 friend class SelectionDAG;
1750
1751 int FI;
1752
1753 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1754 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1755 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1756 }
1757
1758public:
1759 int getIndex() const { return FI; }
1760
1761 static bool classof(const SDNode *N) {
1762 return N->getOpcode() == ISD::FrameIndex ||
1763 N->getOpcode() == ISD::TargetFrameIndex;
1764 }
1765};
1766
1767/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1768/// the offet and size that are started/ended in the underlying FrameIndex.
1769class LifetimeSDNode : public SDNode {
1770 friend class SelectionDAG;
1771 int64_t Size;
1772 int64_t Offset; // -1 if offset is unknown.
1773
1774 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1775 SDVTList VTs, int64_t Size, int64_t Offset)
1776 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1777public:
1778 int64_t getFrameIndex() const {
1779 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1780 }
1781
1782 bool hasOffset() const { return Offset >= 0; }
1783 int64_t getOffset() const {
1784 assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast<
void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1784, __PRETTY_FUNCTION__))
;
1785 return Offset;
1786 }
1787 int64_t getSize() const {
1788 assert(hasOffset() && "offset is unknown")((hasOffset() && "offset is unknown") ? static_cast<
void> (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1788, __PRETTY_FUNCTION__))
;
1789 return Size;
1790 }
1791
1792 // Methods to support isa and dyn_cast
1793 static bool classof(const SDNode *N) {
1794 return N->getOpcode() == ISD::LIFETIME_START ||
1795 N->getOpcode() == ISD::LIFETIME_END;
1796 }
1797};
1798
1799class JumpTableSDNode : public SDNode {
1800 friend class SelectionDAG;
1801
1802 int JTI;
1803 unsigned TargetFlags;
1804
1805 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1806 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1807 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1808 }
1809
1810public:
1811 int getIndex() const { return JTI; }
1812 unsigned getTargetFlags() const { return TargetFlags; }
1813
1814 static bool classof(const SDNode *N) {
1815 return N->getOpcode() == ISD::JumpTable ||
1816 N->getOpcode() == ISD::TargetJumpTable;
1817 }
1818};
1819
1820class ConstantPoolSDNode : public SDNode {
1821 friend class SelectionDAG;
1822
1823 union {
1824 const Constant *ConstVal;
1825 MachineConstantPoolValue *MachineCPVal;
1826 } Val;
1827 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1828 unsigned Alignment; // Minimum alignment requirement of CP (not log2 value).
1829 unsigned TargetFlags;
1830
1831 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1832 unsigned Align, unsigned TF)
1833 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1834 DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
1835 TargetFlags(TF) {
1836 assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1836, __PRETTY_FUNCTION__))
;
1837 Val.ConstVal = c;
1838 }
1839
1840 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v,
1841 EVT VT, int o, unsigned Align, unsigned TF)
1842 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1843 DebugLoc(), getSDVTList(VT)), Offset(o), Alignment(Align),
1844 TargetFlags(TF) {
1845 assert(Offset >= 0 && "Offset is too large")((Offset >= 0 && "Offset is too large") ? static_cast
<void> (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1845, __PRETTY_FUNCTION__))
;
1846 Val.MachineCPVal = v;
1847 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1848 }
1849
1850public:
1851 bool isMachineConstantPoolEntry() const {
1852 return Offset < 0;
1853 }
1854
1855 const Constant *getConstVal() const {
1856 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")((!isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1856, __PRETTY_FUNCTION__))
;
1857 return Val.ConstVal;
1858 }
1859
1860 MachineConstantPoolValue *getMachineCPVal() const {
1861 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")((isMachineConstantPoolEntry() && "Wrong constantpool type"
) ? static_cast<void> (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1861, __PRETTY_FUNCTION__))
;
1862 return Val.MachineCPVal;
1863 }
1864
1865 int getOffset() const {
1866 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1867 }
1868
1869 // Return the alignment of this constant pool object, which is either 0 (for
1870 // default alignment) or the desired value.
1871 unsigned getAlignment() const { return Alignment; }
1872 unsigned getTargetFlags() const { return TargetFlags; }
1873
1874 Type *getType() const;
1875
1876 static bool classof(const SDNode *N) {
1877 return N->getOpcode() == ISD::ConstantPool ||
1878 N->getOpcode() == ISD::TargetConstantPool;
1879 }
1880};
1881
1882/// Completely target-dependent object reference.
1883class TargetIndexSDNode : public SDNode {
1884 friend class SelectionDAG;
1885
1886 unsigned TargetFlags;
1887 int Index;
1888 int64_t Offset;
1889
1890public:
1891 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1892 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1893 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1894
1895 unsigned getTargetFlags() const { return TargetFlags; }
1896 int getIndex() const { return Index; }
1897 int64_t getOffset() const { return Offset; }
1898
1899 static bool classof(const SDNode *N) {
1900 return N->getOpcode() == ISD::TargetIndex;
1901 }
1902};
1903
1904class BasicBlockSDNode : public SDNode {
1905 friend class SelectionDAG;
1906
1907 MachineBasicBlock *MBB;
1908
1909 /// Debug info is meaningful and potentially useful here, but we create
1910 /// blocks out of order when they're jumped to, which makes it a bit
1911 /// harder. Let's see if we need it first.
1912 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1913 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1914 {}
1915
1916public:
1917 MachineBasicBlock *getBasicBlock() const { return MBB; }
1918
1919 static bool classof(const SDNode *N) {
1920 return N->getOpcode() == ISD::BasicBlock;
1921 }
1922};
1923
1924/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1925class BuildVectorSDNode : public SDNode {
1926public:
1927 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1928 explicit BuildVectorSDNode() = delete;
1929
1930 /// Check if this is a constant splat, and if so, find the
1931 /// smallest element size that splats the vector. If MinSplatBits is
1932 /// nonzero, the element size must be at least that large. Note that the
1933 /// splat element may be the entire vector (i.e., a one element vector).
1934 /// Returns the splat element value in SplatValue. Any undefined bits in
1935 /// that value are zero, and the corresponding bits in the SplatUndef mask
1936 /// are set. The SplatBitSize value is set to the splat element size in
1937 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1938 /// undefined. isBigEndian describes the endianness of the target.
1939 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1940 unsigned &SplatBitSize, bool &HasAnyUndefs,
1941 unsigned MinSplatBits = 0,
1942 bool isBigEndian = false) const;
1943
1944 /// Returns the demanded splatted value or a null value if this is not a
1945 /// splat.
1946 ///
1947 /// The DemandedElts mask indicates the elements that must be in the splat.
1948 /// If passed a non-null UndefElements bitvector, it will resize it to match
1949 /// the vector width and set the bits where elements are undef.
1950 SDValue getSplatValue(const APInt &DemandedElts,
1951 BitVector *UndefElements = nullptr) const;
1952
1953 /// Returns the splatted value or a null value if this is not a splat.
1954 ///
1955 /// If passed a non-null UndefElements bitvector, it will resize it to match
1956 /// the vector width and set the bits where elements are undef.
1957 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1958
1959 /// Returns the demanded splatted constant or null if this is not a constant
1960 /// splat.
1961 ///
1962 /// The DemandedElts mask indicates the elements that must be in the splat.
1963 /// If passed a non-null UndefElements bitvector, it will resize it to match
1964 /// the vector width and set the bits where elements are undef.
1965 ConstantSDNode *
1966 getConstantSplatNode(const APInt &DemandedElts,
1967 BitVector *UndefElements = nullptr) const;
1968
1969 /// Returns the splatted constant or null if this is not a constant
1970 /// splat.
1971 ///
1972 /// If passed a non-null UndefElements bitvector, it will resize it to match
1973 /// the vector width and set the bits where elements are undef.
1974 ConstantSDNode *
1975 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
1976
1977 /// Returns the demanded splatted constant FP or null if this is not a
1978 /// constant FP splat.
1979 ///
1980 /// The DemandedElts mask indicates the elements that must be in the splat.
1981 /// If passed a non-null UndefElements bitvector, it will resize it to match
1982 /// the vector width and set the bits where elements are undef.
1983 ConstantFPSDNode *
1984 getConstantFPSplatNode(const APInt &DemandedElts,
1985 BitVector *UndefElements = nullptr) const;
1986
1987 /// Returns the splatted constant FP or null if this is not a constant
1988 /// FP splat.
1989 ///
1990 /// If passed a non-null UndefElements bitvector, it will resize it to match
1991 /// the vector width and set the bits where elements are undef.
1992 ConstantFPSDNode *
1993 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
1994
1995 /// If this is a constant FP splat and the splatted constant FP is an
1996 /// exact power or 2, return the log base 2 integer value. Otherwise,
1997 /// return -1.
1998 ///
1999 /// The BitWidth specifies the necessary bit precision.
2000 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2001 uint32_t BitWidth) const;
2002
2003 bool isConstant() const;
2004
2005 static bool classof(const SDNode *N) {
2006 return N->getOpcode() == ISD::BUILD_VECTOR;
2007 }
2008};
2009
2010/// An SDNode that holds an arbitrary LLVM IR Value. This is
2011/// used when the SelectionDAG needs to make a simple reference to something
2012/// in the LLVM IR representation.
2013///
2014class SrcValueSDNode : public SDNode {
2015 friend class SelectionDAG;
2016
2017 const Value *V;
2018
2019 /// Create a SrcValue for a general value.
2020 explicit SrcValueSDNode(const Value *v)
2021 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2022
2023public:
2024 /// Return the contained Value.
2025 const Value *getValue() const { return V; }
2026
2027 static bool classof(const SDNode *N) {
2028 return N->getOpcode() == ISD::SRCVALUE;
2029 }
2030};
2031
2032class MDNodeSDNode : public SDNode {
2033 friend class SelectionDAG;
2034
2035 const MDNode *MD;
2036
2037 explicit MDNodeSDNode(const MDNode *md)
2038 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2039 {}
2040
2041public:
2042 const MDNode *getMD() const { return MD; }
2043
2044 static bool classof(const SDNode *N) {
2045 return N->getOpcode() == ISD::MDNODE_SDNODE;
2046 }
2047};
2048
2049class RegisterSDNode : public SDNode {
2050 friend class SelectionDAG;
2051
2052 unsigned Reg;
2053
2054 RegisterSDNode(unsigned reg, EVT VT)
2055 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2056
2057public:
2058 unsigned getReg() const { return Reg; }
2059
2060 static bool classof(const SDNode *N) {
2061 return N->getOpcode() == ISD::Register;
2062 }
2063};
2064
2065class RegisterMaskSDNode : public SDNode {
2066 friend class SelectionDAG;
2067
2068 // The memory for RegMask is not owned by the node.
2069 const uint32_t *RegMask;
2070
2071 RegisterMaskSDNode(const uint32_t *mask)
2072 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2073 RegMask(mask) {}
2074
2075public:
2076 const uint32_t *getRegMask() const { return RegMask; }
2077
2078 static bool classof(const SDNode *N) {
2079 return N->getOpcode() == ISD::RegisterMask;
2080 }
2081};
2082
2083class BlockAddressSDNode : public SDNode {
2084 friend class SelectionDAG;
2085
2086 const BlockAddress *BA;
2087 int64_t Offset;
2088 unsigned TargetFlags;
2089
2090 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2091 int64_t o, unsigned Flags)
2092 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2093 BA(ba), Offset(o), TargetFlags(Flags) {}
2094
2095public:
2096 const BlockAddress *getBlockAddress() const { return BA; }
2097 int64_t getOffset() const { return Offset; }
2098 unsigned getTargetFlags() const { return TargetFlags; }
2099
2100 static bool classof(const SDNode *N) {
2101 return N->getOpcode() == ISD::BlockAddress ||
2102 N->getOpcode() == ISD::TargetBlockAddress;
2103 }
2104};
2105
2106class LabelSDNode : public SDNode {
2107 friend class SelectionDAG;
2108
2109 MCSymbol *Label;
2110
2111 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2112 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2113 assert(LabelSDNode::classof(this) && "not a label opcode")((LabelSDNode::classof(this) && "not a label opcode")
? static_cast<void> (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2113, __PRETTY_FUNCTION__))
;
2114 }
2115
2116public:
2117 MCSymbol *getLabel() const { return Label; }
2118
2119 static bool classof(const SDNode *N) {
2120 return N->getOpcode() == ISD::EH_LABEL ||
2121 N->getOpcode() == ISD::ANNOTATION_LABEL;
2122 }
2123};
2124
2125class ExternalSymbolSDNode : public SDNode {
2126 friend class SelectionDAG;
2127
2128 const char *Symbol;
2129 unsigned TargetFlags;
2130
2131 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2132 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2133 DebugLoc(), getSDVTList(VT)),
2134 Symbol(Sym), TargetFlags(TF) {}
2135
2136public:
2137 const char *getSymbol() const { return Symbol; }
2138 unsigned getTargetFlags() const { return TargetFlags; }
2139
2140 static bool classof(const SDNode *N) {
2141 return N->getOpcode() == ISD::ExternalSymbol ||
2142 N->getOpcode() == ISD::TargetExternalSymbol;
2143 }
2144};
2145
2146class MCSymbolSDNode : public SDNode {
2147 friend class SelectionDAG;
2148
2149 MCSymbol *Symbol;
2150
2151 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2152 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2153
2154public:
2155 MCSymbol *getMCSymbol() const { return Symbol; }
2156
2157 static bool classof(const SDNode *N) {
2158 return N->getOpcode() == ISD::MCSymbol;
2159 }
2160};
2161
2162class CondCodeSDNode : public SDNode {
2163 friend class SelectionDAG;
2164
2165 ISD::CondCode Condition;
2166
2167 explicit CondCodeSDNode(ISD::CondCode Cond)
2168 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2169 Condition(Cond) {}
2170
2171public:
2172 ISD::CondCode get() const { return Condition; }
2173
2174 static bool classof(const SDNode *N) {
2175 return N->getOpcode() == ISD::CONDCODE;
2176 }
2177};
2178
2179/// This class is used to represent EVT's, which are used
2180/// to parameterize some operations.
2181class VTSDNode : public SDNode {
2182 friend class SelectionDAG;
2183
2184 EVT ValueType;
2185
2186 explicit VTSDNode(EVT VT)
2187 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2188 ValueType(VT) {}
2189
2190public:
2191 EVT getVT() const { return ValueType; }
2192
2193 static bool classof(const SDNode *N) {
2194 return N->getOpcode() == ISD::VALUETYPE;
2195 }
2196};
2197
2198/// Base class for LoadSDNode and StoreSDNode
2199class LSBaseSDNode : public MemSDNode {
2200public:
2201 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2202 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2203 MachineMemOperand *MMO)
2204 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2205 LSBaseSDNodeBits.AddressingMode = AM;
2206 assert(getAddressingMode() == AM && "Value truncated")((getAddressingMode() == AM && "Value truncated") ? static_cast
<void> (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2206, __PRETTY_FUNCTION__))
;
2207 }
2208
2209 const SDValue &getOffset() const {
2210 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2211 }
2212
2213 /// Return the addressing mode for this load or store:
2214 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2215 ISD::MemIndexedMode getAddressingMode() const {
2216 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2217 }
2218
2219 /// Return true if this is a pre/post inc/dec load/store.
2220 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2221
2222 /// Return true if this is NOT a pre/post inc/dec load/store.
2223 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2224
2225 static bool classof(const SDNode *N) {
2226 return N->getOpcode() == ISD::LOAD ||
2227 N->getOpcode() == ISD::STORE;
2228 }
2229};
2230
2231/// This class is used to represent ISD::LOAD nodes.
2232class LoadSDNode : public LSBaseSDNode {
2233 friend class SelectionDAG;
2234
2235 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2236 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2237 MachineMemOperand *MMO)
2238 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2239 LoadSDNodeBits.ExtTy = ETy;
2240 assert(readMem() && "Load MachineMemOperand is not a load!")((readMem() && "Load MachineMemOperand is not a load!"
) ? static_cast<void> (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2240, __PRETTY_FUNCTION__))
;
2241 assert(!writeMem() && "Load MachineMemOperand is a store!")((!writeMem() && "Load MachineMemOperand is a store!"
) ? static_cast<void> (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2241, __PRETTY_FUNCTION__))
;
2242 }
2243
2244public:
2245 /// Return whether this is a plain node,
2246 /// or one of the varieties of value-extending loads.
2247 ISD::LoadExtType getExtensionType() const {
2248 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2249 }
2250
2251 const SDValue &getBasePtr() const { return getOperand(1); }
2252 const SDValue &getOffset() const { return getOperand(2); }
2253
2254 static bool classof(const SDNode *N) {
2255 return N->getOpcode() == ISD::LOAD;
2256 }
2257};
2258
2259/// This class is used to represent ISD::STORE nodes.
2260class StoreSDNode : public LSBaseSDNode {
2261 friend class SelectionDAG;
2262
2263 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2264 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2265 MachineMemOperand *MMO)
2266 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2267 StoreSDNodeBits.IsTruncating = isTrunc;
2268 assert(!readMem() && "Store MachineMemOperand is a load!")((!readMem() && "Store MachineMemOperand is a load!")
? static_cast<void> (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2268, __PRETTY_FUNCTION__))
;
2269 assert(writeMem() && "Store MachineMemOperand is not a store!")((writeMem() && "Store MachineMemOperand is not a store!"
) ? static_cast<void> (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2269, __PRETTY_FUNCTION__))
;
2270 }
2271
2272public:
2273 /// Return true if the op does a truncation before store.
2274 /// For integers this is the same as doing a TRUNCATE and storing the result.
2275 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2276 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2277 void setTruncatingStore(bool Truncating) {
2278 StoreSDNodeBits.IsTruncating = Truncating;
2279 }
2280
2281 const SDValue &getValue() const { return getOperand(1); }
2282 const SDValue &getBasePtr() const { return getOperand(2); }
2283 const SDValue &getOffset() const { return getOperand(3); }
2284
2285 static bool classof(const SDNode *N) {
2286 return N->getOpcode() == ISD::STORE;
2287 }
2288};
2289
2290/// This base class is used to represent MLOAD and MSTORE nodes
2291class MaskedLoadStoreSDNode : public MemSDNode {
2292public:
2293 friend class SelectionDAG;
2294
2295 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2296 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2297 MachineMemOperand *MMO)
2298 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {}
2299
2300 // MaskedLoadSDNode (Chain, ptr, mask, passthru)
2301 // MaskedStoreSDNode (Chain, data, ptr, mask)
2302 // Mask is a vector of i1 elements
2303 const SDValue &getBasePtr() const {
2304 return getOperand(getOpcode() == ISD::MLOAD ? 1 : 2);
2305 }
2306 const SDValue &getMask() const {
2307 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2308 }
2309
2310 static bool classof(const SDNode *N) {
2311 return N->getOpcode() == ISD::MLOAD ||
2312 N->getOpcode() == ISD::MSTORE;
2313 }
2314};
2315
2316/// This class is used to represent an MLOAD node
2317class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2318public:
2319 friend class SelectionDAG;
2320
2321 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2322 ISD::LoadExtType ETy, bool IsExpanding, EVT MemVT,
2323 MachineMemOperand *MMO)
2324 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) {
2325 LoadSDNodeBits.ExtTy = ETy;
2326 LoadSDNodeBits.IsExpanding = IsExpanding;
2327 }
2328
2329 ISD::LoadExtType getExtensionType() const {
2330 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2331 }
2332
2333 const SDValue &getBasePtr() const { return getOperand(1); }
2334 const SDValue &getMask() const { return getOperand(2); }
2335 const SDValue &getPassThru() const { return getOperand(3); }
2336
2337 static bool classof(const SDNode *N) {
2338 return N->getOpcode() == ISD::MLOAD;
2339 }
2340
2341 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2342};
2343
2344/// This class is used to represent an MSTORE node
2345class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2346public:
2347 friend class SelectionDAG;
2348
2349 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2350 bool isTrunc, bool isCompressing, EVT MemVT,
2351 MachineMemOperand *MMO)
2352 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) {
2353 StoreSDNodeBits.IsTruncating = isTrunc;
2354 StoreSDNodeBits.IsCompressing = isCompressing;
2355 }
2356
2357 /// Return true if the op does a truncation before store.
2358 /// For integers this is the same as doing a TRUNCATE and storing the result.
2359 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2360 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2361
2362 /// Returns true if the op does a compression to the vector before storing.
2363 /// The node contiguously stores the active elements (integers or floats)
2364 /// in src (those with their respective bit set in writemask k) to unaligned
2365 /// memory at base_addr.
2366 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2367
2368 const SDValue &getValue() const { return getOperand(1); }
2369 const SDValue &getBasePtr() const { return getOperand(2); }
2370 const SDValue &getMask() const { return getOperand(3); }
2371
2372 static bool classof(const SDNode *N) {
2373 return N->getOpcode() == ISD::MSTORE;
2374 }
2375};
2376
2377/// This is a base class used to represent
2378/// MGATHER and MSCATTER nodes
2379///
2380class MaskedGatherScatterSDNode : public MemSDNode {
2381public:
2382 friend class SelectionDAG;
2383
2384 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2385 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2386 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2387 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2388 LSBaseSDNodeBits.AddressingMode = IndexType;
2389 assert(getIndexType() == IndexType && "Value truncated")((getIndexType() == IndexType && "Value truncated") ?
static_cast<void> (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-10~svn374877/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2389, __PRETTY_FUNCTION__))
;
2390 }
2391
2392 /// How is Index applied to BasePtr when computing addresses.
2393 ISD::MemIndexType getIndexType() const {
2394 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2395 }
2396 bool isIndexScaled() const {
2397 return (getIndexType() == ISD::SIGNED_SCALED) ||
2398 (getIndexType() == ISD::UNSIGNED_SCALED);
2399 }
2400 bool isIndexSigned() const {
2401 return (getIndexType() == ISD::SIGNED_SCALED) ||
2402 (getIndexType() == ISD::SIGNED_UNSCALED);
2403 }
2404
2405 // In the both nodes address is Op1, mask is Op2:
2406 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2407 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2408 // Mask is a vector of i1 elements
2409 const SDValue &getBasePtr() const { return getOperand(3); }
2410 const SDValue &getIndex() const { return getOperand(4); }
2411 const SDValue &getMask() const { return getOperand(2); }
2412 const SDValue &getScale() const { return getOperand(5); }
2413
2414 static bool classof(const SDNode *N) {
2415 return N->getOpcode() == ISD::MGATHER ||
2416 N->getOpcode() == ISD::MSCATTER;
2417 }
2418};
2419
2420/// This class is used to represent an MGATHER node
2421///
2422class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2423public:
2424 friend class SelectionDAG;
2425
2426 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2427 EVT MemVT, MachineMemOperand *MMO,
2428 ISD::MemIndexType IndexType)
2429 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2430 IndexType) {}
2431
2432 const SDValue &getPassThru() const { return getOperand(1); }
2433
2434 static bool classof(const SDNode *N) {
2435 return N->getOpcode() == ISD::MGATHER;
2436 }
2437};
2438
2439/// This class is used to represent an MSCATTER node
2440///
2441class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2442public:
2443 friend class SelectionDAG;
2444
2445 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2446 EVT MemVT, MachineMemOperand *MMO,
2447 ISD::MemIndexType IndexType)
2448 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2449 IndexType) {}
2450
2451 const SDValue &getValue() const { return getOperand(1); }
2452
2453 static bool classof(const SDNode *N) {