Bug Summary

File:llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
Warning:line 2112, column 9
The result of the left shift is undefined due to shifting by '127', which is greater or equal to the width of type 'long long'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name WebAssemblyISelLowering.cpp -analyzer-store=region -analyzer-opt-analyze-nested-blocks -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/build-llvm -resource-dir /usr/lib/llvm-14/lib/clang/14.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly -I include -I /build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-14/lib/clang/14.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -O2 -Wno-unused-command-line-argument -Wno-unknown-warning-option -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -std=c++14 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/build-llvm -ferror-limit 19 -fvisibility hidden -fvisibility-inlines-hidden -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2021-10-17-004846-21170-1 -x c++ /build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp

1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "Utils/WebAssemblyTypeUtilities.h"
17#include "Utils/WebAssemblyUtilities.h"
18#include "WebAssemblyMachineFunctionInfo.h"
19#include "WebAssemblySubtarget.h"
20#include "WebAssemblyTargetMachine.h"
21#include "llvm/CodeGen/CallingConvLower.h"
22#include "llvm/CodeGen/MachineInstrBuilder.h"
23#include "llvm/CodeGen/MachineJumpTableInfo.h"
24#include "llvm/CodeGen/MachineModuleInfo.h"
25#include "llvm/CodeGen/MachineRegisterInfo.h"
26#include "llvm/CodeGen/SelectionDAG.h"
27#include "llvm/CodeGen/SelectionDAGNodes.h"
28#include "llvm/IR/DiagnosticInfo.h"
29#include "llvm/IR/DiagnosticPrinter.h"
30#include "llvm/IR/Function.h"
31#include "llvm/IR/Intrinsics.h"
32#include "llvm/IR/IntrinsicsWebAssembly.h"
33#include "llvm/Support/Debug.h"
34#include "llvm/Support/ErrorHandling.h"
35#include "llvm/Support/KnownBits.h"
36#include "llvm/Support/MathExtras.h"
37#include "llvm/Support/raw_ostream.h"
38#include "llvm/Target/TargetOptions.h"
39using namespace llvm;
40
41#define DEBUG_TYPE"wasm-lower" "wasm-lower"
42
43WebAssemblyTargetLowering::WebAssemblyTargetLowering(
44 const TargetMachine &TM, const WebAssemblySubtarget &STI)
45 : TargetLowering(TM), Subtarget(&STI) {
46 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
47
48 // Booleans always contain 0 or 1.
49 setBooleanContents(ZeroOrOneBooleanContent);
50 // Except in SIMD vectors
51 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
52 // We don't know the microarchitecture here, so just reduce register pressure.
53 setSchedulingPreference(Sched::RegPressure);
54 // Tell ISel that we have a stack pointer.
55 setStackPointerRegisterToSaveRestore(
56 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
57 // Set up the register classes.
58 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
59 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
60 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
61 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
62 if (Subtarget->hasSIMD128()) {
63 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
64 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
65 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
66 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
67 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
68 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
69 }
70 if (Subtarget->hasReferenceTypes()) {
71 addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
72 addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
73 }
74 // Compute derived properties from the register classes.
75 computeRegisterProperties(Subtarget->getRegisterInfo());
76
77 // Transform loads and stores to pointers in address space 1 to loads and
78 // stores to WebAssembly global variables, outside linear memory.
79 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
80 setOperationAction(ISD::LOAD, T, Custom);
81 setOperationAction(ISD::STORE, T, Custom);
82 }
83 if (Subtarget->hasSIMD128()) {
84 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
85 MVT::v2f64}) {
86 setOperationAction(ISD::LOAD, T, Custom);
87 setOperationAction(ISD::STORE, T, Custom);
88 }
89 }
90 if (Subtarget->hasReferenceTypes()) {
91 for (auto T : {MVT::externref, MVT::funcref}) {
92 setOperationAction(ISD::LOAD, T, Custom);
93 setOperationAction(ISD::STORE, T, Custom);
94 }
95 }
96
97 setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
98 setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom);
99 setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
100 setOperationAction(ISD::JumpTable, MVTPtr, Custom);
101 setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
102 setOperationAction(ISD::BRIND, MVT::Other, Custom);
103
104 // Take the default expansion for va_arg, va_copy, and va_end. There is no
105 // default action for va_start, so we do that custom.
106 setOperationAction(ISD::VASTART, MVT::Other, Custom);
107 setOperationAction(ISD::VAARG, MVT::Other, Expand);
108 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
109 setOperationAction(ISD::VAEND, MVT::Other, Expand);
110
111 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
112 // Don't expand the floating-point types to constant pools.
113 setOperationAction(ISD::ConstantFP, T, Legal);
114 // Expand floating-point comparisons.
115 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
116 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
117 setCondCodeAction(CC, T, Expand);
118 // Expand floating-point library function operators.
119 for (auto Op :
120 {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
121 setOperationAction(Op, T, Expand);
122 // Note supported floating-point library function operators that otherwise
123 // default to expand.
124 for (auto Op :
125 {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
126 setOperationAction(Op, T, Legal);
127 // Support minimum and maximum, which otherwise default to expand.
128 setOperationAction(ISD::FMINIMUM, T, Legal);
129 setOperationAction(ISD::FMAXIMUM, T, Legal);
130 // WebAssembly currently has no builtin f16 support.
131 setOperationAction(ISD::FP16_TO_FP, T, Expand);
132 setOperationAction(ISD::FP_TO_FP16, T, Expand);
133 setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
134 setTruncStoreAction(T, MVT::f16, Expand);
135 }
136
137 // Expand unavailable integer operations.
138 for (auto Op :
139 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
140 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
141 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
142 for (auto T : {MVT::i32, MVT::i64})
143 setOperationAction(Op, T, Expand);
144 if (Subtarget->hasSIMD128())
145 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
146 setOperationAction(Op, T, Expand);
147 }
148
149 if (Subtarget->hasNontrappingFPToInt())
150 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
151 for (auto T : {MVT::i32, MVT::i64})
152 setOperationAction(Op, T, Custom);
153
154 // SIMD-specific configuration
155 if (Subtarget->hasSIMD128()) {
156 // Hoist bitcasts out of shuffles
157 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
158
159 // Combine extends of extract_subvectors into widening ops
160 setTargetDAGCombine(ISD::SIGN_EXTEND);
161 setTargetDAGCombine(ISD::ZERO_EXTEND);
162
163 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
164 // conversions ops
165 setTargetDAGCombine(ISD::SINT_TO_FP);
166 setTargetDAGCombine(ISD::UINT_TO_FP);
167 setTargetDAGCombine(ISD::FP_EXTEND);
168 setTargetDAGCombine(ISD::EXTRACT_SUBVECTOR);
169
170 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
171 // into conversion ops
172 setTargetDAGCombine(ISD::FP_TO_SINT_SAT);
173 setTargetDAGCombine(ISD::FP_TO_UINT_SAT);
174 setTargetDAGCombine(ISD::FP_ROUND);
175 setTargetDAGCombine(ISD::CONCAT_VECTORS);
176
177 // Support saturating add for i8x16 and i16x8
178 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
179 for (auto T : {MVT::v16i8, MVT::v8i16})
180 setOperationAction(Op, T, Legal);
181
182 // Support integer abs
183 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
184 setOperationAction(ISD::ABS, T, Legal);
185
186 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
187 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
188 MVT::v2f64})
189 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
190
191 // We have custom shuffle lowering to expose the shuffle mask
192 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
193 MVT::v2f64})
194 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
195
196 // Custom lowering since wasm shifts must have a scalar shift amount
197 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
198 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
199 setOperationAction(Op, T, Custom);
200
201 // Custom lower lane accesses to expand out variable indices
202 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})
203 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
204 MVT::v2f64})
205 setOperationAction(Op, T, Custom);
206
207 // There is no i8x16.mul instruction
208 setOperationAction(ISD::MUL, MVT::v16i8, Expand);
209
210 // There is no vector conditional select instruction
211 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
212 MVT::v2f64})
213 setOperationAction(ISD::SELECT_CC, T, Expand);
214
215 // Expand integer operations supported for scalars but not SIMD
216 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP, ISD::SDIV, ISD::UDIV,
217 ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})
218 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
219 setOperationAction(Op, T, Expand);
220
221 // But we do have integer min and max operations
222 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
223 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
224 setOperationAction(Op, T, Legal);
225
226 // And we have popcnt for i8x16
227 setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);
228
229 // Expand float operations supported for scalars but not SIMD
230 for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
231 ISD::FEXP, ISD::FEXP2, ISD::FRINT})
232 for (auto T : {MVT::v4f32, MVT::v2f64})
233 setOperationAction(Op, T, Expand);
234
235 // Unsigned comparison operations are unavailable for i64x2 vectors.
236 for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})
237 setCondCodeAction(CC, MVT::v2i64, Custom);
238
239 // 64x2 conversions are not in the spec
240 for (auto Op :
241 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})
242 for (auto T : {MVT::v2i64, MVT::v2f64})
243 setOperationAction(Op, T, Expand);
244
245 // But saturating fp_to_int converstions are
246 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
247 setOperationAction(Op, MVT::v4i32, Custom);
248 }
249
250 // As a special case, these operators use the type to mean the type to
251 // sign-extend from.
252 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
253 if (!Subtarget->hasSignExt()) {
254 // Sign extends are legal only when extending a vector extract
255 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
256 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
257 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
258 }
259 for (auto T : MVT::integer_fixedlen_vector_valuetypes())
260 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
261
262 // Dynamic stack allocation: use the default expansion.
263 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
264 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
265 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);
266
267 setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
268 setOperationAction(ISD::FrameIndex, MVT::i64, Custom);
269 setOperationAction(ISD::CopyToReg, MVT::Other, Custom);
270
271 // Expand these forms; we pattern-match the forms that we can handle in isel.
272 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
273 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
274 setOperationAction(Op, T, Expand);
275
276 // We have custom switch handling.
277 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
278
279 // WebAssembly doesn't have:
280 // - Floating-point extending loads.
281 // - Floating-point truncating stores.
282 // - i1 extending loads.
283 // - truncating SIMD stores and most extending loads
284 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
285 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
286 for (auto T : MVT::integer_valuetypes())
287 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
288 setLoadExtAction(Ext, T, MVT::i1, Promote);
289 if (Subtarget->hasSIMD128()) {
290 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
291 MVT::v2f64}) {
292 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
293 if (MVT(T) != MemT) {
294 setTruncStoreAction(T, MemT, Expand);
295 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
296 setLoadExtAction(Ext, T, MemT, Expand);
297 }
298 }
299 }
300 // But some vector extending loads are legal
301 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
302 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
303 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
304 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
305 }
306 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal);
307 }
308
309 // Don't do anything clever with build_pairs
310 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
311
312 // Trap lowers to wasm unreachable
313 setOperationAction(ISD::TRAP, MVT::Other, Legal);
314 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
315
316 // Exception handling intrinsics
317 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
318 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
319 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
320
321 setMaxAtomicSizeInBitsSupported(64);
322
323 // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
324 // consistent with the f64 and f128 names.
325 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
326 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
327
328 // Define the emscripten name for return address helper.
329 // TODO: when implementing other Wasm backends, make this generic or only do
330 // this on emscripten depending on what they end up doing.
331 setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
332
333 // Always convert switches to br_tables unless there is only one case, which
334 // is equivalent to a simple branch. This reduces code size for wasm, and we
335 // defer possible jump table optimizations to the VM.
336 setMinimumJumpTableEntries(2);
337}
338
339MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL,
340 uint32_t AS) const {
341 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
342 return MVT::externref;
343 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
344 return MVT::funcref;
345 return TargetLowering::getPointerTy(DL, AS);
346}
347
348MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL,
349 uint32_t AS) const {
350 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
351 return MVT::externref;
352 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
353 return MVT::funcref;
354 return TargetLowering::getPointerMemTy(DL, AS);
355}
356
357TargetLowering::AtomicExpansionKind
358WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
359 // We have wasm instructions for these
360 switch (AI->getOperation()) {
361 case AtomicRMWInst::Add:
362 case AtomicRMWInst::Sub:
363 case AtomicRMWInst::And:
364 case AtomicRMWInst::Or:
365 case AtomicRMWInst::Xor:
366 case AtomicRMWInst::Xchg:
367 return AtomicExpansionKind::None;
368 default:
369 break;
370 }
371 return AtomicExpansionKind::CmpXChg;
372}
373
374bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
375 // Implementation copied from X86TargetLowering.
376 unsigned Opc = VecOp.getOpcode();
377
378 // Assume target opcodes can't be scalarized.
379 // TODO - do we have any exceptions?
380 if (Opc >= ISD::BUILTIN_OP_END)
381 return false;
382
383 // If the vector op is not supported, try to convert to scalar.
384 EVT VecVT = VecOp.getValueType();
385 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
386 return true;
387
388 // If the vector op is supported, but the scalar op is not, the transform may
389 // not be worthwhile.
390 EVT ScalarVT = VecVT.getScalarType();
391 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
392}
393
394FastISel *WebAssemblyTargetLowering::createFastISel(
395 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
396 return WebAssembly::createFastISel(FuncInfo, LibInfo);
397}
398
399MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
400 EVT VT) const {
401 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
402 if (BitWidth > 1 && BitWidth < 8)
403 BitWidth = 8;
404
405 if (BitWidth > 64) {
406 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
407 // the count to be an i32.
408 BitWidth = 32;
409 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 410, __extension__ __PRETTY_FUNCTION__))
410 "32-bit shift counts ought to be enough for anyone")(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 410, __extension__ __PRETTY_FUNCTION__))
;
411 }
412
413 MVT Result = MVT::getIntegerVT(BitWidth);
414 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
&& "Unable to represent scalar shift amount type") ?
void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 415, __extension__ __PRETTY_FUNCTION__))
415 "Unable to represent scalar shift amount type")(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
&& "Unable to represent scalar shift amount type") ?
void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 415, __extension__ __PRETTY_FUNCTION__))
;
416 return Result;
417}
418
419// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
420// undefined result on invalid/overflow, to the WebAssembly opcode, which
421// traps on invalid/overflow.
422static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
423 MachineBasicBlock *BB,
424 const TargetInstrInfo &TII,
425 bool IsUnsigned, bool Int64,
426 bool Float64, unsigned LoweredOpcode) {
427 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
428
429 Register OutReg = MI.getOperand(0).getReg();
430 Register InReg = MI.getOperand(1).getReg();
431
432 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
433 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
434 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
435 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
436 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
437 unsigned Eqz = WebAssembly::EQZ_I32;
438 unsigned And = WebAssembly::AND_I32;
439 int64_t Limit = Int64 ? INT64_MIN(-9223372036854775807L -1) : INT32_MIN(-2147483647-1);
440 int64_t Substitute = IsUnsigned ? 0 : Limit;
441 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
442 auto &Context = BB->getParent()->getFunction().getContext();
443 Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);
444
445 const BasicBlock *LLVMBB = BB->getBasicBlock();
446 MachineFunction *F = BB->getParent();
447 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
448 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
449 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
450
451 MachineFunction::iterator It = ++BB->getIterator();
452 F->insert(It, FalseMBB);
453 F->insert(It, TrueMBB);
454 F->insert(It, DoneMBB);
455
456 // Transfer the remainder of BB and its successor edges to DoneMBB.
457 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
458 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
459
460 BB->addSuccessor(TrueMBB);
461 BB->addSuccessor(FalseMBB);
462 TrueMBB->addSuccessor(DoneMBB);
463 FalseMBB->addSuccessor(DoneMBB);
464
465 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
466 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
467 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
468 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
469 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
470 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
471 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
472
473 MI.eraseFromParent();
474 // For signed numbers, we can do a single comparison to determine whether
475 // fabs(x) is within range.
476 if (IsUnsigned) {
477 Tmp0 = InReg;
478 } else {
479 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
480 }
481 BuildMI(BB, DL, TII.get(FConst), Tmp1)
482 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
483 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
484
485 // For unsigned numbers, we have to do a separate comparison with zero.
486 if (IsUnsigned) {
487 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
488 Register SecondCmpReg =
489 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
490 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
491 BuildMI(BB, DL, TII.get(FConst), Tmp1)
492 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
493 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
494 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
495 CmpReg = AndReg;
496 }
497
498 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
499
500 // Create the CFG diamond to select between doing the conversion or using
501 // the substitute value.
502 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
503 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
504 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
505 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
506 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
507 .addReg(FalseReg)
508 .addMBB(FalseMBB)
509 .addReg(TrueReg)
510 .addMBB(TrueMBB);
511
512 return DoneMBB;
513}
514
515static MachineBasicBlock *
516LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
517 const WebAssemblySubtarget *Subtarget,
518 const TargetInstrInfo &TII) {
519 MachineInstr &CallParams = *CallResults.getPrevNode();
520 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS)(static_cast <bool> (CallParams.getOpcode() == WebAssembly
::CALL_PARAMS) ? void (0) : __assert_fail ("CallParams.getOpcode() == WebAssembly::CALL_PARAMS"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 520, __extension__ __PRETTY_FUNCTION__))
;
521 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 522, __extension__ __PRETTY_FUNCTION__))
522 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS)(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 522, __extension__ __PRETTY_FUNCTION__))
;
523
524 bool IsIndirect = CallParams.getOperand(0).isReg();
525 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
526
527 bool IsFuncrefCall = false;
528 if (IsIndirect) {
529 Register Reg = CallParams.getOperand(0).getReg();
530 const MachineFunction *MF = BB->getParent();
531 const MachineRegisterInfo &MRI = MF->getRegInfo();
532 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
533 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
534 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes())(static_cast <bool> (!IsFuncrefCall || Subtarget->hasReferenceTypes
()) ? void (0) : __assert_fail ("!IsFuncrefCall || Subtarget->hasReferenceTypes()"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 534, __extension__ __PRETTY_FUNCTION__))
;
535 }
536
537 unsigned CallOp;
538 if (IsIndirect && IsRetCall) {
539 CallOp = WebAssembly::RET_CALL_INDIRECT;
540 } else if (IsIndirect) {
541 CallOp = WebAssembly::CALL_INDIRECT;
542 } else if (IsRetCall) {
543 CallOp = WebAssembly::RET_CALL;
544 } else {
545 CallOp = WebAssembly::CALL;
546 }
547
548 MachineFunction &MF = *BB->getParent();
549 const MCInstrDesc &MCID = TII.get(CallOp);
550 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));
551
552 // See if we must truncate the function pointer.
553 // CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers
554 // as 64-bit for uniformity with other pointer types.
555 // See also: WebAssemblyFastISel::selectCall
556 if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) {
557 Register Reg32 =
558 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
559 auto &FnPtr = CallParams.getOperand(0);
560 BuildMI(*BB, CallResults.getIterator(), DL,
561 TII.get(WebAssembly::I32_WRAP_I64), Reg32)
562 .addReg(FnPtr.getReg());
563 FnPtr.setReg(Reg32);
564 }
565
566 // Move the function pointer to the end of the arguments for indirect calls
567 if (IsIndirect) {
568 auto FnPtr = CallParams.getOperand(0);
569 CallParams.RemoveOperand(0);
570
571 // For funcrefs, call_indirect is done through __funcref_call_table and the
572 // funcref is always installed in slot 0 of the table, therefore instead of having
573 // the function pointer added at the end of the params list, a zero (the index in
574 // __funcref_call_table is added).
575 if (IsFuncrefCall) {
576 Register RegZero =
577 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
578 MachineInstrBuilder MIBC0 =
579 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
580
581 BB->insert(CallResults.getIterator(), MIBC0);
582 MachineInstrBuilder(MF, CallParams).addReg(RegZero);
583 } else
584 CallParams.addOperand(FnPtr);
585 }
586
587 for (auto Def : CallResults.defs())
588 MIB.add(Def);
589
590 if (IsIndirect) {
591 // Placeholder for the type index.
592 MIB.addImm(0);
593 // The table into which this call_indirect indexes.
594 MCSymbolWasm *Table = IsFuncrefCall
595 ? WebAssembly::getOrCreateFuncrefCallTableSymbol(
596 MF.getContext(), Subtarget)
597 : WebAssembly::getOrCreateFunctionTableSymbol(
598 MF.getContext(), Subtarget);
599 if (Subtarget->hasReferenceTypes()) {
600 MIB.addSym(Table);
601 } else {
602 // For the MVP there is at most one table whose number is 0, but we can't
603 // write a table symbol or issue relocations. Instead we just ensure the
604 // table is live and write a zero.
605 Table->setNoStrip();
606 MIB.addImm(0);
607 }
608 }
609
610 for (auto Use : CallParams.uses())
611 MIB.add(Use);
612
613 BB->insert(CallResults.getIterator(), MIB);
614 CallParams.eraseFromParent();
615 CallResults.eraseFromParent();
616
617 // If this is a funcref call, to avoid hidden GC roots, we need to clear the
618 // table slot with ref.null upon call_indirect return.
619 //
620 // This generates the following code, which comes right after a call_indirect
621 // of a funcref:
622 //
623 // i32.const 0
624 // ref.null func
625 // table.set __funcref_call_table
626 if (IsIndirect && IsFuncrefCall) {
627 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
628 MF.getContext(), Subtarget);
629 Register RegZero =
630 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
631 MachineInstr *Const0 =
632 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
633 BB->insertAfter(MIB.getInstr()->getIterator(), Const0);
634
635 Register RegFuncref =
636 MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
637 MachineInstr *RefNull =
638 BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref)
639 .addImm(static_cast<int32_t>(WebAssembly::HeapType::Funcref));
640 BB->insertAfter(Const0->getIterator(), RefNull);
641
642 MachineInstr *TableSet =
643 BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
644 .addSym(Table)
645 .addReg(RegZero)
646 .addReg(RegFuncref);
647 BB->insertAfter(RefNull->getIterator(), TableSet);
648 }
649
650 return BB;
651}
652
653MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
654 MachineInstr &MI, MachineBasicBlock *BB) const {
655 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
656 DebugLoc DL = MI.getDebugLoc();
657
658 switch (MI.getOpcode()) {
659 default:
660 llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 660)
;
661 case WebAssembly::FP_TO_SINT_I32_F32:
662 return LowerFPToInt(MI, DL, BB, TII, false, false, false,
663 WebAssembly::I32_TRUNC_S_F32);
664 case WebAssembly::FP_TO_UINT_I32_F32:
665 return LowerFPToInt(MI, DL, BB, TII, true, false, false,
666 WebAssembly::I32_TRUNC_U_F32);
667 case WebAssembly::FP_TO_SINT_I64_F32:
668 return LowerFPToInt(MI, DL, BB, TII, false, true, false,
669 WebAssembly::I64_TRUNC_S_F32);
670 case WebAssembly::FP_TO_UINT_I64_F32:
671 return LowerFPToInt(MI, DL, BB, TII, true, true, false,
672 WebAssembly::I64_TRUNC_U_F32);
673 case WebAssembly::FP_TO_SINT_I32_F64:
674 return LowerFPToInt(MI, DL, BB, TII, false, false, true,
675 WebAssembly::I32_TRUNC_S_F64);
676 case WebAssembly::FP_TO_UINT_I32_F64:
677 return LowerFPToInt(MI, DL, BB, TII, true, false, true,
678 WebAssembly::I32_TRUNC_U_F64);
679 case WebAssembly::FP_TO_SINT_I64_F64:
680 return LowerFPToInt(MI, DL, BB, TII, false, true, true,
681 WebAssembly::I64_TRUNC_S_F64);
682 case WebAssembly::FP_TO_UINT_I64_F64:
683 return LowerFPToInt(MI, DL, BB, TII, true, true, true,
684 WebAssembly::I64_TRUNC_U_F64);
685 case WebAssembly::CALL_RESULTS:
686 case WebAssembly::RET_CALL_RESULTS:
687 return LowerCallResults(MI, DL, BB, Subtarget, TII);
688 }
689}
690
691const char *
692WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
693 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
694 case WebAssemblyISD::FIRST_NUMBER:
695 case WebAssemblyISD::FIRST_MEM_OPCODE:
696 break;
697#define HANDLE_NODETYPE(NODE) \
698 case WebAssemblyISD::NODE: \
699 return "WebAssemblyISD::" #NODE;
700#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
701#include "WebAssemblyISD.def"
702#undef HANDLE_MEM_NODETYPE
703#undef HANDLE_NODETYPE
704 }
705 return nullptr;
706}
707
708std::pair<unsigned, const TargetRegisterClass *>
709WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
710 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
711 // First, see if this is a constraint that directly corresponds to a
712 // WebAssembly register class.
713 if (Constraint.size() == 1) {
714 switch (Constraint[0]) {
715 case 'r':
716 assert(VT != MVT::iPTR && "Pointer MVT not expected here")(static_cast <bool> (VT != MVT::iPTR && "Pointer MVT not expected here"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Pointer MVT not expected here\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 716, __extension__ __PRETTY_FUNCTION__))
;
717 if (Subtarget->hasSIMD128() && VT.isVector()) {
718 if (VT.getSizeInBits() == 128)
719 return std::make_pair(0U, &WebAssembly::V128RegClass);
720 }
721 if (VT.isInteger() && !VT.isVector()) {
722 if (VT.getSizeInBits() <= 32)
723 return std::make_pair(0U, &WebAssembly::I32RegClass);
724 if (VT.getSizeInBits() <= 64)
725 return std::make_pair(0U, &WebAssembly::I64RegClass);
726 }
727 if (VT.isFloatingPoint() && !VT.isVector()) {
728 switch (VT.getSizeInBits()) {
729 case 32:
730 return std::make_pair(0U, &WebAssembly::F32RegClass);
731 case 64:
732 return std::make_pair(0U, &WebAssembly::F64RegClass);
733 default:
734 break;
735 }
736 }
737 break;
738 default:
739 break;
740 }
741 }
742
743 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
744}
745
746bool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const {
747 // Assume ctz is a relatively cheap operation.
748 return true;
749}
750
751bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const {
752 // Assume clz is a relatively cheap operation.
753 return true;
754}
755
756bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
757 const AddrMode &AM,
758 Type *Ty, unsigned AS,
759 Instruction *I) const {
760 // WebAssembly offsets are added as unsigned without wrapping. The
761 // isLegalAddressingMode gives us no way to determine if wrapping could be
762 // happening, so we approximate this by accepting only non-negative offsets.
763 if (AM.BaseOffs < 0)
764 return false;
765
766 // WebAssembly has no scale register operands.
767 if (AM.Scale != 0)
768 return false;
769
770 // Everything else is legal.
771 return true;
772}
773
774bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
775 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
776 MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
777 // WebAssembly supports unaligned accesses, though it should be declared
778 // with the p2align attribute on loads and stores which do so, and there
779 // may be a performance impact. We tell LLVM they're "fast" because
780 // for the kinds of things that LLVM uses this for (merging adjacent stores
781 // of constants, etc.), WebAssembly implementations will either want the
782 // unaligned access or they'll split anyway.
783 if (Fast)
784 *Fast = true;
785 return true;
786}
787
788bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
789 AttributeList Attr) const {
790 // The current thinking is that wasm engines will perform this optimization,
791 // so we can save on code size.
792 return true;
793}
794
795bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
796 EVT ExtT = ExtVal.getValueType();
797 EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);
798 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
799 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
800 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
801}
802
803bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
804 const GlobalAddressSDNode *GA) const {
805 // Wasm doesn't support function addresses with offsets
806 const GlobalValue *GV = GA->getGlobal();
807 return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
808}
809
810EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
811 LLVMContext &C,
812 EVT VT) const {
813 if (VT.isVector())
814 return VT.changeVectorElementTypeToInteger();
815
816 // So far, all branch instructions in Wasm take an I32 condition.
817 // The default TargetLowering::getSetCCResultType returns the pointer size,
818 // which would be useful to reduce instruction counts when testing
819 // against 64-bit pointers/values if at some point Wasm supports that.
820 return EVT::getIntegerVT(C, 32);
821}
822
823bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
824 const CallInst &I,
825 MachineFunction &MF,
826 unsigned Intrinsic) const {
827 switch (Intrinsic) {
828 case Intrinsic::wasm_memory_atomic_notify:
829 Info.opc = ISD::INTRINSIC_W_CHAIN;
830 Info.memVT = MVT::i32;
831 Info.ptrVal = I.getArgOperand(0);
832 Info.offset = 0;
833 Info.align = Align(4);
834 // atomic.notify instruction does not really load the memory specified with
835 // this argument, but MachineMemOperand should either be load or store, so
836 // we set this to a load.
837 // FIXME Volatile isn't really correct, but currently all LLVM atomic
838 // instructions are treated as volatiles in the backend, so we should be
839 // consistent. The same applies for wasm_atomic_wait intrinsics too.
840 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
841 return true;
842 case Intrinsic::wasm_memory_atomic_wait32:
843 Info.opc = ISD::INTRINSIC_W_CHAIN;
844 Info.memVT = MVT::i32;
845 Info.ptrVal = I.getArgOperand(0);
846 Info.offset = 0;
847 Info.align = Align(4);
848 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
849 return true;
850 case Intrinsic::wasm_memory_atomic_wait64:
851 Info.opc = ISD::INTRINSIC_W_CHAIN;
852 Info.memVT = MVT::i64;
853 Info.ptrVal = I.getArgOperand(0);
854 Info.offset = 0;
855 Info.align = Align(8);
856 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
857 return true;
858 default:
859 return false;
860 }
861}
862
863void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
864 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
865 const SelectionDAG &DAG, unsigned Depth) const {
866 switch (Op.getOpcode()) {
867 default:
868 break;
869 case ISD::INTRINSIC_WO_CHAIN: {
870 unsigned IntNo = Op.getConstantOperandVal(0);
871 switch (IntNo) {
872 default:
873 break;
874 case Intrinsic::wasm_bitmask: {
875 unsigned BitWidth = Known.getBitWidth();
876 EVT VT = Op.getOperand(1).getSimpleValueType();
877 unsigned PossibleBits = VT.getVectorNumElements();
878 APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);
879 Known.Zero |= ZeroMask;
880 break;
881 }
882 }
883 }
884 }
885}
886
887TargetLoweringBase::LegalizeTypeAction
888WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
889 if (VT.isFixedLengthVector()) {
890 MVT EltVT = VT.getVectorElementType();
891 // We have legal vector types with these lane types, so widening the
892 // vector would let us use some of the lanes directly without having to
893 // extend or truncate values.
894 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
895 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
896 return TypeWidenVector;
897 }
898
899 return TargetLoweringBase::getPreferredVectorAction(VT);
900}
901
902//===----------------------------------------------------------------------===//
903// WebAssembly Lowering private implementation.
904//===----------------------------------------------------------------------===//
905
906//===----------------------------------------------------------------------===//
907// Lowering Code
908//===----------------------------------------------------------------------===//
909
910static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
911 MachineFunction &MF = DAG.getMachineFunction();
912 DAG.getContext()->diagnose(
913 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
914}
915
916// Test whether the given calling convention is supported.
917static bool callingConvSupported(CallingConv::ID CallConv) {
918 // We currently support the language-independent target-independent
919 // conventions. We don't yet have a way to annotate calls with properties like
920 // "cold", and we don't have any call-clobbered registers, so these are mostly
921 // all handled the same.
922 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
923 CallConv == CallingConv::Cold ||
924 CallConv == CallingConv::PreserveMost ||
925 CallConv == CallingConv::PreserveAll ||
926 CallConv == CallingConv::CXX_FAST_TLS ||
927 CallConv == CallingConv::WASM_EmscriptenInvoke ||
928 CallConv == CallingConv::Swift;
929}
930
931SDValue
932WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
933 SmallVectorImpl<SDValue> &InVals) const {
934 SelectionDAG &DAG = CLI.DAG;
935 SDLoc DL = CLI.DL;
936 SDValue Chain = CLI.Chain;
937 SDValue Callee = CLI.Callee;
938 MachineFunction &MF = DAG.getMachineFunction();
939 auto Layout = MF.getDataLayout();
940
941 CallingConv::ID CallConv = CLI.CallConv;
942 if (!callingConvSupported(CallConv))
943 fail(DL, DAG,
944 "WebAssembly doesn't support language-specific or target-specific "
945 "calling conventions yet");
946 if (CLI.IsPatchPoint)
947 fail(DL, DAG, "WebAssembly doesn't support patch point yet");
948
949 if (CLI.IsTailCall) {
950 auto NoTail = [&](const char *Msg) {
951 if (CLI.CB && CLI.CB->isMustTailCall())
952 fail(DL, DAG, Msg);
953 CLI.IsTailCall = false;
954 };
955
956 if (!Subtarget->hasTailCall())
957 NoTail("WebAssembly 'tail-call' feature not enabled");
958
959 // Varargs calls cannot be tail calls because the buffer is on the stack
960 if (CLI.IsVarArg)
961 NoTail("WebAssembly does not support varargs tail calls");
962
963 // Do not tail call unless caller and callee return types match
964 const Function &F = MF.getFunction();
965 const TargetMachine &TM = getTargetMachine();
966 Type *RetTy = F.getReturnType();
967 SmallVector<MVT, 4> CallerRetTys;
968 SmallVector<MVT, 4> CalleeRetTys;
969 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
970 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
971 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
972 std::equal(CallerRetTys.begin(), CallerRetTys.end(),
973 CalleeRetTys.begin());
974 if (!TypesMatch)
975 NoTail("WebAssembly tail call requires caller and callee return types to "
976 "match");
977
978 // If pointers to local stack values are passed, we cannot tail call
979 if (CLI.CB) {
980 for (auto &Arg : CLI.CB->args()) {
981 Value *Val = Arg.get();
982 // Trace the value back through pointer operations
983 while (true) {
984 Value *Src = Val->stripPointerCastsAndAliases();
985 if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
986 Src = GEP->getPointerOperand();
987 if (Val == Src)
988 break;
989 Val = Src;
990 }
991 if (isa<AllocaInst>(Val)) {
992 NoTail(
993 "WebAssembly does not support tail calling with stack arguments");
994 break;
995 }
996 }
997 }
998 }
999
1000 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1001 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1002 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1003
1004 // The generic code may have added an sret argument. If we're lowering an
1005 // invoke function, the ABI requires that the function pointer be the first
1006 // argument, so we may have to swap the arguments.
1007 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
1008 Outs[0].Flags.isSRet()) {
1009 std::swap(Outs[0], Outs[1]);
1010 std::swap(OutVals[0], OutVals[1]);
1011 }
1012
1013 bool HasSwiftSelfArg = false;
1014 bool HasSwiftErrorArg = false;
1015 unsigned NumFixedArgs = 0;
1016 for (unsigned I = 0; I < Outs.size(); ++I) {
1017 const ISD::OutputArg &Out = Outs[I];
1018 SDValue &OutVal = OutVals[I];
1019 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
1020 HasSwiftErrorArg |= Out.Flags.isSwiftError();
1021 if (Out.Flags.isNest())
1022 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1023 if (Out.Flags.isInAlloca())
1024 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1025 if (Out.Flags.isInConsecutiveRegs())
1026 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1027 if (Out.Flags.isInConsecutiveRegsLast())
1028 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1029 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
1030 auto &MFI = MF.getFrameInfo();
1031 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
1032 Out.Flags.getNonZeroByValAlign(),
1033 /*isSS=*/false);
1034 SDValue SizeNode =
1035 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
1036 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1037 Chain = DAG.getMemcpy(
1038 Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
1039 /*isVolatile*/ false, /*AlwaysInline=*/false,
1040 /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
1041 OutVal = FINode;
1042 }
1043 // Count the number of fixed args *after* legalization.
1044 NumFixedArgs += Out.IsFixed;
1045 }
1046
1047 bool IsVarArg = CLI.IsVarArg;
1048 auto PtrVT = getPointerTy(Layout);
1049
1050 // For swiftcc, emit additional swiftself and swifterror arguments
1051 // if there aren't. These additional arguments are also added for callee
1052 // signature They are necessary to match callee and caller signature for
1053 // indirect call.
1054 if (CallConv == CallingConv::Swift) {
1055 if (!HasSwiftSelfArg) {
1056 NumFixedArgs++;
1057 ISD::OutputArg Arg;
1058 Arg.Flags.setSwiftSelf();
1059 CLI.Outs.push_back(Arg);
1060 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1061 CLI.OutVals.push_back(ArgVal);
1062 }
1063 if (!HasSwiftErrorArg) {
1064 NumFixedArgs++;
1065 ISD::OutputArg Arg;
1066 Arg.Flags.setSwiftError();
1067 CLI.Outs.push_back(Arg);
1068 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1069 CLI.OutVals.push_back(ArgVal);
1070 }
1071 }
1072
1073 // Analyze operands of the call, assigning locations to each operand.
1074 SmallVector<CCValAssign, 16> ArgLocs;
1075 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1076
1077 if (IsVarArg) {
1078 // Outgoing non-fixed arguments are placed in a buffer. First
1079 // compute their offsets and the total amount of buffer space needed.
1080 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
1081 const ISD::OutputArg &Out = Outs[I];
1082 SDValue &Arg = OutVals[I];
1083 EVT VT = Arg.getValueType();
1084 assert(VT != MVT::iPTR && "Legalized args should be concrete")(static_cast <bool> (VT != MVT::iPTR && "Legalized args should be concrete"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Legalized args should be concrete\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1084, __extension__ __PRETTY_FUNCTION__))
;
1085 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
1086 Align Alignment =
1087 std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
1088 unsigned Offset =
1089 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
1090 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
1091 Offset, VT.getSimpleVT(),
1092 CCValAssign::Full));
1093 }
1094 }
1095
1096 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1097
1098 SDValue FINode;
1099 if (IsVarArg && NumBytes) {
1100 // For non-fixed arguments, next emit stores to store the argument values
1101 // to the stack buffer at the offsets computed above.
1102 int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
1103 Layout.getStackAlignment(),
1104 /*isSS=*/false);
1105 unsigned ValNo = 0;
1106 SmallVector<SDValue, 8> Chains;
1107 for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
1108 assert(ArgLocs[ValNo].getValNo() == ValNo &&(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
&& "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1109, __extension__ __PRETTY_FUNCTION__))
1109 "ArgLocs should remain in order and only hold varargs args")(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
&& "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1109, __extension__ __PRETTY_FUNCTION__))
;
1110 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1111 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1112 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
1113 DAG.getConstant(Offset, DL, PtrVT));
1114 Chains.push_back(
1115 DAG.getStore(Chain, DL, Arg, Add,
1116 MachinePointerInfo::getFixedStack(MF, FI, Offset)));
1117 }
1118 if (!Chains.empty())
1119 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
1120 } else if (IsVarArg) {
1121 FINode = DAG.getIntPtrConstant(0, DL);
1122 }
1123
1124 if (Callee->getOpcode() == ISD::GlobalAddress) {
1125 // If the callee is a GlobalAddress node (quite common, every direct call
1126 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
1127 // doesn't at MO_GOT which is not needed for direct calls.
1128 GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee);
1129 Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
1130 getPointerTy(DAG.getDataLayout()),
1131 GA->getOffset());
1132 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
1133 getPointerTy(DAG.getDataLayout()), Callee);
1134 }
1135
1136 // Compute the operands for the CALLn node.
1137 SmallVector<SDValue, 16> Ops;
1138 Ops.push_back(Chain);
1139 Ops.push_back(Callee);
1140
1141 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
1142 // isn't reliable.
1143 Ops.append(OutVals.begin(),
1144 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
1145 // Add a pointer to the vararg buffer.
1146 if (IsVarArg)
1147 Ops.push_back(FINode);
1148
1149 SmallVector<EVT, 8> InTys;
1150 for (const auto &In : Ins) {
1151 assert(!In.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!In.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1151, __extension__ __PRETTY_FUNCTION__))
;
1152 assert(!In.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!In.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1152, __extension__ __PRETTY_FUNCTION__))
;
1153 if (In.Flags.isInAlloca())
1154 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
1155 if (In.Flags.isInConsecutiveRegs())
1156 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
1157 if (In.Flags.isInConsecutiveRegsLast())
1158 fail(DL, DAG,
1159 "WebAssembly hasn't implemented cons regs last return values");
1160 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1161 // registers.
1162 InTys.push_back(In.VT);
1163 }
1164
1165 // Lastly, if this is a call to a funcref we need to add an instruction
1166 // table.set to the chain and transform the call.
1167 if (CLI.CB &&
1168 WebAssembly::isFuncrefType(CLI.CB->getCalledOperand()->getType())) {
1169 // In the absence of function references proposal where a funcref call is
1170 // lowered to call_ref, using reference types we generate a table.set to set
1171 // the funcref to a special table used solely for this purpose, followed by
1172 // a call_indirect. Here we just generate the table set, and return the
1173 // SDValue of the table.set so that LowerCall can finalize the lowering by
1174 // generating the call_indirect.
1175 SDValue Chain = Ops[0];
1176
1177 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
1178 MF.getContext(), Subtarget);
1179 SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
1180 SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
1181 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
1182 SDValue TableSet = DAG.getMemIntrinsicNode(
1183 WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
1184 MVT::funcref,
1185 // Machine Mem Operand args
1186 MachinePointerInfo(
1187 WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF),
1188 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),
1189 MachineMemOperand::MOStore);
1190
1191 Ops[0] = TableSet; // The new chain is the TableSet itself
1192 }
1193
1194 if (CLI.IsTailCall) {
1195 // ret_calls do not return values to the current frame
1196 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1197 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
1198 }
1199
1200 InTys.push_back(MVT::Other);
1201 SDVTList InTyList = DAG.getVTList(InTys);
1202 SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);
1203
1204 for (size_t I = 0; I < Ins.size(); ++I)
1205 InVals.push_back(Res.getValue(I));
1206
1207 // Return the chain
1208 return Res.getValue(Ins.size());
1209}
1210
1211bool WebAssemblyTargetLowering::CanLowerReturn(
1212 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
1213 const SmallVectorImpl<ISD::OutputArg> &Outs,
1214 LLVMContext & /*Context*/) const {
1215 // WebAssembly can only handle returning tuples with multivalue enabled
1216 return Subtarget->hasMultivalue() || Outs.size() <= 1;
1217}
1218
1219SDValue WebAssemblyTargetLowering::LowerReturn(
1220 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
1221 const SmallVectorImpl<ISD::OutputArg> &Outs,
1222 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1223 SelectionDAG &DAG) const {
1224 assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1225, __extension__ __PRETTY_FUNCTION__))
1225 "MVP WebAssembly can only return up to one value")(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1225, __extension__ __PRETTY_FUNCTION__))
;
1226 if (!callingConvSupported(CallConv))
1227 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1228
1229 SmallVector<SDValue, 4> RetOps(1, Chain);
1230 RetOps.append(OutVals.begin(), OutVals.end());
1231 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
1232
1233 // Record the number and types of the return values.
1234 for (const ISD::OutputArg &Out : Outs) {
1235 assert(!Out.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!Out.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isByVal() && \"byval is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1235, __extension__ __PRETTY_FUNCTION__))
;
1236 assert(!Out.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!Out.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isNest() && \"nest is not valid for return values\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1236, __extension__ __PRETTY_FUNCTION__))
;
1237 assert(Out.IsFixed && "non-fixed return value is not valid")(static_cast <bool> (Out.IsFixed && "non-fixed return value is not valid"
) ? void (0) : __assert_fail ("Out.IsFixed && \"non-fixed return value is not valid\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1237, __extension__ __PRETTY_FUNCTION__))
;
1238 if (Out.Flags.isInAlloca())
1239 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
1240 if (Out.Flags.isInConsecutiveRegs())
1241 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
1242 if (Out.Flags.isInConsecutiveRegsLast())
1243 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
1244 }
1245
1246 return Chain;
1247}
1248
1249SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1250 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1251 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1252 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1253 if (!callingConvSupported(CallConv))
1254 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1255
1256 MachineFunction &MF = DAG.getMachineFunction();
1257 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
1258
1259 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
1260 // of the incoming values before they're represented by virtual registers.
1261 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
1262
1263 bool HasSwiftErrorArg = false;
1264 bool HasSwiftSelfArg = false;
1265 for (const ISD::InputArg &In : Ins) {
1266 HasSwiftSelfArg |= In.Flags.isSwiftSelf();
1267 HasSwiftErrorArg |= In.Flags.isSwiftError();
1268 if (In.Flags.isInAlloca())
1269 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1270 if (In.Flags.isNest())
1271 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1272 if (In.Flags.isInConsecutiveRegs())
1273 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1274 if (In.Flags.isInConsecutiveRegsLast())
1275 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1276 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1277 // registers.
1278 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
1279 DAG.getTargetConstant(InVals.size(),
1280 DL, MVT::i32))
1281 : DAG.getUNDEF(In.VT));
1282
1283 // Record the number and types of arguments.
1284 MFI->addParam(In.VT);
1285 }
1286
1287 // For swiftcc, emit additional swiftself and swifterror arguments
1288 // if there aren't. These additional arguments are also added for callee
1289 // signature They are necessary to match callee and caller signature for
1290 // indirect call.
1291 auto PtrVT = getPointerTy(MF.getDataLayout());
1292 if (CallConv == CallingConv::Swift) {
1293 if (!HasSwiftSelfArg) {
1294 MFI->addParam(PtrVT);
1295 }
1296 if (!HasSwiftErrorArg) {
1297 MFI->addParam(PtrVT);
1298 }
1299 }
1300 // Varargs are copied into a buffer allocated by the caller, and a pointer to
1301 // the buffer is passed as an argument.
1302 if (IsVarArg) {
1303 MVT PtrVT = getPointerTy(MF.getDataLayout());
1304 Register VarargVreg =
1305 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
1306 MFI->setVarargBufferVreg(VarargVreg);
1307 Chain = DAG.getCopyToReg(
1308 Chain, DL, VarargVreg,
1309 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
1310 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
1311 MFI->addParam(PtrVT);
1312 }
1313
1314 // Record the number and types of arguments and results.
1315 SmallVector<MVT, 4> Params;
1316 SmallVector<MVT, 4> Results;
1317 computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(),
1318 MF.getFunction(), DAG.getTarget(), Params, Results);
1319 for (MVT VT : Results)
1320 MFI->addResult(VT);
1321 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
1322 // the param logic here with ComputeSignatureVTs
1323 assert(MFI->getParams().size() == Params.size() &&(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1325, __extension__ __PRETTY_FUNCTION__))
1324 std::equal(MFI->getParams().begin(), MFI->getParams().end(),(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1325, __extension__ __PRETTY_FUNCTION__))
1325 Params.begin()))(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1325, __extension__ __PRETTY_FUNCTION__))
;
1326
1327 return Chain;
1328}
1329
1330void WebAssemblyTargetLowering::ReplaceNodeResults(
1331 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1332 switch (N->getOpcode()) {
1333 case ISD::SIGN_EXTEND_INREG:
1334 // Do not add any results, signifying that N should not be custom lowered
1335 // after all. This happens because simd128 turns on custom lowering for
1336 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
1337 // illegal type.
1338 break;
1339 default:
1340 llvm_unreachable(::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1341)
1341 "ReplaceNodeResults not implemented for this op for WebAssembly!")::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1341)
;
1342 }
1343}
1344
1345//===----------------------------------------------------------------------===//
1346// Custom lowering hooks.
1347//===----------------------------------------------------------------------===//
1348
1349SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1350 SelectionDAG &DAG) const {
1351 SDLoc DL(Op);
1352 switch (Op.getOpcode()) {
1
Control jumps to 'case BUILD_VECTOR:' at line 1389
1353 default:
1354 llvm_unreachable("unimplemented operation lowering")::llvm::llvm_unreachable_internal("unimplemented operation lowering"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1354)
;
1355 return SDValue();
1356 case ISD::FrameIndex:
1357 return LowerFrameIndex(Op, DAG);
1358 case ISD::GlobalAddress:
1359 return LowerGlobalAddress(Op, DAG);
1360 case ISD::GlobalTLSAddress:
1361 return LowerGlobalTLSAddress(Op, DAG);
1362 case ISD::ExternalSymbol:
1363 return LowerExternalSymbol(Op, DAG);
1364 case ISD::JumpTable:
1365 return LowerJumpTable(Op, DAG);
1366 case ISD::BR_JT:
1367 return LowerBR_JT(Op, DAG);
1368 case ISD::VASTART:
1369 return LowerVASTART(Op, DAG);
1370 case ISD::BlockAddress:
1371 case ISD::BRIND:
1372 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
1373 return SDValue();
1374 case ISD::RETURNADDR:
1375 return LowerRETURNADDR(Op, DAG);
1376 case ISD::FRAMEADDR:
1377 return LowerFRAMEADDR(Op, DAG);
1378 case ISD::CopyToReg:
1379 return LowerCopyToReg(Op, DAG);
1380 case ISD::EXTRACT_VECTOR_ELT:
1381 case ISD::INSERT_VECTOR_ELT:
1382 return LowerAccessVectorElement(Op, DAG);
1383 case ISD::INTRINSIC_VOID:
1384 case ISD::INTRINSIC_WO_CHAIN:
1385 case ISD::INTRINSIC_W_CHAIN:
1386 return LowerIntrinsic(Op, DAG);
1387 case ISD::SIGN_EXTEND_INREG:
1388 return LowerSIGN_EXTEND_INREG(Op, DAG);
1389 case ISD::BUILD_VECTOR:
1390 return LowerBUILD_VECTOR(Op, DAG);
2
Calling 'WebAssemblyTargetLowering::LowerBUILD_VECTOR'
1391 case ISD::VECTOR_SHUFFLE:
1392 return LowerVECTOR_SHUFFLE(Op, DAG);
1393 case ISD::SETCC:
1394 return LowerSETCC(Op, DAG);
1395 case ISD::SHL:
1396 case ISD::SRA:
1397 case ISD::SRL:
1398 return LowerShift(Op, DAG);
1399 case ISD::FP_TO_SINT_SAT:
1400 case ISD::FP_TO_UINT_SAT:
1401 return LowerFP_TO_INT_SAT(Op, DAG);
1402 case ISD::LOAD:
1403 return LowerLoad(Op, DAG);
1404 case ISD::STORE:
1405 return LowerStore(Op, DAG);
1406 }
1407}
1408
1409static bool IsWebAssemblyGlobal(SDValue Op) {
1410 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
1411 return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace());
1412
1413 return false;
1414}
1415
1416static Optional<unsigned> IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG) {
1417 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);
1418 if (!FI)
1419 return None;
1420
1421 auto &MF = DAG.getMachineFunction();
1422 return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex());
1423}
1424
1425SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
1426 SelectionDAG &DAG) const {
1427 SDLoc DL(Op);
1428 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
1429 const SDValue &Value = SN->getValue();
1430 const SDValue &Base = SN->getBasePtr();
1431 const SDValue &Offset = SN->getOffset();
1432
1433 if (IsWebAssemblyGlobal(Base)) {
1434 if (!Offset->isUndef())
1435 report_fatal_error("unexpected offset when storing to webassembly global",
1436 false);
1437
1438 SDVTList Tys = DAG.getVTList(MVT::Other);
1439 SDValue Ops[] = {SN->getChain(), Value, Base};
1440 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
1441 SN->getMemoryVT(), SN->getMemOperand());
1442 }
1443
1444 if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1445 if (!Offset->isUndef())
1446 report_fatal_error("unexpected offset when storing to webassembly local",
1447 false);
1448
1449 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1450 SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
1451 SDValue Ops[] = {SN->getChain(), Idx, Value};
1452 return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
1453 }
1454
1455 return Op;
1456}
1457
1458SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
1459 SelectionDAG &DAG) const {
1460 SDLoc DL(Op);
1461 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
1462 const SDValue &Base = LN->getBasePtr();
1463 const SDValue &Offset = LN->getOffset();
1464
1465 if (IsWebAssemblyGlobal(Base)) {
1466 if (!Offset->isUndef())
1467 report_fatal_error(
1468 "unexpected offset when loading from webassembly global", false);
1469
1470 SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
1471 SDValue Ops[] = {LN->getChain(), Base};
1472 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
1473 LN->getMemoryVT(), LN->getMemOperand());
1474 }
1475
1476 if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1477 if (!Offset->isUndef())
1478 report_fatal_error(
1479 "unexpected offset when loading from webassembly local", false);
1480
1481 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1482 EVT LocalVT = LN->getValueType(0);
1483 SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT,
1484 {LN->getChain(), Idx});
1485 SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL);
1486 assert(Result->getNumValues() == 2 && "Loads must carry a chain!")(static_cast <bool> (Result->getNumValues() == 2 &&
"Loads must carry a chain!") ? void (0) : __assert_fail ("Result->getNumValues() == 2 && \"Loads must carry a chain!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1486, __extension__ __PRETTY_FUNCTION__))
;
1487 return Result;
1488 }
1489
1490 return Op;
1491}
1492
1493SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1494 SelectionDAG &DAG) const {
1495 SDValue Src = Op.getOperand(2);
1496 if (isa<FrameIndexSDNode>(Src.getNode())) {
1497 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1498 // the FI to some LEA-like instruction, but since we don't have that, we
1499 // need to insert some kind of instruction that can take an FI operand and
1500 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1501 // local.copy between Op and its FI operand.
1502 SDValue Chain = Op.getOperand(0);
1503 SDLoc DL(Op);
1504 unsigned Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
1505 EVT VT = Src.getValueType();
1506 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1507 : WebAssembly::COPY_I64,
1508 DL, VT, Src),
1509 0);
1510 return Op.getNode()->getNumValues() == 1
1511 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
1512 : DAG.getCopyToReg(Chain, DL, Reg, Copy,
1513 Op.getNumOperands() == 4 ? Op.getOperand(3)
1514 : SDValue());
1515 }
1516 return SDValue();
1517}
1518
1519SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1520 SelectionDAG &DAG) const {
1521 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
1522 return DAG.getTargetFrameIndex(FI, Op.getValueType());
1523}
1524
1525SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1526 SelectionDAG &DAG) const {
1527 SDLoc DL(Op);
1528
1529 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1530 fail(DL, DAG,
1531 "Non-Emscripten WebAssembly hasn't implemented "
1532 "__builtin_return_address");
1533 return SDValue();
1534 }
1535
1536 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1537 return SDValue();
1538
1539 unsigned Depth = Op.getConstantOperandVal(0);
1540 MakeLibCallOptions CallOptions;
1541 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
1542 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
1543 .first;
1544}
1545
1546SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1547 SelectionDAG &DAG) const {
1548 // Non-zero depths are not supported by WebAssembly currently. Use the
1549 // legalizer's default expansion, which is to return 0 (what this function is
1550 // documented to do).
1551 if (Op.getConstantOperandVal(0) > 0)
1552 return SDValue();
1553
1554 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
1555 EVT VT = Op.getValueType();
1556 Register FP =
1557 Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
1558 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1559}
1560
1561SDValue
1562WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1563 SelectionDAG &DAG) const {
1564 SDLoc DL(Op);
1565 const auto *GA = cast<GlobalAddressSDNode>(Op);
1566
1567 MachineFunction &MF = DAG.getMachineFunction();
1568 if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
1569 report_fatal_error("cannot use thread-local storage without bulk memory",
1570 false);
1571
1572 const GlobalValue *GV = GA->getGlobal();
1573
1574 // Currently Emscripten does not support dynamic linking with threads.
1575 // Therefore, if we have thread-local storage, only the local-exec model
1576 // is possible.
1577 // TODO: remove this and implement proper TLS models once Emscripten
1578 // supports dynamic linking with threads.
1579 if (GV->getThreadLocalMode() != GlobalValue::LocalExecTLSModel &&
1580 !Subtarget->getTargetTriple().isOSEmscripten()) {
1581 report_fatal_error("only -ftls-model=local-exec is supported for now on "
1582 "non-Emscripten OSes: variable " +
1583 GV->getName(),
1584 false);
1585 }
1586
1587 auto model = GV->getThreadLocalMode();
1588
1589 // Unsupported TLS modes
1590 assert(model != GlobalValue::NotThreadLocal)(static_cast <bool> (model != GlobalValue::NotThreadLocal
) ? void (0) : __assert_fail ("model != GlobalValue::NotThreadLocal"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1590, __extension__ __PRETTY_FUNCTION__))
;
1591 assert(model != GlobalValue::InitialExecTLSModel)(static_cast <bool> (model != GlobalValue::InitialExecTLSModel
) ? void (0) : __assert_fail ("model != GlobalValue::InitialExecTLSModel"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1591, __extension__ __PRETTY_FUNCTION__))
;
1592
1593 if (model == GlobalValue::LocalExecTLSModel ||
1594 model == GlobalValue::LocalDynamicTLSModel ||
1595 (model == GlobalValue::GeneralDynamicTLSModel &&
1596 getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))) {
1597 // For DSO-local TLS variables we use offset from __tls_base
1598
1599 MVT PtrVT = getPointerTy(DAG.getDataLayout());
1600 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
1601 : WebAssembly::GLOBAL_GET_I32;
1602 const char *BaseName = MF.createExternalSymbolName("__tls_base");
1603
1604 SDValue BaseAddr(
1605 DAG.getMachineNode(GlobalGet, DL, PtrVT,
1606 DAG.getTargetExternalSymbol(BaseName, PtrVT)),
1607 0);
1608
1609 SDValue TLSOffset = DAG.getTargetGlobalAddress(
1610 GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
1611 SDValue SymOffset =
1612 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);
1613
1614 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);
1615 }
1616
1617 assert(model == GlobalValue::GeneralDynamicTLSModel)(static_cast <bool> (model == GlobalValue::GeneralDynamicTLSModel
) ? void (0) : __assert_fail ("model == GlobalValue::GeneralDynamicTLSModel"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1617, __extension__ __PRETTY_FUNCTION__))
;
1618
1619 EVT VT = Op.getValueType();
1620 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1621 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1622 GA->getOffset(),
1623 WebAssemblyII::MO_GOT_TLS));
1624}
1625
1626SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1627 SelectionDAG &DAG) const {
1628 SDLoc DL(Op);
1629 const auto *GA = cast<GlobalAddressSDNode>(Op);
1630 EVT VT = Op.getValueType();
1631 assert(GA->getTargetFlags() == 0 &&(static_cast <bool> (GA->getTargetFlags() == 0 &&
"Unexpected target flags on generic GlobalAddressSDNode") ? void
(0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1632, __extension__ __PRETTY_FUNCTION__))
1632 "Unexpected target flags on generic GlobalAddressSDNode")(static_cast <bool> (GA->getTargetFlags() == 0 &&
"Unexpected target flags on generic GlobalAddressSDNode") ? void
(0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1632, __extension__ __PRETTY_FUNCTION__))
;
1633 if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace()))
1634 fail(DL, DAG, "Invalid address space for WebAssembly target");
1635
1636 unsigned OperandFlags = 0;
1637 if (isPositionIndependent()) {
1638 const GlobalValue *GV = GA->getGlobal();
1639 if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
1640 MachineFunction &MF = DAG.getMachineFunction();
1641 MVT PtrVT = getPointerTy(MF.getDataLayout());
1642 const char *BaseName;
1643 if (GV->getValueType()->isFunctionTy()) {
1644 BaseName = MF.createExternalSymbolName("__table_base");
1645 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
1646 }
1647 else {
1648 BaseName = MF.createExternalSymbolName("__memory_base");
1649 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
1650 }
1651 SDValue BaseAddr =
1652 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1653 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1654
1655 SDValue SymAddr = DAG.getNode(
1656 WebAssemblyISD::WrapperREL, DL, VT,
1657 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
1658 OperandFlags));
1659
1660 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
1661 }
1662 OperandFlags = WebAssemblyII::MO_GOT;
1663 }
1664
1665 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1666 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1667 GA->getOffset(), OperandFlags));
1668}
1669
1670SDValue
1671WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
1672 SelectionDAG &DAG) const {
1673 SDLoc DL(Op);
1674 const auto *ES = cast<ExternalSymbolSDNode>(Op);
1675 EVT VT = Op.getValueType();
1676 assert(ES->getTargetFlags() == 0 &&(static_cast <bool> (ES->getTargetFlags() == 0 &&
"Unexpected target flags on generic ExternalSymbolSDNode") ?
void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1677, __extension__ __PRETTY_FUNCTION__))
1677 "Unexpected target flags on generic ExternalSymbolSDNode")(static_cast <bool> (ES->getTargetFlags() == 0 &&
"Unexpected target flags on generic ExternalSymbolSDNode") ?
void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1677, __extension__ __PRETTY_FUNCTION__))
;
1678 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1679 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1680}
1681
1682SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1683 SelectionDAG &DAG) const {
1684 // There's no need for a Wrapper node because we always incorporate a jump
1685 // table operand into a BR_TABLE instruction, rather than ever
1686 // materializing it in a register.
1687 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
1688 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
1689 JT->getTargetFlags());
1690}
1691
1692SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1693 SelectionDAG &DAG) const {
1694 SDLoc DL(Op);
1695 SDValue Chain = Op.getOperand(0);
1696 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
1697 SDValue Index = Op.getOperand(2);
1698 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags")(static_cast <bool> (JT->getTargetFlags() == 0 &&
"WebAssembly doesn't set target flags") ? void (0) : __assert_fail
("JT->getTargetFlags() == 0 && \"WebAssembly doesn't set target flags\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1698, __extension__ __PRETTY_FUNCTION__))
;
1699
1700 SmallVector<SDValue, 8> Ops;
1701 Ops.push_back(Chain);
1702 Ops.push_back(Index);
1703
1704 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
1705 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
1706
1707 // Add an operand for each case.
1708 for (auto MBB : MBBs)
1709 Ops.push_back(DAG.getBasicBlock(MBB));
1710
1711 // Add the first MBB as a dummy default target for now. This will be replaced
1712 // with the proper default target (and the preceding range check eliminated)
1713 // if possible by WebAssemblyFixBrTableDefaults.
1714 Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
1715 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1716}
1717
1718SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1719 SelectionDAG &DAG) const {
1720 SDLoc DL(Op);
1721 EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());
1722
1723 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
1724 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
1725
1726 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
1727 MFI->getVarargBufferVreg(), PtrVT);
1728 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
1729 MachinePointerInfo(SV));
1730}
1731
1732SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1733 SelectionDAG &DAG) const {
1734 MachineFunction &MF = DAG.getMachineFunction();
1735 unsigned IntNo;
1736 switch (Op.getOpcode()) {
1737 case ISD::INTRINSIC_VOID:
1738 case ISD::INTRINSIC_W_CHAIN:
1739 IntNo = Op.getConstantOperandVal(1);
1740 break;
1741 case ISD::INTRINSIC_WO_CHAIN:
1742 IntNo = Op.getConstantOperandVal(0);
1743 break;
1744 default:
1745 llvm_unreachable("Invalid intrinsic")::llvm::llvm_unreachable_internal("Invalid intrinsic", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1745)
;
1746 }
1747 SDLoc DL(Op);
1748
1749 switch (IntNo) {
1750 default:
1751 return SDValue(); // Don't custom lower most intrinsics.
1752
1753 case Intrinsic::wasm_lsda: {
1754 auto PtrVT = getPointerTy(MF.getDataLayout());
1755 const char *SymName = MF.createExternalSymbolName(
1756 "GCC_except_table" + std::to_string(MF.getFunctionNumber()));
1757 if (isPositionIndependent()) {
1758 SDValue Node = DAG.getTargetExternalSymbol(
1759 SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);
1760 const char *BaseName = MF.createExternalSymbolName("__memory_base");
1761 SDValue BaseAddr =
1762 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1763 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1764 SDValue SymAddr =
1765 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);
1766 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
1767 }
1768 SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);
1769 return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);
1770 }
1771
1772 case Intrinsic::wasm_shuffle: {
1773 // Drop in-chain and replace undefs, but otherwise pass through unchanged
1774 SDValue Ops[18];
1775 size_t OpIdx = 0;
1776 Ops[OpIdx++] = Op.getOperand(1);
1777 Ops[OpIdx++] = Op.getOperand(2);
1778 while (OpIdx < 18) {
1779 const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
1780 if (MaskIdx.isUndef() ||
1781 cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) {
1782 Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32);
1783 } else {
1784 Ops[OpIdx++] = MaskIdx;
1785 }
1786 }
1787 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
1788 }
1789 }
1790}
1791
1792SDValue
1793WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1794 SelectionDAG &DAG) const {
1795 SDLoc DL(Op);
1796 // If sign extension operations are disabled, allow sext_inreg only if operand
1797 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
1798 // extension operations, but allowing sext_inreg in this context lets us have
1799 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
1800 // everywhere would be simpler in this file, but would necessitate large and
1801 // brittle patterns to undo the expansion and select extract_lane_s
1802 // instructions.
1803 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128())(static_cast <bool> (!Subtarget->hasSignExt() &&
Subtarget->hasSIMD128()) ? void (0) : __assert_fail ("!Subtarget->hasSignExt() && Subtarget->hasSIMD128()"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1803, __extension__ __PRETTY_FUNCTION__))
;
1804 if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1805 return SDValue();
1806
1807 const SDValue &Extract = Op.getOperand(0);
1808 MVT VecT = Extract.getOperand(0).getSimpleValueType();
1809 if (VecT.getVectorElementType().getSizeInBits() > 32)
1810 return SDValue();
1811 MVT ExtractedLaneT =
1812 cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
1813 MVT ExtractedVecT =
1814 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
1815 if (ExtractedVecT == VecT)
1816 return Op;
1817
1818 // Bitcast vector to appropriate type to ensure ISel pattern coverage
1819 const SDNode *Index = Extract.getOperand(1).getNode();
1820 if (!isa<ConstantSDNode>(Index))
1821 return SDValue();
1822 unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue();
1823 unsigned Scale =
1824 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
1825 assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail
("Scale > 1", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1825, __extension__ __PRETTY_FUNCTION__))
;
1826 SDValue NewIndex =
1827 DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
1828 SDValue NewExtract = DAG.getNode(
1829 ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
1830 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
1831 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
1832 Op.getOperand(1));
1833}
1834
1835static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) {
1836 SDLoc DL(Op);
1837 if (Op.getValueType() != MVT::v2f64)
1838 return SDValue();
1839
1840 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
1841 unsigned &Index) -> bool {
1842 switch (Op.getOpcode()) {
1843 case ISD::SINT_TO_FP:
1844 Opcode = WebAssemblyISD::CONVERT_LOW_S;
1845 break;
1846 case ISD::UINT_TO_FP:
1847 Opcode = WebAssemblyISD::CONVERT_LOW_U;
1848 break;
1849 case ISD::FP_EXTEND:
1850 Opcode = WebAssemblyISD::PROMOTE_LOW;
1851 break;
1852 default:
1853 return false;
1854 }
1855
1856 auto ExtractVector = Op.getOperand(0);
1857 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1858 return false;
1859
1860 if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))
1861 return false;
1862
1863 SrcVec = ExtractVector.getOperand(0);
1864 Index = ExtractVector.getConstantOperandVal(1);
1865 return true;
1866 };
1867
1868 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
1869 SDValue LHSSrcVec, RHSSrcVec;
1870 if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||
1871 !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))
1872 return SDValue();
1873
1874 if (LHSOpcode != RHSOpcode)
1875 return SDValue();
1876
1877 MVT ExpectedSrcVT;
1878 switch (LHSOpcode) {
1879 case WebAssemblyISD::CONVERT_LOW_S:
1880 case WebAssemblyISD::CONVERT_LOW_U:
1881 ExpectedSrcVT = MVT::v4i32;
1882 break;
1883 case WebAssemblyISD::PROMOTE_LOW:
1884 ExpectedSrcVT = MVT::v4f32;
1885 break;
1886 }
1887 if (LHSSrcVec.getValueType() != ExpectedSrcVT)
1888 return SDValue();
1889
1890 auto Src = LHSSrcVec;
1891 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
1892 // Shuffle the source vector so that the converted lanes are the low lanes.
1893 Src = DAG.getVectorShuffle(
1894 ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec,
1895 {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});
1896 }
1897 return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src);
1898}
1899
1900SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1901 SelectionDAG &DAG) const {
1902 if (auto ConvertLow = LowerConvertLow(Op, DAG))
3
Calling 'SDValue::operator bool'
5
Returning from 'SDValue::operator bool'
6
Taking false branch
1903 return ConvertLow;
1904
1905 SDLoc DL(Op);
1906 const EVT VecT = Op.getValueType();
1907 const EVT LaneT = Op.getOperand(0).getValueType();
1908 const size_t Lanes = Op.getNumOperands();
1909 bool CanSwizzle = VecT == MVT::v16i8;
1910
1911 // BUILD_VECTORs are lowered to the instruction that initializes the highest
1912 // possible number of lanes at once followed by a sequence of replace_lane
1913 // instructions to individually initialize any remaining lanes.
1914
1915 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
1916 // swizzled lanes should be given greater weight.
1917
1918 // TODO: Investigate looping rather than always extracting/replacing specific
1919 // lanes to fill gaps.
1920
1921 auto IsConstant = [](const SDValue &V) {
1922 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
27
Assuming the condition is true
28
Returning the value 1, which participates in a condition later
1923 };
1924
1925 // Returns the source vector and index vector pair if they exist. Checks for:
1926 // (extract_vector_elt
1927 // $src,
1928 // (sign_extend_inreg (extract_vector_elt $indices, $i))
1929 // )
1930 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
1931 auto Bail = std::make_pair(SDValue(), SDValue());
1932 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1933 return Bail;
1934 const SDValue &SwizzleSrc = Lane->getOperand(0);
1935 const SDValue &IndexExt = Lane->getOperand(1);
1936 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
1937 return Bail;
1938 const SDValue &Index = IndexExt->getOperand(0);
1939 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1940 return Bail;
1941 const SDValue &SwizzleIndices = Index->getOperand(0);
1942 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
1943 SwizzleIndices.getValueType() != MVT::v16i8 ||
1944 Index->getOperand(1)->getOpcode() != ISD::Constant ||
1945 Index->getConstantOperandVal(1) != I)
1946 return Bail;
1947 return std::make_pair(SwizzleSrc, SwizzleIndices);
1948 };
1949
1950 // If the lane is extracted from another vector at a constant index, return
1951 // that vector. The source vector must not have more lanes than the dest
1952 // because the shufflevector indices are in terms of the destination lanes and
1953 // would not be able to address the smaller individual source lanes.
1954 auto GetShuffleSrc = [&](const SDValue &Lane) {
1955 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1956 return SDValue();
1957 if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
1958 return SDValue();
1959 if (Lane->getOperand(0).getValueType().getVectorNumElements() >
1960 VecT.getVectorNumElements())
1961 return SDValue();
1962 return Lane->getOperand(0);
1963 };
1964
1965 using ValueEntry = std::pair<SDValue, size_t>;
1966 SmallVector<ValueEntry, 16> SplatValueCounts;
1967
1968 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
1969 SmallVector<SwizzleEntry, 16> SwizzleCounts;
1970
1971 using ShuffleEntry = std::pair<SDValue, size_t>;
1972 SmallVector<ShuffleEntry, 16> ShuffleCounts;
1973
1974 auto AddCount = [](auto &Counts, const auto &Val) {
1975 auto CountIt =
1976 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
1977 if (CountIt == Counts.end()) {
1978 Counts.emplace_back(Val, 1);
1979 } else {
1980 CountIt->second++;
1981 }
1982 };
1983
1984 auto GetMostCommon = [](auto &Counts) {
1985 auto CommonIt =
1986 std::max_element(Counts.begin(), Counts.end(),
1987 [](auto A, auto B) { return A.second < B.second; });
1988 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector")(static_cast <bool> (CommonIt != Counts.end() &&
"Unexpected all-undef build_vector") ? void (0) : __assert_fail
("CommonIt != Counts.end() && \"Unexpected all-undef build_vector\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1988, __extension__ __PRETTY_FUNCTION__))
;
1989 return *CommonIt;
1990 };
1991
1992 size_t NumConstantLanes = 0;
1993
1994 // Count eligible lanes for each type of vector creation op
1995 for (size_t I = 0; I
6.1
'I' is < 'Lanes'
6.1
'I' is < 'Lanes'
< Lanes
; ++I) {
7
Loop condition is true. Entering loop body
13
Assuming 'I' is >= 'Lanes'
14
Loop condition is false. Execution continues on line 2013
1996 const SDValue &Lane = Op->getOperand(I);
1997 if (Lane.isUndef())
8
Taking false branch
1998 continue;
1999
2000 AddCount(SplatValueCounts, Lane);
2001
2002 if (IsConstant(Lane))
9
Taking false branch
2003 NumConstantLanes++;
2004 if (auto ShuffleSrc = GetShuffleSrc(Lane))
10
Taking false branch
2005 AddCount(ShuffleCounts, ShuffleSrc);
2006 if (CanSwizzle
10.1
'CanSwizzle' is true
10.1
'CanSwizzle' is true
) {
11
Taking true branch
2007 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
2008 if (SwizzleSrcs.first)
12
Taking false branch
2009 AddCount(SwizzleCounts, SwizzleSrcs);
2010 }
2011 }
2012
2013 SDValue SplatValue;
2014 size_t NumSplatLanes;
2015 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
2016
2017 SDValue SwizzleSrc;
2018 SDValue SwizzleIndices;
2019 size_t NumSwizzleLanes = 0;
2020 if (SwizzleCounts.size())
15
Assuming the condition is false
16
Taking false branch
2021 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
2022 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2023
2024 // Shuffles can draw from up to two vectors, so find the two most common
2025 // sources.
2026 SDValue ShuffleSrc1, ShuffleSrc2;
2027 size_t NumShuffleLanes = 0;
2028 if (ShuffleCounts.size()) {
17
Assuming the condition is true
18
Taking true branch
2029 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2030 ShuffleCounts.erase(std::remove_if(ShuffleCounts.begin(),
2031 ShuffleCounts.end(),
2032 [&](const auto &Pair) {
2033 return Pair.first == ShuffleSrc1;
2034 }),
2035 ShuffleCounts.end());
2036 }
2037 if (ShuffleCounts.size()) {
19
Assuming the condition is false
20
Taking false branch
2038 size_t AdditionalShuffleLanes;
2039 std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
2040 GetMostCommon(ShuffleCounts);
2041 NumShuffleLanes += AdditionalShuffleLanes;
2042 }
2043
2044 // Predicate returning true if the lane is properly initialized by the
2045 // original instruction
2046 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
2047 SDValue Result;
2048 // Prefer swizzles over shuffles over vector consts over splats
2049 if (NumSwizzleLanes >= NumShuffleLanes &&
21
Assuming 'NumSwizzleLanes' is < 'NumShuffleLanes'
2050 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2051 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
2052 SwizzleIndices);
2053 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
2054 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
2055 return Swizzled == GetSwizzleSrcs(I, Lane);
2056 };
2057 } else if (NumShuffleLanes
21.1
'NumShuffleLanes' is >= 'NumConstantLanes'
21.1
'NumShuffleLanes' is >= 'NumConstantLanes'
>= NumConstantLanes &&
23
Taking false branch
2058 NumShuffleLanes >= NumSplatLanes) {
22
Assuming 'NumShuffleLanes' is < 'NumSplatLanes'
2059 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
2060 size_t DestLaneCount = VecT.getVectorNumElements();
2061 size_t Scale1 = 1;
2062 size_t Scale2 = 1;
2063 SDValue Src1 = ShuffleSrc1;
2064 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
2065 if (Src1.getValueType() != VecT) {
2066 size_t LaneSize =
2067 Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2068 assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
(0) : __assert_fail ("LaneSize > DestLaneSize", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2068, __extension__ __PRETTY_FUNCTION__))
;
2069 Scale1 = LaneSize / DestLaneSize;
2070 Src1 = DAG.getBitcast(VecT, Src1);
2071 }
2072 if (Src2.getValueType() != VecT) {
2073 size_t LaneSize =
2074 Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2075 assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
(0) : __assert_fail ("LaneSize > DestLaneSize", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2075, __extension__ __PRETTY_FUNCTION__))
;
2076 Scale2 = LaneSize / DestLaneSize;
2077 Src2 = DAG.getBitcast(VecT, Src2);
2078 }
2079
2080 int Mask[16];
2081 assert(DestLaneCount <= 16)(static_cast <bool> (DestLaneCount <= 16) ? void (0)
: __assert_fail ("DestLaneCount <= 16", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2081, __extension__ __PRETTY_FUNCTION__))
;
2082 for (size_t I = 0; I < DestLaneCount; ++I) {
2083 const SDValue &Lane = Op->getOperand(I);
2084 SDValue Src = GetShuffleSrc(Lane);
2085 if (Src == ShuffleSrc1) {
2086 Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
2087 } else if (Src && Src == ShuffleSrc2) {
2088 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
2089 } else {
2090 Mask[I] = -1;
2091 }
2092 }
2093 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2094 Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
2095 IsLaneConstructed = [&](size_t, const SDValue &Lane) {
2096 auto Src = GetShuffleSrc(Lane);
2097 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2098 };
2099 } else if (NumConstantLanes >= NumSplatLanes) {
24
Assuming 'NumConstantLanes' is >= 'NumSplatLanes'
25
Taking true branch
2100 SmallVector<SDValue, 16> ConstLanes;
2101 for (const SDValue &Lane : Op->op_values()) {
2102 if (IsConstant(Lane)) {
26
Calling 'operator()'
29
Returning from 'operator()'
30
Taking true branch
2103 // Values may need to be fixed so that they will sign extend to be
2104 // within the expected range during ISel. Check whether the value is in
2105 // bounds based on the lane bit width and if it is out of bounds, lop
2106 // off the extra bits and subtract 2^n to reflect giving the high bit
2107 // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it
2108 // cannot possibly be out of range.
2109 auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode());
31
Assuming the object is not a 'ConstantSDNode'
2110 int64_t Val = Const
31.1
'Const' is null
31.1
'Const' is null
? Const->getSExtValue() : 0;
32
'?' condition is false
2111 uint64_t LaneBits = 128 / Lanes;
2112 assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll
<< (LaneBits - 1))) && "Unexpected out of bounds negative value"
) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2113, __extension__ __PRETTY_FUNCTION__))
33
The result of the left shift is undefined due to shifting by '127', which is greater or equal to the width of type 'long long'
2113 "Unexpected out of bounds negative value")(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll
<< (LaneBits - 1))) && "Unexpected out of bounds negative value"
) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2113, __extension__ __PRETTY_FUNCTION__))
;
2114 if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) {
2115 auto NewVal = ((uint64_t)Val % (1ll << LaneBits)) - (1ll << LaneBits);
2116 ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT));
2117 } else {
2118 ConstLanes.push_back(Lane);
2119 }
2120 } else if (LaneT.isFloatingPoint()) {
2121 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
2122 } else {
2123 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
2124 }
2125 }
2126 Result = DAG.getBuildVector(VecT, DL, ConstLanes);
2127 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
2128 return IsConstant(Lane);
2129 };
2130 } else {
2131 // Use a splat, but possibly a load_splat
2132 LoadSDNode *SplattedLoad;
2133 if ((SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
2134 SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
2135 Result = DAG.getMemIntrinsicNode(
2136 WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT),
2137 {SplattedLoad->getChain(), SplattedLoad->getBasePtr(),
2138 SplattedLoad->getOffset()},
2139 SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand());
2140 } else {
2141 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
2142 }
2143 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
2144 return Lane == SplatValue;
2145 };
2146 }
2147
2148 assert(Result)(static_cast <bool> (Result) ? void (0) : __assert_fail
("Result", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2148, __extension__ __PRETTY_FUNCTION__))
;
2149 assert(IsLaneConstructed)(static_cast <bool> (IsLaneConstructed) ? void (0) : __assert_fail
("IsLaneConstructed", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2149, __extension__ __PRETTY_FUNCTION__))
;
2150
2151 // Add replace_lane instructions for any unhandled values
2152 for (size_t I = 0; I < Lanes; ++I) {
2153 const SDValue &Lane = Op->getOperand(I);
2154 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
2155 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
2156 DAG.getConstant(I, DL, MVT::i32));
2157 }
2158
2159 return Result;
2160}
2161
2162SDValue
2163WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2164 SelectionDAG &DAG) const {
2165 SDLoc DL(Op);
2166 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
2167 MVT VecType = Op.getOperand(0).getSimpleValueType();
2168 assert(VecType.is128BitVector() && "Unexpected shuffle vector type")(static_cast <bool> (VecType.is128BitVector() &&
"Unexpected shuffle vector type") ? void (0) : __assert_fail
("VecType.is128BitVector() && \"Unexpected shuffle vector type\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2168, __extension__ __PRETTY_FUNCTION__))
;
2169 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
2170
2171 // Space for two vector args and sixteen mask indices
2172 SDValue Ops[18];
2173 size_t OpIdx = 0;
2174 Ops[OpIdx++] = Op.getOperand(0);
2175 Ops[OpIdx++] = Op.getOperand(1);
2176
2177 // Expand mask indices to byte indices and materialize them as operands
2178 for (int M : Mask) {
2179 for (size_t J = 0; J < LaneBytes; ++J) {
2180 // Lower undefs (represented by -1 in mask) to zero
2181 uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J;
2182 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
2183 }
2184 }
2185
2186 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2187}
2188
2189SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
2190 SelectionDAG &DAG) const {
2191 SDLoc DL(Op);
2192 // The legalizer does not know how to expand the unsupported comparison modes
2193 // of i64x2 vectors, so we manually unroll them here.
2194 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64)(static_cast <bool> (Op->getOperand(0)->getSimpleValueType
(0) == MVT::v2i64) ? void (0) : __assert_fail ("Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2194, __extension__ __PRETTY_FUNCTION__))
;
2195 SmallVector<SDValue, 2> LHS, RHS;
2196 DAG.ExtractVectorElements(Op->getOperand(0), LHS);
2197 DAG.ExtractVectorElements(Op->getOperand(1), RHS);
2198 const SDValue &CC = Op->getOperand(2);
2199 auto MakeLane = [&](unsigned I) {
2200 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
2201 DAG.getConstant(uint64_t(-1), DL, MVT::i64),
2202 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
2203 };
2204 return DAG.getBuildVector(Op->getValueType(0), DL,
2205 {MakeLane(0), MakeLane(1)});
2206}
2207
2208SDValue
2209WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
2210 SelectionDAG &DAG) const {
2211 // Allow constant lane indices, expand variable lane indices
2212 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
2213 if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
2214 return Op;
2215 else
2216 // Perform default expansion
2217 return SDValue();
2218}
2219
2220static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
2221 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
2222 // 32-bit and 64-bit unrolled shifts will have proper semantics
2223 if (LaneT.bitsGE(MVT::i32))
2224 return DAG.UnrollVectorOp(Op.getNode());
2225 // Otherwise mask the shift value to get proper semantics from 32-bit shift
2226 SDLoc DL(Op);
2227 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
2228 SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
2229 unsigned ShiftOpcode = Op.getOpcode();
2230 SmallVector<SDValue, 16> ShiftedElements;
2231 DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
2232 SmallVector<SDValue, 16> ShiftElements;
2233 DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
2234 SmallVector<SDValue, 16> UnrolledOps;
2235 for (size_t i = 0; i < NumLanes; ++i) {
2236 SDValue MaskedShiftValue =
2237 DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
2238 SDValue ShiftedValue = ShiftedElements[i];
2239 if (ShiftOpcode == ISD::SRA)
2240 ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
2241 ShiftedValue, DAG.getValueType(LaneT));
2242 UnrolledOps.push_back(
2243 DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
2244 }
2245 return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
2246}
2247
2248SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
2249 SelectionDAG &DAG) const {
2250 SDLoc DL(Op);
2251
2252 // Only manually lower vector shifts
2253 assert(Op.getSimpleValueType().isVector())(static_cast <bool> (Op.getSimpleValueType().isVector()
) ? void (0) : __assert_fail ("Op.getSimpleValueType().isVector()"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2253, __extension__ __PRETTY_FUNCTION__))
;
2254
2255 auto ShiftVal = DAG.getSplatValue(Op.getOperand(1));
2256 if (!ShiftVal)
2257 return unrollVectorShift(Op, DAG);
2258
2259 // Use anyext because none of the high bits can affect the shift
2260 ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);
2261
2262 unsigned Opcode;
2263 switch (Op.getOpcode()) {
2264 case ISD::SHL:
2265 Opcode = WebAssemblyISD::VEC_SHL;
2266 break;
2267 case ISD::SRA:
2268 Opcode = WebAssemblyISD::VEC_SHR_S;
2269 break;
2270 case ISD::SRL:
2271 Opcode = WebAssemblyISD::VEC_SHR_U;
2272 break;
2273 default:
2274 llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2274)
;
2275 }
2276
2277 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
2278}
2279
2280SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
2281 SelectionDAG &DAG) const {
2282 SDLoc DL(Op);
2283 EVT ResT = Op.getValueType();
2284 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2285
2286 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2287 (SatVT == MVT::i32 || SatVT == MVT::i64))
2288 return Op;
2289
2290 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2291 return Op;
2292
2293 return SDValue();
2294}
2295
2296//===----------------------------------------------------------------------===//
2297// Custom DAG combine hooks
2298//===----------------------------------------------------------------------===//
2299static SDValue
2300performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
2301 auto &DAG = DCI.DAG;
2302 auto Shuffle = cast<ShuffleVectorSDNode>(N);
2303
2304 // Hoist vector bitcasts that don't change the number of lanes out of unary
2305 // shuffles, where they are less likely to get in the way of other combines.
2306 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
2307 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
2308 SDValue Bitcast = N->getOperand(0);
2309 if (Bitcast.getOpcode() != ISD::BITCAST)
2310 return SDValue();
2311 if (!N->getOperand(1).isUndef())
2312 return SDValue();
2313 SDValue CastOp = Bitcast.getOperand(0);
2314 MVT SrcType = CastOp.getSimpleValueType();
2315 MVT DstType = Bitcast.getSimpleValueType();
2316 if (!SrcType.is128BitVector() ||
2317 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
2318 return SDValue();
2319 SDValue NewShuffle = DAG.getVectorShuffle(
2320 SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
2321 return DAG.getBitcast(DstType, NewShuffle);
2322}
2323
2324static SDValue
2325performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
2326 auto &DAG = DCI.DAG;
2327 assert(N->getOpcode() == ISD::SIGN_EXTEND ||(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
|| N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__))
2328 N->getOpcode() == ISD::ZERO_EXTEND)(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
|| N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2328, __extension__ __PRETTY_FUNCTION__))
;
2329
2330 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
2331 // possible before the extract_subvector can be expanded.
2332 auto Extract = N->getOperand(0);
2333 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
2334 return SDValue();
2335 auto Source = Extract.getOperand(0);
2336 auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
2337 if (IndexNode == nullptr)
2338 return SDValue();
2339 auto Index = IndexNode->getZExtValue();
2340
2341 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
2342 // extracted subvector is the low or high half of its source.
2343 EVT ResVT = N->getValueType(0);
2344 if (ResVT == MVT::v8i16) {
2345 if (Extract.getValueType() != MVT::v8i8 ||
2346 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
2347 return SDValue();
2348 } else if (ResVT == MVT::v4i32) {
2349 if (Extract.getValueType() != MVT::v4i16 ||
2350 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
2351 return SDValue();
2352 } else if (ResVT == MVT::v2i64) {
2353 if (Extract.getValueType() != MVT::v2i32 ||
2354 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
2355 return SDValue();
2356 } else {
2357 return SDValue();
2358 }
2359
2360 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
2361 bool IsLow = Index == 0;
2362
2363 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
2364 : WebAssemblyISD::EXTEND_HIGH_S)
2365 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
2366 : WebAssemblyISD::EXTEND_HIGH_U);
2367
2368 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2369}
2370
2371static SDValue
2372performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
2373 auto &DAG = DCI.DAG;
2374
2375 auto GetWasmConversionOp = [](unsigned Op) {
2376 switch (Op) {
2377 case ISD::FP_TO_SINT_SAT:
2378 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
2379 case ISD::FP_TO_UINT_SAT:
2380 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
2381 case ISD::FP_ROUND:
2382 return WebAssemblyISD::DEMOTE_ZERO;
2383 }
2384 llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2384)
;
2385 };
2386
2387 auto IsZeroSplat = [](SDValue SplatVal) {
2388 auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
2389 APInt SplatValue, SplatUndef;
2390 unsigned SplatBitSize;
2391 bool HasAnyUndefs;
2392 return Splat &&
2393 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2394 HasAnyUndefs) &&
2395 SplatValue == 0;
2396 };
2397
2398 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
2399 // Combine this:
2400 //
2401 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
2402 //
2403 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2404 //
2405 // Or this:
2406 //
2407 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
2408 //
2409 // into (f32x4.demote_zero_f64x2 $x).
2410 EVT ResVT;
2411 EVT ExpectedConversionType;
2412 auto Conversion = N->getOperand(0);
2413 auto ConversionOp = Conversion.getOpcode();
2414 switch (ConversionOp) {
2415 case ISD::FP_TO_SINT_SAT:
2416 case ISD::FP_TO_UINT_SAT:
2417 ResVT = MVT::v4i32;
2418 ExpectedConversionType = MVT::v2i32;
2419 break;
2420 case ISD::FP_ROUND:
2421 ResVT = MVT::v4f32;
2422 ExpectedConversionType = MVT::v2f32;
2423 break;
2424 default:
2425 return SDValue();
2426 }
2427
2428 if (N->getValueType(0) != ResVT)
2429 return SDValue();
2430
2431 if (Conversion.getValueType() != ExpectedConversionType)
2432 return SDValue();
2433
2434 auto Source = Conversion.getOperand(0);
2435 if (Source.getValueType() != MVT::v2f64)
2436 return SDValue();
2437
2438 if (!IsZeroSplat(N->getOperand(1)) ||
2439 N->getOperand(1).getValueType() != ExpectedConversionType)
2440 return SDValue();
2441
2442 unsigned Op = GetWasmConversionOp(ConversionOp);
2443 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2444 }
2445
2446 // Combine this:
2447 //
2448 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
2449 //
2450 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2451 //
2452 // Or this:
2453 //
2454 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
2455 //
2456 // into (f32x4.demote_zero_f64x2 $x).
2457 EVT ResVT;
2458 auto ConversionOp = N->getOpcode();
2459 switch (ConversionOp) {
2460 case ISD::FP_TO_SINT_SAT:
2461 case ISD::FP_TO_UINT_SAT:
2462 ResVT = MVT::v4i32;
2463 break;
2464 case ISD::FP_ROUND:
2465 ResVT = MVT::v4f32;
2466 break;
2467 default:
2468 llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2468)
;
2469 }
2470
2471 if (N->getValueType(0) != ResVT)
2472 return SDValue();
2473
2474 auto Concat = N->getOperand(0);
2475 if (Concat.getValueType() != MVT::v4f64)
2476 return SDValue();
2477
2478 auto Source = Concat.getOperand(0);
2479 if (Source.getValueType() != MVT::v2f64)
2480 return SDValue();
2481
2482 if (!IsZeroSplat(Concat.getOperand(1)) ||
2483 Concat.getOperand(1).getValueType() != MVT::v2f64)
2484 return SDValue();
2485
2486 unsigned Op = GetWasmConversionOp(ConversionOp);
2487 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2488}
2489
2490SDValue
2491WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
2492 DAGCombinerInfo &DCI) const {
2493 switch (N->getOpcode()) {
2494 default:
2495 return SDValue();
2496 case ISD::VECTOR_SHUFFLE:
2497 return performVECTOR_SHUFFLECombine(N, DCI);
2498 case ISD::SIGN_EXTEND:
2499 case ISD::ZERO_EXTEND:
2500 return performVectorExtendCombine(N, DCI);
2501 case ISD::FP_TO_SINT_SAT:
2502 case ISD::FP_TO_UINT_SAT:
2503 case ISD::FP_ROUND:
2504 case ISD::CONCAT_VECTORS:
2505 return performVectorTruncZeroCombine(N, DCI);
2506 }
2507}

/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h

1//===- llvm/CodeGen/SelectionDAGNodes.h - SelectionDAG Nodes ----*- C++ -*-===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file declares the SDNode class and derived classes, which are used to
10// represent the nodes and operations present in a SelectionDAG. These nodes
11// and operations are machine code level operations, with some similarities to
12// the GCC RTL representation.
13//
14// Clients should include the SelectionDAG.h file instead of this file directly.
15//
16//===----------------------------------------------------------------------===//
17
18#ifndef LLVM_CODEGEN_SELECTIONDAGNODES_H
19#define LLVM_CODEGEN_SELECTIONDAGNODES_H
20
21#include "llvm/ADT/APFloat.h"
22#include "llvm/ADT/ArrayRef.h"
23#include "llvm/ADT/BitVector.h"
24#include "llvm/ADT/FoldingSet.h"
25#include "llvm/ADT/GraphTraits.h"
26#include "llvm/ADT/SmallPtrSet.h"
27#include "llvm/ADT/SmallVector.h"
28#include "llvm/ADT/ilist_node.h"
29#include "llvm/ADT/iterator.h"
30#include "llvm/ADT/iterator_range.h"
31#include "llvm/CodeGen/ISDOpcodes.h"
32#include "llvm/CodeGen/MachineMemOperand.h"
33#include "llvm/CodeGen/Register.h"
34#include "llvm/CodeGen/ValueTypes.h"
35#include "llvm/IR/Constants.h"
36#include "llvm/IR/DebugLoc.h"
37#include "llvm/IR/Instruction.h"
38#include "llvm/IR/Instructions.h"
39#include "llvm/IR/Metadata.h"
40#include "llvm/IR/Operator.h"
41#include "llvm/Support/AlignOf.h"
42#include "llvm/Support/AtomicOrdering.h"
43#include "llvm/Support/Casting.h"
44#include "llvm/Support/ErrorHandling.h"
45#include "llvm/Support/MachineValueType.h"
46#include "llvm/Support/TypeSize.h"
47#include <algorithm>
48#include <cassert>
49#include <climits>
50#include <cstddef>
51#include <cstdint>
52#include <cstring>
53#include <iterator>
54#include <string>
55#include <tuple>
56
57namespace llvm {
58
59class APInt;
60class Constant;
61template <typename T> struct DenseMapInfo;
62class GlobalValue;
63class MachineBasicBlock;
64class MachineConstantPoolValue;
65class MCSymbol;
66class raw_ostream;
67class SDNode;
68class SelectionDAG;
69class Type;
70class Value;
71
72void checkForCycles(const SDNode *N, const SelectionDAG *DAG = nullptr,
73 bool force = false);
74
75/// This represents a list of ValueType's that has been intern'd by
76/// a SelectionDAG. Instances of this simple value class are returned by
77/// SelectionDAG::getVTList(...).
78///
79struct SDVTList {
80 const EVT *VTs;
81 unsigned int NumVTs;
82};
83
84namespace ISD {
85
86 /// Node predicates
87
88/// If N is a BUILD_VECTOR or SPLAT_VECTOR node whose elements are all the
89/// same constant or undefined, return true and return the constant value in
90/// \p SplatValue.
91bool isConstantSplatVector(const SDNode *N, APInt &SplatValue);
92
93/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
94/// all of the elements are ~0 or undef. If \p BuildVectorOnly is set to
95/// true, it only checks BUILD_VECTOR.
96bool isConstantSplatVectorAllOnes(const SDNode *N,
97 bool BuildVectorOnly = false);
98
99/// Return true if the specified node is a BUILD_VECTOR or SPLAT_VECTOR where
100/// all of the elements are 0 or undef. If \p BuildVectorOnly is set to true, it
101/// only checks BUILD_VECTOR.
102bool isConstantSplatVectorAllZeros(const SDNode *N,
103 bool BuildVectorOnly = false);
104
105/// Return true if the specified node is a BUILD_VECTOR where all of the
106/// elements are ~0 or undef.
107bool isBuildVectorAllOnes(const SDNode *N);
108
109/// Return true if the specified node is a BUILD_VECTOR where all of the
110/// elements are 0 or undef.
111bool isBuildVectorAllZeros(const SDNode *N);
112
113/// Return true if the specified node is a BUILD_VECTOR node of all
114/// ConstantSDNode or undef.
115bool isBuildVectorOfConstantSDNodes(const SDNode *N);
116
117/// Return true if the specified node is a BUILD_VECTOR node of all
118/// ConstantFPSDNode or undef.
119bool isBuildVectorOfConstantFPSDNodes(const SDNode *N);
120
121/// Return true if the node has at least one operand and all operands of the
122/// specified node are ISD::UNDEF.
123bool allOperandsUndef(const SDNode *N);
124
125} // end namespace ISD
126
127//===----------------------------------------------------------------------===//
128/// Unlike LLVM values, Selection DAG nodes may return multiple
129/// values as the result of a computation. Many nodes return multiple values,
130/// from loads (which define a token and a return value) to ADDC (which returns
131/// a result and a carry value), to calls (which may return an arbitrary number
132/// of values).
133///
134/// As such, each use of a SelectionDAG computation must indicate the node that
135/// computes it as well as which return value to use from that node. This pair
136/// of information is represented with the SDValue value type.
137///
138class SDValue {
139 friend struct DenseMapInfo<SDValue>;
140
141 SDNode *Node = nullptr; // The node defining the value we are using.
142 unsigned ResNo = 0; // Which return value of the node we are using.
143
144public:
145 SDValue() = default;
146 SDValue(SDNode *node, unsigned resno);
147
148 /// get the index which selects a specific result in the SDNode
149 unsigned getResNo() const { return ResNo; }
150
151 /// get the SDNode which holds the desired result
152 SDNode *getNode() const { return Node; }
153
154 /// set the SDNode
155 void setNode(SDNode *N) { Node = N; }
156
157 inline SDNode *operator->() const { return Node; }
158
159 bool operator==(const SDValue &O) const {
160 return Node == O.Node && ResNo == O.ResNo;
161 }
162 bool operator!=(const SDValue &O) const {
163 return !operator==(O);
164 }
165 bool operator<(const SDValue &O) const {
166 return std::tie(Node, ResNo) < std::tie(O.Node, O.ResNo);
167 }
168 explicit operator bool() const {
169 return Node != nullptr;
4
Returning zero, which participates in a condition later
170 }
171
172 SDValue getValue(unsigned R) const {
173 return SDValue(Node, R);
174 }
175
176 /// Return true if this node is an operand of N.
177 bool isOperandOf(const SDNode *N) const;
178
179 /// Return the ValueType of the referenced return value.
180 inline EVT getValueType() const;
181
182 /// Return the simple ValueType of the referenced return value.
183 MVT getSimpleValueType() const {
184 return getValueType().getSimpleVT();
185 }
186
187 /// Returns the size of the value in bits.
188 ///
189 /// If the value type is a scalable vector type, the scalable property will
190 /// be set and the runtime size will be a positive integer multiple of the
191 /// base size.
192 TypeSize getValueSizeInBits() const {
193 return getValueType().getSizeInBits();
194 }
195
196 uint64_t getScalarValueSizeInBits() const {
197 return getValueType().getScalarType().getFixedSizeInBits();
198 }
199
200 // Forwarding methods - These forward to the corresponding methods in SDNode.
201 inline unsigned getOpcode() const;
202 inline unsigned getNumOperands() const;
203 inline const SDValue &getOperand(unsigned i) const;
204 inline uint64_t getConstantOperandVal(unsigned i) const;
205 inline const APInt &getConstantOperandAPInt(unsigned i) const;
206 inline bool isTargetMemoryOpcode() const;
207 inline bool isTargetOpcode() const;
208 inline bool isMachineOpcode() const;
209 inline bool isUndef() const;
210 inline unsigned getMachineOpcode() const;
211 inline const DebugLoc &getDebugLoc() const;
212 inline void dump() const;
213 inline void dump(const SelectionDAG *G) const;
214 inline void dumpr() const;
215 inline void dumpr(const SelectionDAG *G) const;
216
217 /// Return true if this operand (which must be a chain) reaches the
218 /// specified operand without crossing any side-effecting instructions.
219 /// In practice, this looks through token factors and non-volatile loads.
220 /// In order to remain efficient, this only
221 /// looks a couple of nodes in, it does not do an exhaustive search.
222 bool reachesChainWithoutSideEffects(SDValue Dest,
223 unsigned Depth = 2) const;
224
225 /// Return true if there are no nodes using value ResNo of Node.
226 inline bool use_empty() const;
227
228 /// Return true if there is exactly one node using value ResNo of Node.
229 inline bool hasOneUse() const;
230};
231
232template<> struct DenseMapInfo<SDValue> {
233 static inline SDValue getEmptyKey() {
234 SDValue V;
235 V.ResNo = -1U;
236 return V;
237 }
238
239 static inline SDValue getTombstoneKey() {
240 SDValue V;
241 V.ResNo = -2U;
242 return V;
243 }
244
245 static unsigned getHashValue(const SDValue &Val) {
246 return ((unsigned)((uintptr_t)Val.getNode() >> 4) ^
247 (unsigned)((uintptr_t)Val.getNode() >> 9)) + Val.getResNo();
248 }
249
250 static bool isEqual(const SDValue &LHS, const SDValue &RHS) {
251 return LHS == RHS;
252 }
253};
254
255/// Allow casting operators to work directly on
256/// SDValues as if they were SDNode*'s.
257template<> struct simplify_type<SDValue> {
258 using SimpleType = SDNode *;
259
260 static SimpleType getSimplifiedValue(SDValue &Val) {
261 return Val.getNode();
262 }
263};
264template<> struct simplify_type<const SDValue> {
265 using SimpleType = /*const*/ SDNode *;
266
267 static SimpleType getSimplifiedValue(const SDValue &Val) {
268 return Val.getNode();
269 }
270};
271
272/// Represents a use of a SDNode. This class holds an SDValue,
273/// which records the SDNode being used and the result number, a
274/// pointer to the SDNode using the value, and Next and Prev pointers,
275/// which link together all the uses of an SDNode.
276///
277class SDUse {
278 /// Val - The value being used.
279 SDValue Val;
280 /// User - The user of this value.
281 SDNode *User = nullptr;
282 /// Prev, Next - Pointers to the uses list of the SDNode referred by
283 /// this operand.
284 SDUse **Prev = nullptr;
285 SDUse *Next = nullptr;
286
287public:
288 SDUse() = default;
289 SDUse(const SDUse &U) = delete;
290 SDUse &operator=(const SDUse &) = delete;
291
292 /// Normally SDUse will just implicitly convert to an SDValue that it holds.
293 operator const SDValue&() const { return Val; }
294
295 /// If implicit conversion to SDValue doesn't work, the get() method returns
296 /// the SDValue.
297 const SDValue &get() const { return Val; }
298
299 /// This returns the SDNode that contains this Use.
300 SDNode *getUser() { return User; }
301
302 /// Get the next SDUse in the use list.
303 SDUse *getNext() const { return Next; }
304
305 /// Convenience function for get().getNode().
306 SDNode *getNode() const { return Val.getNode(); }
307 /// Convenience function for get().getResNo().
308 unsigned getResNo() const { return Val.getResNo(); }
309 /// Convenience function for get().getValueType().
310 EVT getValueType() const { return Val.getValueType(); }
311
312 /// Convenience function for get().operator==
313 bool operator==(const SDValue &V) const {
314 return Val == V;
315 }
316
317 /// Convenience function for get().operator!=
318 bool operator!=(const SDValue &V) const {
319 return Val != V;
320 }
321
322 /// Convenience function for get().operator<
323 bool operator<(const SDValue &V) const {
324 return Val < V;
325 }
326
327private:
328 friend class SelectionDAG;
329 friend class SDNode;
330 // TODO: unfriend HandleSDNode once we fix its operand handling.
331 friend class HandleSDNode;
332
333 void setUser(SDNode *p) { User = p; }
334
335 /// Remove this use from its existing use list, assign it the
336 /// given value, and add it to the new value's node's use list.
337 inline void set(const SDValue &V);
338 /// Like set, but only supports initializing a newly-allocated
339 /// SDUse with a non-null value.
340 inline void setInitial(const SDValue &V);
341 /// Like set, but only sets the Node portion of the value,
342 /// leaving the ResNo portion unmodified.
343 inline void setNode(SDNode *N);
344
345 void addToList(SDUse **List) {
346 Next = *List;
347 if (Next) Next->Prev = &Next;
348 Prev = List;
349 *List = this;
350 }
351
352 void removeFromList() {
353 *Prev = Next;
354 if (Next) Next->Prev = Prev;
355 }
356};
357
358/// simplify_type specializations - Allow casting operators to work directly on
359/// SDValues as if they were SDNode*'s.
360template<> struct simplify_type<SDUse> {
361 using SimpleType = SDNode *;
362
363 static SimpleType getSimplifiedValue(SDUse &Val) {
364 return Val.getNode();
365 }
366};
367
368/// These are IR-level optimization flags that may be propagated to SDNodes.
369/// TODO: This data structure should be shared by the IR optimizer and the
370/// the backend.
371struct SDNodeFlags {
372private:
373 bool NoUnsignedWrap : 1;
374 bool NoSignedWrap : 1;
375 bool Exact : 1;
376 bool NoNaNs : 1;
377 bool NoInfs : 1;
378 bool NoSignedZeros : 1;
379 bool AllowReciprocal : 1;
380 bool AllowContract : 1;
381 bool ApproximateFuncs : 1;
382 bool AllowReassociation : 1;
383
384 // We assume instructions do not raise floating-point exceptions by default,
385 // and only those marked explicitly may do so. We could choose to represent
386 // this via a positive "FPExcept" flags like on the MI level, but having a
387 // negative "NoFPExcept" flag here (that defaults to true) makes the flag
388 // intersection logic more straightforward.
389 bool NoFPExcept : 1;
390
391public:
392 /// Default constructor turns off all optimization flags.
393 SDNodeFlags()
394 : NoUnsignedWrap(false), NoSignedWrap(false), Exact(false), NoNaNs(false),
395 NoInfs(false), NoSignedZeros(false), AllowReciprocal(false),
396 AllowContract(false), ApproximateFuncs(false),
397 AllowReassociation(false), NoFPExcept(false) {}
398
399 /// Propagate the fast-math-flags from an IR FPMathOperator.
400 void copyFMF(const FPMathOperator &FPMO) {
401 setNoNaNs(FPMO.hasNoNaNs());
402 setNoInfs(FPMO.hasNoInfs());
403 setNoSignedZeros(FPMO.hasNoSignedZeros());
404 setAllowReciprocal(FPMO.hasAllowReciprocal());
405 setAllowContract(FPMO.hasAllowContract());
406 setApproximateFuncs(FPMO.hasApproxFunc());
407 setAllowReassociation(FPMO.hasAllowReassoc());
408 }
409
410 // These are mutators for each flag.
411 void setNoUnsignedWrap(bool b) { NoUnsignedWrap = b; }
412 void setNoSignedWrap(bool b) { NoSignedWrap = b; }
413 void setExact(bool b) { Exact = b; }
414 void setNoNaNs(bool b) { NoNaNs = b; }
415 void setNoInfs(bool b) { NoInfs = b; }
416 void setNoSignedZeros(bool b) { NoSignedZeros = b; }
417 void setAllowReciprocal(bool b) { AllowReciprocal = b; }
418 void setAllowContract(bool b) { AllowContract = b; }
419 void setApproximateFuncs(bool b) { ApproximateFuncs = b; }
420 void setAllowReassociation(bool b) { AllowReassociation = b; }
421 void setNoFPExcept(bool b) { NoFPExcept = b; }
422
423 // These are accessors for each flag.
424 bool hasNoUnsignedWrap() const { return NoUnsignedWrap; }
425 bool hasNoSignedWrap() const { return NoSignedWrap; }
426 bool hasExact() const { return Exact; }
427 bool hasNoNaNs() const { return NoNaNs; }
428 bool hasNoInfs() const { return NoInfs; }
429 bool hasNoSignedZeros() const { return NoSignedZeros; }
430 bool hasAllowReciprocal() const { return AllowReciprocal; }
431 bool hasAllowContract() const { return AllowContract; }
432 bool hasApproximateFuncs() const { return ApproximateFuncs; }
433 bool hasAllowReassociation() const { return AllowReassociation; }
434 bool hasNoFPExcept() const { return NoFPExcept; }
435
436 /// Clear any flags in this flag set that aren't also set in Flags. All
437 /// flags will be cleared if Flags are undefined.
438 void intersectWith(const SDNodeFlags Flags) {
439 NoUnsignedWrap &= Flags.NoUnsignedWrap;
440 NoSignedWrap &= Flags.NoSignedWrap;
441 Exact &= Flags.Exact;
442 NoNaNs &= Flags.NoNaNs;
443 NoInfs &= Flags.NoInfs;
444 NoSignedZeros &= Flags.NoSignedZeros;
445 AllowReciprocal &= Flags.AllowReciprocal;
446 AllowContract &= Flags.AllowContract;
447 ApproximateFuncs &= Flags.ApproximateFuncs;
448 AllowReassociation &= Flags.AllowReassociation;
449 NoFPExcept &= Flags.NoFPExcept;
450 }
451};
452
453/// Represents one node in the SelectionDAG.
454///
455class SDNode : public FoldingSetNode, public ilist_node<SDNode> {
456private:
457 /// The operation that this node performs.
458 int16_t NodeType;
459
460protected:
461 // We define a set of mini-helper classes to help us interpret the bits in our
462 // SubclassData. These are designed to fit within a uint16_t so they pack
463 // with NodeType.
464
465#if defined(_AIX) && (!defined(__GNUC__4) || defined(__clang__1))
466// Except for GCC; by default, AIX compilers store bit-fields in 4-byte words
467// and give the `pack` pragma push semantics.
468#define BEGIN_TWO_BYTE_PACK() _Pragma("pack(2)")pack(2)
469#define END_TWO_BYTE_PACK() _Pragma("pack(pop)")pack(pop)
470#else
471#define BEGIN_TWO_BYTE_PACK()
472#define END_TWO_BYTE_PACK()
473#endif
474
475BEGIN_TWO_BYTE_PACK()
476 class SDNodeBitfields {
477 friend class SDNode;
478 friend class MemIntrinsicSDNode;
479 friend class MemSDNode;
480 friend class SelectionDAG;
481
482 uint16_t HasDebugValue : 1;
483 uint16_t IsMemIntrinsic : 1;
484 uint16_t IsDivergent : 1;
485 };
486 enum { NumSDNodeBits = 3 };
487
488 class ConstantSDNodeBitfields {
489 friend class ConstantSDNode;
490
491 uint16_t : NumSDNodeBits;
492
493 uint16_t IsOpaque : 1;
494 };
495
496 class MemSDNodeBitfields {
497 friend class MemSDNode;
498 friend class MemIntrinsicSDNode;
499 friend class AtomicSDNode;
500
501 uint16_t : NumSDNodeBits;
502
503 uint16_t IsVolatile : 1;
504 uint16_t IsNonTemporal : 1;
505 uint16_t IsDereferenceable : 1;
506 uint16_t IsInvariant : 1;
507 };
508 enum { NumMemSDNodeBits = NumSDNodeBits + 4 };
509
510 class LSBaseSDNodeBitfields {
511 friend class LSBaseSDNode;
512 friend class VPLoadStoreSDNode;
513 friend class MaskedLoadStoreSDNode;
514 friend class MaskedGatherScatterSDNode;
515 friend class VPGatherScatterSDNode;
516
517 uint16_t : NumMemSDNodeBits;
518
519 // This storage is shared between disparate class hierarchies to hold an
520 // enumeration specific to the class hierarchy in use.
521 // LSBaseSDNode => enum ISD::MemIndexedMode
522 // VPLoadStoreBaseSDNode => enum ISD::MemIndexedMode
523 // MaskedLoadStoreBaseSDNode => enum ISD::MemIndexedMode
524 // VPGatherScatterSDNode => enum ISD::MemIndexType
525 // MaskedGatherScatterSDNode => enum ISD::MemIndexType
526 uint16_t AddressingMode : 3;
527 };
528 enum { NumLSBaseSDNodeBits = NumMemSDNodeBits + 3 };
529
530 class LoadSDNodeBitfields {
531 friend class LoadSDNode;
532 friend class VPLoadSDNode;
533 friend class MaskedLoadSDNode;
534 friend class MaskedGatherSDNode;
535 friend class VPGatherSDNode;
536
537 uint16_t : NumLSBaseSDNodeBits;
538
539 uint16_t ExtTy : 2; // enum ISD::LoadExtType
540 uint16_t IsExpanding : 1;
541 };
542
543 class StoreSDNodeBitfields {
544 friend class StoreSDNode;
545 friend class VPStoreSDNode;
546 friend class MaskedStoreSDNode;
547 friend class MaskedScatterSDNode;
548 friend class VPScatterSDNode;
549
550 uint16_t : NumLSBaseSDNodeBits;
551
552 uint16_t IsTruncating : 1;
553 uint16_t IsCompressing : 1;
554 };
555
556 union {
557 char RawSDNodeBits[sizeof(uint16_t)];
558 SDNodeBitfields SDNodeBits;
559 ConstantSDNodeBitfields ConstantSDNodeBits;
560 MemSDNodeBitfields MemSDNodeBits;
561 LSBaseSDNodeBitfields LSBaseSDNodeBits;
562 LoadSDNodeBitfields LoadSDNodeBits;
563 StoreSDNodeBitfields StoreSDNodeBits;
564 };
565END_TWO_BYTE_PACK()
566#undef BEGIN_TWO_BYTE_PACK
567#undef END_TWO_BYTE_PACK
568
569 // RawSDNodeBits must cover the entirety of the union. This means that all of
570 // the union's members must have size <= RawSDNodeBits. We write the RHS as
571 // "2" instead of sizeof(RawSDNodeBits) because MSVC can't handle the latter.
572 static_assert(sizeof(SDNodeBitfields) <= 2, "field too wide");
573 static_assert(sizeof(ConstantSDNodeBitfields) <= 2, "field too wide");
574 static_assert(sizeof(MemSDNodeBitfields) <= 2, "field too wide");
575 static_assert(sizeof(LSBaseSDNodeBitfields) <= 2, "field too wide");
576 static_assert(sizeof(LoadSDNodeBitfields) <= 2, "field too wide");
577 static_assert(sizeof(StoreSDNodeBitfields) <= 2, "field too wide");
578
579private:
580 friend class SelectionDAG;
581 // TODO: unfriend HandleSDNode once we fix its operand handling.
582 friend class HandleSDNode;
583
584 /// Unique id per SDNode in the DAG.
585 int NodeId = -1;
586
587 /// The values that are used by this operation.
588 SDUse *OperandList = nullptr;
589
590 /// The types of the values this node defines. SDNode's may
591 /// define multiple values simultaneously.
592 const EVT *ValueList;
593
594 /// List of uses for this SDNode.
595 SDUse *UseList = nullptr;
596
597 /// The number of entries in the Operand/Value list.
598 unsigned short NumOperands = 0;
599 unsigned short NumValues;
600
601 // The ordering of the SDNodes. It roughly corresponds to the ordering of the
602 // original LLVM instructions.
603 // This is used for turning off scheduling, because we'll forgo
604 // the normal scheduling algorithms and output the instructions according to
605 // this ordering.
606 unsigned IROrder;
607
608 /// Source line information.
609 DebugLoc debugLoc;
610
611 /// Return a pointer to the specified value type.
612 static const EVT *getValueTypeList(EVT VT);
613
614 SDNodeFlags Flags;
615
616public:
617 /// Unique and persistent id per SDNode in the DAG.
618 /// Used for debug printing.
619 uint16_t PersistentId;
620
621 //===--------------------------------------------------------------------===//
622 // Accessors
623 //
624
625 /// Return the SelectionDAG opcode value for this node. For
626 /// pre-isel nodes (those for which isMachineOpcode returns false), these
627 /// are the opcode values in the ISD and <target>ISD namespaces. For
628 /// post-isel opcodes, see getMachineOpcode.
629 unsigned getOpcode() const { return (unsigned short)NodeType; }
630
631 /// Test if this node has a target-specific opcode (in the
632 /// \<target\>ISD namespace).
633 bool isTargetOpcode() const { return NodeType >= ISD::BUILTIN_OP_END; }
634
635 /// Test if this node has a target-specific opcode that may raise
636 /// FP exceptions (in the \<target\>ISD namespace and greater than
637 /// FIRST_TARGET_STRICTFP_OPCODE). Note that all target memory
638 /// opcode are currently automatically considered to possibly raise
639 /// FP exceptions as well.
640 bool isTargetStrictFPOpcode() const {
641 return NodeType >= ISD::FIRST_TARGET_STRICTFP_OPCODE;
642 }
643
644 /// Test if this node has a target-specific
645 /// memory-referencing opcode (in the \<target\>ISD namespace and
646 /// greater than FIRST_TARGET_MEMORY_OPCODE).
647 bool isTargetMemoryOpcode() const {
648 return NodeType >= ISD::FIRST_TARGET_MEMORY_OPCODE;
649 }
650
651 /// Return true if the type of the node type undefined.
652 bool isUndef() const { return NodeType == ISD::UNDEF; }
653
654 /// Test if this node is a memory intrinsic (with valid pointer information).
655 /// INTRINSIC_W_CHAIN and INTRINSIC_VOID nodes are sometimes created for
656 /// non-memory intrinsics (with chains) that are not really instances of
657 /// MemSDNode. For such nodes, we need some extra state to determine the
658 /// proper classof relationship.
659 bool isMemIntrinsic() const {
660 return (NodeType == ISD::INTRINSIC_W_CHAIN ||
661 NodeType == ISD::INTRINSIC_VOID) &&
662 SDNodeBits.IsMemIntrinsic;
663 }
664
665 /// Test if this node is a strict floating point pseudo-op.
666 bool isStrictFPOpcode() {
667 switch (NodeType) {
668 default:
669 return false;
670 case ISD::STRICT_FP16_TO_FP:
671 case ISD::STRICT_FP_TO_FP16:
672#define DAG_INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC, DAGN) \
673 case ISD::STRICT_##DAGN:
674#include "llvm/IR/ConstrainedOps.def"
675 return true;
676 }
677 }
678
679 /// Test if this node has a post-isel opcode, directly
680 /// corresponding to a MachineInstr opcode.
681 bool isMachineOpcode() const { return NodeType < 0; }
682
683 /// This may only be called if isMachineOpcode returns
684 /// true. It returns the MachineInstr opcode value that the node's opcode
685 /// corresponds to.
686 unsigned getMachineOpcode() const {
687 assert(isMachineOpcode() && "Not a MachineInstr opcode!")(static_cast <bool> (isMachineOpcode() && "Not a MachineInstr opcode!"
) ? void (0) : __assert_fail ("isMachineOpcode() && \"Not a MachineInstr opcode!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 687, __extension__ __PRETTY_FUNCTION__))
;
688 return ~NodeType;
689 }
690
691 bool getHasDebugValue() const { return SDNodeBits.HasDebugValue; }
692 void setHasDebugValue(bool b) { SDNodeBits.HasDebugValue = b; }
693
694 bool isDivergent() const { return SDNodeBits.IsDivergent; }
695
696 /// Return true if there are no uses of this node.
697 bool use_empty() const { return UseList == nullptr; }
698
699 /// Return true if there is exactly one use of this node.
700 bool hasOneUse() const { return hasSingleElement(uses()); }
701
702 /// Return the number of uses of this node. This method takes
703 /// time proportional to the number of uses.
704 size_t use_size() const { return std::distance(use_begin(), use_end()); }
705
706 /// Return the unique node id.
707 int getNodeId() const { return NodeId; }
708
709 /// Set unique node id.
710 void setNodeId(int Id) { NodeId = Id; }
711
712 /// Return the node ordering.
713 unsigned getIROrder() const { return IROrder; }
714
715 /// Set the node ordering.
716 void setIROrder(unsigned Order) { IROrder = Order; }
717
718 /// Return the source location info.
719 const DebugLoc &getDebugLoc() const { return debugLoc; }
720
721 /// Set source location info. Try to avoid this, putting
722 /// it in the constructor is preferable.
723 void setDebugLoc(DebugLoc dl) { debugLoc = std::move(dl); }
724
725 /// This class provides iterator support for SDUse
726 /// operands that use a specific SDNode.
727 class use_iterator {
728 friend class SDNode;
729
730 SDUse *Op = nullptr;
731
732 explicit use_iterator(SDUse *op) : Op(op) {}
733
734 public:
735 using iterator_category = std::forward_iterator_tag;
736 using value_type = SDUse;
737 using difference_type = std::ptrdiff_t;
738 using pointer = value_type *;
739 using reference = value_type &;
740
741 use_iterator() = default;
742 use_iterator(const use_iterator &I) : Op(I.Op) {}
743
744 bool operator==(const use_iterator &x) const {
745 return Op == x.Op;
746 }
747 bool operator!=(const use_iterator &x) const {
748 return !operator==(x);
749 }
750
751 /// Return true if this iterator is at the end of uses list.
752 bool atEnd() const { return Op == nullptr; }
753
754 // Iterator traversal: forward iteration only.
755 use_iterator &operator++() { // Preincrement
756 assert(Op && "Cannot increment end iterator!")(static_cast <bool> (Op && "Cannot increment end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot increment end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 756, __extension__ __PRETTY_FUNCTION__))
;
757 Op = Op->getNext();
758 return *this;
759 }
760
761 use_iterator operator++(int) { // Postincrement
762 use_iterator tmp = *this; ++*this; return tmp;
763 }
764
765 /// Retrieve a pointer to the current user node.
766 SDNode *operator*() const {
767 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 767, __extension__ __PRETTY_FUNCTION__))
;
768 return Op->getUser();
769 }
770
771 SDNode *operator->() const { return operator*(); }
772
773 SDUse &getUse() const { return *Op; }
774
775 /// Retrieve the operand # of this use in its user.
776 unsigned getOperandNo() const {
777 assert(Op && "Cannot dereference end iterator!")(static_cast <bool> (Op && "Cannot dereference end iterator!"
) ? void (0) : __assert_fail ("Op && \"Cannot dereference end iterator!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 777, __extension__ __PRETTY_FUNCTION__))
;
778 return (unsigned)(Op - Op->getUser()->OperandList);
779 }
780 };
781
782 /// Provide iteration support to walk over all uses of an SDNode.
783 use_iterator use_begin() const {
784 return use_iterator(UseList);
785 }
786
787 static use_iterator use_end() { return use_iterator(nullptr); }
788
789 inline iterator_range<use_iterator> uses() {
790 return make_range(use_begin(), use_end());
791 }
792 inline iterator_range<use_iterator> uses() const {
793 return make_range(use_begin(), use_end());
794 }
795
796 /// Return true if there are exactly NUSES uses of the indicated value.
797 /// This method ignores uses of other values defined by this operation.
798 bool hasNUsesOfValue(unsigned NUses, unsigned Value) const;
799
800 /// Return true if there are any use of the indicated value.
801 /// This method ignores uses of other values defined by this operation.
802 bool hasAnyUseOfValue(unsigned Value) const;
803
804 /// Return true if this node is the only use of N.
805 bool isOnlyUserOf(const SDNode *N) const;
806
807 /// Return true if this node is an operand of N.
808 bool isOperandOf(const SDNode *N) const;
809
810 /// Return true if this node is a predecessor of N.
811 /// NOTE: Implemented on top of hasPredecessor and every bit as
812 /// expensive. Use carefully.
813 bool isPredecessorOf(const SDNode *N) const {
814 return N->hasPredecessor(this);
815 }
816
817 /// Return true if N is a predecessor of this node.
818 /// N is either an operand of this node, or can be reached by recursively
819 /// traversing up the operands.
820 /// NOTE: This is an expensive method. Use it carefully.
821 bool hasPredecessor(const SDNode *N) const;
822
823 /// Returns true if N is a predecessor of any node in Worklist. This
824 /// helper keeps Visited and Worklist sets externally to allow unions
825 /// searches to be performed in parallel, caching of results across
826 /// queries and incremental addition to Worklist. Stops early if N is
827 /// found but will resume. Remember to clear Visited and Worklists
828 /// if DAG changes. MaxSteps gives a maximum number of nodes to visit before
829 /// giving up. The TopologicalPrune flag signals that positive NodeIds are
830 /// topologically ordered (Operands have strictly smaller node id) and search
831 /// can be pruned leveraging this.
832 static bool hasPredecessorHelper(const SDNode *N,
833 SmallPtrSetImpl<const SDNode *> &Visited,
834 SmallVectorImpl<const SDNode *> &Worklist,
835 unsigned int MaxSteps = 0,
836 bool TopologicalPrune = false) {
837 SmallVector<const SDNode *, 8> DeferredNodes;
838 if (Visited.count(N))
839 return true;
840
841 // Node Id's are assigned in three places: As a topological
842 // ordering (> 0), during legalization (results in values set to
843 // 0), new nodes (set to -1). If N has a topolgical id then we
844 // know that all nodes with ids smaller than it cannot be
845 // successors and we need not check them. Filter out all node
846 // that can't be matches. We add them to the worklist before exit
847 // in case of multiple calls. Note that during selection the topological id
848 // may be violated if a node's predecessor is selected before it. We mark
849 // this at selection negating the id of unselected successors and
850 // restricting topological pruning to positive ids.
851
852 int NId = N->getNodeId();
853 // If we Invalidated the Id, reconstruct original NId.
854 if (NId < -1)
855 NId = -(NId + 1);
856
857 bool Found = false;
858 while (!Worklist.empty()) {
859 const SDNode *M = Worklist.pop_back_val();
860 int MId = M->getNodeId();
861 if (TopologicalPrune && M->getOpcode() != ISD::TokenFactor && (NId > 0) &&
862 (MId > 0) && (MId < NId)) {
863 DeferredNodes.push_back(M);
864 continue;
865 }
866 for (const SDValue &OpV : M->op_values()) {
867 SDNode *Op = OpV.getNode();
868 if (Visited.insert(Op).second)
869 Worklist.push_back(Op);
870 if (Op == N)
871 Found = true;
872 }
873 if (Found)
874 break;
875 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
876 break;
877 }
878 // Push deferred nodes back on worklist.
879 Worklist.append(DeferredNodes.begin(), DeferredNodes.end());
880 // If we bailed early, conservatively return found.
881 if (MaxSteps != 0 && Visited.size() >= MaxSteps)
882 return true;
883 return Found;
884 }
885
886 /// Return true if all the users of N are contained in Nodes.
887 /// NOTE: Requires at least one match, but doesn't require them all.
888 static bool areOnlyUsersOf(ArrayRef<const SDNode *> Nodes, const SDNode *N);
889
890 /// Return the number of values used by this operation.
891 unsigned getNumOperands() const { return NumOperands; }
892
893 /// Return the maximum number of operands that a SDNode can hold.
894 static constexpr size_t getMaxNumOperands() {
895 return std::numeric_limits<decltype(SDNode::NumOperands)>::max();
896 }
897
898 /// Helper method returns the integer value of a ConstantSDNode operand.
899 inline uint64_t getConstantOperandVal(unsigned Num) const;
900
901 /// Helper method returns the APInt of a ConstantSDNode operand.
902 inline const APInt &getConstantOperandAPInt(unsigned Num) const;
903
904 const SDValue &getOperand(unsigned Num) const {
905 assert(Num < NumOperands && "Invalid child # of SDNode!")(static_cast <bool> (Num < NumOperands && "Invalid child # of SDNode!"
) ? void (0) : __assert_fail ("Num < NumOperands && \"Invalid child # of SDNode!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 905, __extension__ __PRETTY_FUNCTION__))
;
906 return OperandList[Num];
907 }
908
909 using op_iterator = SDUse *;
910
911 op_iterator op_begin() const { return OperandList; }
912 op_iterator op_end() const { return OperandList+NumOperands; }
913 ArrayRef<SDUse> ops() const { return makeArrayRef(op_begin(), op_end()); }
914
915 /// Iterator for directly iterating over the operand SDValue's.
916 struct value_op_iterator
917 : iterator_adaptor_base<value_op_iterator, op_iterator,
918 std::random_access_iterator_tag, SDValue,
919 ptrdiff_t, value_op_iterator *,
920 value_op_iterator *> {
921 explicit value_op_iterator(SDUse *U = nullptr)
922 : iterator_adaptor_base(U) {}
923
924 const SDValue &operator*() const { return I->get(); }
925 };
926
927 iterator_range<value_op_iterator> op_values() const {
928 return make_range(value_op_iterator(op_begin()),
929 value_op_iterator(op_end()));
930 }
931
932 SDVTList getVTList() const {
933 SDVTList X = { ValueList, NumValues };
934 return X;
935 }
936
937 /// If this node has a glue operand, return the node
938 /// to which the glue operand points. Otherwise return NULL.
939 SDNode *getGluedNode() const {
940 if (getNumOperands() != 0 &&
941 getOperand(getNumOperands()-1).getValueType() == MVT::Glue)
942 return getOperand(getNumOperands()-1).getNode();
943 return nullptr;
944 }
945
946 /// If this node has a glue value with a user, return
947 /// the user (there is at most one). Otherwise return NULL.
948 SDNode *getGluedUser() const {
949 for (use_iterator UI = use_begin(), UE = use_end(); UI != UE; ++UI)
950 if (UI.getUse().get().getValueType() == MVT::Glue)
951 return *UI;
952 return nullptr;
953 }
954
955 SDNodeFlags getFlags() const { return Flags; }
956 void setFlags(SDNodeFlags NewFlags) { Flags = NewFlags; }
957
958 /// Clear any flags in this node that aren't also set in Flags.
959 /// If Flags is not in a defined state then this has no effect.
960 void intersectFlagsWith(const SDNodeFlags Flags);
961
962 /// Return the number of values defined/returned by this operator.
963 unsigned getNumValues() const { return NumValues; }
964
965 /// Return the type of a specified result.
966 EVT getValueType(unsigned ResNo) const {
967 assert(ResNo < NumValues && "Illegal result number!")(static_cast <bool> (ResNo < NumValues && "Illegal result number!"
) ? void (0) : __assert_fail ("ResNo < NumValues && \"Illegal result number!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 967, __extension__ __PRETTY_FUNCTION__))
;
968 return ValueList[ResNo];
969 }
970
971 /// Return the type of a specified result as a simple type.
972 MVT getSimpleValueType(unsigned ResNo) const {
973 return getValueType(ResNo).getSimpleVT();
974 }
975
976 /// Returns MVT::getSizeInBits(getValueType(ResNo)).
977 ///
978 /// If the value type is a scalable vector type, the scalable property will
979 /// be set and the runtime size will be a positive integer multiple of the
980 /// base size.
981 TypeSize getValueSizeInBits(unsigned ResNo) const {
982 return getValueType(ResNo).getSizeInBits();
983 }
984
985 using value_iterator = const EVT *;
986
987 value_iterator value_begin() const { return ValueList; }
988 value_iterator value_end() const { return ValueList+NumValues; }
989 iterator_range<value_iterator> values() const {
990 return llvm::make_range(value_begin(), value_end());
991 }
992
993 /// Return the opcode of this operation for printing.
994 std::string getOperationName(const SelectionDAG *G = nullptr) const;
995 static const char* getIndexedModeName(ISD::MemIndexedMode AM);
996 void print_types(raw_ostream &OS, const SelectionDAG *G) const;
997 void print_details(raw_ostream &OS, const SelectionDAG *G) const;
998 void print(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
999 void printr(raw_ostream &OS, const SelectionDAG *G = nullptr) const;
1000
1001 /// Print a SelectionDAG node and all children down to
1002 /// the leaves. The given SelectionDAG allows target-specific nodes
1003 /// to be printed in human-readable form. Unlike printr, this will
1004 /// print the whole DAG, including children that appear multiple
1005 /// times.
1006 ///
1007 void printrFull(raw_ostream &O, const SelectionDAG *G = nullptr) const;
1008
1009 /// Print a SelectionDAG node and children up to
1010 /// depth "depth." The given SelectionDAG allows target-specific
1011 /// nodes to be printed in human-readable form. Unlike printr, this
1012 /// will print children that appear multiple times wherever they are
1013 /// used.
1014 ///
1015 void printrWithDepth(raw_ostream &O, const SelectionDAG *G = nullptr,
1016 unsigned depth = 100) const;
1017
1018 /// Dump this node, for debugging.
1019 void dump() const;
1020
1021 /// Dump (recursively) this node and its use-def subgraph.
1022 void dumpr() const;
1023
1024 /// Dump this node, for debugging.
1025 /// The given SelectionDAG allows target-specific nodes to be printed
1026 /// in human-readable form.
1027 void dump(const SelectionDAG *G) const;
1028
1029 /// Dump (recursively) this node and its use-def subgraph.
1030 /// The given SelectionDAG allows target-specific nodes to be printed
1031 /// in human-readable form.
1032 void dumpr(const SelectionDAG *G) const;
1033
1034 /// printrFull to dbgs(). The given SelectionDAG allows
1035 /// target-specific nodes to be printed in human-readable form.
1036 /// Unlike dumpr, this will print the whole DAG, including children
1037 /// that appear multiple times.
1038 void dumprFull(const SelectionDAG *G = nullptr) const;
1039
1040 /// printrWithDepth to dbgs(). The given
1041 /// SelectionDAG allows target-specific nodes to be printed in
1042 /// human-readable form. Unlike dumpr, this will print children
1043 /// that appear multiple times wherever they are used.
1044 ///
1045 void dumprWithDepth(const SelectionDAG *G = nullptr,
1046 unsigned depth = 100) const;
1047
1048 /// Gather unique data for the node.
1049 void Profile(FoldingSetNodeID &ID) const;
1050
1051 /// This method should only be used by the SDUse class.
1052 void addUse(SDUse &U) { U.addToList(&UseList); }
1053
1054protected:
1055 static SDVTList getSDVTList(EVT VT) {
1056 SDVTList Ret = { getValueTypeList(VT), 1 };
1057 return Ret;
1058 }
1059
1060 /// Create an SDNode.
1061 ///
1062 /// SDNodes are created without any operands, and never own the operand
1063 /// storage. To add operands, see SelectionDAG::createOperands.
1064 SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs)
1065 : NodeType(Opc), ValueList(VTs.VTs), NumValues(VTs.NumVTs),
1066 IROrder(Order), debugLoc(std::move(dl)) {
1067 memset(&RawSDNodeBits, 0, sizeof(RawSDNodeBits));
1068 assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor")(static_cast <bool> (debugLoc.hasTrivialDestructor() &&
"Expected trivial destructor") ? void (0) : __assert_fail ("debugLoc.hasTrivialDestructor() && \"Expected trivial destructor\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1068, __extension__ __PRETTY_FUNCTION__))
;
1069 assert(NumValues == VTs.NumVTs &&(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__))
1070 "NumValues wasn't wide enough for its operands!")(static_cast <bool> (NumValues == VTs.NumVTs &&
"NumValues wasn't wide enough for its operands!") ? void (0)
: __assert_fail ("NumValues == VTs.NumVTs && \"NumValues wasn't wide enough for its operands!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1070, __extension__ __PRETTY_FUNCTION__))
;
1071 }
1072
1073 /// Release the operands and set this node to have zero operands.
1074 void DropOperands();
1075};
1076
1077/// Wrapper class for IR location info (IR ordering and DebugLoc) to be passed
1078/// into SDNode creation functions.
1079/// When an SDNode is created from the DAGBuilder, the DebugLoc is extracted
1080/// from the original Instruction, and IROrder is the ordinal position of
1081/// the instruction.
1082/// When an SDNode is created after the DAG is being built, both DebugLoc and
1083/// the IROrder are propagated from the original SDNode.
1084/// So SDLoc class provides two constructors besides the default one, one to
1085/// be used by the DAGBuilder, the other to be used by others.
1086class SDLoc {
1087private:
1088 DebugLoc DL;
1089 int IROrder = 0;
1090
1091public:
1092 SDLoc() = default;
1093 SDLoc(const SDNode *N) : DL(N->getDebugLoc()), IROrder(N->getIROrder()) {}
1094 SDLoc(const SDValue V) : SDLoc(V.getNode()) {}
1095 SDLoc(const Instruction *I, int Order) : IROrder(Order) {
1096 assert(Order >= 0 && "bad IROrder")(static_cast <bool> (Order >= 0 && "bad IROrder"
) ? void (0) : __assert_fail ("Order >= 0 && \"bad IROrder\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1096, __extension__ __PRETTY_FUNCTION__))
;
1097 if (I)
1098 DL = I->getDebugLoc();
1099 }
1100
1101 unsigned getIROrder() const { return IROrder; }
1102 const DebugLoc &getDebugLoc() const { return DL; }
1103};
1104
1105// Define inline functions from the SDValue class.
1106
1107inline SDValue::SDValue(SDNode *node, unsigned resno)
1108 : Node(node), ResNo(resno) {
1109 // Explicitly check for !ResNo to avoid use-after-free, because there are
1110 // callers that use SDValue(N, 0) with a deleted N to indicate successful
1111 // combines.
1112 assert((!Node || !ResNo || ResNo < Node->getNumValues()) &&(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__))
1113 "Invalid result number for the given node!")(static_cast <bool> ((!Node || !ResNo || ResNo < Node
->getNumValues()) && "Invalid result number for the given node!"
) ? void (0) : __assert_fail ("(!Node || !ResNo || ResNo < Node->getNumValues()) && \"Invalid result number for the given node!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1113, __extension__ __PRETTY_FUNCTION__))
;
1114 assert(ResNo < -2U && "Cannot use result numbers reserved for DenseMaps.")(static_cast <bool> (ResNo < -2U && "Cannot use result numbers reserved for DenseMaps."
) ? void (0) : __assert_fail ("ResNo < -2U && \"Cannot use result numbers reserved for DenseMaps.\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1114, __extension__ __PRETTY_FUNCTION__))
;
1115}
1116
1117inline unsigned SDValue::getOpcode() const {
1118 return Node->getOpcode();
1119}
1120
1121inline EVT SDValue::getValueType() const {
1122 return Node->getValueType(ResNo);
1123}
1124
1125inline unsigned SDValue::getNumOperands() const {
1126 return Node->getNumOperands();
1127}
1128
1129inline const SDValue &SDValue::getOperand(unsigned i) const {
1130 return Node->getOperand(i);
1131}
1132
1133inline uint64_t SDValue::getConstantOperandVal(unsigned i) const {
1134 return Node->getConstantOperandVal(i);
1135}
1136
1137inline const APInt &SDValue::getConstantOperandAPInt(unsigned i) const {
1138 return Node->getConstantOperandAPInt(i);
1139}
1140
1141inline bool SDValue::isTargetOpcode() const {
1142 return Node->isTargetOpcode();
1143}
1144
1145inline bool SDValue::isTargetMemoryOpcode() const {
1146 return Node->isTargetMemoryOpcode();
1147}
1148
1149inline bool SDValue::isMachineOpcode() const {
1150 return Node->isMachineOpcode();
1151}
1152
1153inline unsigned SDValue::getMachineOpcode() const {
1154 return Node->getMachineOpcode();
1155}
1156
1157inline bool SDValue::isUndef() const {
1158 return Node->isUndef();
1159}
1160
1161inline bool SDValue::use_empty() const {
1162 return !Node->hasAnyUseOfValue(ResNo);
1163}
1164
1165inline bool SDValue::hasOneUse() const {
1166 return Node->hasNUsesOfValue(1, ResNo);
1167}
1168
1169inline const DebugLoc &SDValue::getDebugLoc() const {
1170 return Node->getDebugLoc();
1171}
1172
1173inline void SDValue::dump() const {
1174 return Node->dump();
1175}
1176
1177inline void SDValue::dump(const SelectionDAG *G) const {
1178 return Node->dump(G);
1179}
1180
1181inline void SDValue::dumpr() const {
1182 return Node->dumpr();
1183}
1184
1185inline void SDValue::dumpr(const SelectionDAG *G) const {
1186 return Node->dumpr(G);
1187}
1188
1189// Define inline functions from the SDUse class.
1190
1191inline void SDUse::set(const SDValue &V) {
1192 if (Val.getNode()) removeFromList();
1193 Val = V;
1194 if (V.getNode()) V.getNode()->addUse(*this);
1195}
1196
1197inline void SDUse::setInitial(const SDValue &V) {
1198 Val = V;
1199 V.getNode()->addUse(*this);
1200}
1201
1202inline void SDUse::setNode(SDNode *N) {
1203 if (Val.getNode()) removeFromList();
1204 Val.setNode(N);
1205 if (N) N->addUse(*this);
1206}
1207
1208/// This class is used to form a handle around another node that
1209/// is persistent and is updated across invocations of replaceAllUsesWith on its
1210/// operand. This node should be directly created by end-users and not added to
1211/// the AllNodes list.
1212class HandleSDNode : public SDNode {
1213 SDUse Op;
1214
1215public:
1216 explicit HandleSDNode(SDValue X)
1217 : SDNode(ISD::HANDLENODE, 0, DebugLoc(), getSDVTList(MVT::Other)) {
1218 // HandleSDNodes are never inserted into the DAG, so they won't be
1219 // auto-numbered. Use ID 65535 as a sentinel.
1220 PersistentId = 0xffff;
1221
1222 // Manually set up the operand list. This node type is special in that it's
1223 // always stack allocated and SelectionDAG does not manage its operands.
1224 // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not
1225 // be so special.
1226 Op.setUser(this);
1227 Op.setInitial(X);
1228 NumOperands = 1;
1229 OperandList = &Op;
1230 }
1231 ~HandleSDNode();
1232
1233 const SDValue &getValue() const { return Op; }
1234};
1235
1236class AddrSpaceCastSDNode : public SDNode {
1237private:
1238 unsigned SrcAddrSpace;
1239 unsigned DestAddrSpace;
1240
1241public:
1242 AddrSpaceCastSDNode(unsigned Order, const DebugLoc &dl, EVT VT,
1243 unsigned SrcAS, unsigned DestAS);
1244
1245 unsigned getSrcAddressSpace() const { return SrcAddrSpace; }
1246 unsigned getDestAddressSpace() const { return DestAddrSpace; }
1247
1248 static bool classof(const SDNode *N) {
1249 return N->getOpcode() == ISD::ADDRSPACECAST;
1250 }
1251};
1252
1253/// This is an abstract virtual class for memory operations.
1254class MemSDNode : public SDNode {
1255private:
1256 // VT of in-memory value.
1257 EVT MemoryVT;
1258
1259protected:
1260 /// Memory reference information.
1261 MachineMemOperand *MMO;
1262
1263public:
1264 MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTs,
1265 EVT memvt, MachineMemOperand *MMO);
1266
1267 bool readMem() const { return MMO->isLoad(); }
1268 bool writeMem() const { return MMO->isStore(); }
1269
1270 /// Returns alignment and volatility of the memory access
1271 Align getOriginalAlign() const { return MMO->getBaseAlign(); }
1272 Align getAlign() const { return MMO->getAlign(); }
1273 // FIXME: Remove once transition to getAlign is over.
1274 unsigned getAlignment() const { return MMO->getAlign().value(); }
1275
1276 /// Return the SubclassData value, without HasDebugValue. This contains an
1277 /// encoding of the volatile flag, as well as bits used by subclasses. This
1278 /// function should only be used to compute a FoldingSetNodeID value.
1279 /// The HasDebugValue bit is masked out because CSE map needs to match
1280 /// nodes with debug info with nodes without debug info. Same is about
1281 /// isDivergent bit.
1282 unsigned getRawSubclassData() const {
1283 uint16_t Data;
1284 union {
1285 char RawSDNodeBits[sizeof(uint16_t)];
1286 SDNodeBitfields SDNodeBits;
1287 };
1288 memcpy(&RawSDNodeBits, &this->RawSDNodeBits, sizeof(this->RawSDNodeBits));
1289 SDNodeBits.HasDebugValue = 0;
1290 SDNodeBits.IsDivergent = false;
1291 memcpy(&Data, &RawSDNodeBits, sizeof(RawSDNodeBits));
1292 return Data;
1293 }
1294
1295 bool isVolatile() const { return MemSDNodeBits.IsVolatile; }
1296 bool isNonTemporal() const { return MemSDNodeBits.IsNonTemporal; }
1297 bool isDereferenceable() const { return MemSDNodeBits.IsDereferenceable; }
1298 bool isInvariant() const { return MemSDNodeBits.IsInvariant; }
1299
1300 // Returns the offset from the location of the access.
1301 int64_t getSrcValueOffset() const { return MMO->getOffset(); }
1302
1303 /// Returns the AA info that describes the dereference.
1304 AAMDNodes getAAInfo() const { return MMO->getAAInfo(); }
1305
1306 /// Returns the Ranges that describes the dereference.
1307 const MDNode *getRanges() const { return MMO->getRanges(); }
1308
1309 /// Returns the synchronization scope ID for this memory operation.
1310 SyncScope::ID getSyncScopeID() const { return MMO->getSyncScopeID(); }
1311
1312 /// Return the atomic ordering requirements for this memory operation. For
1313 /// cmpxchg atomic operations, return the atomic ordering requirements when
1314 /// store occurs.
1315 AtomicOrdering getSuccessOrdering() const {
1316 return MMO->getSuccessOrdering();
1317 }
1318
1319 /// Return a single atomic ordering that is at least as strong as both the
1320 /// success and failure orderings for an atomic operation. (For operations
1321 /// other than cmpxchg, this is equivalent to getSuccessOrdering().)
1322 AtomicOrdering getMergedOrdering() const { return MMO->getMergedOrdering(); }
1323
1324 /// Return true if the memory operation ordering is Unordered or higher.
1325 bool isAtomic() const { return MMO->isAtomic(); }
1326
1327 /// Returns true if the memory operation doesn't imply any ordering
1328 /// constraints on surrounding memory operations beyond the normal memory
1329 /// aliasing rules.
1330 bool isUnordered() const { return MMO->isUnordered(); }
1331
1332 /// Returns true if the memory operation is neither atomic or volatile.
1333 bool isSimple() const { return !isAtomic() && !isVolatile(); }
1334
1335 /// Return the type of the in-memory value.
1336 EVT getMemoryVT() const { return MemoryVT; }
1337
1338 /// Return a MachineMemOperand object describing the memory
1339 /// reference performed by operation.
1340 MachineMemOperand *getMemOperand() const { return MMO; }
1341
1342 const MachinePointerInfo &getPointerInfo() const {
1343 return MMO->getPointerInfo();
1344 }
1345
1346 /// Return the address space for the associated pointer
1347 unsigned getAddressSpace() const {
1348 return getPointerInfo().getAddrSpace();
1349 }
1350
1351 /// Update this MemSDNode's MachineMemOperand information
1352 /// to reflect the alignment of NewMMO, if it has a greater alignment.
1353 /// This must only be used when the new alignment applies to all users of
1354 /// this MachineMemOperand.
1355 void refineAlignment(const MachineMemOperand *NewMMO) {
1356 MMO->refineAlignment(NewMMO);
1357 }
1358
1359 const SDValue &getChain() const { return getOperand(0); }
1360
1361 const SDValue &getBasePtr() const {
1362 switch (getOpcode()) {
1363 case ISD::STORE:
1364 case ISD::VP_STORE:
1365 case ISD::MSTORE:
1366 case ISD::VP_SCATTER:
1367 return getOperand(2);
1368 case ISD::MGATHER:
1369 case ISD::MSCATTER:
1370 return getOperand(3);
1371 default:
1372 return getOperand(1);
1373 }
1374 }
1375
1376 // Methods to support isa and dyn_cast
1377 static bool classof(const SDNode *N) {
1378 // For some targets, we lower some target intrinsics to a MemIntrinsicNode
1379 // with either an intrinsic or a target opcode.
1380 switch (N->getOpcode()) {
1381 case ISD::LOAD:
1382 case ISD::STORE:
1383 case ISD::PREFETCH:
1384 case ISD::ATOMIC_CMP_SWAP:
1385 case ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS:
1386 case ISD::ATOMIC_SWAP:
1387 case ISD::ATOMIC_LOAD_ADD:
1388 case ISD::ATOMIC_LOAD_SUB:
1389 case ISD::ATOMIC_LOAD_AND:
1390 case ISD::ATOMIC_LOAD_CLR:
1391 case ISD::ATOMIC_LOAD_OR:
1392 case ISD::ATOMIC_LOAD_XOR:
1393 case ISD::ATOMIC_LOAD_NAND:
1394 case ISD::ATOMIC_LOAD_MIN:
1395 case ISD::ATOMIC_LOAD_MAX:
1396 case ISD::ATOMIC_LOAD_UMIN:
1397 case ISD::ATOMIC_LOAD_UMAX:
1398 case ISD::ATOMIC_LOAD_FADD:
1399 case ISD::ATOMIC_LOAD_FSUB:
1400 case ISD::ATOMIC_LOAD:
1401 case ISD::ATOMIC_STORE:
1402 case ISD::MLOAD:
1403 case ISD::MSTORE:
1404 case ISD::MGATHER:
1405 case ISD::MSCATTER:
1406 case ISD::VP_LOAD:
1407 case ISD::VP_STORE:
1408 case ISD::VP_GATHER:
1409 case ISD::VP_SCATTER:
1410 return true;
1411 default:
1412 return N->isMemIntrinsic() || N->isTargetMemoryOpcode();
1413 }
1414 }
1415};
1416
1417/// This is an SDNode representing atomic operations.
1418class AtomicSDNode : public MemSDNode {
1419public:
1420 AtomicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, SDVTList VTL,
1421 EVT MemVT, MachineMemOperand *MMO)
1422 : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) {
1423 assert(((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) ||(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__))
1424 MMO->isAtomic()) && "then why are we using an AtomicSDNode?")(static_cast <bool> (((Opc != ISD::ATOMIC_LOAD &&
Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && "then why are we using an AtomicSDNode?"
) ? void (0) : __assert_fail ("((Opc != ISD::ATOMIC_LOAD && Opc != ISD::ATOMIC_STORE) || MMO->isAtomic()) && \"then why are we using an AtomicSDNode?\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1424, __extension__ __PRETTY_FUNCTION__))
;
1425 }
1426
1427 const SDValue &getBasePtr() const { return getOperand(1); }
1428 const SDValue &getVal() const { return getOperand(2); }
1429
1430 /// Returns true if this SDNode represents cmpxchg atomic operation, false
1431 /// otherwise.
1432 bool isCompareAndSwap() const {
1433 unsigned Op = getOpcode();
1434 return Op == ISD::ATOMIC_CMP_SWAP ||
1435 Op == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS;
1436 }
1437
1438 /// For cmpxchg atomic operations, return the atomic ordering requirements
1439 /// when store does not occur.
1440 AtomicOrdering getFailureOrdering() const {
1441 assert(isCompareAndSwap() && "Must be cmpxchg operation")(static_cast <bool> (isCompareAndSwap() && "Must be cmpxchg operation"
) ? void (0) : __assert_fail ("isCompareAndSwap() && \"Must be cmpxchg operation\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1441, __extension__ __PRETTY_FUNCTION__))
;
1442 return MMO->getFailureOrdering();
1443 }
1444
1445 // Methods to support isa and dyn_cast
1446 static bool classof(const SDNode *N) {
1447 return N->getOpcode() == ISD::ATOMIC_CMP_SWAP ||
1448 N->getOpcode() == ISD::ATOMIC_CMP_SWAP_WITH_SUCCESS ||
1449 N->getOpcode() == ISD::ATOMIC_SWAP ||
1450 N->getOpcode() == ISD::ATOMIC_LOAD_ADD ||
1451 N->getOpcode() == ISD::ATOMIC_LOAD_SUB ||
1452 N->getOpcode() == ISD::ATOMIC_LOAD_AND ||
1453 N->getOpcode() == ISD::ATOMIC_LOAD_CLR ||
1454 N->getOpcode() == ISD::ATOMIC_LOAD_OR ||
1455 N->getOpcode() == ISD::ATOMIC_LOAD_XOR ||
1456 N->getOpcode() == ISD::ATOMIC_LOAD_NAND ||
1457 N->getOpcode() == ISD::ATOMIC_LOAD_MIN ||
1458 N->getOpcode() == ISD::ATOMIC_LOAD_MAX ||
1459 N->getOpcode() == ISD::ATOMIC_LOAD_UMIN ||
1460 N->getOpcode() == ISD::ATOMIC_LOAD_UMAX ||
1461 N->getOpcode() == ISD::ATOMIC_LOAD_FADD ||
1462 N->getOpcode() == ISD::ATOMIC_LOAD_FSUB ||
1463 N->getOpcode() == ISD::ATOMIC_LOAD ||
1464 N->getOpcode() == ISD::ATOMIC_STORE;
1465 }
1466};
1467
1468/// This SDNode is used for target intrinsics that touch
1469/// memory and need an associated MachineMemOperand. Its opcode may be
1470/// INTRINSIC_VOID, INTRINSIC_W_CHAIN, PREFETCH, or a target-specific opcode
1471/// with a value not less than FIRST_TARGET_MEMORY_OPCODE.
1472class MemIntrinsicSDNode : public MemSDNode {
1473public:
1474 MemIntrinsicSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl,
1475 SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO)
1476 : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) {
1477 SDNodeBits.IsMemIntrinsic = true;
1478 }
1479
1480 // Methods to support isa and dyn_cast
1481 static bool classof(const SDNode *N) {
1482 // We lower some target intrinsics to their target opcode
1483 // early a node with a target opcode can be of this class
1484 return N->isMemIntrinsic() ||
1485 N->getOpcode() == ISD::PREFETCH ||
1486 N->isTargetMemoryOpcode();
1487 }
1488};
1489
1490/// This SDNode is used to implement the code generator
1491/// support for the llvm IR shufflevector instruction. It combines elements
1492/// from two input vectors into a new input vector, with the selection and
1493/// ordering of elements determined by an array of integers, referred to as
1494/// the shuffle mask. For input vectors of width N, mask indices of 0..N-1
1495/// refer to elements from the LHS input, and indices from N to 2N-1 the RHS.
1496/// An index of -1 is treated as undef, such that the code generator may put
1497/// any value in the corresponding element of the result.
1498class ShuffleVectorSDNode : public SDNode {
1499 // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and
1500 // is freed when the SelectionDAG object is destroyed.
1501 const int *Mask;
1502
1503protected:
1504 friend class SelectionDAG;
1505
1506 ShuffleVectorSDNode(EVT VT, unsigned Order, const DebugLoc &dl, const int *M)
1507 : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {}
1508
1509public:
1510 ArrayRef<int> getMask() const {
1511 EVT VT = getValueType(0);
1512 return makeArrayRef(Mask, VT.getVectorNumElements());
1513 }
1514
1515 int getMaskElt(unsigned Idx) const {
1516 assert(Idx < getValueType(0).getVectorNumElements() && "Idx out of range!")(static_cast <bool> (Idx < getValueType(0).getVectorNumElements
() && "Idx out of range!") ? void (0) : __assert_fail
("Idx < getValueType(0).getVectorNumElements() && \"Idx out of range!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1516, __extension__ __PRETTY_FUNCTION__))
;
1517 return Mask[Idx];
1518 }
1519
1520 bool isSplat() const { return isSplatMask(Mask, getValueType(0)); }
1521
1522 int getSplatIndex() const {
1523 assert(isSplat() && "Cannot get splat index for non-splat!")(static_cast <bool> (isSplat() && "Cannot get splat index for non-splat!"
) ? void (0) : __assert_fail ("isSplat() && \"Cannot get splat index for non-splat!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1523, __extension__ __PRETTY_FUNCTION__))
;
1524 EVT VT = getValueType(0);
1525 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i)
1526 if (Mask[i] >= 0)
1527 return Mask[i];
1528
1529 // We can choose any index value here and be correct because all elements
1530 // are undefined. Return 0 for better potential for callers to simplify.
1531 return 0;
1532 }
1533
1534 static bool isSplatMask(const int *Mask, EVT VT);
1535
1536 /// Change values in a shuffle permute mask assuming
1537 /// the two vector operands have swapped position.
1538 static void commuteMask(MutableArrayRef<int> Mask) {
1539 unsigned NumElems = Mask.size();
1540 for (unsigned i = 0; i != NumElems; ++i) {
1541 int idx = Mask[i];
1542 if (idx < 0)
1543 continue;
1544 else if (idx < (int)NumElems)
1545 Mask[i] = idx + NumElems;
1546 else
1547 Mask[i] = idx - NumElems;
1548 }
1549 }
1550
1551 static bool classof(const SDNode *N) {
1552 return N->getOpcode() == ISD::VECTOR_SHUFFLE;
1553 }
1554};
1555
1556class ConstantSDNode : public SDNode {
1557 friend class SelectionDAG;
1558
1559 const ConstantInt *Value;
1560
1561 ConstantSDNode(bool isTarget, bool isOpaque, const ConstantInt *val, EVT VT)
1562 : SDNode(isTarget ? ISD::TargetConstant : ISD::Constant, 0, DebugLoc(),
1563 getSDVTList(VT)),
1564 Value(val) {
1565 ConstantSDNodeBits.IsOpaque = isOpaque;
1566 }
1567
1568public:
1569 const ConstantInt *getConstantIntValue() const { return Value; }
1570 const APInt &getAPIntValue() const { return Value->getValue(); }
1571 uint64_t getZExtValue() const { return Value->getZExtValue(); }
1572 int64_t getSExtValue() const { return Value->getSExtValue(); }
1573 uint64_t getLimitedValue(uint64_t Limit = UINT64_MAX(18446744073709551615UL)) {
1574 return Value->getLimitedValue(Limit);
1575 }
1576 MaybeAlign getMaybeAlignValue() const { return Value->getMaybeAlignValue(); }
1577 Align getAlignValue() const { return Value->getAlignValue(); }
1578
1579 bool isOne() const { return Value->isOne(); }
1580 bool isZero() const { return Value->isZero(); }
1581 // NOTE: This is soft-deprecated. Please use `isZero()` instead.
1582 bool isNullValue() const { return isZero(); }
1583 bool isAllOnes() const { return Value->isMinusOne(); }
1584 // NOTE: This is soft-deprecated. Please use `isAllOnes()` instead.
1585 bool isAllOnesValue() const { return isAllOnes(); }
1586 bool isMaxSignedValue() const { return Value->isMaxValue(true); }
1587 bool isMinSignedValue() const { return Value->isMinValue(true); }
1588
1589 bool isOpaque() const { return ConstantSDNodeBits.IsOpaque; }
1590
1591 static bool classof(const SDNode *N) {
1592 return N->getOpcode() == ISD::Constant ||
1593 N->getOpcode() == ISD::TargetConstant;
1594 }
1595};
1596
1597uint64_t SDNode::getConstantOperandVal(unsigned Num) const {
1598 return cast<ConstantSDNode>(getOperand(Num))->getZExtValue();
1599}
1600
1601const APInt &SDNode::getConstantOperandAPInt(unsigned Num) const {
1602 return cast<ConstantSDNode>(getOperand(Num))->getAPIntValue();
1603}
1604
1605class ConstantFPSDNode : public SDNode {
1606 friend class SelectionDAG;
1607
1608 const ConstantFP *Value;
1609
1610 ConstantFPSDNode(bool isTarget, const ConstantFP *val, EVT VT)
1611 : SDNode(isTarget ? ISD::TargetConstantFP : ISD::ConstantFP, 0,
1612 DebugLoc(), getSDVTList(VT)),
1613 Value(val) {}
1614
1615public:
1616 const APFloat& getValueAPF() const { return Value->getValueAPF(); }
1617 const ConstantFP *getConstantFPValue() const { return Value; }
1618
1619 /// Return true if the value is positive or negative zero.
1620 bool isZero() const { return Value->isZero(); }
1621
1622 /// Return true if the value is a NaN.
1623 bool isNaN() const { return Value->isNaN(); }
1624
1625 /// Return true if the value is an infinity
1626 bool isInfinity() const { return Value->isInfinity(); }
1627
1628 /// Return true if the value is negative.
1629 bool isNegative() const { return Value->isNegative(); }
1630
1631 /// We don't rely on operator== working on double values, as
1632 /// it returns true for things that are clearly not equal, like -0.0 and 0.0.
1633 /// As such, this method can be used to do an exact bit-for-bit comparison of
1634 /// two floating point values.
1635
1636 /// We leave the version with the double argument here because it's just so
1637 /// convenient to write "2.0" and the like. Without this function we'd
1638 /// have to duplicate its logic everywhere it's called.
1639 bool isExactlyValue(double V) const {
1640 return Value->getValueAPF().isExactlyValue(V);
1641 }
1642 bool isExactlyValue(const APFloat& V) const;
1643
1644 static bool isValueValidForType(EVT VT, const APFloat& Val);
1645
1646 static bool classof(const SDNode *N) {
1647 return N->getOpcode() == ISD::ConstantFP ||
1648 N->getOpcode() == ISD::TargetConstantFP;
1649 }
1650};
1651
1652/// Returns true if \p V is a constant integer zero.
1653bool isNullConstant(SDValue V);
1654
1655/// Returns true if \p V is an FP constant with a value of positive zero.
1656bool isNullFPConstant(SDValue V);
1657
1658/// Returns true if \p V is an integer constant with all bits set.
1659bool isAllOnesConstant(SDValue V);
1660
1661/// Returns true if \p V is a constant integer one.
1662bool isOneConstant(SDValue V);
1663
1664/// Return the non-bitcasted source operand of \p V if it exists.
1665/// If \p V is not a bitcasted value, it is returned as-is.
1666SDValue peekThroughBitcasts(SDValue V);
1667
1668/// Return the non-bitcasted and one-use source operand of \p V if it exists.
1669/// If \p V is not a bitcasted one-use value, it is returned as-is.
1670SDValue peekThroughOneUseBitcasts(SDValue V);
1671
1672/// Return the non-extracted vector source operand of \p V if it exists.
1673/// If \p V is not an extracted subvector, it is returned as-is.
1674SDValue peekThroughExtractSubvectors(SDValue V);
1675
1676/// Returns true if \p V is a bitwise not operation. Assumes that an all ones
1677/// constant is canonicalized to be operand 1.
1678bool isBitwiseNot(SDValue V, bool AllowUndefs = false);
1679
1680/// Returns the SDNode if it is a constant splat BuildVector or constant int.
1681ConstantSDNode *isConstOrConstSplat(SDValue N, bool AllowUndefs = false,
1682 bool AllowTruncation = false);
1683
1684/// Returns the SDNode if it is a demanded constant splat BuildVector or
1685/// constant int.
1686ConstantSDNode *isConstOrConstSplat(SDValue N, const APInt &DemandedElts,
1687 bool AllowUndefs = false,
1688 bool AllowTruncation = false);
1689
1690/// Returns the SDNode if it is a constant splat BuildVector or constant float.
1691ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, bool AllowUndefs = false);
1692
1693/// Returns the SDNode if it is a demanded constant splat BuildVector or
1694/// constant float.
1695ConstantFPSDNode *isConstOrConstSplatFP(SDValue N, const APInt &DemandedElts,
1696 bool AllowUndefs = false);
1697
1698/// Return true if the value is a constant 0 integer or a splatted vector of
1699/// a constant 0 integer (with no undefs by default).
1700/// Build vector implicit truncation is not an issue for null values.
1701bool isNullOrNullSplat(SDValue V, bool AllowUndefs = false);
1702
1703/// Return true if the value is a constant 1 integer or a splatted vector of a
1704/// constant 1 integer (with no undefs).
1705/// Does not permit build vector implicit truncation.
1706bool isOneOrOneSplat(SDValue V, bool AllowUndefs = false);
1707
1708/// Return true if the value is a constant -1 integer or a splatted vector of a
1709/// constant -1 integer (with no undefs).
1710/// Does not permit build vector implicit truncation.
1711bool isAllOnesOrAllOnesSplat(SDValue V, bool AllowUndefs = false);
1712
1713/// Return true if \p V is either a integer or FP constant.
1714inline bool isIntOrFPConstant(SDValue V) {
1715 return isa<ConstantSDNode>(V) || isa<ConstantFPSDNode>(V);
1716}
1717
1718class GlobalAddressSDNode : public SDNode {
1719 friend class SelectionDAG;
1720
1721 const GlobalValue *TheGlobal;
1722 int64_t Offset;
1723 unsigned TargetFlags;
1724
1725 GlobalAddressSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL,
1726 const GlobalValue *GA, EVT VT, int64_t o,
1727 unsigned TF);
1728
1729public:
1730 const GlobalValue *getGlobal() const { return TheGlobal; }
1731 int64_t getOffset() const { return Offset; }
1732 unsigned getTargetFlags() const { return TargetFlags; }
1733 // Return the address space this GlobalAddress belongs to.
1734 unsigned getAddressSpace() const;
1735
1736 static bool classof(const SDNode *N) {
1737 return N->getOpcode() == ISD::GlobalAddress ||
1738 N->getOpcode() == ISD::TargetGlobalAddress ||
1739 N->getOpcode() == ISD::GlobalTLSAddress ||
1740 N->getOpcode() == ISD::TargetGlobalTLSAddress;
1741 }
1742};
1743
1744class FrameIndexSDNode : public SDNode {
1745 friend class SelectionDAG;
1746
1747 int FI;
1748
1749 FrameIndexSDNode(int fi, EVT VT, bool isTarg)
1750 : SDNode(isTarg ? ISD::TargetFrameIndex : ISD::FrameIndex,
1751 0, DebugLoc(), getSDVTList(VT)), FI(fi) {
1752 }
1753
1754public:
1755 int getIndex() const { return FI; }
1756
1757 static bool classof(const SDNode *N) {
1758 return N->getOpcode() == ISD::FrameIndex ||
1759 N->getOpcode() == ISD::TargetFrameIndex;
1760 }
1761};
1762
1763/// This SDNode is used for LIFETIME_START/LIFETIME_END values, which indicate
1764/// the offet and size that are started/ended in the underlying FrameIndex.
1765class LifetimeSDNode : public SDNode {
1766 friend class SelectionDAG;
1767 int64_t Size;
1768 int64_t Offset; // -1 if offset is unknown.
1769
1770 LifetimeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl,
1771 SDVTList VTs, int64_t Size, int64_t Offset)
1772 : SDNode(Opcode, Order, dl, VTs), Size(Size), Offset(Offset) {}
1773public:
1774 int64_t getFrameIndex() const {
1775 return cast<FrameIndexSDNode>(getOperand(1))->getIndex();
1776 }
1777
1778 bool hasOffset() const { return Offset >= 0; }
1779 int64_t getOffset() const {
1780 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1780, __extension__ __PRETTY_FUNCTION__))
;
1781 return Offset;
1782 }
1783 int64_t getSize() const {
1784 assert(hasOffset() && "offset is unknown")(static_cast <bool> (hasOffset() && "offset is unknown"
) ? void (0) : __assert_fail ("hasOffset() && \"offset is unknown\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1784, __extension__ __PRETTY_FUNCTION__))
;
1785 return Size;
1786 }
1787
1788 // Methods to support isa and dyn_cast
1789 static bool classof(const SDNode *N) {
1790 return N->getOpcode() == ISD::LIFETIME_START ||
1791 N->getOpcode() == ISD::LIFETIME_END;
1792 }
1793};
1794
1795/// This SDNode is used for PSEUDO_PROBE values, which are the function guid and
1796/// the index of the basic block being probed. A pseudo probe serves as a place
1797/// holder and will be removed at the end of compilation. It does not have any
1798/// operand because we do not want the instruction selection to deal with any.
1799class PseudoProbeSDNode : public SDNode {
1800 friend class SelectionDAG;
1801 uint64_t Guid;
1802 uint64_t Index;
1803 uint32_t Attributes;
1804
1805 PseudoProbeSDNode(unsigned Opcode, unsigned Order, const DebugLoc &Dl,
1806 SDVTList VTs, uint64_t Guid, uint64_t Index, uint32_t Attr)
1807 : SDNode(Opcode, Order, Dl, VTs), Guid(Guid), Index(Index),
1808 Attributes(Attr) {}
1809
1810public:
1811 uint64_t getGuid() const { return Guid; }
1812 uint64_t getIndex() const { return Index; }
1813 uint32_t getAttributes() const { return Attributes; }
1814
1815 // Methods to support isa and dyn_cast
1816 static bool classof(const SDNode *N) {
1817 return N->getOpcode() == ISD::PSEUDO_PROBE;
1818 }
1819};
1820
1821class JumpTableSDNode : public SDNode {
1822 friend class SelectionDAG;
1823
1824 int JTI;
1825 unsigned TargetFlags;
1826
1827 JumpTableSDNode(int jti, EVT VT, bool isTarg, unsigned TF)
1828 : SDNode(isTarg ? ISD::TargetJumpTable : ISD::JumpTable,
1829 0, DebugLoc(), getSDVTList(VT)), JTI(jti), TargetFlags(TF) {
1830 }
1831
1832public:
1833 int getIndex() const { return JTI; }
1834 unsigned getTargetFlags() const { return TargetFlags; }
1835
1836 static bool classof(const SDNode *N) {
1837 return N->getOpcode() == ISD::JumpTable ||
1838 N->getOpcode() == ISD::TargetJumpTable;
1839 }
1840};
1841
1842class ConstantPoolSDNode : public SDNode {
1843 friend class SelectionDAG;
1844
1845 union {
1846 const Constant *ConstVal;
1847 MachineConstantPoolValue *MachineCPVal;
1848 } Val;
1849 int Offset; // It's a MachineConstantPoolValue if top bit is set.
1850 Align Alignment; // Minimum alignment requirement of CP.
1851 unsigned TargetFlags;
1852
1853 ConstantPoolSDNode(bool isTarget, const Constant *c, EVT VT, int o,
1854 Align Alignment, unsigned TF)
1855 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1856 DebugLoc(), getSDVTList(VT)),
1857 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1858 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1858, __extension__ __PRETTY_FUNCTION__))
;
1859 Val.ConstVal = c;
1860 }
1861
1862 ConstantPoolSDNode(bool isTarget, MachineConstantPoolValue *v, EVT VT, int o,
1863 Align Alignment, unsigned TF)
1864 : SDNode(isTarget ? ISD::TargetConstantPool : ISD::ConstantPool, 0,
1865 DebugLoc(), getSDVTList(VT)),
1866 Offset(o), Alignment(Alignment), TargetFlags(TF) {
1867 assert(Offset >= 0 && "Offset is too large")(static_cast <bool> (Offset >= 0 && "Offset is too large"
) ? void (0) : __assert_fail ("Offset >= 0 && \"Offset is too large\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1867, __extension__ __PRETTY_FUNCTION__))
;
1868 Val.MachineCPVal = v;
1869 Offset |= 1 << (sizeof(unsigned)*CHAR_BIT8-1);
1870 }
1871
1872public:
1873 bool isMachineConstantPoolEntry() const {
1874 return Offset < 0;
1875 }
1876
1877 const Constant *getConstVal() const {
1878 assert(!isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (!isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("!isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1878, __extension__ __PRETTY_FUNCTION__))
;
1879 return Val.ConstVal;
1880 }
1881
1882 MachineConstantPoolValue *getMachineCPVal() const {
1883 assert(isMachineConstantPoolEntry() && "Wrong constantpool type")(static_cast <bool> (isMachineConstantPoolEntry() &&
"Wrong constantpool type") ? void (0) : __assert_fail ("isMachineConstantPoolEntry() && \"Wrong constantpool type\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 1883, __extension__ __PRETTY_FUNCTION__))
;
1884 return Val.MachineCPVal;
1885 }
1886
1887 int getOffset() const {
1888 return Offset & ~(1 << (sizeof(unsigned)*CHAR_BIT8-1));
1889 }
1890
1891 // Return the alignment of this constant pool object, which is either 0 (for
1892 // default alignment) or the desired value.
1893 Align getAlign() const { return Alignment; }
1894 unsigned getTargetFlags() const { return TargetFlags; }
1895
1896 Type *getType() const;
1897
1898 static bool classof(const SDNode *N) {
1899 return N->getOpcode() == ISD::ConstantPool ||
1900 N->getOpcode() == ISD::TargetConstantPool;
1901 }
1902};
1903
1904/// Completely target-dependent object reference.
1905class TargetIndexSDNode : public SDNode {
1906 friend class SelectionDAG;
1907
1908 unsigned TargetFlags;
1909 int Index;
1910 int64_t Offset;
1911
1912public:
1913 TargetIndexSDNode(int Idx, EVT VT, int64_t Ofs, unsigned TF)
1914 : SDNode(ISD::TargetIndex, 0, DebugLoc(), getSDVTList(VT)),
1915 TargetFlags(TF), Index(Idx), Offset(Ofs) {}
1916
1917 unsigned getTargetFlags() const { return TargetFlags; }
1918 int getIndex() const { return Index; }
1919 int64_t getOffset() const { return Offset; }
1920
1921 static bool classof(const SDNode *N) {
1922 return N->getOpcode() == ISD::TargetIndex;
1923 }
1924};
1925
1926class BasicBlockSDNode : public SDNode {
1927 friend class SelectionDAG;
1928
1929 MachineBasicBlock *MBB;
1930
1931 /// Debug info is meaningful and potentially useful here, but we create
1932 /// blocks out of order when they're jumped to, which makes it a bit
1933 /// harder. Let's see if we need it first.
1934 explicit BasicBlockSDNode(MachineBasicBlock *mbb)
1935 : SDNode(ISD::BasicBlock, 0, DebugLoc(), getSDVTList(MVT::Other)), MBB(mbb)
1936 {}
1937
1938public:
1939 MachineBasicBlock *getBasicBlock() const { return MBB; }
1940
1941 static bool classof(const SDNode *N) {
1942 return N->getOpcode() == ISD::BasicBlock;
1943 }
1944};
1945
1946/// A "pseudo-class" with methods for operating on BUILD_VECTORs.
1947class BuildVectorSDNode : public SDNode {
1948public:
1949 // These are constructed as SDNodes and then cast to BuildVectorSDNodes.
1950 explicit BuildVectorSDNode() = delete;
1951
1952 /// Check if this is a constant splat, and if so, find the
1953 /// smallest element size that splats the vector. If MinSplatBits is
1954 /// nonzero, the element size must be at least that large. Note that the
1955 /// splat element may be the entire vector (i.e., a one element vector).
1956 /// Returns the splat element value in SplatValue. Any undefined bits in
1957 /// that value are zero, and the corresponding bits in the SplatUndef mask
1958 /// are set. The SplatBitSize value is set to the splat element size in
1959 /// bits. HasAnyUndefs is set to true if any bits in the vector are
1960 /// undefined. isBigEndian describes the endianness of the target.
1961 bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef,
1962 unsigned &SplatBitSize, bool &HasAnyUndefs,
1963 unsigned MinSplatBits = 0,
1964 bool isBigEndian = false) const;
1965
1966 /// Returns the demanded splatted value or a null value if this is not a
1967 /// splat.
1968 ///
1969 /// The DemandedElts mask indicates the elements that must be in the splat.
1970 /// If passed a non-null UndefElements bitvector, it will resize it to match
1971 /// the vector width and set the bits where elements are undef.
1972 SDValue getSplatValue(const APInt &DemandedElts,
1973 BitVector *UndefElements = nullptr) const;
1974
1975 /// Returns the splatted value or a null value if this is not a splat.
1976 ///
1977 /// If passed a non-null UndefElements bitvector, it will resize it to match
1978 /// the vector width and set the bits where elements are undef.
1979 SDValue getSplatValue(BitVector *UndefElements = nullptr) const;
1980
1981 /// Find the shortest repeating sequence of values in the build vector.
1982 ///
1983 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1984 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
1985 ///
1986 /// Currently this must be a power-of-2 build vector.
1987 /// The DemandedElts mask indicates the elements that must be present,
1988 /// undemanded elements in Sequence may be null (SDValue()). If passed a
1989 /// non-null UndefElements bitvector, it will resize it to match the original
1990 /// vector width and set the bits where elements are undef. If result is
1991 /// false, Sequence will be empty.
1992 bool getRepeatedSequence(const APInt &DemandedElts,
1993 SmallVectorImpl<SDValue> &Sequence,
1994 BitVector *UndefElements = nullptr) const;
1995
1996 /// Find the shortest repeating sequence of values in the build vector.
1997 ///
1998 /// e.g. { u, X, u, X, u, u, X, u } -> { X }
1999 /// { X, Y, u, Y, u, u, X, u } -> { X, Y }
2000 ///
2001 /// Currently this must be a power-of-2 build vector.
2002 /// If passed a non-null UndefElements bitvector, it will resize it to match
2003 /// the original vector width and set the bits where elements are undef.
2004 /// If result is false, Sequence will be empty.
2005 bool getRepeatedSequence(SmallVectorImpl<SDValue> &Sequence,
2006 BitVector *UndefElements = nullptr) const;
2007
2008 /// Returns the demanded splatted constant or null if this is not a constant
2009 /// splat.
2010 ///
2011 /// The DemandedElts mask indicates the elements that must be in the splat.
2012 /// If passed a non-null UndefElements bitvector, it will resize it to match
2013 /// the vector width and set the bits where elements are undef.
2014 ConstantSDNode *
2015 getConstantSplatNode(const APInt &DemandedElts,
2016 BitVector *UndefElements = nullptr) const;
2017
2018 /// Returns the splatted constant or null if this is not a constant
2019 /// splat.
2020 ///
2021 /// If passed a non-null UndefElements bitvector, it will resize it to match
2022 /// the vector width and set the bits where elements are undef.
2023 ConstantSDNode *
2024 getConstantSplatNode(BitVector *UndefElements = nullptr) const;
2025
2026 /// Returns the demanded splatted constant FP or null if this is not a
2027 /// constant FP splat.
2028 ///
2029 /// The DemandedElts mask indicates the elements that must be in the splat.
2030 /// If passed a non-null UndefElements bitvector, it will resize it to match
2031 /// the vector width and set the bits where elements are undef.
2032 ConstantFPSDNode *
2033 getConstantFPSplatNode(const APInt &DemandedElts,
2034 BitVector *UndefElements = nullptr) const;
2035
2036 /// Returns the splatted constant FP or null if this is not a constant
2037 /// FP splat.
2038 ///
2039 /// If passed a non-null UndefElements bitvector, it will resize it to match
2040 /// the vector width and set the bits where elements are undef.
2041 ConstantFPSDNode *
2042 getConstantFPSplatNode(BitVector *UndefElements = nullptr) const;
2043
2044 /// If this is a constant FP splat and the splatted constant FP is an
2045 /// exact power or 2, return the log base 2 integer value. Otherwise,
2046 /// return -1.
2047 ///
2048 /// The BitWidth specifies the necessary bit precision.
2049 int32_t getConstantFPSplatPow2ToLog2Int(BitVector *UndefElements,
2050 uint32_t BitWidth) const;
2051
2052 bool isConstant() const;
2053
2054 static bool classof(const SDNode *N) {
2055 return N->getOpcode() == ISD::BUILD_VECTOR;
2056 }
2057};
2058
2059/// An SDNode that holds an arbitrary LLVM IR Value. This is
2060/// used when the SelectionDAG needs to make a simple reference to something
2061/// in the LLVM IR representation.
2062///
2063class SrcValueSDNode : public SDNode {
2064 friend class SelectionDAG;
2065
2066 const Value *V;
2067
2068 /// Create a SrcValue for a general value.
2069 explicit SrcValueSDNode(const Value *v)
2070 : SDNode(ISD::SRCVALUE, 0, DebugLoc(), getSDVTList(MVT::Other)), V(v) {}
2071
2072public:
2073 /// Return the contained Value.
2074 const Value *getValue() const { return V; }
2075
2076 static bool classof(const SDNode *N) {
2077 return N->getOpcode() == ISD::SRCVALUE;
2078 }
2079};
2080
2081class MDNodeSDNode : public SDNode {
2082 friend class SelectionDAG;
2083
2084 const MDNode *MD;
2085
2086 explicit MDNodeSDNode(const MDNode *md)
2087 : SDNode(ISD::MDNODE_SDNODE, 0, DebugLoc(), getSDVTList(MVT::Other)), MD(md)
2088 {}
2089
2090public:
2091 const MDNode *getMD() const { return MD; }
2092
2093 static bool classof(const SDNode *N) {
2094 return N->getOpcode() == ISD::MDNODE_SDNODE;
2095 }
2096};
2097
2098class RegisterSDNode : public SDNode {
2099 friend class SelectionDAG;
2100
2101 Register Reg;
2102
2103 RegisterSDNode(Register reg, EVT VT)
2104 : SDNode(ISD::Register, 0, DebugLoc(), getSDVTList(VT)), Reg(reg) {}
2105
2106public:
2107 Register getReg() const { return Reg; }
2108
2109 static bool classof(const SDNode *N) {
2110 return N->getOpcode() == ISD::Register;
2111 }
2112};
2113
2114class RegisterMaskSDNode : public SDNode {
2115 friend class SelectionDAG;
2116
2117 // The memory for RegMask is not owned by the node.
2118 const uint32_t *RegMask;
2119
2120 RegisterMaskSDNode(const uint32_t *mask)
2121 : SDNode(ISD::RegisterMask, 0, DebugLoc(), getSDVTList(MVT::Untyped)),
2122 RegMask(mask) {}
2123
2124public:
2125 const uint32_t *getRegMask() const { return RegMask; }
2126
2127 static bool classof(const SDNode *N) {
2128 return N->getOpcode() == ISD::RegisterMask;
2129 }
2130};
2131
2132class BlockAddressSDNode : public SDNode {
2133 friend class SelectionDAG;
2134
2135 const BlockAddress *BA;
2136 int64_t Offset;
2137 unsigned TargetFlags;
2138
2139 BlockAddressSDNode(unsigned NodeTy, EVT VT, const BlockAddress *ba,
2140 int64_t o, unsigned Flags)
2141 : SDNode(NodeTy, 0, DebugLoc(), getSDVTList(VT)),
2142 BA(ba), Offset(o), TargetFlags(Flags) {}
2143
2144public:
2145 const BlockAddress *getBlockAddress() const { return BA; }
2146 int64_t getOffset() const { return Offset; }
2147 unsigned getTargetFlags() const { return TargetFlags; }
2148
2149 static bool classof(const SDNode *N) {
2150 return N->getOpcode() == ISD::BlockAddress ||
2151 N->getOpcode() == ISD::TargetBlockAddress;
2152 }
2153};
2154
2155class LabelSDNode : public SDNode {
2156 friend class SelectionDAG;
2157
2158 MCSymbol *Label;
2159
2160 LabelSDNode(unsigned Opcode, unsigned Order, const DebugLoc &dl, MCSymbol *L)
2161 : SDNode(Opcode, Order, dl, getSDVTList(MVT::Other)), Label(L) {
2162 assert(LabelSDNode::classof(this) && "not a label opcode")(static_cast <bool> (LabelSDNode::classof(this) &&
"not a label opcode") ? void (0) : __assert_fail ("LabelSDNode::classof(this) && \"not a label opcode\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2162, __extension__ __PRETTY_FUNCTION__))
;
2163 }
2164
2165public:
2166 MCSymbol *getLabel() const { return Label; }
2167
2168 static bool classof(const SDNode *N) {
2169 return N->getOpcode() == ISD::EH_LABEL ||
2170 N->getOpcode() == ISD::ANNOTATION_LABEL;
2171 }
2172};
2173
2174class ExternalSymbolSDNode : public SDNode {
2175 friend class SelectionDAG;
2176
2177 const char *Symbol;
2178 unsigned TargetFlags;
2179
2180 ExternalSymbolSDNode(bool isTarget, const char *Sym, unsigned TF, EVT VT)
2181 : SDNode(isTarget ? ISD::TargetExternalSymbol : ISD::ExternalSymbol, 0,
2182 DebugLoc(), getSDVTList(VT)),
2183 Symbol(Sym), TargetFlags(TF) {}
2184
2185public:
2186 const char *getSymbol() const { return Symbol; }
2187 unsigned getTargetFlags() const { return TargetFlags; }
2188
2189 static bool classof(const SDNode *N) {
2190 return N->getOpcode() == ISD::ExternalSymbol ||
2191 N->getOpcode() == ISD::TargetExternalSymbol;
2192 }
2193};
2194
2195class MCSymbolSDNode : public SDNode {
2196 friend class SelectionDAG;
2197
2198 MCSymbol *Symbol;
2199
2200 MCSymbolSDNode(MCSymbol *Symbol, EVT VT)
2201 : SDNode(ISD::MCSymbol, 0, DebugLoc(), getSDVTList(VT)), Symbol(Symbol) {}
2202
2203public:
2204 MCSymbol *getMCSymbol() const { return Symbol; }
2205
2206 static bool classof(const SDNode *N) {
2207 return N->getOpcode() == ISD::MCSymbol;
2208 }
2209};
2210
2211class CondCodeSDNode : public SDNode {
2212 friend class SelectionDAG;
2213
2214 ISD::CondCode Condition;
2215
2216 explicit CondCodeSDNode(ISD::CondCode Cond)
2217 : SDNode(ISD::CONDCODE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2218 Condition(Cond) {}
2219
2220public:
2221 ISD::CondCode get() const { return Condition; }
2222
2223 static bool classof(const SDNode *N) {
2224 return N->getOpcode() == ISD::CONDCODE;
2225 }
2226};
2227
2228/// This class is used to represent EVT's, which are used
2229/// to parameterize some operations.
2230class VTSDNode : public SDNode {
2231 friend class SelectionDAG;
2232
2233 EVT ValueType;
2234
2235 explicit VTSDNode(EVT VT)
2236 : SDNode(ISD::VALUETYPE, 0, DebugLoc(), getSDVTList(MVT::Other)),
2237 ValueType(VT) {}
2238
2239public:
2240 EVT getVT() const { return ValueType; }
2241
2242 static bool classof(const SDNode *N) {
2243 return N->getOpcode() == ISD::VALUETYPE;
2244 }
2245};
2246
2247/// Base class for LoadSDNode and StoreSDNode
2248class LSBaseSDNode : public MemSDNode {
2249public:
2250 LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2251 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2252 MachineMemOperand *MMO)
2253 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2254 LSBaseSDNodeBits.AddressingMode = AM;
2255 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2255, __extension__ __PRETTY_FUNCTION__))
;
2256 }
2257
2258 const SDValue &getOffset() const {
2259 return getOperand(getOpcode() == ISD::LOAD ? 2 : 3);
2260 }
2261
2262 /// Return the addressing mode for this load or store:
2263 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2264 ISD::MemIndexedMode getAddressingMode() const {
2265 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2266 }
2267
2268 /// Return true if this is a pre/post inc/dec load/store.
2269 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2270
2271 /// Return true if this is NOT a pre/post inc/dec load/store.
2272 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2273
2274 static bool classof(const SDNode *N) {
2275 return N->getOpcode() == ISD::LOAD ||
2276 N->getOpcode() == ISD::STORE;
2277 }
2278};
2279
2280/// This class is used to represent ISD::LOAD nodes.
2281class LoadSDNode : public LSBaseSDNode {
2282 friend class SelectionDAG;
2283
2284 LoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2285 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT,
2286 MachineMemOperand *MMO)
2287 : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2288 LoadSDNodeBits.ExtTy = ETy;
2289 assert(readMem() && "Load MachineMemOperand is not a load!")(static_cast <bool> (readMem() && "Load MachineMemOperand is not a load!"
) ? void (0) : __assert_fail ("readMem() && \"Load MachineMemOperand is not a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2289, __extension__ __PRETTY_FUNCTION__))
;
2290 assert(!writeMem() && "Load MachineMemOperand is a store!")(static_cast <bool> (!writeMem() && "Load MachineMemOperand is a store!"
) ? void (0) : __assert_fail ("!writeMem() && \"Load MachineMemOperand is a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2290, __extension__ __PRETTY_FUNCTION__))
;
2291 }
2292
2293public:
2294 /// Return whether this is a plain node,
2295 /// or one of the varieties of value-extending loads.
2296 ISD::LoadExtType getExtensionType() const {
2297 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2298 }
2299
2300 const SDValue &getBasePtr() const { return getOperand(1); }
2301 const SDValue &getOffset() const { return getOperand(2); }
2302
2303 static bool classof(const SDNode *N) {
2304 return N->getOpcode() == ISD::LOAD;
2305 }
2306};
2307
2308/// This class is used to represent ISD::STORE nodes.
2309class StoreSDNode : public LSBaseSDNode {
2310 friend class SelectionDAG;
2311
2312 StoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2313 ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT,
2314 MachineMemOperand *MMO)
2315 : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) {
2316 StoreSDNodeBits.IsTruncating = isTrunc;
2317 assert(!readMem() && "Store MachineMemOperand is a load!")(static_cast <bool> (!readMem() && "Store MachineMemOperand is a load!"
) ? void (0) : __assert_fail ("!readMem() && \"Store MachineMemOperand is a load!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2317, __extension__ __PRETTY_FUNCTION__))
;
2318 assert(writeMem() && "Store MachineMemOperand is not a store!")(static_cast <bool> (writeMem() && "Store MachineMemOperand is not a store!"
) ? void (0) : __assert_fail ("writeMem() && \"Store MachineMemOperand is not a store!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2318, __extension__ __PRETTY_FUNCTION__))
;
2319 }
2320
2321public:
2322 /// Return true if the op does a truncation before store.
2323 /// For integers this is the same as doing a TRUNCATE and storing the result.
2324 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2325 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2326 void setTruncatingStore(bool Truncating) {
2327 StoreSDNodeBits.IsTruncating = Truncating;
2328 }
2329
2330 const SDValue &getValue() const { return getOperand(1); }
2331 const SDValue &getBasePtr() const { return getOperand(2); }
2332 const SDValue &getOffset() const { return getOperand(3); }
2333
2334 static bool classof(const SDNode *N) {
2335 return N->getOpcode() == ISD::STORE;
2336 }
2337};
2338
2339/// This base class is used to represent VP_LOAD and VP_STORE nodes
2340class VPLoadStoreSDNode : public MemSDNode {
2341public:
2342 friend class SelectionDAG;
2343
2344 VPLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, const DebugLoc &dl,
2345 SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT,
2346 MachineMemOperand *MMO)
2347 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2348 LSBaseSDNodeBits.AddressingMode = AM;
2349 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2349, __extension__ __PRETTY_FUNCTION__))
;
2350 }
2351
2352 // VPLoadSDNode (Chain, Ptr, Offset, Mask, EVL)
2353 // VPStoreSDNode (Chain, Data, Ptr, Offset, Mask, EVL)
2354 // Mask is a vector of i1 elements;
2355 // the type of EVL is TLI.getVPExplicitVectorLengthTy().
2356 const SDValue &getOffset() const {
2357 return getOperand(getOpcode() == ISD::VP_LOAD ? 2 : 3);
2358 }
2359 const SDValue &getBasePtr() const {
2360 return getOperand(getOpcode() == ISD::VP_LOAD ? 1 : 2);
2361 }
2362 const SDValue &getMask() const {
2363 return getOperand(getOpcode() == ISD::VP_LOAD ? 3 : 4);
2364 }
2365 const SDValue &getVectorLength() const {
2366 return getOperand(getOpcode() == ISD::VP_LOAD ? 4 : 5);
2367 }
2368
2369 /// Return the addressing mode for this load or store:
2370 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2371 ISD::MemIndexedMode getAddressingMode() const {
2372 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2373 }
2374
2375 /// Return true if this is a pre/post inc/dec load/store.
2376 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2377
2378 /// Return true if this is NOT a pre/post inc/dec load/store.
2379 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2380
2381 static bool classof(const SDNode *N) {
2382 return N->getOpcode() == ISD::VP_LOAD || N->getOpcode() == ISD::VP_STORE;
2383 }
2384};
2385
2386/// This class is used to represent a VP_LOAD node
2387class VPLoadSDNode : public VPLoadStoreSDNode {
2388public:
2389 friend class SelectionDAG;
2390
2391 VPLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2392 ISD::MemIndexedMode AM, ISD::LoadExtType ETy, bool isExpanding,
2393 EVT MemVT, MachineMemOperand *MMO)
2394 : VPLoadStoreSDNode(ISD::VP_LOAD, Order, dl, VTs, AM, MemVT, MMO) {
2395 LoadSDNodeBits.ExtTy = ETy;
2396 LoadSDNodeBits.IsExpanding = isExpanding;
2397 }
2398
2399 ISD::LoadExtType getExtensionType() const {
2400 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2401 }
2402
2403 const SDValue &getBasePtr() const { return getOperand(1); }
2404 const SDValue &getOffset() const { return getOperand(2); }
2405 const SDValue &getMask() const { return getOperand(3); }
2406 const SDValue &getVectorLength() const { return getOperand(4); }
2407
2408 static bool classof(const SDNode *N) {
2409 return N->getOpcode() == ISD::VP_LOAD;
2410 }
2411 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2412};
2413
2414/// This class is used to represent a VP_STORE node
2415class VPStoreSDNode : public VPLoadStoreSDNode {
2416public:
2417 friend class SelectionDAG;
2418
2419 VPStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2420 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2421 EVT MemVT, MachineMemOperand *MMO)
2422 : VPLoadStoreSDNode(ISD::VP_STORE, Order, dl, VTs, AM, MemVT, MMO) {
2423 StoreSDNodeBits.IsTruncating = isTrunc;
2424 StoreSDNodeBits.IsCompressing = isCompressing;
2425 }
2426
2427 /// Return true if this is a truncating store.
2428 /// For integers this is the same as doing a TRUNCATE and storing the result.
2429 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2430 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2431
2432 /// Returns true if the op does a compression to the vector before storing.
2433 /// The node contiguously stores the active elements (integers or floats)
2434 /// in src (those with their respective bit set in writemask k) to unaligned
2435 /// memory at base_addr.
2436 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2437
2438 const SDValue &getValue() const { return getOperand(1); }
2439 const SDValue &getBasePtr() const { return getOperand(2); }
2440 const SDValue &getOffset() const { return getOperand(3); }
2441 const SDValue &getMask() const { return getOperand(4); }
2442 const SDValue &getVectorLength() const { return getOperand(5); }
2443
2444 static bool classof(const SDNode *N) {
2445 return N->getOpcode() == ISD::VP_STORE;
2446 }
2447};
2448
2449/// This base class is used to represent MLOAD and MSTORE nodes
2450class MaskedLoadStoreSDNode : public MemSDNode {
2451public:
2452 friend class SelectionDAG;
2453
2454 MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order,
2455 const DebugLoc &dl, SDVTList VTs,
2456 ISD::MemIndexedMode AM, EVT MemVT,
2457 MachineMemOperand *MMO)
2458 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2459 LSBaseSDNodeBits.AddressingMode = AM;
2460 assert(getAddressingMode() == AM && "Value truncated")(static_cast <bool> (getAddressingMode() == AM &&
"Value truncated") ? void (0) : __assert_fail ("getAddressingMode() == AM && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2460, __extension__ __PRETTY_FUNCTION__))
;
2461 }
2462
2463 // MaskedLoadSDNode (Chain, ptr, offset, mask, passthru)
2464 // MaskedStoreSDNode (Chain, data, ptr, offset, mask)
2465 // Mask is a vector of i1 elements
2466 const SDValue &getOffset() const {
2467 return getOperand(getOpcode() == ISD::MLOAD ? 2 : 3);
2468 }
2469 const SDValue &getMask() const {
2470 return getOperand(getOpcode() == ISD::MLOAD ? 3 : 4);
2471 }
2472
2473 /// Return the addressing mode for this load or store:
2474 /// unindexed, pre-inc, pre-dec, post-inc, or post-dec.
2475 ISD::MemIndexedMode getAddressingMode() const {
2476 return static_cast<ISD::MemIndexedMode>(LSBaseSDNodeBits.AddressingMode);
2477 }
2478
2479 /// Return true if this is a pre/post inc/dec load/store.
2480 bool isIndexed() const { return getAddressingMode() != ISD::UNINDEXED; }
2481
2482 /// Return true if this is NOT a pre/post inc/dec load/store.
2483 bool isUnindexed() const { return getAddressingMode() == ISD::UNINDEXED; }
2484
2485 static bool classof(const SDNode *N) {
2486 return N->getOpcode() == ISD::MLOAD ||
2487 N->getOpcode() == ISD::MSTORE;
2488 }
2489};
2490
2491/// This class is used to represent an MLOAD node
2492class MaskedLoadSDNode : public MaskedLoadStoreSDNode {
2493public:
2494 friend class SelectionDAG;
2495
2496 MaskedLoadSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2497 ISD::MemIndexedMode AM, ISD::LoadExtType ETy,
2498 bool IsExpanding, EVT MemVT, MachineMemOperand *MMO)
2499 : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, AM, MemVT, MMO) {
2500 LoadSDNodeBits.ExtTy = ETy;
2501 LoadSDNodeBits.IsExpanding = IsExpanding;
2502 }
2503
2504 ISD::LoadExtType getExtensionType() const {
2505 return static_cast<ISD::LoadExtType>(LoadSDNodeBits.ExtTy);
2506 }
2507
2508 const SDValue &getBasePtr() const { return getOperand(1); }
2509 const SDValue &getOffset() const { return getOperand(2); }
2510 const SDValue &getMask() const { return getOperand(3); }
2511 const SDValue &getPassThru() const { return getOperand(4); }
2512
2513 static bool classof(const SDNode *N) {
2514 return N->getOpcode() == ISD::MLOAD;
2515 }
2516
2517 bool isExpandingLoad() const { return LoadSDNodeBits.IsExpanding; }
2518};
2519
2520/// This class is used to represent an MSTORE node
2521class MaskedStoreSDNode : public MaskedLoadStoreSDNode {
2522public:
2523 friend class SelectionDAG;
2524
2525 MaskedStoreSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2526 ISD::MemIndexedMode AM, bool isTrunc, bool isCompressing,
2527 EVT MemVT, MachineMemOperand *MMO)
2528 : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, AM, MemVT, MMO) {
2529 StoreSDNodeBits.IsTruncating = isTrunc;
2530 StoreSDNodeBits.IsCompressing = isCompressing;
2531 }
2532
2533 /// Return true if the op does a truncation before store.
2534 /// For integers this is the same as doing a TRUNCATE and storing the result.
2535 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2536 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2537
2538 /// Returns true if the op does a compression to the vector before storing.
2539 /// The node contiguously stores the active elements (integers or floats)
2540 /// in src (those with their respective bit set in writemask k) to unaligned
2541 /// memory at base_addr.
2542 bool isCompressingStore() const { return StoreSDNodeBits.IsCompressing; }
2543
2544 const SDValue &getValue() const { return getOperand(1); }
2545 const SDValue &getBasePtr() const { return getOperand(2); }
2546 const SDValue &getOffset() const { return getOperand(3); }
2547 const SDValue &getMask() const { return getOperand(4); }
2548
2549 static bool classof(const SDNode *N) {
2550 return N->getOpcode() == ISD::MSTORE;
2551 }
2552};
2553
2554/// This is a base class used to represent
2555/// VP_GATHER and VP_SCATTER nodes
2556///
2557class VPGatherScatterSDNode : public MemSDNode {
2558public:
2559 friend class SelectionDAG;
2560
2561 VPGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2562 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2563 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2564 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2565 LSBaseSDNodeBits.AddressingMode = IndexType;
2566 assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
"Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2566, __extension__ __PRETTY_FUNCTION__))
;
2567 }
2568
2569 /// How is Index applied to BasePtr when computing addresses.
2570 ISD::MemIndexType getIndexType() const {
2571 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2572 }
2573 bool isIndexScaled() const {
2574 return (getIndexType() == ISD::SIGNED_SCALED) ||
2575 (getIndexType() == ISD::UNSIGNED_SCALED);
2576 }
2577 bool isIndexSigned() const {
2578 return (getIndexType() == ISD::SIGNED_SCALED) ||
2579 (getIndexType() == ISD::SIGNED_UNSCALED);
2580 }
2581
2582 // In the both nodes address is Op1, mask is Op2:
2583 // VPGatherSDNode (Chain, base, index, scale, mask, vlen)
2584 // VPScatterSDNode (Chain, value, base, index, scale, mask, vlen)
2585 // Mask is a vector of i1 elements
2586 const SDValue &getBasePtr() const {
2587 return getOperand((getOpcode() == ISD::VP_GATHER) ? 1 : 2);
2588 }
2589 const SDValue &getIndex() const {
2590 return getOperand((getOpcode() == ISD::VP_GATHER) ? 2 : 3);
2591 }
2592 const SDValue &getScale() const {
2593 return getOperand((getOpcode() == ISD::VP_GATHER) ? 3 : 4);
2594 }
2595 const SDValue &getMask() const {
2596 return getOperand((getOpcode() == ISD::VP_GATHER) ? 4 : 5);
2597 }
2598 const SDValue &getVectorLength() const {
2599 return getOperand((getOpcode() == ISD::VP_GATHER) ? 5 : 6);
2600 }
2601
2602 static bool classof(const SDNode *N) {
2603 return N->getOpcode() == ISD::VP_GATHER ||
2604 N->getOpcode() == ISD::VP_SCATTER;
2605 }
2606};
2607
2608/// This class is used to represent an VP_GATHER node
2609///
2610class VPGatherSDNode : public VPGatherScatterSDNode {
2611public:
2612 friend class SelectionDAG;
2613
2614 VPGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2615 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2616 : VPGatherScatterSDNode(ISD::VP_GATHER, Order, dl, VTs, MemVT, MMO,
2617 IndexType) {}
2618
2619 static bool classof(const SDNode *N) {
2620 return N->getOpcode() == ISD::VP_GATHER;
2621 }
2622};
2623
2624/// This class is used to represent an VP_SCATTER node
2625///
2626class VPScatterSDNode : public VPGatherScatterSDNode {
2627public:
2628 friend class SelectionDAG;
2629
2630 VPScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2631 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2632 : VPGatherScatterSDNode(ISD::VP_SCATTER, Order, dl, VTs, MemVT, MMO,
2633 IndexType) {}
2634
2635 const SDValue &getValue() const { return getOperand(1); }
2636
2637 static bool classof(const SDNode *N) {
2638 return N->getOpcode() == ISD::VP_SCATTER;
2639 }
2640};
2641
2642/// This is a base class used to represent
2643/// MGATHER and MSCATTER nodes
2644///
2645class MaskedGatherScatterSDNode : public MemSDNode {
2646public:
2647 friend class SelectionDAG;
2648
2649 MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order,
2650 const DebugLoc &dl, SDVTList VTs, EVT MemVT,
2651 MachineMemOperand *MMO, ISD::MemIndexType IndexType)
2652 : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {
2653 LSBaseSDNodeBits.AddressingMode = IndexType;
2654 assert(getIndexType() == IndexType && "Value truncated")(static_cast <bool> (getIndexType() == IndexType &&
"Value truncated") ? void (0) : __assert_fail ("getIndexType() == IndexType && \"Value truncated\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2654, __extension__ __PRETTY_FUNCTION__))
;
2655 }
2656
2657 /// How is Index applied to BasePtr when computing addresses.
2658 ISD::MemIndexType getIndexType() const {
2659 return static_cast<ISD::MemIndexType>(LSBaseSDNodeBits.AddressingMode);
2660 }
2661 void setIndexType(ISD::MemIndexType IndexType) {
2662 LSBaseSDNodeBits.AddressingMode = IndexType;
2663 }
2664 bool isIndexScaled() const {
2665 return (getIndexType() == ISD::SIGNED_SCALED) ||
2666 (getIndexType() == ISD::UNSIGNED_SCALED);
2667 }
2668 bool isIndexSigned() const {
2669 return (getIndexType() == ISD::SIGNED_SCALED) ||
2670 (getIndexType() == ISD::SIGNED_UNSCALED);
2671 }
2672
2673 // In the both nodes address is Op1, mask is Op2:
2674 // MaskedGatherSDNode (Chain, passthru, mask, base, index, scale)
2675 // MaskedScatterSDNode (Chain, value, mask, base, index, scale)
2676 // Mask is a vector of i1 elements
2677 const SDValue &getBasePtr() const { return getOperand(3); }
2678 const SDValue &getIndex() const { return getOperand(4); }
2679 const SDValue &getMask() const { return getOperand(2); }
2680 const SDValue &getScale() const { return getOperand(5); }
2681
2682 static bool classof(const SDNode *N) {
2683 return N->getOpcode() == ISD::MGATHER ||
2684 N->getOpcode() == ISD::MSCATTER;
2685 }
2686};
2687
2688/// This class is used to represent an MGATHER node
2689///
2690class MaskedGatherSDNode : public MaskedGatherScatterSDNode {
2691public:
2692 friend class SelectionDAG;
2693
2694 MaskedGatherSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2695 EVT MemVT, MachineMemOperand *MMO,
2696 ISD::MemIndexType IndexType, ISD::LoadExtType ETy)
2697 : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO,
2698 IndexType) {
2699 LoadSDNodeBits.ExtTy = ETy;
2700 }
2701
2702 const SDValue &getPassThru() const { return getOperand(1); }
2703
2704 ISD::LoadExtType getExtensionType() const {
2705 return ISD::LoadExtType(LoadSDNodeBits.ExtTy);
2706 }
2707
2708 static bool classof(const SDNode *N) {
2709 return N->getOpcode() == ISD::MGATHER;
2710 }
2711};
2712
2713/// This class is used to represent an MSCATTER node
2714///
2715class MaskedScatterSDNode : public MaskedGatherScatterSDNode {
2716public:
2717 friend class SelectionDAG;
2718
2719 MaskedScatterSDNode(unsigned Order, const DebugLoc &dl, SDVTList VTs,
2720 EVT MemVT, MachineMemOperand *MMO,
2721 ISD::MemIndexType IndexType, bool IsTrunc)
2722 : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO,
2723 IndexType) {
2724 StoreSDNodeBits.IsTruncating = IsTrunc;
2725 }
2726
2727 /// Return true if the op does a truncation before store.
2728 /// For integers this is the same as doing a TRUNCATE and storing the result.
2729 /// For floats, it is the same as doing an FP_ROUND and storing the result.
2730 bool isTruncatingStore() const { return StoreSDNodeBits.IsTruncating; }
2731
2732 const SDValue &getValue() const { return getOperand(1); }
2733
2734 static bool classof(const SDNode *N) {
2735 return N->getOpcode() == ISD::MSCATTER;
2736 }
2737};
2738
2739/// An SDNode that represents everything that will be needed
2740/// to construct a MachineInstr. These nodes are created during the
2741/// instruction selection proper phase.
2742///
2743/// Note that the only supported way to set the `memoperands` is by calling the
2744/// `SelectionDAG::setNodeMemRefs` function as the memory management happens
2745/// inside the DAG rather than in the node.
2746class MachineSDNode : public SDNode {
2747private:
2748 friend class SelectionDAG;
2749
2750 MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs)
2751 : SDNode(Opc, Order, DL, VTs) {}
2752
2753 // We use a pointer union between a single `MachineMemOperand` pointer and
2754 // a pointer to an array of `MachineMemOperand` pointers. This is null when
2755 // the number of these is zero, the single pointer variant used when the
2756 // number is one, and the array is used for larger numbers.
2757 //
2758 // The array is allocated via the `SelectionDAG`'s allocator and so will
2759 // always live until the DAG is cleaned up and doesn't require ownership here.
2760 //
2761 // We can't use something simpler like `TinyPtrVector` here because `SDNode`
2762 // subclasses aren't managed in a conforming C++ manner. See the comments on
2763 // `SelectionDAG::MorphNodeTo` which details what all goes on, but the
2764 // constraint here is that these don't manage memory with their constructor or
2765 // destructor and can be initialized to a good state even if they start off
2766 // uninitialized.
2767 PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {};
2768
2769 // Note that this could be folded into the above `MemRefs` member if doing so
2770 // is advantageous at some point. We don't need to store this in most cases.
2771 // However, at the moment this doesn't appear to make the allocation any
2772 // smaller and makes the code somewhat simpler to read.
2773 int NumMemRefs = 0;
2774
2775public:
2776 using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator;
2777
2778 ArrayRef<MachineMemOperand *> memoperands() const {
2779 // Special case the common cases.
2780 if (NumMemRefs == 0)
2781 return {};
2782 if (NumMemRefs == 1)
2783 return makeArrayRef(MemRefs.getAddrOfPtr1(), 1);
2784
2785 // Otherwise we have an actual array.
2786 return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs);
2787 }
2788 mmo_iterator memoperands_begin() const { return memoperands().begin(); }
2789 mmo_iterator memoperands_end() const { return memoperands().end(); }
2790 bool memoperands_empty() const { return memoperands().empty(); }
2791
2792 /// Clear out the memory reference descriptor list.
2793 void clearMemRefs() {
2794 MemRefs = nullptr;
2795 NumMemRefs = 0;
2796 }
2797
2798 static bool classof(const SDNode *N) {
2799 return N->isMachineOpcode();
2800 }
2801};
2802
2803/// An SDNode that records if a register contains a value that is guaranteed to
2804/// be aligned accordingly.
2805class AssertAlignSDNode : public SDNode {
2806 Align Alignment;
2807
2808public:
2809 AssertAlignSDNode(unsigned Order, const DebugLoc &DL, EVT VT, Align A)
2810 : SDNode(ISD::AssertAlign, Order, DL, getSDVTList(VT)), Alignment(A) {}
2811
2812 Align getAlign() const { return Alignment; }
2813
2814 static bool classof(const SDNode *N) {
2815 return N->getOpcode() == ISD::AssertAlign;
2816 }
2817};
2818
2819class SDNodeIterator {
2820 const SDNode *Node;
2821 unsigned Operand;
2822
2823 SDNodeIterator(const SDNode *N, unsigned Op) : Node(N), Operand(Op) {}
2824
2825public:
2826 using iterator_category = std::forward_iterator_tag;
2827 using value_type = SDNode;
2828 using difference_type = std::ptrdiff_t;
2829 using pointer = value_type *;
2830 using reference = value_type &;
2831
2832 bool operator==(const SDNodeIterator& x) const {
2833 return Operand == x.Operand;
2834 }
2835 bool operator!=(const SDNodeIterator& x) const { return !operator==(x); }
2836
2837 pointer operator*() const {
2838 return Node->getOperand(Operand).getNode();
2839 }
2840 pointer operator->() const { return operator*(); }
2841
2842 SDNodeIterator& operator++() { // Preincrement
2843 ++Operand;
2844 return *this;
2845 }
2846 SDNodeIterator operator++(int) { // Postincrement
2847 SDNodeIterator tmp = *this; ++*this; return tmp;
2848 }
2849 size_t operator-(SDNodeIterator Other) const {
2850 assert(Node == Other.Node &&(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2851, __extension__ __PRETTY_FUNCTION__))
2851 "Cannot compare iterators of two different nodes!")(static_cast <bool> (Node == Other.Node && "Cannot compare iterators of two different nodes!"
) ? void (0) : __assert_fail ("Node == Other.Node && \"Cannot compare iterators of two different nodes!\""
, "/build/llvm-toolchain-snapshot-14~++20211016100712+8e1d532707fd/llvm/include/llvm/CodeGen/SelectionDAGNodes.h"
, 2851, __extension__ __PRETTY_FUNCTION__))
;
2852 return Operand - Other.Operand;
2853 }
2854
2855 static SDNodeIterator begin(const SDNode *N) { return SDNodeIterator(N, 0); }
2856 static SDNodeIterator end (const SDNode *N) {
2857 return SDNodeIterator(N, N->getNumOperands());
2858 }
2859
2860 unsigned getOperand() const { return Operand; }
2861 const SDNode *getNode() const { return Node; }
2862};
2863
2864template <> struct GraphTraits<SDNode*> {
2865 using NodeRef = SDNode *;
2866 using ChildIteratorType = SDNodeIterator;
2867
2868 static NodeRef getEntryNode(SDNode *N) { return N; }
2869
2870 static ChildIteratorType child_begin(NodeRef N) {
2871 return SDNodeIterator::begin(N);
2872 }
2873
2874 static ChildIteratorType child_end(NodeRef N) {
2875 return SDNodeIterator::end(N);
2876 }
2877};
2878
2879/// A representation of the largest SDNode, for use in sizeof().
2880///
2881/// This needs to be a union because the largest node differs on 32 bit systems
2882/// with 4 and 8 byte pointer alignment, respectively.
2883using LargestSDNode = AlignedCharArrayUnion<AtomicSDNode, TargetIndexSDNode,
2884 BlockAddressSDNode,
2885 GlobalAddressSDNode,
2886 PseudoProbeSDNode>;
2887
2888/// The SDNode class with the greatest alignment requirement.
2889using MostAlignedSDNode = GlobalAddressSDNode;
2890
2891namespace ISD {
2892
2893 /// Returns true if the specified node is a non-extending and unindexed load.
2894 inline bool isNormalLoad(const SDNode *N) {
2895 const LoadSDNode *Ld = dyn_cast<LoadSDNode>(N);
2896 return Ld && Ld->getExtensionType() == ISD::NON_EXTLOAD &&
2897 Ld->getAddressingMode() == ISD::UNINDEXED;
2898 }
2899
2900 /// Returns true if the specified node is a non-extending load.
2901 inline bool isNON_EXTLoad(const SDNode *N) {
2902 return isa<LoadSDNode>(N) &&
2903 cast<LoadSDNode>(N)->getExtensionType() == ISD::NON_EXTLOAD;
2904 }
2905
2906 /// Returns true if the specified node is a EXTLOAD.
2907 inline bool isEXTLoad(const SDNode *N) {
2908 return isa<LoadSDNode>(N) &&
2909 cast<LoadSDNode>(N)->getExtensionType() == ISD::EXTLOAD;
2910 }
2911
2912 /// Returns true if the specified node is a SEXTLOAD.
2913 inline bool isSEXTLoad(const SDNode *N) {
2914 return isa<LoadSDNode>(N) &&
2915 cast<LoadSDNode>(N)->getExtensionType() == ISD::SEXTLOAD;
2916 }
2917
2918 /// Returns true if the specified node is a ZEXTLOAD.
2919 inline bool isZEXTLoad(const SDNode *N) {
2920 return isa<LoadSDNode>(N) &&
2921 cast<LoadSDNode>(N)->getExtensionType() == ISD::ZEXTLOAD;
2922 }
2923
2924 /// Returns true if the specified node is an unindexed load.
2925 inline bool isUNINDEXEDLoad(const SDNode *N) {
2926 return isa<LoadSDNode>(N) &&
2927 cast<LoadSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2928 }
2929
2930 /// Returns true if the specified node is a non-truncating
2931 /// and unindexed store.
2932 inline bool isNormalStore(const SDNode *N) {
2933 const StoreSDNode *St = dyn_cast<StoreSDNode>(N);
2934 return St && !St->isTruncatingStore() &&
2935 St->getAddressingMode() == ISD::UNINDEXED;
2936 }
2937
2938 /// Returns true if the specified node is an unindexed store.
2939 inline bool isUNINDEXEDStore(const SDNode *N) {
2940 return isa<StoreSDNode>(N) &&
2941 cast<StoreSDNode>(N)->getAddressingMode() == ISD::UNINDEXED;
2942 }
2943
2944 /// Attempt to match a unary predicate against a scalar/splat constant or
2945 /// every element of a constant BUILD_VECTOR.
2946 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2947 bool matchUnaryPredicate(SDValue Op,
2948 std::function<bool(ConstantSDNode *)> Match,
2949 bool AllowUndefs = false);
2950
2951 /// Attempt to match a binary predicate against a pair of scalar/splat
2952 /// constants or every element of a pair of constant BUILD_VECTORs.
2953 /// If AllowUndef is true, then UNDEF elements will pass nullptr to Match.
2954 /// If AllowTypeMismatch is true then RetType + ArgTypes don't need to match.
2955 bool matchBinaryPredicate(
2956 SDValue LHS, SDValue RHS,
2957 std::function<bool(ConstantSDNode *, ConstantSDNode *)> Match,
2958 bool AllowUndefs = false, bool AllowTypeMismatch = false);
2959
2960 /// Returns true if the specified value is the overflow result from one
2961 /// of the overflow intrinsic nodes.
2962 inline bool isOverflowIntrOpRes(SDValue Op) {
2963 unsigned Opc = Op.getOpcode();
2964 return (Op.getResNo() == 1 &&
2965 (Opc == ISD::SADDO || Opc == ISD::UADDO || Opc == ISD::SSUBO ||
2966 Opc == ISD::USUBO || Opc == ISD::SMULO || Opc == ISD::UMULO));
2967 }
2968
2969} // end namespace ISD
2970
2971} // end namespace llvm
2972
2973#endif // LLVM_CODEGEN_SELECTIONDAGNODES_H