Bug Summary

File:build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
Warning:line 2144, column 9
The result of the left shift is undefined due to shifting by '127', which is greater or equal to the width of type 'long long'

Annotated Source Code

Press '?' to see keyboard shortcuts

clang -cc1 -cc1 -triple x86_64-pc-linux-gnu -analyze -disable-free -clear-ast-before-backend -disable-llvm-verifier -discard-value-names -main-file-name WebAssemblyISelLowering.cpp -analyzer-checker=core -analyzer-checker=apiModeling -analyzer-checker=unix -analyzer-checker=deadcode -analyzer-checker=cplusplus -analyzer-checker=security.insecureAPI.UncheckedReturn -analyzer-checker=security.insecureAPI.getpw -analyzer-checker=security.insecureAPI.gets -analyzer-checker=security.insecureAPI.mktemp -analyzer-checker=security.insecureAPI.mkstemp -analyzer-checker=security.insecureAPI.vfork -analyzer-checker=nullability.NullPassedToNonnull -analyzer-checker=nullability.NullReturnedFromNonnull -analyzer-output plist -w -setup-static-analyzer -analyzer-config-compatibility-mode=true -mrelocation-model pic -pic-level 2 -mframe-pointer=none -fmath-errno -ffp-contract=on -fno-rounding-math -mconstructor-aliases -funwind-tables=2 -target-cpu x86-64 -tune-cpu generic -debugger-tuning=gdb -ffunction-sections -fdata-sections -fcoverage-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins -resource-dir /usr/lib/llvm-16/lib/clang/16.0.0 -D _DEBUG -D _GNU_SOURCE -D __STDC_CONSTANT_MACROS -D __STDC_FORMAT_MACROS -D __STDC_LIMIT_MACROS -I lib/Target/WebAssembly -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/WebAssembly -I include -I /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/include -D _FORTIFY_SOURCE=2 -D NDEBUG -U NDEBUG -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/x86_64-linux-gnu/c++/10 -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../include/c++/10/backward -internal-isystem /usr/lib/llvm-16/lib/clang/16.0.0/include -internal-isystem /usr/local/include -internal-isystem /usr/lib/gcc/x86_64-linux-gnu/10/../../../../x86_64-linux-gnu/include -internal-externc-isystem /usr/include/x86_64-linux-gnu -internal-externc-isystem /include -internal-externc-isystem /usr/include -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fmacro-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fcoverage-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -O2 -Wno-unused-command-line-argument -Wno-unused-parameter -Wwrite-strings -Wno-missing-field-initializers -Wno-long-long -Wno-maybe-uninitialized -Wno-class-memaccess -Wno-redundant-move -Wno-pessimizing-move -Wno-noexcept-type -Wno-comment -Wno-misleading-indentation -std=c++17 -fdeprecated-macro -fdebug-compilation-dir=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/build-llvm/tools/clang/stage2-bins=build-llvm/tools/clang/stage2-bins -fdebug-prefix-map=/build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/= -ferror-limit 19 -fvisibility=hidden -fvisibility-inlines-hidden -stack-protector 2 -fgnuc-version=4.2.1 -fcolor-diagnostics -vectorize-loops -vectorize-slp -analyzer-output=html -analyzer-config stable-report-filename=true -faddrsig -D__GCC_HAVE_DWARF2_CFI_ASM=1 -o /tmp/scan-build-2022-10-03-140002-15933-1 -x c++ /build/llvm-toolchain-snapshot-16~++20221003111214+1fa2019828ca/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp
1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
14#include "WebAssemblyISelLowering.h"
15#include "MCTargetDesc/WebAssemblyMCTargetDesc.h"
16#include "Utils/WebAssemblyTypeUtilities.h"
17#include "Utils/WebAssemblyUtilities.h"
18#include "WebAssemblyMachineFunctionInfo.h"
19#include "WebAssemblySubtarget.h"
20#include "WebAssemblyTargetMachine.h"
21#include "llvm/CodeGen/CallingConvLower.h"
22#include "llvm/CodeGen/MachineFrameInfo.h"
23#include "llvm/CodeGen/MachineFunctionPass.h"
24#include "llvm/CodeGen/MachineInstrBuilder.h"
25#include "llvm/CodeGen/MachineJumpTableInfo.h"
26#include "llvm/CodeGen/MachineModuleInfo.h"
27#include "llvm/CodeGen/MachineRegisterInfo.h"
28#include "llvm/CodeGen/SelectionDAG.h"
29#include "llvm/CodeGen/SelectionDAGNodes.h"
30#include "llvm/IR/DiagnosticInfo.h"
31#include "llvm/IR/DiagnosticPrinter.h"
32#include "llvm/IR/Function.h"
33#include "llvm/IR/Intrinsics.h"
34#include "llvm/IR/IntrinsicsWebAssembly.h"
35#include "llvm/Support/Debug.h"
36#include "llvm/Support/ErrorHandling.h"
37#include "llvm/Support/KnownBits.h"
38#include "llvm/Support/MathExtras.h"
39#include "llvm/Support/raw_ostream.h"
40#include "llvm/Target/TargetOptions.h"
41using namespace llvm;
42
43#define DEBUG_TYPE"wasm-lower" "wasm-lower"
44
45WebAssemblyTargetLowering::WebAssemblyTargetLowering(
46 const TargetMachine &TM, const WebAssemblySubtarget &STI)
47 : TargetLowering(TM), Subtarget(&STI) {
48 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
49
50 // Booleans always contain 0 or 1.
51 setBooleanContents(ZeroOrOneBooleanContent);
52 // Except in SIMD vectors
53 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
54 // We don't know the microarchitecture here, so just reduce register pressure.
55 setSchedulingPreference(Sched::RegPressure);
56 // Tell ISel that we have a stack pointer.
57 setStackPointerRegisterToSaveRestore(
58 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
59 // Set up the register classes.
60 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
61 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
62 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
63 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
64 if (Subtarget->hasSIMD128()) {
65 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
66 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
67 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
68 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
69 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
70 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
71 }
72 if (Subtarget->hasReferenceTypes()) {
73 addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
74 addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
75 }
76 // Compute derived properties from the register classes.
77 computeRegisterProperties(Subtarget->getRegisterInfo());
78
79 // Transform loads and stores to pointers in address space 1 to loads and
80 // stores to WebAssembly global variables, outside linear memory.
81 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
82 setOperationAction(ISD::LOAD, T, Custom);
83 setOperationAction(ISD::STORE, T, Custom);
84 }
85 if (Subtarget->hasSIMD128()) {
86 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
87 MVT::v2f64}) {
88 setOperationAction(ISD::LOAD, T, Custom);
89 setOperationAction(ISD::STORE, T, Custom);
90 }
91 }
92 if (Subtarget->hasReferenceTypes()) {
93 // We need custom load and store lowering for both externref, funcref and
94 // Other. The MVT::Other here represents tables of reference types.
95 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
96 setOperationAction(ISD::LOAD, T, Custom);
97 setOperationAction(ISD::STORE, T, Custom);
98 }
99 }
100
101 setOperationAction(ISD::GlobalAddress, MVTPtr, Custom);
102 setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom);
103 setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom);
104 setOperationAction(ISD::JumpTable, MVTPtr, Custom);
105 setOperationAction(ISD::BlockAddress, MVTPtr, Custom);
106 setOperationAction(ISD::BRIND, MVT::Other, Custom);
107
108 // Take the default expansion for va_arg, va_copy, and va_end. There is no
109 // default action for va_start, so we do that custom.
110 setOperationAction(ISD::VASTART, MVT::Other, Custom);
111 setOperationAction(ISD::VAARG, MVT::Other, Expand);
112 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
113 setOperationAction(ISD::VAEND, MVT::Other, Expand);
114
115 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) {
116 // Don't expand the floating-point types to constant pools.
117 setOperationAction(ISD::ConstantFP, T, Legal);
118 // Expand floating-point comparisons.
119 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
120 ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE})
121 setCondCodeAction(CC, T, Expand);
122 // Expand floating-point library function operators.
123 for (auto Op :
124 {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
125 setOperationAction(Op, T, Expand);
126 // Note supported floating-point library function operators that otherwise
127 // default to expand.
128 for (auto Op :
129 {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT})
130 setOperationAction(Op, T, Legal);
131 // Support minimum and maximum, which otherwise default to expand.
132 setOperationAction(ISD::FMINIMUM, T, Legal);
133 setOperationAction(ISD::FMAXIMUM, T, Legal);
134 // WebAssembly currently has no builtin f16 support.
135 setOperationAction(ISD::FP16_TO_FP, T, Expand);
136 setOperationAction(ISD::FP_TO_FP16, T, Expand);
137 setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand);
138 setTruncStoreAction(T, MVT::f16, Expand);
139 }
140
141 // Expand unavailable integer operations.
142 for (auto Op :
143 {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU,
144 ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS,
145 ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) {
146 for (auto T : {MVT::i32, MVT::i64})
147 setOperationAction(Op, T, Expand);
148 if (Subtarget->hasSIMD128())
149 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
150 setOperationAction(Op, T, Expand);
151 }
152
153 if (Subtarget->hasNontrappingFPToInt())
154 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
155 for (auto T : {MVT::i32, MVT::i64})
156 setOperationAction(Op, T, Custom);
157
158 // SIMD-specific configuration
159 if (Subtarget->hasSIMD128()) {
160 // Hoist bitcasts out of shuffles
161 setTargetDAGCombine(ISD::VECTOR_SHUFFLE);
162
163 // Combine extends of extract_subvectors into widening ops
164 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND});
165
166 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
167 // conversions ops
168 setTargetDAGCombine({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_EXTEND,
169 ISD::EXTRACT_SUBVECTOR});
170
171 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
172 // into conversion ops
173 setTargetDAGCombine({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT,
174 ISD::FP_ROUND, ISD::CONCAT_VECTORS});
175
176 setTargetDAGCombine(ISD::TRUNCATE);
177
178 // Support saturating add for i8x16 and i16x8
179 for (auto Op : {ISD::SADDSAT, ISD::UADDSAT})
180 for (auto T : {MVT::v16i8, MVT::v8i16})
181 setOperationAction(Op, T, Legal);
182
183 // Support integer abs
184 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
185 setOperationAction(ISD::ABS, T, Legal);
186
187 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
188 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
189 MVT::v2f64})
190 setOperationAction(ISD::BUILD_VECTOR, T, Custom);
191
192 // We have custom shuffle lowering to expose the shuffle mask
193 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
194 MVT::v2f64})
195 setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom);
196
197 // Custom lowering since wasm shifts must have a scalar shift amount
198 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
199 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
200 setOperationAction(Op, T, Custom);
201
202 // Custom lower lane accesses to expand out variable indices
203 for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT})
204 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
205 MVT::v2f64})
206 setOperationAction(Op, T, Custom);
207
208 // There is no i8x16.mul instruction
209 setOperationAction(ISD::MUL, MVT::v16i8, Expand);
210
211 // There is no vector conditional select instruction
212 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
213 MVT::v2f64})
214 setOperationAction(ISD::SELECT_CC, T, Expand);
215
216 // Expand integer operations supported for scalars but not SIMD
217 for (auto Op :
218 {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR})
219 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
220 setOperationAction(Op, T, Expand);
221
222 // But we do have integer min and max operations
223 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
224 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
225 setOperationAction(Op, T, Legal);
226
227 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.
228 setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);
229 setOperationAction(ISD::CTLZ, MVT::v16i8, Expand);
230 setOperationAction(ISD::CTTZ, MVT::v16i8, Expand);
231
232 // Custom lower bit counting operations for other types to scalarize them.
233 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})
234 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
235 setOperationAction(Op, T, Custom);
236
237 // Expand float operations supported for scalars but not SIMD
238 for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
239 ISD::FEXP, ISD::FEXP2, ISD::FRINT})
240 for (auto T : {MVT::v4f32, MVT::v2f64})
241 setOperationAction(Op, T, Expand);
242
243 // Unsigned comparison operations are unavailable for i64x2 vectors.
244 for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE})
245 setCondCodeAction(CC, MVT::v2i64, Custom);
246
247 // 64x2 conversions are not in the spec
248 for (auto Op :
249 {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT})
250 for (auto T : {MVT::v2i64, MVT::v2f64})
251 setOperationAction(Op, T, Expand);
252
253 // But saturating fp_to_int converstions are
254 for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT})
255 setOperationAction(Op, MVT::v4i32, Custom);
256 }
257
258 // As a special case, these operators use the type to mean the type to
259 // sign-extend from.
260 setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand);
261 if (!Subtarget->hasSignExt()) {
262 // Sign extends are legal only when extending a vector extract
263 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
264 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
265 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action);
266 }
267 for (auto T : MVT::integer_fixedlen_vector_valuetypes())
268 setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand);
269
270 // Dynamic stack allocation: use the default expansion.
271 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
272 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
273 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);
274
275 setOperationAction(ISD::FrameIndex, MVT::i32, Custom);
276 setOperationAction(ISD::FrameIndex, MVT::i64, Custom);
277 setOperationAction(ISD::CopyToReg, MVT::Other, Custom);
278
279 // Expand these forms; we pattern-match the forms that we can handle in isel.
280 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
281 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
282 setOperationAction(Op, T, Expand);
283
284 // We have custom switch handling.
285 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
286
287 // WebAssembly doesn't have:
288 // - Floating-point extending loads.
289 // - Floating-point truncating stores.
290 // - i1 extending loads.
291 // - truncating SIMD stores and most extending loads
292 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
293 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
294 for (auto T : MVT::integer_valuetypes())
295 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
296 setLoadExtAction(Ext, T, MVT::i1, Promote);
297 if (Subtarget->hasSIMD128()) {
298 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
299 MVT::v2f64}) {
300 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
301 if (MVT(T) != MemT) {
302 setTruncStoreAction(T, MemT, Expand);
303 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
304 setLoadExtAction(Ext, T, MemT, Expand);
305 }
306 }
307 }
308 // But some vector extending loads are legal
309 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
310 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
311 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
312 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
313 }
314 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal);
315 }
316
317 // Don't do anything clever with build_pairs
318 setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand);
319
320 // Trap lowers to wasm unreachable
321 setOperationAction(ISD::TRAP, MVT::Other, Legal);
322 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
323
324 // Exception handling intrinsics
325 setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
326 setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
327 setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom);
328
329 setMaxAtomicSizeInBitsSupported(64);
330
331 // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is
332 // consistent with the f64 and f128 names.
333 setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2");
334 setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2");
335
336 // Define the emscripten name for return address helper.
337 // TODO: when implementing other Wasm backends, make this generic or only do
338 // this on emscripten depending on what they end up doing.
339 setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address");
340
341 // Always convert switches to br_tables unless there is only one case, which
342 // is equivalent to a simple branch. This reduces code size for wasm, and we
343 // defer possible jump table optimizations to the VM.
344 setMinimumJumpTableEntries(2);
345}
346
347MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL,
348 uint32_t AS) const {
349 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
350 return MVT::externref;
351 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
352 return MVT::funcref;
353 return TargetLowering::getPointerTy(DL, AS);
354}
355
356MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL,
357 uint32_t AS) const {
358 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF)
359 return MVT::externref;
360 if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF)
361 return MVT::funcref;
362 return TargetLowering::getPointerMemTy(DL, AS);
363}
364
365TargetLowering::AtomicExpansionKind
366WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
367 // We have wasm instructions for these
368 switch (AI->getOperation()) {
369 case AtomicRMWInst::Add:
370 case AtomicRMWInst::Sub:
371 case AtomicRMWInst::And:
372 case AtomicRMWInst::Or:
373 case AtomicRMWInst::Xor:
374 case AtomicRMWInst::Xchg:
375 return AtomicExpansionKind::None;
376 default:
377 break;
378 }
379 return AtomicExpansionKind::CmpXChg;
380}
381
382bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
383 // Implementation copied from X86TargetLowering.
384 unsigned Opc = VecOp.getOpcode();
385
386 // Assume target opcodes can't be scalarized.
387 // TODO - do we have any exceptions?
388 if (Opc >= ISD::BUILTIN_OP_END)
389 return false;
390
391 // If the vector op is not supported, try to convert to scalar.
392 EVT VecVT = VecOp.getValueType();
393 if (!isOperationLegalOrCustomOrPromote(Opc, VecVT))
394 return true;
395
396 // If the vector op is supported, but the scalar op is not, the transform may
397 // not be worthwhile.
398 EVT ScalarVT = VecVT.getScalarType();
399 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
400}
401
402FastISel *WebAssemblyTargetLowering::createFastISel(
403 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
404 return WebAssembly::createFastISel(FuncInfo, LibInfo);
405}
406
407MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
408 EVT VT) const {
409 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
410 if (BitWidth > 1 && BitWidth < 8)
411 BitWidth = 8;
412
413 if (BitWidth > 64) {
414 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
415 // the count to be an i32.
416 BitWidth = 32;
417 assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 418
, __extension__ __PRETTY_FUNCTION__))
418 "32-bit shift counts ought to be enough for anyone")(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits
()) && "32-bit shift counts ought to be enough for anyone"
) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 418
, __extension__ __PRETTY_FUNCTION__))
;
419 }
420
421 MVT Result = MVT::getIntegerVT(BitWidth);
422 assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
&& "Unable to represent scalar shift amount type") ?
void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 423
, __extension__ __PRETTY_FUNCTION__))
423 "Unable to represent scalar shift amount type")(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE
&& "Unable to represent scalar shift amount type") ?
void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 423
, __extension__ __PRETTY_FUNCTION__))
;
424 return Result;
425}
426
427// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
428// undefined result on invalid/overflow, to the WebAssembly opcode, which
429// traps on invalid/overflow.
430static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL,
431 MachineBasicBlock *BB,
432 const TargetInstrInfo &TII,
433 bool IsUnsigned, bool Int64,
434 bool Float64, unsigned LoweredOpcode) {
435 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
436
437 Register OutReg = MI.getOperand(0).getReg();
438 Register InReg = MI.getOperand(1).getReg();
439
440 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
441 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
442 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
443 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
444 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
445 unsigned Eqz = WebAssembly::EQZ_I32;
446 unsigned And = WebAssembly::AND_I32;
447 int64_t Limit = Int64 ? INT64_MIN(-9223372036854775807L -1) : INT32_MIN(-2147483647-1);
448 int64_t Substitute = IsUnsigned ? 0 : Limit;
449 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
450 auto &Context = BB->getParent()->getFunction().getContext();
451 Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);
452
453 const BasicBlock *LLVMBB = BB->getBasicBlock();
454 MachineFunction *F = BB->getParent();
455 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
456 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
457 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
458
459 MachineFunction::iterator It = ++BB->getIterator();
460 F->insert(It, FalseMBB);
461 F->insert(It, TrueMBB);
462 F->insert(It, DoneMBB);
463
464 // Transfer the remainder of BB and its successor edges to DoneMBB.
465 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
466 DoneMBB->transferSuccessorsAndUpdatePHIs(BB);
467
468 BB->addSuccessor(TrueMBB);
469 BB->addSuccessor(FalseMBB);
470 TrueMBB->addSuccessor(DoneMBB);
471 FalseMBB->addSuccessor(DoneMBB);
472
473 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
474 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
475 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
476 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
477 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
478 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
479 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
480
481 MI.eraseFromParent();
482 // For signed numbers, we can do a single comparison to determine whether
483 // fabs(x) is within range.
484 if (IsUnsigned) {
485 Tmp0 = InReg;
486 } else {
487 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
488 }
489 BuildMI(BB, DL, TII.get(FConst), Tmp1)
490 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
491 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
492
493 // For unsigned numbers, we have to do a separate comparison with zero.
494 if (IsUnsigned) {
495 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
496 Register SecondCmpReg =
497 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
498 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
499 BuildMI(BB, DL, TII.get(FConst), Tmp1)
500 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
501 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
502 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
503 CmpReg = AndReg;
504 }
505
506 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
507
508 // Create the CFG diamond to select between doing the conversion or using
509 // the substitute value.
510 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
511 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
512 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
513 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
514 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
515 .addReg(FalseReg)
516 .addMBB(FalseMBB)
517 .addReg(TrueReg)
518 .addMBB(TrueMBB);
519
520 return DoneMBB;
521}
522
523static MachineBasicBlock *
524LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB,
525 const WebAssemblySubtarget *Subtarget,
526 const TargetInstrInfo &TII) {
527 MachineInstr &CallParams = *CallResults.getPrevNode();
528 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS)(static_cast <bool> (CallParams.getOpcode() == WebAssembly
::CALL_PARAMS) ? void (0) : __assert_fail ("CallParams.getOpcode() == WebAssembly::CALL_PARAMS"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 528
, __extension__ __PRETTY_FUNCTION__))
;
529 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 530
, __extension__ __PRETTY_FUNCTION__))
530 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS)(static_cast <bool> (CallResults.getOpcode() == WebAssembly
::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS
) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 530
, __extension__ __PRETTY_FUNCTION__))
;
531
532 bool IsIndirect = CallParams.getOperand(0).isReg();
533 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
534
535 bool IsFuncrefCall = false;
536 if (IsIndirect) {
537 Register Reg = CallParams.getOperand(0).getReg();
538 const MachineFunction *MF = BB->getParent();
539 const MachineRegisterInfo &MRI = MF->getRegInfo();
540 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
541 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
542 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes())(static_cast <bool> (!IsFuncrefCall || Subtarget->hasReferenceTypes
()) ? void (0) : __assert_fail ("!IsFuncrefCall || Subtarget->hasReferenceTypes()"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 542
, __extension__ __PRETTY_FUNCTION__))
;
543 }
544
545 unsigned CallOp;
546 if (IsIndirect && IsRetCall) {
547 CallOp = WebAssembly::RET_CALL_INDIRECT;
548 } else if (IsIndirect) {
549 CallOp = WebAssembly::CALL_INDIRECT;
550 } else if (IsRetCall) {
551 CallOp = WebAssembly::RET_CALL;
552 } else {
553 CallOp = WebAssembly::CALL;
554 }
555
556 MachineFunction &MF = *BB->getParent();
557 const MCInstrDesc &MCID = TII.get(CallOp);
558 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));
559
560 // See if we must truncate the function pointer.
561 // CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers
562 // as 64-bit for uniformity with other pointer types.
563 // See also: WebAssemblyFastISel::selectCall
564 if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) {
565 Register Reg32 =
566 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
567 auto &FnPtr = CallParams.getOperand(0);
568 BuildMI(*BB, CallResults.getIterator(), DL,
569 TII.get(WebAssembly::I32_WRAP_I64), Reg32)
570 .addReg(FnPtr.getReg());
571 FnPtr.setReg(Reg32);
572 }
573
574 // Move the function pointer to the end of the arguments for indirect calls
575 if (IsIndirect) {
576 auto FnPtr = CallParams.getOperand(0);
577 CallParams.removeOperand(0);
578
579 // For funcrefs, call_indirect is done through __funcref_call_table and the
580 // funcref is always installed in slot 0 of the table, therefore instead of
581 // having the function pointer added at the end of the params list, a zero
582 // (the index in
583 // __funcref_call_table is added).
584 if (IsFuncrefCall) {
585 Register RegZero =
586 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
587 MachineInstrBuilder MIBC0 =
588 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
589
590 BB->insert(CallResults.getIterator(), MIBC0);
591 MachineInstrBuilder(MF, CallParams).addReg(RegZero);
592 } else
593 CallParams.addOperand(FnPtr);
594 }
595
596 for (auto Def : CallResults.defs())
597 MIB.add(Def);
598
599 if (IsIndirect) {
600 // Placeholder for the type index.
601 MIB.addImm(0);
602 // The table into which this call_indirect indexes.
603 MCSymbolWasm *Table = IsFuncrefCall
604 ? WebAssembly::getOrCreateFuncrefCallTableSymbol(
605 MF.getContext(), Subtarget)
606 : WebAssembly::getOrCreateFunctionTableSymbol(
607 MF.getContext(), Subtarget);
608 if (Subtarget->hasReferenceTypes()) {
609 MIB.addSym(Table);
610 } else {
611 // For the MVP there is at most one table whose number is 0, but we can't
612 // write a table symbol or issue relocations. Instead we just ensure the
613 // table is live and write a zero.
614 Table->setNoStrip();
615 MIB.addImm(0);
616 }
617 }
618
619 for (auto Use : CallParams.uses())
620 MIB.add(Use);
621
622 BB->insert(CallResults.getIterator(), MIB);
623 CallParams.eraseFromParent();
624 CallResults.eraseFromParent();
625
626 // If this is a funcref call, to avoid hidden GC roots, we need to clear the
627 // table slot with ref.null upon call_indirect return.
628 //
629 // This generates the following code, which comes right after a call_indirect
630 // of a funcref:
631 //
632 // i32.const 0
633 // ref.null func
634 // table.set __funcref_call_table
635 if (IsIndirect && IsFuncrefCall) {
636 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
637 MF.getContext(), Subtarget);
638 Register RegZero =
639 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
640 MachineInstr *Const0 =
641 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
642 BB->insertAfter(MIB.getInstr()->getIterator(), Const0);
643
644 Register RegFuncref =
645 MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
646 MachineInstr *RefNull =
647 BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);
648 BB->insertAfter(Const0->getIterator(), RefNull);
649
650 MachineInstr *TableSet =
651 BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
652 .addSym(Table)
653 .addReg(RegZero)
654 .addReg(RegFuncref);
655 BB->insertAfter(RefNull->getIterator(), TableSet);
656 }
657
658 return BB;
659}
660
661MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
662 MachineInstr &MI, MachineBasicBlock *BB) const {
663 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
664 DebugLoc DL = MI.getDebugLoc();
665
666 switch (MI.getOpcode()) {
667 default:
668 llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 668
)
;
669 case WebAssembly::FP_TO_SINT_I32_F32:
670 return LowerFPToInt(MI, DL, BB, TII, false, false, false,
671 WebAssembly::I32_TRUNC_S_F32);
672 case WebAssembly::FP_TO_UINT_I32_F32:
673 return LowerFPToInt(MI, DL, BB, TII, true, false, false,
674 WebAssembly::I32_TRUNC_U_F32);
675 case WebAssembly::FP_TO_SINT_I64_F32:
676 return LowerFPToInt(MI, DL, BB, TII, false, true, false,
677 WebAssembly::I64_TRUNC_S_F32);
678 case WebAssembly::FP_TO_UINT_I64_F32:
679 return LowerFPToInt(MI, DL, BB, TII, true, true, false,
680 WebAssembly::I64_TRUNC_U_F32);
681 case WebAssembly::FP_TO_SINT_I32_F64:
682 return LowerFPToInt(MI, DL, BB, TII, false, false, true,
683 WebAssembly::I32_TRUNC_S_F64);
684 case WebAssembly::FP_TO_UINT_I32_F64:
685 return LowerFPToInt(MI, DL, BB, TII, true, false, true,
686 WebAssembly::I32_TRUNC_U_F64);
687 case WebAssembly::FP_TO_SINT_I64_F64:
688 return LowerFPToInt(MI, DL, BB, TII, false, true, true,
689 WebAssembly::I64_TRUNC_S_F64);
690 case WebAssembly::FP_TO_UINT_I64_F64:
691 return LowerFPToInt(MI, DL, BB, TII, true, true, true,
692 WebAssembly::I64_TRUNC_U_F64);
693 case WebAssembly::CALL_RESULTS:
694 case WebAssembly::RET_CALL_RESULTS:
695 return LowerCallResults(MI, DL, BB, Subtarget, TII);
696 }
697}
698
699const char *
700WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
701 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
702 case WebAssemblyISD::FIRST_NUMBER:
703 case WebAssemblyISD::FIRST_MEM_OPCODE:
704 break;
705#define HANDLE_NODETYPE(NODE) \
706 case WebAssemblyISD::NODE: \
707 return "WebAssemblyISD::" #NODE;
708#define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE)
709#include "WebAssemblyISD.def"
710#undef HANDLE_MEM_NODETYPE
711#undef HANDLE_NODETYPE
712 }
713 return nullptr;
714}
715
716std::pair<unsigned, const TargetRegisterClass *>
717WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
718 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
719 // First, see if this is a constraint that directly corresponds to a
720 // WebAssembly register class.
721 if (Constraint.size() == 1) {
722 switch (Constraint[0]) {
723 case 'r':
724 assert(VT != MVT::iPTR && "Pointer MVT not expected here")(static_cast <bool> (VT != MVT::iPTR && "Pointer MVT not expected here"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Pointer MVT not expected here\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 724
, __extension__ __PRETTY_FUNCTION__))
;
725 if (Subtarget->hasSIMD128() && VT.isVector()) {
726 if (VT.getSizeInBits() == 128)
727 return std::make_pair(0U, &WebAssembly::V128RegClass);
728 }
729 if (VT.isInteger() && !VT.isVector()) {
730 if (VT.getSizeInBits() <= 32)
731 return std::make_pair(0U, &WebAssembly::I32RegClass);
732 if (VT.getSizeInBits() <= 64)
733 return std::make_pair(0U, &WebAssembly::I64RegClass);
734 }
735 if (VT.isFloatingPoint() && !VT.isVector()) {
736 switch (VT.getSizeInBits()) {
737 case 32:
738 return std::make_pair(0U, &WebAssembly::F32RegClass);
739 case 64:
740 return std::make_pair(0U, &WebAssembly::F64RegClass);
741 default:
742 break;
743 }
744 }
745 break;
746 default:
747 break;
748 }
749 }
750
751 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
752}
753
754bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
755 // Assume ctz is a relatively cheap operation.
756 return true;
757}
758
759bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
760 // Assume clz is a relatively cheap operation.
761 return true;
762}
763
764bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
765 const AddrMode &AM,
766 Type *Ty, unsigned AS,
767 Instruction *I) const {
768 // WebAssembly offsets are added as unsigned without wrapping. The
769 // isLegalAddressingMode gives us no way to determine if wrapping could be
770 // happening, so we approximate this by accepting only non-negative offsets.
771 if (AM.BaseOffs < 0)
772 return false;
773
774 // WebAssembly has no scale register operands.
775 if (AM.Scale != 0)
776 return false;
777
778 // Everything else is legal.
779 return true;
780}
781
782bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
783 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
784 MachineMemOperand::Flags /*Flags*/, bool *Fast) const {
785 // WebAssembly supports unaligned accesses, though it should be declared
786 // with the p2align attribute on loads and stores which do so, and there
787 // may be a performance impact. We tell LLVM they're "fast" because
788 // for the kinds of things that LLVM uses this for (merging adjacent stores
789 // of constants, etc.), WebAssembly implementations will either want the
790 // unaligned access or they'll split anyway.
791 if (Fast)
792 *Fast = true;
793 return true;
794}
795
796bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
797 AttributeList Attr) const {
798 // The current thinking is that wasm engines will perform this optimization,
799 // so we can save on code size.
800 return true;
801}
802
803bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
804 EVT ExtT = ExtVal.getValueType();
805 EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);
806 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
807 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
808 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
809}
810
811bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
812 const GlobalAddressSDNode *GA) const {
813 // Wasm doesn't support function addresses with offsets
814 const GlobalValue *GV = GA->getGlobal();
815 return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA);
816}
817
818EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
819 LLVMContext &C,
820 EVT VT) const {
821 if (VT.isVector())
822 return VT.changeVectorElementTypeToInteger();
823
824 // So far, all branch instructions in Wasm take an I32 condition.
825 // The default TargetLowering::getSetCCResultType returns the pointer size,
826 // which would be useful to reduce instruction counts when testing
827 // against 64-bit pointers/values if at some point Wasm supports that.
828 return EVT::getIntegerVT(C, 32);
829}
830
831bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
832 const CallInst &I,
833 MachineFunction &MF,
834 unsigned Intrinsic) const {
835 switch (Intrinsic) {
836 case Intrinsic::wasm_memory_atomic_notify:
837 Info.opc = ISD::INTRINSIC_W_CHAIN;
838 Info.memVT = MVT::i32;
839 Info.ptrVal = I.getArgOperand(0);
840 Info.offset = 0;
841 Info.align = Align(4);
842 // atomic.notify instruction does not really load the memory specified with
843 // this argument, but MachineMemOperand should either be load or store, so
844 // we set this to a load.
845 // FIXME Volatile isn't really correct, but currently all LLVM atomic
846 // instructions are treated as volatiles in the backend, so we should be
847 // consistent. The same applies for wasm_atomic_wait intrinsics too.
848 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
849 return true;
850 case Intrinsic::wasm_memory_atomic_wait32:
851 Info.opc = ISD::INTRINSIC_W_CHAIN;
852 Info.memVT = MVT::i32;
853 Info.ptrVal = I.getArgOperand(0);
854 Info.offset = 0;
855 Info.align = Align(4);
856 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
857 return true;
858 case Intrinsic::wasm_memory_atomic_wait64:
859 Info.opc = ISD::INTRINSIC_W_CHAIN;
860 Info.memVT = MVT::i64;
861 Info.ptrVal = I.getArgOperand(0);
862 Info.offset = 0;
863 Info.align = Align(8);
864 Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad;
865 return true;
866 default:
867 return false;
868 }
869}
870
871void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
872 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
873 const SelectionDAG &DAG, unsigned Depth) const {
874 switch (Op.getOpcode()) {
875 default:
876 break;
877 case ISD::INTRINSIC_WO_CHAIN: {
878 unsigned IntNo = Op.getConstantOperandVal(0);
879 switch (IntNo) {
880 default:
881 break;
882 case Intrinsic::wasm_bitmask: {
883 unsigned BitWidth = Known.getBitWidth();
884 EVT VT = Op.getOperand(1).getSimpleValueType();
885 unsigned PossibleBits = VT.getVectorNumElements();
886 APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);
887 Known.Zero |= ZeroMask;
888 break;
889 }
890 }
891 }
892 }
893}
894
895TargetLoweringBase::LegalizeTypeAction
896WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
897 if (VT.isFixedLengthVector()) {
898 MVT EltVT = VT.getVectorElementType();
899 // We have legal vector types with these lane types, so widening the
900 // vector would let us use some of the lanes directly without having to
901 // extend or truncate values.
902 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
903 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
904 return TypeWidenVector;
905 }
906
907 return TargetLoweringBase::getPreferredVectorAction(VT);
908}
909
910bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
911 SDValue Op, const TargetLoweringOpt &TLO) const {
912 // ISel process runs DAGCombiner after legalization; this step is called
913 // SelectionDAG optimization phase. This post-legalization combining process
914 // runs DAGCombiner on each node, and if there was a change to be made,
915 // re-runs legalization again on it and its user nodes to make sure
916 // everythiing is in a legalized state.
917 //
918 // The legalization calls lowering routines, and we do our custom lowering for
919 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements
920 // into zeros. But there is a set of routines in DAGCombiner that turns unused
921 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts
922 // turns unused vector elements into undefs. But this routine does not work
923 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This
924 // combination can result in a infinite loop, in which undefs are converted to
925 // zeros in legalization and back to undefs in combining.
926 //
927 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from
928 // running for build_vectors.
929 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)
930 return false;
931 return true;
932}
933
934//===----------------------------------------------------------------------===//
935// WebAssembly Lowering private implementation.
936//===----------------------------------------------------------------------===//
937
938//===----------------------------------------------------------------------===//
939// Lowering Code
940//===----------------------------------------------------------------------===//
941
942static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
943 MachineFunction &MF = DAG.getMachineFunction();
944 DAG.getContext()->diagnose(
945 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
946}
947
948// Test whether the given calling convention is supported.
949static bool callingConvSupported(CallingConv::ID CallConv) {
950 // We currently support the language-independent target-independent
951 // conventions. We don't yet have a way to annotate calls with properties like
952 // "cold", and we don't have any call-clobbered registers, so these are mostly
953 // all handled the same.
954 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
955 CallConv == CallingConv::Cold ||
956 CallConv == CallingConv::PreserveMost ||
957 CallConv == CallingConv::PreserveAll ||
958 CallConv == CallingConv::CXX_FAST_TLS ||
959 CallConv == CallingConv::WASM_EmscriptenInvoke ||
960 CallConv == CallingConv::Swift;
961}
962
963SDValue
964WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
965 SmallVectorImpl<SDValue> &InVals) const {
966 SelectionDAG &DAG = CLI.DAG;
967 SDLoc DL = CLI.DL;
968 SDValue Chain = CLI.Chain;
969 SDValue Callee = CLI.Callee;
970 MachineFunction &MF = DAG.getMachineFunction();
971 auto Layout = MF.getDataLayout();
972
973 CallingConv::ID CallConv = CLI.CallConv;
974 if (!callingConvSupported(CallConv))
975 fail(DL, DAG,
976 "WebAssembly doesn't support language-specific or target-specific "
977 "calling conventions yet");
978 if (CLI.IsPatchPoint)
979 fail(DL, DAG, "WebAssembly doesn't support patch point yet");
980
981 if (CLI.IsTailCall) {
982 auto NoTail = [&](const char *Msg) {
983 if (CLI.CB && CLI.CB->isMustTailCall())
984 fail(DL, DAG, Msg);
985 CLI.IsTailCall = false;
986 };
987
988 if (!Subtarget->hasTailCall())
989 NoTail("WebAssembly 'tail-call' feature not enabled");
990
991 // Varargs calls cannot be tail calls because the buffer is on the stack
992 if (CLI.IsVarArg)
993 NoTail("WebAssembly does not support varargs tail calls");
994
995 // Do not tail call unless caller and callee return types match
996 const Function &F = MF.getFunction();
997 const TargetMachine &TM = getTargetMachine();
998 Type *RetTy = F.getReturnType();
999 SmallVector<MVT, 4> CallerRetTys;
1000 SmallVector<MVT, 4> CalleeRetTys;
1001 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
1002 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
1003 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
1004 std::equal(CallerRetTys.begin(), CallerRetTys.end(),
1005 CalleeRetTys.begin());
1006 if (!TypesMatch)
1007 NoTail("WebAssembly tail call requires caller and callee return types to "
1008 "match");
1009
1010 // If pointers to local stack values are passed, we cannot tail call
1011 if (CLI.CB) {
1012 for (auto &Arg : CLI.CB->args()) {
1013 Value *Val = Arg.get();
1014 // Trace the value back through pointer operations
1015 while (true) {
1016 Value *Src = Val->stripPointerCastsAndAliases();
1017 if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
1018 Src = GEP->getPointerOperand();
1019 if (Val == Src)
1020 break;
1021 Val = Src;
1022 }
1023 if (isa<AllocaInst>(Val)) {
1024 NoTail(
1025 "WebAssembly does not support tail calling with stack arguments");
1026 break;
1027 }
1028 }
1029 }
1030 }
1031
1032 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1033 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1034 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1035
1036 // The generic code may have added an sret argument. If we're lowering an
1037 // invoke function, the ABI requires that the function pointer be the first
1038 // argument, so we may have to swap the arguments.
1039 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
1040 Outs[0].Flags.isSRet()) {
1041 std::swap(Outs[0], Outs[1]);
1042 std::swap(OutVals[0], OutVals[1]);
1043 }
1044
1045 bool HasSwiftSelfArg = false;
1046 bool HasSwiftErrorArg = false;
1047 unsigned NumFixedArgs = 0;
1048 for (unsigned I = 0; I < Outs.size(); ++I) {
1049 const ISD::OutputArg &Out = Outs[I];
1050 SDValue &OutVal = OutVals[I];
1051 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
1052 HasSwiftErrorArg |= Out.Flags.isSwiftError();
1053 if (Out.Flags.isNest())
1054 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1055 if (Out.Flags.isInAlloca())
1056 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1057 if (Out.Flags.isInConsecutiveRegs())
1058 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1059 if (Out.Flags.isInConsecutiveRegsLast())
1060 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1061 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
1062 auto &MFI = MF.getFrameInfo();
1063 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
1064 Out.Flags.getNonZeroByValAlign(),
1065 /*isSS=*/false);
1066 SDValue SizeNode =
1067 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
1068 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1069 Chain = DAG.getMemcpy(
1070 Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(),
1071 /*isVolatile*/ false, /*AlwaysInline=*/false,
1072 /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo());
1073 OutVal = FINode;
1074 }
1075 // Count the number of fixed args *after* legalization.
1076 NumFixedArgs += Out.IsFixed;
1077 }
1078
1079 bool IsVarArg = CLI.IsVarArg;
1080 auto PtrVT = getPointerTy(Layout);
1081
1082 // For swiftcc, emit additional swiftself and swifterror arguments
1083 // if there aren't. These additional arguments are also added for callee
1084 // signature They are necessary to match callee and caller signature for
1085 // indirect call.
1086 if (CallConv == CallingConv::Swift) {
1087 if (!HasSwiftSelfArg) {
1088 NumFixedArgs++;
1089 ISD::OutputArg Arg;
1090 Arg.Flags.setSwiftSelf();
1091 CLI.Outs.push_back(Arg);
1092 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1093 CLI.OutVals.push_back(ArgVal);
1094 }
1095 if (!HasSwiftErrorArg) {
1096 NumFixedArgs++;
1097 ISD::OutputArg Arg;
1098 Arg.Flags.setSwiftError();
1099 CLI.Outs.push_back(Arg);
1100 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1101 CLI.OutVals.push_back(ArgVal);
1102 }
1103 }
1104
1105 // Analyze operands of the call, assigning locations to each operand.
1106 SmallVector<CCValAssign, 16> ArgLocs;
1107 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1108
1109 if (IsVarArg) {
1110 // Outgoing non-fixed arguments are placed in a buffer. First
1111 // compute their offsets and the total amount of buffer space needed.
1112 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
1113 const ISD::OutputArg &Out = Outs[I];
1114 SDValue &Arg = OutVals[I];
1115 EVT VT = Arg.getValueType();
1116 assert(VT != MVT::iPTR && "Legalized args should be concrete")(static_cast <bool> (VT != MVT::iPTR && "Legalized args should be concrete"
) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Legalized args should be concrete\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1116
, __extension__ __PRETTY_FUNCTION__))
;
1117 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
1118 Align Alignment =
1119 std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
1120 unsigned Offset =
1121 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
1122 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
1123 Offset, VT.getSimpleVT(),
1124 CCValAssign::Full));
1125 }
1126 }
1127
1128 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1129
1130 SDValue FINode;
1131 if (IsVarArg && NumBytes) {
1132 // For non-fixed arguments, next emit stores to store the argument values
1133 // to the stack buffer at the offsets computed above.
1134 int FI = MF.getFrameInfo().CreateStackObject(NumBytes,
1135 Layout.getStackAlignment(),
1136 /*isSS=*/false);
1137 unsigned ValNo = 0;
1138 SmallVector<SDValue, 8> Chains;
1139 for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
1140 assert(ArgLocs[ValNo].getValNo() == ValNo &&(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
&& "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1141
, __extension__ __PRETTY_FUNCTION__))
1141 "ArgLocs should remain in order and only hold varargs args")(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo
&& "ArgLocs should remain in order and only hold varargs args"
) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1141
, __extension__ __PRETTY_FUNCTION__))
;
1142 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1143 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1144 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
1145 DAG.getConstant(Offset, DL, PtrVT));
1146 Chains.push_back(
1147 DAG.getStore(Chain, DL, Arg, Add,
1148 MachinePointerInfo::getFixedStack(MF, FI, Offset)));
1149 }
1150 if (!Chains.empty())
1151 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
1152 } else if (IsVarArg) {
1153 FINode = DAG.getIntPtrConstant(0, DL);
1154 }
1155
1156 if (Callee->getOpcode() == ISD::GlobalAddress) {
1157 // If the callee is a GlobalAddress node (quite common, every direct call
1158 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
1159 // doesn't at MO_GOT which is not needed for direct calls.
1160 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Callee);
1161 Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL,
1162 getPointerTy(DAG.getDataLayout()),
1163 GA->getOffset());
1164 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
1165 getPointerTy(DAG.getDataLayout()), Callee);
1166 }
1167
1168 // Compute the operands for the CALLn node.
1169 SmallVector<SDValue, 16> Ops;
1170 Ops.push_back(Chain);
1171 Ops.push_back(Callee);
1172
1173 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
1174 // isn't reliable.
1175 Ops.append(OutVals.begin(),
1176 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
1177 // Add a pointer to the vararg buffer.
1178 if (IsVarArg)
1179 Ops.push_back(FINode);
1180
1181 SmallVector<EVT, 8> InTys;
1182 for (const auto &In : Ins) {
1183 assert(!In.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!In.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isByVal() && \"byval is not valid for return values\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1183
, __extension__ __PRETTY_FUNCTION__))
;
1184 assert(!In.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!In.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!In.Flags.isNest() && \"nest is not valid for return values\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1184
, __extension__ __PRETTY_FUNCTION__))
;
1185 if (In.Flags.isInAlloca())
1186 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
1187 if (In.Flags.isInConsecutiveRegs())
1188 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
1189 if (In.Flags.isInConsecutiveRegsLast())
1190 fail(DL, DAG,
1191 "WebAssembly hasn't implemented cons regs last return values");
1192 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1193 // registers.
1194 InTys.push_back(In.VT);
1195 }
1196
1197 // Lastly, if this is a call to a funcref we need to add an instruction
1198 // table.set to the chain and transform the call.
1199 if (CLI.CB &&
1200 WebAssembly::isFuncrefType(CLI.CB->getCalledOperand()->getType())) {
1201 // In the absence of function references proposal where a funcref call is
1202 // lowered to call_ref, using reference types we generate a table.set to set
1203 // the funcref to a special table used solely for this purpose, followed by
1204 // a call_indirect. Here we just generate the table set, and return the
1205 // SDValue of the table.set so that LowerCall can finalize the lowering by
1206 // generating the call_indirect.
1207 SDValue Chain = Ops[0];
1208
1209 MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol(
1210 MF.getContext(), Subtarget);
1211 SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
1212 SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
1213 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
1214 SDValue TableSet = DAG.getMemIntrinsicNode(
1215 WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
1216 MVT::funcref,
1217 // Machine Mem Operand args
1218 MachinePointerInfo(
1219 WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF),
1220 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),
1221 MachineMemOperand::MOStore);
1222
1223 Ops[0] = TableSet; // The new chain is the TableSet itself
1224 }
1225
1226 if (CLI.IsTailCall) {
1227 // ret_calls do not return values to the current frame
1228 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1229 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
1230 }
1231
1232 InTys.push_back(MVT::Other);
1233 SDVTList InTyList = DAG.getVTList(InTys);
1234 SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);
1235
1236 for (size_t I = 0; I < Ins.size(); ++I)
1237 InVals.push_back(Res.getValue(I));
1238
1239 // Return the chain
1240 return Res.getValue(Ins.size());
1241}
1242
1243bool WebAssemblyTargetLowering::CanLowerReturn(
1244 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
1245 const SmallVectorImpl<ISD::OutputArg> &Outs,
1246 LLVMContext & /*Context*/) const {
1247 // WebAssembly can only handle returning tuples with multivalue enabled
1248 return Subtarget->hasMultivalue() || Outs.size() <= 1;
1249}
1250
1251SDValue WebAssemblyTargetLowering::LowerReturn(
1252 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
1253 const SmallVectorImpl<ISD::OutputArg> &Outs,
1254 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1255 SelectionDAG &DAG) const {
1256 assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1257
, __extension__ __PRETTY_FUNCTION__))
1257 "MVP WebAssembly can only return up to one value")(static_cast <bool> ((Subtarget->hasMultivalue() || Outs
.size() <= 1) && "MVP WebAssembly can only return up to one value"
) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1257
, __extension__ __PRETTY_FUNCTION__))
;
1258 if (!callingConvSupported(CallConv))
1259 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1260
1261 SmallVector<SDValue, 4> RetOps(1, Chain);
1262 RetOps.append(OutVals.begin(), OutVals.end());
1263 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
1264
1265 // Record the number and types of the return values.
1266 for (const ISD::OutputArg &Out : Outs) {
1267 assert(!Out.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!Out.Flags.isByVal() && "byval is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isByVal() && \"byval is not valid for return values\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1267
, __extension__ __PRETTY_FUNCTION__))
;
1268 assert(!Out.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!Out.Flags.isNest() && "nest is not valid for return values"
) ? void (0) : __assert_fail ("!Out.Flags.isNest() && \"nest is not valid for return values\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1268
, __extension__ __PRETTY_FUNCTION__))
;
1269 assert(Out.IsFixed && "non-fixed return value is not valid")(static_cast <bool> (Out.IsFixed && "non-fixed return value is not valid"
) ? void (0) : __assert_fail ("Out.IsFixed && \"non-fixed return value is not valid\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1269
, __extension__ __PRETTY_FUNCTION__))
;
1270 if (Out.Flags.isInAlloca())
1271 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
1272 if (Out.Flags.isInConsecutiveRegs())
1273 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
1274 if (Out.Flags.isInConsecutiveRegsLast())
1275 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
1276 }
1277
1278 return Chain;
1279}
1280
1281SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1282 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1283 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1284 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1285 if (!callingConvSupported(CallConv))
1286 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1287
1288 MachineFunction &MF = DAG.getMachineFunction();
1289 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
1290
1291 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
1292 // of the incoming values before they're represented by virtual registers.
1293 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
1294
1295 bool HasSwiftErrorArg = false;
1296 bool HasSwiftSelfArg = false;
1297 for (const ISD::InputArg &In : Ins) {
1298 HasSwiftSelfArg |= In.Flags.isSwiftSelf();
1299 HasSwiftErrorArg |= In.Flags.isSwiftError();
1300 if (In.Flags.isInAlloca())
1301 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1302 if (In.Flags.isNest())
1303 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1304 if (In.Flags.isInConsecutiveRegs())
1305 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1306 if (In.Flags.isInConsecutiveRegsLast())
1307 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1308 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1309 // registers.
1310 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
1311 DAG.getTargetConstant(InVals.size(),
1312 DL, MVT::i32))
1313 : DAG.getUNDEF(In.VT));
1314
1315 // Record the number and types of arguments.
1316 MFI->addParam(In.VT);
1317 }
1318
1319 // For swiftcc, emit additional swiftself and swifterror arguments
1320 // if there aren't. These additional arguments are also added for callee
1321 // signature They are necessary to match callee and caller signature for
1322 // indirect call.
1323 auto PtrVT = getPointerTy(MF.getDataLayout());
1324 if (CallConv == CallingConv::Swift) {
1325 if (!HasSwiftSelfArg) {
1326 MFI->addParam(PtrVT);
1327 }
1328 if (!HasSwiftErrorArg) {
1329 MFI->addParam(PtrVT);
1330 }
1331 }
1332 // Varargs are copied into a buffer allocated by the caller, and a pointer to
1333 // the buffer is passed as an argument.
1334 if (IsVarArg) {
1335 MVT PtrVT = getPointerTy(MF.getDataLayout());
1336 Register VarargVreg =
1337 MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT));
1338 MFI->setVarargBufferVreg(VarargVreg);
1339 Chain = DAG.getCopyToReg(
1340 Chain, DL, VarargVreg,
1341 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
1342 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
1343 MFI->addParam(PtrVT);
1344 }
1345
1346 // Record the number and types of arguments and results.
1347 SmallVector<MVT, 4> Params;
1348 SmallVector<MVT, 4> Results;
1349 computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(),
1350 MF.getFunction(), DAG.getTarget(), Params, Results);
1351 for (MVT VT : Results)
1352 MFI->addResult(VT);
1353 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
1354 // the param logic here with ComputeSignatureVTs
1355 assert(MFI->getParams().size() == Params.size() &&(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1357
, __extension__ __PRETTY_FUNCTION__))
1356 std::equal(MFI->getParams().begin(), MFI->getParams().end(),(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1357
, __extension__ __PRETTY_FUNCTION__))
1357 Params.begin()))(static_cast <bool> (MFI->getParams().size() == Params
.size() && std::equal(MFI->getParams().begin(), MFI
->getParams().end(), Params.begin())) ? void (0) : __assert_fail
("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1357
, __extension__ __PRETTY_FUNCTION__))
;
1358
1359 return Chain;
1360}
1361
1362void WebAssemblyTargetLowering::ReplaceNodeResults(
1363 SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const {
1364 switch (N->getOpcode()) {
1365 case ISD::SIGN_EXTEND_INREG:
1366 // Do not add any results, signifying that N should not be custom lowered
1367 // after all. This happens because simd128 turns on custom lowering for
1368 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
1369 // illegal type.
1370 break;
1371 default:
1372 llvm_unreachable(::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1373
)
1373 "ReplaceNodeResults not implemented for this op for WebAssembly!")::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1373
)
;
1374 }
1375}
1376
1377//===----------------------------------------------------------------------===//
1378// Custom lowering hooks.
1379//===----------------------------------------------------------------------===//
1380
1381SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1382 SelectionDAG &DAG) const {
1383 SDLoc DL(Op);
1384 switch (Op.getOpcode()) {
1
Control jumps to 'case BUILD_VECTOR:' at line 1421
1385 default:
1386 llvm_unreachable("unimplemented operation lowering")::llvm::llvm_unreachable_internal("unimplemented operation lowering"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1386
)
;
1387 return SDValue();
1388 case ISD::FrameIndex:
1389 return LowerFrameIndex(Op, DAG);
1390 case ISD::GlobalAddress:
1391 return LowerGlobalAddress(Op, DAG);
1392 case ISD::GlobalTLSAddress:
1393 return LowerGlobalTLSAddress(Op, DAG);
1394 case ISD::ExternalSymbol:
1395 return LowerExternalSymbol(Op, DAG);
1396 case ISD::JumpTable:
1397 return LowerJumpTable(Op, DAG);
1398 case ISD::BR_JT:
1399 return LowerBR_JT(Op, DAG);
1400 case ISD::VASTART:
1401 return LowerVASTART(Op, DAG);
1402 case ISD::BlockAddress:
1403 case ISD::BRIND:
1404 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
1405 return SDValue();
1406 case ISD::RETURNADDR:
1407 return LowerRETURNADDR(Op, DAG);
1408 case ISD::FRAMEADDR:
1409 return LowerFRAMEADDR(Op, DAG);
1410 case ISD::CopyToReg:
1411 return LowerCopyToReg(Op, DAG);
1412 case ISD::EXTRACT_VECTOR_ELT:
1413 case ISD::INSERT_VECTOR_ELT:
1414 return LowerAccessVectorElement(Op, DAG);
1415 case ISD::INTRINSIC_VOID:
1416 case ISD::INTRINSIC_WO_CHAIN:
1417 case ISD::INTRINSIC_W_CHAIN:
1418 return LowerIntrinsic(Op, DAG);
1419 case ISD::SIGN_EXTEND_INREG:
1420 return LowerSIGN_EXTEND_INREG(Op, DAG);
1421 case ISD::BUILD_VECTOR:
1422 return LowerBUILD_VECTOR(Op, DAG);
2
Calling 'WebAssemblyTargetLowering::LowerBUILD_VECTOR'
1423 case ISD::VECTOR_SHUFFLE:
1424 return LowerVECTOR_SHUFFLE(Op, DAG);
1425 case ISD::SETCC:
1426 return LowerSETCC(Op, DAG);
1427 case ISD::SHL:
1428 case ISD::SRA:
1429 case ISD::SRL:
1430 return LowerShift(Op, DAG);
1431 case ISD::FP_TO_SINT_SAT:
1432 case ISD::FP_TO_UINT_SAT:
1433 return LowerFP_TO_INT_SAT(Op, DAG);
1434 case ISD::LOAD:
1435 return LowerLoad(Op, DAG);
1436 case ISD::STORE:
1437 return LowerStore(Op, DAG);
1438 case ISD::CTPOP:
1439 case ISD::CTLZ:
1440 case ISD::CTTZ:
1441 return DAG.UnrollVectorOp(Op.getNode());
1442 }
1443}
1444
1445static bool IsWebAssemblyGlobal(SDValue Op) {
1446 if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op))
1447 return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace());
1448
1449 return false;
1450}
1451
1452static Optional<unsigned> IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG) {
1453 const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op);
1454 if (!FI)
1455 return None;
1456
1457 auto &MF = DAG.getMachineFunction();
1458 return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex());
1459}
1460
1461SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
1462 SelectionDAG &DAG) const {
1463 SDLoc DL(Op);
1464 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
1465 const SDValue &Value = SN->getValue();
1466 const SDValue &Base = SN->getBasePtr();
1467 const SDValue &Offset = SN->getOffset();
1468
1469 if (IsWebAssemblyGlobal(Base)) {
1470 if (!Offset->isUndef())
1471 report_fatal_error("unexpected offset when storing to webassembly global",
1472 false);
1473
1474 SDVTList Tys = DAG.getVTList(MVT::Other);
1475 SDValue Ops[] = {SN->getChain(), Value, Base};
1476 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
1477 SN->getMemoryVT(), SN->getMemOperand());
1478 }
1479
1480 if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1481 if (!Offset->isUndef())
1482 report_fatal_error("unexpected offset when storing to webassembly local",
1483 false);
1484
1485 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1486 SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
1487 SDValue Ops[] = {SN->getChain(), Idx, Value};
1488 return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
1489 }
1490
1491 if (WebAssembly::isWasmVarAddressSpace(SN->getAddressSpace()))
1492 report_fatal_error(
1493 "Encountered an unlowerable store to the wasm_var address space",
1494 false);
1495
1496 return Op;
1497}
1498
1499SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
1500 SelectionDAG &DAG) const {
1501 SDLoc DL(Op);
1502 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
1503 const SDValue &Base = LN->getBasePtr();
1504 const SDValue &Offset = LN->getOffset();
1505
1506 if (IsWebAssemblyGlobal(Base)) {
1507 if (!Offset->isUndef())
1508 report_fatal_error(
1509 "unexpected offset when loading from webassembly global", false);
1510
1511 SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
1512 SDValue Ops[] = {LN->getChain(), Base};
1513 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
1514 LN->getMemoryVT(), LN->getMemOperand());
1515 }
1516
1517 if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1518 if (!Offset->isUndef())
1519 report_fatal_error(
1520 "unexpected offset when loading from webassembly local", false);
1521
1522 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1523 EVT LocalVT = LN->getValueType(0);
1524 SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT,
1525 {LN->getChain(), Idx});
1526 SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL);
1527 assert(Result->getNumValues() == 2 && "Loads must carry a chain!")(static_cast <bool> (Result->getNumValues() == 2 &&
"Loads must carry a chain!") ? void (0) : __assert_fail ("Result->getNumValues() == 2 && \"Loads must carry a chain!\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1527
, __extension__ __PRETTY_FUNCTION__))
;
1528 return Result;
1529 }
1530
1531 if (WebAssembly::isWasmVarAddressSpace(LN->getAddressSpace()))
1532 report_fatal_error(
1533 "Encountered an unlowerable load from the wasm_var address space",
1534 false);
1535
1536 return Op;
1537}
1538
1539SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1540 SelectionDAG &DAG) const {
1541 SDValue Src = Op.getOperand(2);
1542 if (isa<FrameIndexSDNode>(Src.getNode())) {
1543 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1544 // the FI to some LEA-like instruction, but since we don't have that, we
1545 // need to insert some kind of instruction that can take an FI operand and
1546 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1547 // local.copy between Op and its FI operand.
1548 SDValue Chain = Op.getOperand(0);
1549 SDLoc DL(Op);
1550 Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
1551 EVT VT = Src.getValueType();
1552 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1553 : WebAssembly::COPY_I64,
1554 DL, VT, Src),
1555 0);
1556 return Op.getNode()->getNumValues() == 1
1557 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
1558 : DAG.getCopyToReg(Chain, DL, Reg, Copy,
1559 Op.getNumOperands() == 4 ? Op.getOperand(3)
1560 : SDValue());
1561 }
1562 return SDValue();
1563}
1564
1565SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1566 SelectionDAG &DAG) const {
1567 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
1568 return DAG.getTargetFrameIndex(FI, Op.getValueType());
1569}
1570
1571SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1572 SelectionDAG &DAG) const {
1573 SDLoc DL(Op);
1574
1575 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1576 fail(DL, DAG,
1577 "Non-Emscripten WebAssembly hasn't implemented "
1578 "__builtin_return_address");
1579 return SDValue();
1580 }
1581
1582 if (verifyReturnAddressArgumentIsConstant(Op, DAG))
1583 return SDValue();
1584
1585 unsigned Depth = Op.getConstantOperandVal(0);
1586 MakeLibCallOptions CallOptions;
1587 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
1588 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
1589 .first;
1590}
1591
1592SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1593 SelectionDAG &DAG) const {
1594 // Non-zero depths are not supported by WebAssembly currently. Use the
1595 // legalizer's default expansion, which is to return 0 (what this function is
1596 // documented to do).
1597 if (Op.getConstantOperandVal(0) > 0)
1598 return SDValue();
1599
1600 DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true);
1601 EVT VT = Op.getValueType();
1602 Register FP =
1603 Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
1604 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1605}
1606
1607SDValue
1608WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1609 SelectionDAG &DAG) const {
1610 SDLoc DL(Op);
1611 const auto *GA = cast<GlobalAddressSDNode>(Op);
1612
1613 MachineFunction &MF = DAG.getMachineFunction();
1614 if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
1615 report_fatal_error("cannot use thread-local storage without bulk memory",
1616 false);
1617
1618 const GlobalValue *GV = GA->getGlobal();
1619
1620 // Currently only Emscripten supports dynamic linking with threads. Therefore,
1621 // on other targets, if we have thread-local storage, only the local-exec
1622 // model is possible.
1623 auto model = Subtarget->getTargetTriple().isOSEmscripten()
1624 ? GV->getThreadLocalMode()
1625 : GlobalValue::LocalExecTLSModel;
1626
1627 // Unsupported TLS modes
1628 assert(model != GlobalValue::NotThreadLocal)(static_cast <bool> (model != GlobalValue::NotThreadLocal
) ? void (0) : __assert_fail ("model != GlobalValue::NotThreadLocal"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1628
, __extension__ __PRETTY_FUNCTION__))
;
1629 assert(model != GlobalValue::InitialExecTLSModel)(static_cast <bool> (model != GlobalValue::InitialExecTLSModel
) ? void (0) : __assert_fail ("model != GlobalValue::InitialExecTLSModel"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1629
, __extension__ __PRETTY_FUNCTION__))
;
1630
1631 if (model == GlobalValue::LocalExecTLSModel ||
1632 model == GlobalValue::LocalDynamicTLSModel ||
1633 (model == GlobalValue::GeneralDynamicTLSModel &&
1634 getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))) {
1635 // For DSO-local TLS variables we use offset from __tls_base
1636
1637 MVT PtrVT = getPointerTy(DAG.getDataLayout());
1638 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
1639 : WebAssembly::GLOBAL_GET_I32;
1640 const char *BaseName = MF.createExternalSymbolName("__tls_base");
1641
1642 SDValue BaseAddr(
1643 DAG.getMachineNode(GlobalGet, DL, PtrVT,
1644 DAG.getTargetExternalSymbol(BaseName, PtrVT)),
1645 0);
1646
1647 SDValue TLSOffset = DAG.getTargetGlobalAddress(
1648 GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
1649 SDValue SymOffset =
1650 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);
1651
1652 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);
1653 }
1654
1655 assert(model == GlobalValue::GeneralDynamicTLSModel)(static_cast <bool> (model == GlobalValue::GeneralDynamicTLSModel
) ? void (0) : __assert_fail ("model == GlobalValue::GeneralDynamicTLSModel"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1655
, __extension__ __PRETTY_FUNCTION__))
;
1656
1657 EVT VT = Op.getValueType();
1658 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1659 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1660 GA->getOffset(),
1661 WebAssemblyII::MO_GOT_TLS));
1662}
1663
1664SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1665 SelectionDAG &DAG) const {
1666 SDLoc DL(Op);
1667 const auto *GA = cast<GlobalAddressSDNode>(Op);
1668 EVT VT = Op.getValueType();
1669 assert(GA->getTargetFlags() == 0 &&(static_cast <bool> (GA->getTargetFlags() == 0 &&
"Unexpected target flags on generic GlobalAddressSDNode") ? void
(0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1670
, __extension__ __PRETTY_FUNCTION__))
1670 "Unexpected target flags on generic GlobalAddressSDNode")(static_cast <bool> (GA->getTargetFlags() == 0 &&
"Unexpected target flags on generic GlobalAddressSDNode") ? void
(0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1670
, __extension__ __PRETTY_FUNCTION__))
;
1671 if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace()))
1672 fail(DL, DAG, "Invalid address space for WebAssembly target");
1673
1674 unsigned OperandFlags = 0;
1675 if (isPositionIndependent()) {
1676 const GlobalValue *GV = GA->getGlobal();
1677 if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) {
1678 MachineFunction &MF = DAG.getMachineFunction();
1679 MVT PtrVT = getPointerTy(MF.getDataLayout());
1680 const char *BaseName;
1681 if (GV->getValueType()->isFunctionTy()) {
1682 BaseName = MF.createExternalSymbolName("__table_base");
1683 OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL;
1684 } else {
1685 BaseName = MF.createExternalSymbolName("__memory_base");
1686 OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL;
1687 }
1688 SDValue BaseAddr =
1689 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1690 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1691
1692 SDValue SymAddr = DAG.getNode(
1693 WebAssemblyISD::WrapperREL, DL, VT,
1694 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
1695 OperandFlags));
1696
1697 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
1698 }
1699 OperandFlags = WebAssemblyII::MO_GOT;
1700 }
1701
1702 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1703 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1704 GA->getOffset(), OperandFlags));
1705}
1706
1707SDValue
1708WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
1709 SelectionDAG &DAG) const {
1710 SDLoc DL(Op);
1711 const auto *ES = cast<ExternalSymbolSDNode>(Op);
1712 EVT VT = Op.getValueType();
1713 assert(ES->getTargetFlags() == 0 &&(static_cast <bool> (ES->getTargetFlags() == 0 &&
"Unexpected target flags on generic ExternalSymbolSDNode") ?
void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1714
, __extension__ __PRETTY_FUNCTION__))
1714 "Unexpected target flags on generic ExternalSymbolSDNode")(static_cast <bool> (ES->getTargetFlags() == 0 &&
"Unexpected target flags on generic ExternalSymbolSDNode") ?
void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1714
, __extension__ __PRETTY_FUNCTION__))
;
1715 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1716 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
1717}
1718
1719SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
1720 SelectionDAG &DAG) const {
1721 // There's no need for a Wrapper node because we always incorporate a jump
1722 // table operand into a BR_TABLE instruction, rather than ever
1723 // materializing it in a register.
1724 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
1725 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
1726 JT->getTargetFlags());
1727}
1728
1729SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
1730 SelectionDAG &DAG) const {
1731 SDLoc DL(Op);
1732 SDValue Chain = Op.getOperand(0);
1733 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
1734 SDValue Index = Op.getOperand(2);
1735 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags")(static_cast <bool> (JT->getTargetFlags() == 0 &&
"WebAssembly doesn't set target flags") ? void (0) : __assert_fail
("JT->getTargetFlags() == 0 && \"WebAssembly doesn't set target flags\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1735
, __extension__ __PRETTY_FUNCTION__))
;
1736
1737 SmallVector<SDValue, 8> Ops;
1738 Ops.push_back(Chain);
1739 Ops.push_back(Index);
1740
1741 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
1742 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
1743
1744 // Add an operand for each case.
1745 for (auto *MBB : MBBs)
1746 Ops.push_back(DAG.getBasicBlock(MBB));
1747
1748 // Add the first MBB as a dummy default target for now. This will be replaced
1749 // with the proper default target (and the preceding range check eliminated)
1750 // if possible by WebAssemblyFixBrTableDefaults.
1751 Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
1752 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
1753}
1754
1755SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
1756 SelectionDAG &DAG) const {
1757 SDLoc DL(Op);
1758 EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());
1759
1760 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
1761 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
1762
1763 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
1764 MFI->getVarargBufferVreg(), PtrVT);
1765 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
1766 MachinePointerInfo(SV));
1767}
1768
1769SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
1770 SelectionDAG &DAG) const {
1771 MachineFunction &MF = DAG.getMachineFunction();
1772 unsigned IntNo;
1773 switch (Op.getOpcode()) {
1774 case ISD::INTRINSIC_VOID:
1775 case ISD::INTRINSIC_W_CHAIN:
1776 IntNo = Op.getConstantOperandVal(1);
1777 break;
1778 case ISD::INTRINSIC_WO_CHAIN:
1779 IntNo = Op.getConstantOperandVal(0);
1780 break;
1781 default:
1782 llvm_unreachable("Invalid intrinsic")::llvm::llvm_unreachable_internal("Invalid intrinsic", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1782)
;
1783 }
1784 SDLoc DL(Op);
1785
1786 switch (IntNo) {
1787 default:
1788 return SDValue(); // Don't custom lower most intrinsics.
1789
1790 case Intrinsic::wasm_lsda: {
1791 auto PtrVT = getPointerTy(MF.getDataLayout());
1792 const char *SymName = MF.createExternalSymbolName(
1793 "GCC_except_table" + std::to_string(MF.getFunctionNumber()));
1794 if (isPositionIndependent()) {
1795 SDValue Node = DAG.getTargetExternalSymbol(
1796 SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);
1797 const char *BaseName = MF.createExternalSymbolName("__memory_base");
1798 SDValue BaseAddr =
1799 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
1800 DAG.getTargetExternalSymbol(BaseName, PtrVT));
1801 SDValue SymAddr =
1802 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);
1803 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
1804 }
1805 SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);
1806 return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);
1807 }
1808
1809 case Intrinsic::wasm_shuffle: {
1810 // Drop in-chain and replace undefs, but otherwise pass through unchanged
1811 SDValue Ops[18];
1812 size_t OpIdx = 0;
1813 Ops[OpIdx++] = Op.getOperand(1);
1814 Ops[OpIdx++] = Op.getOperand(2);
1815 while (OpIdx < 18) {
1816 const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
1817 if (MaskIdx.isUndef() ||
1818 cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) {
1819 Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32);
1820 } else {
1821 Ops[OpIdx++] = MaskIdx;
1822 }
1823 }
1824 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
1825 }
1826 }
1827}
1828
1829SDValue
1830WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
1831 SelectionDAG &DAG) const {
1832 SDLoc DL(Op);
1833 // If sign extension operations are disabled, allow sext_inreg only if operand
1834 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
1835 // extension operations, but allowing sext_inreg in this context lets us have
1836 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
1837 // everywhere would be simpler in this file, but would necessitate large and
1838 // brittle patterns to undo the expansion and select extract_lane_s
1839 // instructions.
1840 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128())(static_cast <bool> (!Subtarget->hasSignExt() &&
Subtarget->hasSIMD128()) ? void (0) : __assert_fail ("!Subtarget->hasSignExt() && Subtarget->hasSIMD128()"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1840
, __extension__ __PRETTY_FUNCTION__))
;
1841 if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1842 return SDValue();
1843
1844 const SDValue &Extract = Op.getOperand(0);
1845 MVT VecT = Extract.getOperand(0).getSimpleValueType();
1846 if (VecT.getVectorElementType().getSizeInBits() > 32)
1847 return SDValue();
1848 MVT ExtractedLaneT =
1849 cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
1850 MVT ExtractedVecT =
1851 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
1852 if (ExtractedVecT == VecT)
1853 return Op;
1854
1855 // Bitcast vector to appropriate type to ensure ISel pattern coverage
1856 const SDNode *Index = Extract.getOperand(1).getNode();
1857 if (!isa<ConstantSDNode>(Index))
1858 return SDValue();
1859 unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue();
1860 unsigned Scale =
1861 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
1862 assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail
("Scale > 1", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 1862, __extension__ __PRETTY_FUNCTION__))
;
1863 SDValue NewIndex =
1864 DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
1865 SDValue NewExtract = DAG.getNode(
1866 ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(),
1867 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
1868 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
1869 Op.getOperand(1));
1870}
1871
1872static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) {
1873 SDLoc DL(Op);
1874 if (Op.getValueType() != MVT::v2f64)
1875 return SDValue();
1876
1877 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
1878 unsigned &Index) -> bool {
1879 switch (Op.getOpcode()) {
1880 case ISD::SINT_TO_FP:
1881 Opcode = WebAssemblyISD::CONVERT_LOW_S;
1882 break;
1883 case ISD::UINT_TO_FP:
1884 Opcode = WebAssemblyISD::CONVERT_LOW_U;
1885 break;
1886 case ISD::FP_EXTEND:
1887 Opcode = WebAssemblyISD::PROMOTE_LOW;
1888 break;
1889 default:
1890 return false;
1891 }
1892
1893 auto ExtractVector = Op.getOperand(0);
1894 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1895 return false;
1896
1897 if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))
1898 return false;
1899
1900 SrcVec = ExtractVector.getOperand(0);
1901 Index = ExtractVector.getConstantOperandVal(1);
1902 return true;
1903 };
1904
1905 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
1906 SDValue LHSSrcVec, RHSSrcVec;
1907 if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||
1908 !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))
1909 return SDValue();
1910
1911 if (LHSOpcode != RHSOpcode)
1912 return SDValue();
1913
1914 MVT ExpectedSrcVT;
1915 switch (LHSOpcode) {
1916 case WebAssemblyISD::CONVERT_LOW_S:
1917 case WebAssemblyISD::CONVERT_LOW_U:
1918 ExpectedSrcVT = MVT::v4i32;
1919 break;
1920 case WebAssemblyISD::PROMOTE_LOW:
1921 ExpectedSrcVT = MVT::v4f32;
1922 break;
1923 }
1924 if (LHSSrcVec.getValueType() != ExpectedSrcVT)
1925 return SDValue();
1926
1927 auto Src = LHSSrcVec;
1928 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
1929 // Shuffle the source vector so that the converted lanes are the low lanes.
1930 Src = DAG.getVectorShuffle(
1931 ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec,
1932 {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});
1933 }
1934 return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src);
1935}
1936
1937SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
1938 SelectionDAG &DAG) const {
1939 if (auto ConvertLow = LowerConvertLow(Op, DAG))
3
Taking false branch
1940 return ConvertLow;
1941
1942 SDLoc DL(Op);
1943 const EVT VecT = Op.getValueType();
1944 const EVT LaneT = Op.getOperand(0).getValueType();
1945 const size_t Lanes = Op.getNumOperands();
1946 bool CanSwizzle = VecT == MVT::v16i8;
1947
1948 // BUILD_VECTORs are lowered to the instruction that initializes the highest
1949 // possible number of lanes at once followed by a sequence of replace_lane
1950 // instructions to individually initialize any remaining lanes.
1951
1952 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
1953 // swizzled lanes should be given greater weight.
1954
1955 // TODO: Investigate looping rather than always extracting/replacing specific
1956 // lanes to fill gaps.
1957
1958 auto IsConstant = [](const SDValue &V) {
1959 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
1960 };
1961
1962 // Returns the source vector and index vector pair if they exist. Checks for:
1963 // (extract_vector_elt
1964 // $src,
1965 // (sign_extend_inreg (extract_vector_elt $indices, $i))
1966 // )
1967 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
1968 auto Bail = std::make_pair(SDValue(), SDValue());
1969 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1970 return Bail;
1971 const SDValue &SwizzleSrc = Lane->getOperand(0);
1972 const SDValue &IndexExt = Lane->getOperand(1);
1973 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
1974 return Bail;
1975 const SDValue &Index = IndexExt->getOperand(0);
1976 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1977 return Bail;
1978 const SDValue &SwizzleIndices = Index->getOperand(0);
1979 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
1980 SwizzleIndices.getValueType() != MVT::v16i8 ||
1981 Index->getOperand(1)->getOpcode() != ISD::Constant ||
1982 Index->getConstantOperandVal(1) != I)
1983 return Bail;
1984 return std::make_pair(SwizzleSrc, SwizzleIndices);
1985 };
1986
1987 // If the lane is extracted from another vector at a constant index, return
1988 // that vector. The source vector must not have more lanes than the dest
1989 // because the shufflevector indices are in terms of the destination lanes and
1990 // would not be able to address the smaller individual source lanes.
1991 auto GetShuffleSrc = [&](const SDValue &Lane) {
1992 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
1993 return SDValue();
1994 if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
1995 return SDValue();
1996 if (Lane->getOperand(0).getValueType().getVectorNumElements() >
1997 VecT.getVectorNumElements())
1998 return SDValue();
1999 return Lane->getOperand(0);
2000 };
2001
2002 using ValueEntry = std::pair<SDValue, size_t>;
2003 SmallVector<ValueEntry, 16> SplatValueCounts;
2004
2005 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
2006 SmallVector<SwizzleEntry, 16> SwizzleCounts;
2007
2008 using ShuffleEntry = std::pair<SDValue, size_t>;
2009 SmallVector<ShuffleEntry, 16> ShuffleCounts;
2010
2011 auto AddCount = [](auto &Counts, const auto &Val) {
2012 auto CountIt =
2013 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
2014 if (CountIt == Counts.end()) {
2015 Counts.emplace_back(Val, 1);
2016 } else {
2017 CountIt->second++;
2018 }
2019 };
2020
2021 auto GetMostCommon = [](auto &Counts) {
2022 auto CommonIt =
2023 std::max_element(Counts.begin(), Counts.end(), llvm::less_second());
2024 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector")(static_cast <bool> (CommonIt != Counts.end() &&
"Unexpected all-undef build_vector") ? void (0) : __assert_fail
("CommonIt != Counts.end() && \"Unexpected all-undef build_vector\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2024
, __extension__ __PRETTY_FUNCTION__))
;
2025 return *CommonIt;
2026 };
2027
2028 size_t NumConstantLanes = 0;
2029
2030 // Count eligible lanes for each type of vector creation op
2031 for (size_t I = 0; I
3.1
'I' is < 'Lanes'
< Lanes
; ++I) {
4
Loop condition is true. Entering loop body
10
Assuming 'I' is >= 'Lanes'
11
Loop condition is false. Execution continues on line 2049
2032 const SDValue &Lane = Op->getOperand(I);
2033 if (Lane.isUndef())
5
Taking false branch
2034 continue;
2035
2036 AddCount(SplatValueCounts, Lane);
2037
2038 if (IsConstant(Lane))
6
Taking false branch
2039 NumConstantLanes++;
2040 if (auto ShuffleSrc = GetShuffleSrc(Lane))
7
Taking false branch
2041 AddCount(ShuffleCounts, ShuffleSrc);
2042 if (CanSwizzle
7.1
'CanSwizzle' is true
) {
8
Taking true branch
2043 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
2044 if (SwizzleSrcs.first)
9
Taking false branch
2045 AddCount(SwizzleCounts, SwizzleSrcs);
2046 }
2047 }
2048
2049 SDValue SplatValue;
2050 size_t NumSplatLanes;
2051 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
2052
2053 SDValue SwizzleSrc;
2054 SDValue SwizzleIndices;
2055 size_t NumSwizzleLanes = 0;
2056 if (SwizzleCounts.size())
12
Assuming the condition is false
13
Taking false branch
2057 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
2058 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2059
2060 // Shuffles can draw from up to two vectors, so find the two most common
2061 // sources.
2062 SDValue ShuffleSrc1, ShuffleSrc2;
2063 size_t NumShuffleLanes = 0;
2064 if (ShuffleCounts.size()) {
14
Assuming the condition is true
15
Taking true branch
2065 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2066 llvm::erase_if(ShuffleCounts,
2067 [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });
2068 }
2069 if (ShuffleCounts.size()) {
16
Assuming the condition is false
17
Taking false branch
2070 size_t AdditionalShuffleLanes;
2071 std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
2072 GetMostCommon(ShuffleCounts);
2073 NumShuffleLanes += AdditionalShuffleLanes;
2074 }
2075
2076 // Predicate returning true if the lane is properly initialized by the
2077 // original instruction
2078 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
2079 SDValue Result;
2080 // Prefer swizzles over shuffles over vector consts over splats
2081 if (NumSwizzleLanes >= NumShuffleLanes &&
18
Assuming 'NumSwizzleLanes' is < 'NumShuffleLanes'
2082 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2083 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
2084 SwizzleIndices);
2085 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
2086 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
2087 return Swizzled == GetSwizzleSrcs(I, Lane);
2088 };
2089 } else if (NumShuffleLanes
18.1
'NumShuffleLanes' is >= 'NumConstantLanes'
>= NumConstantLanes &&
20
Taking false branch
2090 NumShuffleLanes >= NumSplatLanes) {
19
Assuming 'NumShuffleLanes' is < 'NumSplatLanes'
2091 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
2092 size_t DestLaneCount = VecT.getVectorNumElements();
2093 size_t Scale1 = 1;
2094 size_t Scale2 = 1;
2095 SDValue Src1 = ShuffleSrc1;
2096 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
2097 if (Src1.getValueType() != VecT) {
2098 size_t LaneSize =
2099 Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2100 assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
(0) : __assert_fail ("LaneSize > DestLaneSize", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2100, __extension__ __PRETTY_FUNCTION__))
;
2101 Scale1 = LaneSize / DestLaneSize;
2102 Src1 = DAG.getBitcast(VecT, Src1);
2103 }
2104 if (Src2.getValueType() != VecT) {
2105 size_t LaneSize =
2106 Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8;
2107 assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void
(0) : __assert_fail ("LaneSize > DestLaneSize", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2107, __extension__ __PRETTY_FUNCTION__))
;
2108 Scale2 = LaneSize / DestLaneSize;
2109 Src2 = DAG.getBitcast(VecT, Src2);
2110 }
2111
2112 int Mask[16];
2113 assert(DestLaneCount <= 16)(static_cast <bool> (DestLaneCount <= 16) ? void (0)
: __assert_fail ("DestLaneCount <= 16", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2113, __extension__ __PRETTY_FUNCTION__))
;
2114 for (size_t I = 0; I < DestLaneCount; ++I) {
2115 const SDValue &Lane = Op->getOperand(I);
2116 SDValue Src = GetShuffleSrc(Lane);
2117 if (Src == ShuffleSrc1) {
2118 Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
2119 } else if (Src && Src == ShuffleSrc2) {
2120 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
2121 } else {
2122 Mask[I] = -1;
2123 }
2124 }
2125 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2126 Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
2127 IsLaneConstructed = [&](size_t, const SDValue &Lane) {
2128 auto Src = GetShuffleSrc(Lane);
2129 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2130 };
2131 } else if (NumConstantLanes >= NumSplatLanes) {
21
Assuming 'NumConstantLanes' is >= 'NumSplatLanes'
22
Taking true branch
2132 SmallVector<SDValue, 16> ConstLanes;
2133 for (const SDValue &Lane : Op->op_values()) {
2134 if (IsConstant(Lane)) {
23
Taking true branch
2135 // Values may need to be fixed so that they will sign extend to be
2136 // within the expected range during ISel. Check whether the value is in
2137 // bounds based on the lane bit width and if it is out of bounds, lop
2138 // off the extra bits and subtract 2^n to reflect giving the high bit
2139 // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it
2140 // cannot possibly be out of range.
2141 auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode());
24
Assuming the object is not a 'CastReturnType'
2142 int64_t Val = Const
24.1
'Const' is null
? Const->getSExtValue() : 0;
25
'?' condition is false
2143 uint64_t LaneBits = 128 / Lanes;
2144 assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll
<< (LaneBits - 1))) && "Unexpected out of bounds negative value"
) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2145
, __extension__ __PRETTY_FUNCTION__))
26
The result of the left shift is undefined due to shifting by '127', which is greater or equal to the width of type 'long long'
2145 "Unexpected out of bounds negative value")(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll
<< (LaneBits - 1))) && "Unexpected out of bounds negative value"
) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2145
, __extension__ __PRETTY_FUNCTION__))
;
2146 if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) {
2147 auto NewVal = ((uint64_t)Val % (1ll << LaneBits)) - (1ll << LaneBits);
2148 ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT));
2149 } else {
2150 ConstLanes.push_back(Lane);
2151 }
2152 } else if (LaneT.isFloatingPoint()) {
2153 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
2154 } else {
2155 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
2156 }
2157 }
2158 Result = DAG.getBuildVector(VecT, DL, ConstLanes);
2159 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
2160 return IsConstant(Lane);
2161 };
2162 } else {
2163 // Use a splat, but possibly a load_splat
2164 LoadSDNode *SplattedLoad;
2165 if ((SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) &&
2166 SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) {
2167 Result = DAG.getMemIntrinsicNode(
2168 WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT),
2169 {SplattedLoad->getChain(), SplattedLoad->getBasePtr(),
2170 SplattedLoad->getOffset()},
2171 SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand());
2172 } else {
2173 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
2174 }
2175 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
2176 return Lane == SplatValue;
2177 };
2178 }
2179
2180 assert(Result)(static_cast <bool> (Result) ? void (0) : __assert_fail
("Result", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2180, __extension__ __PRETTY_FUNCTION__))
;
2181 assert(IsLaneConstructed)(static_cast <bool> (IsLaneConstructed) ? void (0) : __assert_fail
("IsLaneConstructed", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2181, __extension__ __PRETTY_FUNCTION__))
;
2182
2183 // Add replace_lane instructions for any unhandled values
2184 for (size_t I = 0; I < Lanes; ++I) {
2185 const SDValue &Lane = Op->getOperand(I);
2186 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
2187 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
2188 DAG.getConstant(I, DL, MVT::i32));
2189 }
2190
2191 return Result;
2192}
2193
2194SDValue
2195WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2196 SelectionDAG &DAG) const {
2197 SDLoc DL(Op);
2198 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
2199 MVT VecType = Op.getOperand(0).getSimpleValueType();
2200 assert(VecType.is128BitVector() && "Unexpected shuffle vector type")(static_cast <bool> (VecType.is128BitVector() &&
"Unexpected shuffle vector type") ? void (0) : __assert_fail
("VecType.is128BitVector() && \"Unexpected shuffle vector type\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2200
, __extension__ __PRETTY_FUNCTION__))
;
2201 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
2202
2203 // Space for two vector args and sixteen mask indices
2204 SDValue Ops[18];
2205 size_t OpIdx = 0;
2206 Ops[OpIdx++] = Op.getOperand(0);
2207 Ops[OpIdx++] = Op.getOperand(1);
2208
2209 // Expand mask indices to byte indices and materialize them as operands
2210 for (int M : Mask) {
2211 for (size_t J = 0; J < LaneBytes; ++J) {
2212 // Lower undefs (represented by -1 in mask) to {0..J}, which use a
2213 // whole lane of vector input, to allow further reduction at VM. E.g.
2214 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.
2215 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;
2216 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
2217 }
2218 }
2219
2220 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2221}
2222
2223SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
2224 SelectionDAG &DAG) const {
2225 SDLoc DL(Op);
2226 // The legalizer does not know how to expand the unsupported comparison modes
2227 // of i64x2 vectors, so we manually unroll them here.
2228 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64)(static_cast <bool> (Op->getOperand(0)->getSimpleValueType
(0) == MVT::v2i64) ? void (0) : __assert_fail ("Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2228
, __extension__ __PRETTY_FUNCTION__))
;
2229 SmallVector<SDValue, 2> LHS, RHS;
2230 DAG.ExtractVectorElements(Op->getOperand(0), LHS);
2231 DAG.ExtractVectorElements(Op->getOperand(1), RHS);
2232 const SDValue &CC = Op->getOperand(2);
2233 auto MakeLane = [&](unsigned I) {
2234 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
2235 DAG.getConstant(uint64_t(-1), DL, MVT::i64),
2236 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
2237 };
2238 return DAG.getBuildVector(Op->getValueType(0), DL,
2239 {MakeLane(0), MakeLane(1)});
2240}
2241
2242SDValue
2243WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
2244 SelectionDAG &DAG) const {
2245 // Allow constant lane indices, expand variable lane indices
2246 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
2247 if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef())
2248 return Op;
2249 else
2250 // Perform default expansion
2251 return SDValue();
2252}
2253
2254static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) {
2255 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
2256 // 32-bit and 64-bit unrolled shifts will have proper semantics
2257 if (LaneT.bitsGE(MVT::i32))
2258 return DAG.UnrollVectorOp(Op.getNode());
2259 // Otherwise mask the shift value to get proper semantics from 32-bit shift
2260 SDLoc DL(Op);
2261 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
2262 SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
2263 unsigned ShiftOpcode = Op.getOpcode();
2264 SmallVector<SDValue, 16> ShiftedElements;
2265 DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
2266 SmallVector<SDValue, 16> ShiftElements;
2267 DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
2268 SmallVector<SDValue, 16> UnrolledOps;
2269 for (size_t i = 0; i < NumLanes; ++i) {
2270 SDValue MaskedShiftValue =
2271 DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
2272 SDValue ShiftedValue = ShiftedElements[i];
2273 if (ShiftOpcode == ISD::SRA)
2274 ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
2275 ShiftedValue, DAG.getValueType(LaneT));
2276 UnrolledOps.push_back(
2277 DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
2278 }
2279 return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
2280}
2281
2282SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
2283 SelectionDAG &DAG) const {
2284 SDLoc DL(Op);
2285
2286 // Only manually lower vector shifts
2287 assert(Op.getSimpleValueType().isVector())(static_cast <bool> (Op.getSimpleValueType().isVector()
) ? void (0) : __assert_fail ("Op.getSimpleValueType().isVector()"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2287
, __extension__ __PRETTY_FUNCTION__))
;
2288
2289 auto ShiftVal = DAG.getSplatValue(Op.getOperand(1));
2290 if (!ShiftVal)
2291 return unrollVectorShift(Op, DAG);
2292
2293 // Use anyext because none of the high bits can affect the shift
2294 ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);
2295
2296 unsigned Opcode;
2297 switch (Op.getOpcode()) {
2298 case ISD::SHL:
2299 Opcode = WebAssemblyISD::VEC_SHL;
2300 break;
2301 case ISD::SRA:
2302 Opcode = WebAssemblyISD::VEC_SHR_S;
2303 break;
2304 case ISD::SRL:
2305 Opcode = WebAssemblyISD::VEC_SHR_U;
2306 break;
2307 default:
2308 llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2308)
;
2309 }
2310
2311 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
2312}
2313
2314SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
2315 SelectionDAG &DAG) const {
2316 SDLoc DL(Op);
2317 EVT ResT = Op.getValueType();
2318 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2319
2320 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2321 (SatVT == MVT::i32 || SatVT == MVT::i64))
2322 return Op;
2323
2324 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2325 return Op;
2326
2327 return SDValue();
2328}
2329
2330//===----------------------------------------------------------------------===//
2331// Custom DAG combine hooks
2332//===----------------------------------------------------------------------===//
2333static SDValue
2334performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
2335 auto &DAG = DCI.DAG;
2336 auto Shuffle = cast<ShuffleVectorSDNode>(N);
2337
2338 // Hoist vector bitcasts that don't change the number of lanes out of unary
2339 // shuffles, where they are less likely to get in the way of other combines.
2340 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
2341 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
2342 SDValue Bitcast = N->getOperand(0);
2343 if (Bitcast.getOpcode() != ISD::BITCAST)
2344 return SDValue();
2345 if (!N->getOperand(1).isUndef())
2346 return SDValue();
2347 SDValue CastOp = Bitcast.getOperand(0);
2348 MVT SrcType = CastOp.getSimpleValueType();
2349 MVT DstType = Bitcast.getSimpleValueType();
2350 if (!SrcType.is128BitVector() ||
2351 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
2352 return SDValue();
2353 SDValue NewShuffle = DAG.getVectorShuffle(
2354 SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
2355 return DAG.getBitcast(DstType, NewShuffle);
2356}
2357
2358static SDValue
2359performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
2360 auto &DAG = DCI.DAG;
2361 assert(N->getOpcode() == ISD::SIGN_EXTEND ||(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
|| N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2362
, __extension__ __PRETTY_FUNCTION__))
2362 N->getOpcode() == ISD::ZERO_EXTEND)(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND
|| N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail
("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND"
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2362
, __extension__ __PRETTY_FUNCTION__))
;
2363
2364 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
2365 // possible before the extract_subvector can be expanded.
2366 auto Extract = N->getOperand(0);
2367 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
2368 return SDValue();
2369 auto Source = Extract.getOperand(0);
2370 auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
2371 if (IndexNode == nullptr)
2372 return SDValue();
2373 auto Index = IndexNode->getZExtValue();
2374
2375 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
2376 // extracted subvector is the low or high half of its source.
2377 EVT ResVT = N->getValueType(0);
2378 if (ResVT == MVT::v8i16) {
2379 if (Extract.getValueType() != MVT::v8i8 ||
2380 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
2381 return SDValue();
2382 } else if (ResVT == MVT::v4i32) {
2383 if (Extract.getValueType() != MVT::v4i16 ||
2384 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
2385 return SDValue();
2386 } else if (ResVT == MVT::v2i64) {
2387 if (Extract.getValueType() != MVT::v2i32 ||
2388 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
2389 return SDValue();
2390 } else {
2391 return SDValue();
2392 }
2393
2394 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
2395 bool IsLow = Index == 0;
2396
2397 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
2398 : WebAssemblyISD::EXTEND_HIGH_S)
2399 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
2400 : WebAssemblyISD::EXTEND_HIGH_U);
2401
2402 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2403}
2404
2405static SDValue
2406performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) {
2407 auto &DAG = DCI.DAG;
2408
2409 auto GetWasmConversionOp = [](unsigned Op) {
2410 switch (Op) {
2411 case ISD::FP_TO_SINT_SAT:
2412 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
2413 case ISD::FP_TO_UINT_SAT:
2414 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
2415 case ISD::FP_ROUND:
2416 return WebAssemblyISD::DEMOTE_ZERO;
2417 }
2418 llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2418)
;
2419 };
2420
2421 auto IsZeroSplat = [](SDValue SplatVal) {
2422 auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
2423 APInt SplatValue, SplatUndef;
2424 unsigned SplatBitSize;
2425 bool HasAnyUndefs;
2426 return Splat &&
2427 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2428 HasAnyUndefs) &&
2429 SplatValue == 0;
2430 };
2431
2432 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
2433 // Combine this:
2434 //
2435 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
2436 //
2437 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2438 //
2439 // Or this:
2440 //
2441 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
2442 //
2443 // into (f32x4.demote_zero_f64x2 $x).
2444 EVT ResVT;
2445 EVT ExpectedConversionType;
2446 auto Conversion = N->getOperand(0);
2447 auto ConversionOp = Conversion.getOpcode();
2448 switch (ConversionOp) {
2449 case ISD::FP_TO_SINT_SAT:
2450 case ISD::FP_TO_UINT_SAT:
2451 ResVT = MVT::v4i32;
2452 ExpectedConversionType = MVT::v2i32;
2453 break;
2454 case ISD::FP_ROUND:
2455 ResVT = MVT::v4f32;
2456 ExpectedConversionType = MVT::v2f32;
2457 break;
2458 default:
2459 return SDValue();
2460 }
2461
2462 if (N->getValueType(0) != ResVT)
2463 return SDValue();
2464
2465 if (Conversion.getValueType() != ExpectedConversionType)
2466 return SDValue();
2467
2468 auto Source = Conversion.getOperand(0);
2469 if (Source.getValueType() != MVT::v2f64)
2470 return SDValue();
2471
2472 if (!IsZeroSplat(N->getOperand(1)) ||
2473 N->getOperand(1).getValueType() != ExpectedConversionType)
2474 return SDValue();
2475
2476 unsigned Op = GetWasmConversionOp(ConversionOp);
2477 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2478 }
2479
2480 // Combine this:
2481 //
2482 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
2483 //
2484 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2485 //
2486 // Or this:
2487 //
2488 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
2489 //
2490 // into (f32x4.demote_zero_f64x2 $x).
2491 EVT ResVT;
2492 auto ConversionOp = N->getOpcode();
2493 switch (ConversionOp) {
2494 case ISD::FP_TO_SINT_SAT:
2495 case ISD::FP_TO_UINT_SAT:
2496 ResVT = MVT::v4i32;
2497 break;
2498 case ISD::FP_ROUND:
2499 ResVT = MVT::v4f32;
2500 break;
2501 default:
2502 llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp"
, 2502)
;
2503 }
2504
2505 if (N->getValueType(0) != ResVT)
2506 return SDValue();
2507
2508 auto Concat = N->getOperand(0);
2509 if (Concat.getValueType() != MVT::v4f64)
2510 return SDValue();
2511
2512 auto Source = Concat.getOperand(0);
2513 if (Source.getValueType() != MVT::v2f64)
2514 return SDValue();
2515
2516 if (!IsZeroSplat(Concat.getOperand(1)) ||
2517 Concat.getOperand(1).getValueType() != MVT::v2f64)
2518 return SDValue();
2519
2520 unsigned Op = GetWasmConversionOp(ConversionOp);
2521 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2522}
2523
2524// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
2525static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
2526 const SDLoc &DL, unsigned VectorWidth) {
2527 EVT VT = Vec.getValueType();
2528 EVT ElVT = VT.getVectorElementType();
2529 unsigned Factor = VT.getSizeInBits() / VectorWidth;
2530 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
2531 VT.getVectorNumElements() / Factor);
2532
2533 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR
2534 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
2535 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2")(static_cast <bool> (isPowerOf2_32(ElemsPerChunk) &&
"Elements per chunk not power of 2") ? void (0) : __assert_fail
("isPowerOf2_32(ElemsPerChunk) && \"Elements per chunk not power of 2\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2535
, __extension__ __PRETTY_FUNCTION__))
;
2536
2537 // This is the index of the first element of the VectorWidth-bit chunk
2538 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
2539 IdxVal &= ~(ElemsPerChunk - 1);
2540
2541 // If the input is a buildvector just emit a smaller one.
2542 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
2543 return DAG.getBuildVector(ResultVT, DL,
2544 Vec->ops().slice(IdxVal, ElemsPerChunk));
2545
2546 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL);
2547 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx);
2548}
2549
2550// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
2551// is the expected destination value type after recursion. In is the initial
2552// input. Note that the input should have enough leading zero bits to prevent
2553// NARROW_U from saturating results.
2554static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL,
2555 SelectionDAG &DAG) {
2556 EVT SrcVT = In.getValueType();
2557
2558 // No truncation required, we might get here due to recursive calls.
2559 if (SrcVT == DstVT)
2560 return In;
2561
2562 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
2563 unsigned NumElems = SrcVT.getVectorNumElements();
2564 if (!isPowerOf2_32(NumElems))
2565 return SDValue();
2566 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation")(static_cast <bool> (DstVT.getVectorNumElements() == NumElems
&& "Illegal truncation") ? void (0) : __assert_fail (
"DstVT.getVectorNumElements() == NumElems && \"Illegal truncation\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2566
, __extension__ __PRETTY_FUNCTION__))
;
2567 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation")(static_cast <bool> (SrcSizeInBits > DstVT.getSizeInBits
() && "Illegal truncation") ? void (0) : __assert_fail
("SrcSizeInBits > DstVT.getSizeInBits() && \"Illegal truncation\""
, "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2567
, __extension__ __PRETTY_FUNCTION__))
;
2568
2569 LLVMContext &Ctx = *DAG.getContext();
2570 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
2571
2572 // Narrow to the largest type possible:
2573 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
2574 EVT InVT = MVT::i16, OutVT = MVT::i8;
2575 if (SrcVT.getScalarSizeInBits() > 16) {
2576 InVT = MVT::i32;
2577 OutVT = MVT::i16;
2578 }
2579 unsigned SubSizeInBits = SrcSizeInBits / 2;
2580 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
2581 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
2582
2583 // Split lower/upper subvectors.
2584 SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits);
2585 SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits);
2586
2587 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
2588 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
2589 Lo = DAG.getBitcast(InVT, Lo);
2590 Hi = DAG.getBitcast(InVT, Hi);
2591 SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi);
2592 return DAG.getBitcast(DstVT, Res);
2593 }
2594
2595 // Recursively narrow lower/upper subvectors, concat result and narrow again.
2596 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
2597 Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG);
2598 Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG);
2599
2600 PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
2601 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
2602 return truncateVectorWithNARROW(DstVT, Res, DL, DAG);
2603}
2604
2605static SDValue performTruncateCombine(SDNode *N,
2606 TargetLowering::DAGCombinerInfo &DCI) {
2607 auto &DAG = DCI.DAG;
2608
2609 SDValue In = N->getOperand(0);
2610 EVT InVT = In.getValueType();
2611 if (!InVT.isSimple())
2612 return SDValue();
2613
2614 EVT OutVT = N->getValueType(0);
2615 if (!OutVT.isVector())
2616 return SDValue();
2617
2618 EVT OutSVT = OutVT.getVectorElementType();
2619 EVT InSVT = InVT.getVectorElementType();
2620 // Currently only cover truncate to v16i8 or v8i16.
2621 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
2622 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
2623 return SDValue();
2624
2625 SDLoc DL(N);
2626 APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(),
2627 OutVT.getScalarSizeInBits());
2628 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
2629 return truncateVectorWithNARROW(OutVT, In, DL, DAG);
2630}
2631
2632SDValue
2633WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
2634 DAGCombinerInfo &DCI) const {
2635 switch (N->getOpcode()) {
2636 default:
2637 return SDValue();
2638 case ISD::VECTOR_SHUFFLE:
2639 return performVECTOR_SHUFFLECombine(N, DCI);
2640 case ISD::SIGN_EXTEND:
2641 case ISD::ZERO_EXTEND:
2642 return performVectorExtendCombine(N, DCI);
2643 case ISD::FP_TO_SINT_SAT:
2644 case ISD::FP_TO_UINT_SAT:
2645 case ISD::FP_ROUND:
2646 case ISD::CONCAT_VECTORS:
2647 return performVectorTruncZeroCombine(N, DCI);
2648 case ISD::TRUNCATE:
2649 return performTruncateCombine(N, DCI);
2650 }
2651}