File: | build/llvm-toolchain-snapshot-15~++20220420111733+e13d2efed663/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp |
Warning: | line 2256, column 9 The result of the left shift is undefined due to shifting by '127', which is greater or equal to the width of type 'long long' |
Press '?' to see keyboard shortcuts
Keyboard shortcuts:
1 | //=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==// | |||
2 | // | |||
3 | // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. | |||
4 | // See https://llvm.org/LICENSE.txt for license information. | |||
5 | // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |||
6 | // | |||
7 | //===----------------------------------------------------------------------===// | |||
8 | /// | |||
9 | /// \file | |||
10 | /// This file implements the WebAssemblyTargetLowering class. | |||
11 | /// | |||
12 | //===----------------------------------------------------------------------===// | |||
13 | ||||
14 | #include "WebAssemblyISelLowering.h" | |||
15 | #include "MCTargetDesc/WebAssemblyMCTargetDesc.h" | |||
16 | #include "Utils/WebAssemblyTypeUtilities.h" | |||
17 | #include "Utils/WebAssemblyUtilities.h" | |||
18 | #include "WebAssemblyMachineFunctionInfo.h" | |||
19 | #include "WebAssemblySubtarget.h" | |||
20 | #include "WebAssemblyTargetMachine.h" | |||
21 | #include "llvm/CodeGen/CallingConvLower.h" | |||
22 | #include "llvm/CodeGen/MachineFrameInfo.h" | |||
23 | #include "llvm/CodeGen/MachineFunctionPass.h" | |||
24 | #include "llvm/CodeGen/MachineInstrBuilder.h" | |||
25 | #include "llvm/CodeGen/MachineJumpTableInfo.h" | |||
26 | #include "llvm/CodeGen/MachineModuleInfo.h" | |||
27 | #include "llvm/CodeGen/MachineRegisterInfo.h" | |||
28 | #include "llvm/CodeGen/SelectionDAG.h" | |||
29 | #include "llvm/CodeGen/SelectionDAGNodes.h" | |||
30 | #include "llvm/IR/DiagnosticInfo.h" | |||
31 | #include "llvm/IR/DiagnosticPrinter.h" | |||
32 | #include "llvm/IR/Function.h" | |||
33 | #include "llvm/IR/Intrinsics.h" | |||
34 | #include "llvm/IR/IntrinsicsWebAssembly.h" | |||
35 | #include "llvm/Support/Debug.h" | |||
36 | #include "llvm/Support/ErrorHandling.h" | |||
37 | #include "llvm/Support/KnownBits.h" | |||
38 | #include "llvm/Support/MathExtras.h" | |||
39 | #include "llvm/Support/raw_ostream.h" | |||
40 | #include "llvm/Target/TargetOptions.h" | |||
41 | using namespace llvm; | |||
42 | ||||
43 | #define DEBUG_TYPE"wasm-lower" "wasm-lower" | |||
44 | ||||
45 | WebAssemblyTargetLowering::WebAssemblyTargetLowering( | |||
46 | const TargetMachine &TM, const WebAssemblySubtarget &STI) | |||
47 | : TargetLowering(TM), Subtarget(&STI) { | |||
48 | auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32; | |||
49 | ||||
50 | // Booleans always contain 0 or 1. | |||
51 | setBooleanContents(ZeroOrOneBooleanContent); | |||
52 | // Except in SIMD vectors | |||
53 | setBooleanVectorContents(ZeroOrNegativeOneBooleanContent); | |||
54 | // We don't know the microarchitecture here, so just reduce register pressure. | |||
55 | setSchedulingPreference(Sched::RegPressure); | |||
56 | // Tell ISel that we have a stack pointer. | |||
57 | setStackPointerRegisterToSaveRestore( | |||
58 | Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32); | |||
59 | // Set up the register classes. | |||
60 | addRegisterClass(MVT::i32, &WebAssembly::I32RegClass); | |||
61 | addRegisterClass(MVT::i64, &WebAssembly::I64RegClass); | |||
62 | addRegisterClass(MVT::f32, &WebAssembly::F32RegClass); | |||
63 | addRegisterClass(MVT::f64, &WebAssembly::F64RegClass); | |||
64 | if (Subtarget->hasSIMD128()) { | |||
65 | addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass); | |||
66 | addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass); | |||
67 | addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass); | |||
68 | addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass); | |||
69 | addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass); | |||
70 | addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass); | |||
71 | } | |||
72 | if (Subtarget->hasReferenceTypes()) { | |||
73 | addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass); | |||
74 | addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass); | |||
75 | } | |||
76 | // Compute derived properties from the register classes. | |||
77 | computeRegisterProperties(Subtarget->getRegisterInfo()); | |||
78 | ||||
79 | // Transform loads and stores to pointers in address space 1 to loads and | |||
80 | // stores to WebAssembly global variables, outside linear memory. | |||
81 | for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) { | |||
82 | setOperationAction(ISD::LOAD, T, Custom); | |||
83 | setOperationAction(ISD::STORE, T, Custom); | |||
84 | } | |||
85 | if (Subtarget->hasSIMD128()) { | |||
86 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, | |||
87 | MVT::v2f64}) { | |||
88 | setOperationAction(ISD::LOAD, T, Custom); | |||
89 | setOperationAction(ISD::STORE, T, Custom); | |||
90 | } | |||
91 | } | |||
92 | if (Subtarget->hasReferenceTypes()) { | |||
93 | // We need custom load and store lowering for both externref, funcref and | |||
94 | // Other. The MVT::Other here represents tables of reference types. | |||
95 | for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) { | |||
96 | setOperationAction(ISD::LOAD, T, Custom); | |||
97 | setOperationAction(ISD::STORE, T, Custom); | |||
98 | } | |||
99 | } | |||
100 | ||||
101 | setOperationAction(ISD::GlobalAddress, MVTPtr, Custom); | |||
102 | setOperationAction(ISD::GlobalTLSAddress, MVTPtr, Custom); | |||
103 | setOperationAction(ISD::ExternalSymbol, MVTPtr, Custom); | |||
104 | setOperationAction(ISD::JumpTable, MVTPtr, Custom); | |||
105 | setOperationAction(ISD::BlockAddress, MVTPtr, Custom); | |||
106 | setOperationAction(ISD::BRIND, MVT::Other, Custom); | |||
107 | ||||
108 | // Take the default expansion for va_arg, va_copy, and va_end. There is no | |||
109 | // default action for va_start, so we do that custom. | |||
110 | setOperationAction(ISD::VASTART, MVT::Other, Custom); | |||
111 | setOperationAction(ISD::VAARG, MVT::Other, Expand); | |||
112 | setOperationAction(ISD::VACOPY, MVT::Other, Expand); | |||
113 | setOperationAction(ISD::VAEND, MVT::Other, Expand); | |||
114 | ||||
115 | for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64}) { | |||
116 | // Don't expand the floating-point types to constant pools. | |||
117 | setOperationAction(ISD::ConstantFP, T, Legal); | |||
118 | // Expand floating-point comparisons. | |||
119 | for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE, | |||
120 | ISD::SETULT, ISD::SETULE, ISD::SETUGT, ISD::SETUGE}) | |||
121 | setCondCodeAction(CC, T, Expand); | |||
122 | // Expand floating-point library function operators. | |||
123 | for (auto Op : | |||
124 | {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA}) | |||
125 | setOperationAction(Op, T, Expand); | |||
126 | // Note supported floating-point library function operators that otherwise | |||
127 | // default to expand. | |||
128 | for (auto Op : | |||
129 | {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT, ISD::FRINT}) | |||
130 | setOperationAction(Op, T, Legal); | |||
131 | // Support minimum and maximum, which otherwise default to expand. | |||
132 | setOperationAction(ISD::FMINIMUM, T, Legal); | |||
133 | setOperationAction(ISD::FMAXIMUM, T, Legal); | |||
134 | // WebAssembly currently has no builtin f16 support. | |||
135 | setOperationAction(ISD::FP16_TO_FP, T, Expand); | |||
136 | setOperationAction(ISD::FP_TO_FP16, T, Expand); | |||
137 | setLoadExtAction(ISD::EXTLOAD, T, MVT::f16, Expand); | |||
138 | setTruncStoreAction(T, MVT::f16, Expand); | |||
139 | } | |||
140 | ||||
141 | // Expand unavailable integer operations. | |||
142 | for (auto Op : | |||
143 | {ISD::BSWAP, ISD::SMUL_LOHI, ISD::UMUL_LOHI, ISD::MULHS, ISD::MULHU, | |||
144 | ISD::SDIVREM, ISD::UDIVREM, ISD::SHL_PARTS, ISD::SRA_PARTS, | |||
145 | ISD::SRL_PARTS, ISD::ADDC, ISD::ADDE, ISD::SUBC, ISD::SUBE}) { | |||
146 | for (auto T : {MVT::i32, MVT::i64}) | |||
147 | setOperationAction(Op, T, Expand); | |||
148 | if (Subtarget->hasSIMD128()) | |||
149 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) | |||
150 | setOperationAction(Op, T, Expand); | |||
151 | } | |||
152 | ||||
153 | if (Subtarget->hasNontrappingFPToInt()) | |||
154 | for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) | |||
155 | for (auto T : {MVT::i32, MVT::i64}) | |||
156 | setOperationAction(Op, T, Custom); | |||
157 | ||||
158 | // SIMD-specific configuration | |||
159 | if (Subtarget->hasSIMD128()) { | |||
160 | // Hoist bitcasts out of shuffles | |||
161 | setTargetDAGCombine(ISD::VECTOR_SHUFFLE); | |||
162 | ||||
163 | // Combine extends of extract_subvectors into widening ops | |||
164 | setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}); | |||
165 | ||||
166 | // Combine int_to_fp or fp_extend of extract_vectors and vice versa into | |||
167 | // conversions ops | |||
168 | setTargetDAGCombine({ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_EXTEND, | |||
169 | ISD::EXTRACT_SUBVECTOR}); | |||
170 | ||||
171 | // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa | |||
172 | // into conversion ops | |||
173 | setTargetDAGCombine({ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT, | |||
174 | ISD::FP_ROUND, ISD::CONCAT_VECTORS}); | |||
175 | ||||
176 | setTargetDAGCombine(ISD::TRUNCATE); | |||
177 | ||||
178 | // Support saturating add for i8x16 and i16x8 | |||
179 | for (auto Op : {ISD::SADDSAT, ISD::UADDSAT}) | |||
180 | for (auto T : {MVT::v16i8, MVT::v8i16}) | |||
181 | setOperationAction(Op, T, Legal); | |||
182 | ||||
183 | // Support integer abs | |||
184 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) | |||
185 | setOperationAction(ISD::ABS, T, Legal); | |||
186 | ||||
187 | // Custom lower BUILD_VECTORs to minimize number of replace_lanes | |||
188 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, | |||
189 | MVT::v2f64}) | |||
190 | setOperationAction(ISD::BUILD_VECTOR, T, Custom); | |||
191 | ||||
192 | // We have custom shuffle lowering to expose the shuffle mask | |||
193 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, | |||
194 | MVT::v2f64}) | |||
195 | setOperationAction(ISD::VECTOR_SHUFFLE, T, Custom); | |||
196 | ||||
197 | // Custom lowering since wasm shifts must have a scalar shift amount | |||
198 | for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL}) | |||
199 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) | |||
200 | setOperationAction(Op, T, Custom); | |||
201 | ||||
202 | // Custom lower lane accesses to expand out variable indices | |||
203 | for (auto Op : {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}) | |||
204 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, | |||
205 | MVT::v2f64}) | |||
206 | setOperationAction(Op, T, Custom); | |||
207 | ||||
208 | // There is no i8x16.mul instruction | |||
209 | setOperationAction(ISD::MUL, MVT::v16i8, Expand); | |||
210 | ||||
211 | // There is no vector conditional select instruction | |||
212 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64, | |||
213 | MVT::v2f64}) | |||
214 | setOperationAction(ISD::SELECT_CC, T, Expand); | |||
215 | ||||
216 | // Expand integer operations supported for scalars but not SIMD | |||
217 | for (auto Op : | |||
218 | {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM, ISD::ROTL, ISD::ROTR}) | |||
219 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) | |||
220 | setOperationAction(Op, T, Expand); | |||
221 | ||||
222 | // But we do have integer min and max operations | |||
223 | for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}) | |||
224 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32}) | |||
225 | setOperationAction(Op, T, Legal); | |||
226 | ||||
227 | // And we have popcnt for i8x16. It can be used to expand ctlz/cttz. | |||
228 | setOperationAction(ISD::CTPOP, MVT::v16i8, Legal); | |||
229 | setOperationAction(ISD::CTLZ, MVT::v16i8, Expand); | |||
230 | setOperationAction(ISD::CTTZ, MVT::v16i8, Expand); | |||
231 | ||||
232 | // Custom lower bit counting operations for other types to scalarize them. | |||
233 | for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}) | |||
234 | for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64}) | |||
235 | setOperationAction(Op, T, Custom); | |||
236 | ||||
237 | // Expand float operations supported for scalars but not SIMD | |||
238 | for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10, | |||
239 | ISD::FEXP, ISD::FEXP2, ISD::FRINT}) | |||
240 | for (auto T : {MVT::v4f32, MVT::v2f64}) | |||
241 | setOperationAction(Op, T, Expand); | |||
242 | ||||
243 | // Unsigned comparison operations are unavailable for i64x2 vectors. | |||
244 | for (auto CC : {ISD::SETUGT, ISD::SETUGE, ISD::SETULT, ISD::SETULE}) | |||
245 | setCondCodeAction(CC, MVT::v2i64, Custom); | |||
246 | ||||
247 | // 64x2 conversions are not in the spec | |||
248 | for (auto Op : | |||
249 | {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}) | |||
250 | for (auto T : {MVT::v2i64, MVT::v2f64}) | |||
251 | setOperationAction(Op, T, Expand); | |||
252 | ||||
253 | // But saturating fp_to_int converstions are | |||
254 | for (auto Op : {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}) | |||
255 | setOperationAction(Op, MVT::v4i32, Custom); | |||
256 | } | |||
257 | ||||
258 | // As a special case, these operators use the type to mean the type to | |||
259 | // sign-extend from. | |||
260 | setOperationAction(ISD::SIGN_EXTEND_INREG, MVT::i1, Expand); | |||
261 | if (!Subtarget->hasSignExt()) { | |||
262 | // Sign extends are legal only when extending a vector extract | |||
263 | auto Action = Subtarget->hasSIMD128() ? Custom : Expand; | |||
264 | for (auto T : {MVT::i8, MVT::i16, MVT::i32}) | |||
265 | setOperationAction(ISD::SIGN_EXTEND_INREG, T, Action); | |||
266 | } | |||
267 | for (auto T : MVT::integer_fixedlen_vector_valuetypes()) | |||
268 | setOperationAction(ISD::SIGN_EXTEND_INREG, T, Expand); | |||
269 | ||||
270 | // Dynamic stack allocation: use the default expansion. | |||
271 | setOperationAction(ISD::STACKSAVE, MVT::Other, Expand); | |||
272 | setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand); | |||
273 | setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand); | |||
274 | ||||
275 | setOperationAction(ISD::FrameIndex, MVT::i32, Custom); | |||
276 | setOperationAction(ISD::FrameIndex, MVT::i64, Custom); | |||
277 | setOperationAction(ISD::CopyToReg, MVT::Other, Custom); | |||
278 | ||||
279 | // Expand these forms; we pattern-match the forms that we can handle in isel. | |||
280 | for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) | |||
281 | for (auto Op : {ISD::BR_CC, ISD::SELECT_CC}) | |||
282 | setOperationAction(Op, T, Expand); | |||
283 | ||||
284 | // We have custom switch handling. | |||
285 | setOperationAction(ISD::BR_JT, MVT::Other, Custom); | |||
286 | ||||
287 | // WebAssembly doesn't have: | |||
288 | // - Floating-point extending loads. | |||
289 | // - Floating-point truncating stores. | |||
290 | // - i1 extending loads. | |||
291 | // - truncating SIMD stores and most extending loads | |||
292 | setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand); | |||
293 | setTruncStoreAction(MVT::f64, MVT::f32, Expand); | |||
294 | for (auto T : MVT::integer_valuetypes()) | |||
295 | for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}) | |||
296 | setLoadExtAction(Ext, T, MVT::i1, Promote); | |||
297 | if (Subtarget->hasSIMD128()) { | |||
298 | for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, | |||
299 | MVT::v2f64}) { | |||
300 | for (auto MemT : MVT::fixedlen_vector_valuetypes()) { | |||
301 | if (MVT(T) != MemT) { | |||
302 | setTruncStoreAction(T, MemT, Expand); | |||
303 | for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD}) | |||
304 | setLoadExtAction(Ext, T, MemT, Expand); | |||
305 | } | |||
306 | } | |||
307 | } | |||
308 | // But some vector extending loads are legal | |||
309 | for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) { | |||
310 | setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal); | |||
311 | setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal); | |||
312 | setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal); | |||
313 | } | |||
314 | setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal); | |||
315 | } | |||
316 | ||||
317 | // Don't do anything clever with build_pairs | |||
318 | setOperationAction(ISD::BUILD_PAIR, MVT::i64, Expand); | |||
319 | ||||
320 | // Trap lowers to wasm unreachable | |||
321 | setOperationAction(ISD::TRAP, MVT::Other, Legal); | |||
322 | setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal); | |||
323 | ||||
324 | // Exception handling intrinsics | |||
325 | setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom); | |||
326 | setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom); | |||
327 | setOperationAction(ISD::INTRINSIC_VOID, MVT::Other, Custom); | |||
328 | ||||
329 | setMaxAtomicSizeInBitsSupported(64); | |||
330 | ||||
331 | // Override the __gnu_f2h_ieee/__gnu_h2f_ieee names so that the f32 name is | |||
332 | // consistent with the f64 and f128 names. | |||
333 | setLibcallName(RTLIB::FPEXT_F16_F32, "__extendhfsf2"); | |||
334 | setLibcallName(RTLIB::FPROUND_F32_F16, "__truncsfhf2"); | |||
335 | ||||
336 | // Define the emscripten name for return address helper. | |||
337 | // TODO: when implementing other Wasm backends, make this generic or only do | |||
338 | // this on emscripten depending on what they end up doing. | |||
339 | setLibcallName(RTLIB::RETURN_ADDRESS, "emscripten_return_address"); | |||
340 | ||||
341 | // Always convert switches to br_tables unless there is only one case, which | |||
342 | // is equivalent to a simple branch. This reduces code size for wasm, and we | |||
343 | // defer possible jump table optimizations to the VM. | |||
344 | setMinimumJumpTableEntries(2); | |||
345 | } | |||
346 | ||||
347 | MVT WebAssemblyTargetLowering::getPointerTy(const DataLayout &DL, | |||
348 | uint32_t AS) const { | |||
349 | if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF) | |||
350 | return MVT::externref; | |||
351 | if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF) | |||
352 | return MVT::funcref; | |||
353 | return TargetLowering::getPointerTy(DL, AS); | |||
354 | } | |||
355 | ||||
356 | MVT WebAssemblyTargetLowering::getPointerMemTy(const DataLayout &DL, | |||
357 | uint32_t AS) const { | |||
358 | if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_EXTERNREF) | |||
359 | return MVT::externref; | |||
360 | if (AS == WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF) | |||
361 | return MVT::funcref; | |||
362 | return TargetLowering::getPointerMemTy(DL, AS); | |||
363 | } | |||
364 | ||||
365 | TargetLowering::AtomicExpansionKind | |||
366 | WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { | |||
367 | // We have wasm instructions for these | |||
368 | switch (AI->getOperation()) { | |||
369 | case AtomicRMWInst::Add: | |||
370 | case AtomicRMWInst::Sub: | |||
371 | case AtomicRMWInst::And: | |||
372 | case AtomicRMWInst::Or: | |||
373 | case AtomicRMWInst::Xor: | |||
374 | case AtomicRMWInst::Xchg: | |||
375 | return AtomicExpansionKind::None; | |||
376 | default: | |||
377 | break; | |||
378 | } | |||
379 | return AtomicExpansionKind::CmpXChg; | |||
380 | } | |||
381 | ||||
382 | bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const { | |||
383 | // Implementation copied from X86TargetLowering. | |||
384 | unsigned Opc = VecOp.getOpcode(); | |||
385 | ||||
386 | // Assume target opcodes can't be scalarized. | |||
387 | // TODO - do we have any exceptions? | |||
388 | if (Opc >= ISD::BUILTIN_OP_END) | |||
389 | return false; | |||
390 | ||||
391 | // If the vector op is not supported, try to convert to scalar. | |||
392 | EVT VecVT = VecOp.getValueType(); | |||
393 | if (!isOperationLegalOrCustomOrPromote(Opc, VecVT)) | |||
394 | return true; | |||
395 | ||||
396 | // If the vector op is supported, but the scalar op is not, the transform may | |||
397 | // not be worthwhile. | |||
398 | EVT ScalarVT = VecVT.getScalarType(); | |||
399 | return isOperationLegalOrCustomOrPromote(Opc, ScalarVT); | |||
400 | } | |||
401 | ||||
402 | FastISel *WebAssemblyTargetLowering::createFastISel( | |||
403 | FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const { | |||
404 | return WebAssembly::createFastISel(FuncInfo, LibInfo); | |||
405 | } | |||
406 | ||||
407 | MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/, | |||
408 | EVT VT) const { | |||
409 | unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1); | |||
410 | if (BitWidth > 1 && BitWidth < 8) | |||
411 | BitWidth = 8; | |||
412 | ||||
413 | if (BitWidth > 64) { | |||
414 | // The shift will be lowered to a libcall, and compiler-rt libcalls expect | |||
415 | // the count to be an i32. | |||
416 | BitWidth = 32; | |||
417 | assert(BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) &&(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits ()) && "32-bit shift counts ought to be enough for anyone" ) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 418 , __extension__ __PRETTY_FUNCTION__)) | |||
418 | "32-bit shift counts ought to be enough for anyone")(static_cast <bool> (BitWidth >= Log2_32_Ceil(VT.getSizeInBits ()) && "32-bit shift counts ought to be enough for anyone" ) ? void (0) : __assert_fail ("BitWidth >= Log2_32_Ceil(VT.getSizeInBits()) && \"32-bit shift counts ought to be enough for anyone\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 418 , __extension__ __PRETTY_FUNCTION__)); | |||
419 | } | |||
420 | ||||
421 | MVT Result = MVT::getIntegerVT(BitWidth); | |||
422 | assert(Result != MVT::INVALID_SIMPLE_VALUE_TYPE &&(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE && "Unable to represent scalar shift amount type") ? void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 423 , __extension__ __PRETTY_FUNCTION__)) | |||
423 | "Unable to represent scalar shift amount type")(static_cast <bool> (Result != MVT::INVALID_SIMPLE_VALUE_TYPE && "Unable to represent scalar shift amount type") ? void (0) : __assert_fail ("Result != MVT::INVALID_SIMPLE_VALUE_TYPE && \"Unable to represent scalar shift amount type\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 423 , __extension__ __PRETTY_FUNCTION__)); | |||
424 | return Result; | |||
425 | } | |||
426 | ||||
427 | // Lower an fp-to-int conversion operator from the LLVM opcode, which has an | |||
428 | // undefined result on invalid/overflow, to the WebAssembly opcode, which | |||
429 | // traps on invalid/overflow. | |||
430 | static MachineBasicBlock *LowerFPToInt(MachineInstr &MI, DebugLoc DL, | |||
431 | MachineBasicBlock *BB, | |||
432 | const TargetInstrInfo &TII, | |||
433 | bool IsUnsigned, bool Int64, | |||
434 | bool Float64, unsigned LoweredOpcode) { | |||
435 | MachineRegisterInfo &MRI = BB->getParent()->getRegInfo(); | |||
436 | ||||
437 | Register OutReg = MI.getOperand(0).getReg(); | |||
438 | Register InReg = MI.getOperand(1).getReg(); | |||
439 | ||||
440 | unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32; | |||
441 | unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32; | |||
442 | unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32; | |||
443 | unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32; | |||
444 | unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32; | |||
445 | unsigned Eqz = WebAssembly::EQZ_I32; | |||
446 | unsigned And = WebAssembly::AND_I32; | |||
447 | int64_t Limit = Int64 ? INT64_MIN(-9223372036854775807L -1) : INT32_MIN(-2147483647-1); | |||
448 | int64_t Substitute = IsUnsigned ? 0 : Limit; | |||
449 | double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit; | |||
450 | auto &Context = BB->getParent()->getFunction().getContext(); | |||
451 | Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context); | |||
452 | ||||
453 | const BasicBlock *LLVMBB = BB->getBasicBlock(); | |||
454 | MachineFunction *F = BB->getParent(); | |||
455 | MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB); | |||
456 | MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB); | |||
457 | MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB); | |||
458 | ||||
459 | MachineFunction::iterator It = ++BB->getIterator(); | |||
460 | F->insert(It, FalseMBB); | |||
461 | F->insert(It, TrueMBB); | |||
462 | F->insert(It, DoneMBB); | |||
463 | ||||
464 | // Transfer the remainder of BB and its successor edges to DoneMBB. | |||
465 | DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end()); | |||
466 | DoneMBB->transferSuccessorsAndUpdatePHIs(BB); | |||
467 | ||||
468 | BB->addSuccessor(TrueMBB); | |||
469 | BB->addSuccessor(FalseMBB); | |||
470 | TrueMBB->addSuccessor(DoneMBB); | |||
471 | FalseMBB->addSuccessor(DoneMBB); | |||
472 | ||||
473 | unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg; | |||
474 | Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); | |||
475 | Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); | |||
476 | CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); | |||
477 | EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); | |||
478 | FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg)); | |||
479 | TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg)); | |||
480 | ||||
481 | MI.eraseFromParent(); | |||
482 | // For signed numbers, we can do a single comparison to determine whether | |||
483 | // fabs(x) is within range. | |||
484 | if (IsUnsigned) { | |||
485 | Tmp0 = InReg; | |||
486 | } else { | |||
487 | BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg); | |||
488 | } | |||
489 | BuildMI(BB, DL, TII.get(FConst), Tmp1) | |||
490 | .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal))); | |||
491 | BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1); | |||
492 | ||||
493 | // For unsigned numbers, we have to do a separate comparison with zero. | |||
494 | if (IsUnsigned) { | |||
495 | Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg)); | |||
496 | Register SecondCmpReg = | |||
497 | MRI.createVirtualRegister(&WebAssembly::I32RegClass); | |||
498 | Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass); | |||
499 | BuildMI(BB, DL, TII.get(FConst), Tmp1) | |||
500 | .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0))); | |||
501 | BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1); | |||
502 | BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg); | |||
503 | CmpReg = AndReg; | |||
504 | } | |||
505 | ||||
506 | BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg); | |||
507 | ||||
508 | // Create the CFG diamond to select between doing the conversion or using | |||
509 | // the substitute value. | |||
510 | BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg); | |||
511 | BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg); | |||
512 | BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB); | |||
513 | BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute); | |||
514 | BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg) | |||
515 | .addReg(FalseReg) | |||
516 | .addMBB(FalseMBB) | |||
517 | .addReg(TrueReg) | |||
518 | .addMBB(TrueMBB); | |||
519 | ||||
520 | return DoneMBB; | |||
521 | } | |||
522 | ||||
523 | static MachineBasicBlock * | |||
524 | LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, | |||
525 | const WebAssemblySubtarget *Subtarget, | |||
526 | const TargetInstrInfo &TII) { | |||
527 | MachineInstr &CallParams = *CallResults.getPrevNode(); | |||
528 | assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS)(static_cast <bool> (CallParams.getOpcode() == WebAssembly ::CALL_PARAMS) ? void (0) : __assert_fail ("CallParams.getOpcode() == WebAssembly::CALL_PARAMS" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 528 , __extension__ __PRETTY_FUNCTION__)); | |||
529 | assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||(static_cast <bool> (CallResults.getOpcode() == WebAssembly ::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS ) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 530 , __extension__ __PRETTY_FUNCTION__)) | |||
530 | CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS)(static_cast <bool> (CallResults.getOpcode() == WebAssembly ::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS ) ? void (0) : __assert_fail ("CallResults.getOpcode() == WebAssembly::CALL_RESULTS || CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 530 , __extension__ __PRETTY_FUNCTION__)); | |||
531 | ||||
532 | bool IsIndirect = CallParams.getOperand(0).isReg(); | |||
533 | bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS; | |||
534 | ||||
535 | bool IsFuncrefCall = false; | |||
536 | if (IsIndirect) { | |||
537 | Register Reg = CallParams.getOperand(0).getReg(); | |||
538 | const MachineFunction *MF = BB->getParent(); | |||
539 | const MachineRegisterInfo &MRI = MF->getRegInfo(); | |||
540 | const TargetRegisterClass *TRC = MRI.getRegClass(Reg); | |||
541 | IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass); | |||
542 | assert(!IsFuncrefCall || Subtarget->hasReferenceTypes())(static_cast <bool> (!IsFuncrefCall || Subtarget->hasReferenceTypes ()) ? void (0) : __assert_fail ("!IsFuncrefCall || Subtarget->hasReferenceTypes()" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 542 , __extension__ __PRETTY_FUNCTION__)); | |||
543 | } | |||
544 | ||||
545 | unsigned CallOp; | |||
546 | if (IsIndirect && IsRetCall) { | |||
547 | CallOp = WebAssembly::RET_CALL_INDIRECT; | |||
548 | } else if (IsIndirect) { | |||
549 | CallOp = WebAssembly::CALL_INDIRECT; | |||
550 | } else if (IsRetCall) { | |||
551 | CallOp = WebAssembly::RET_CALL; | |||
552 | } else { | |||
553 | CallOp = WebAssembly::CALL; | |||
554 | } | |||
555 | ||||
556 | MachineFunction &MF = *BB->getParent(); | |||
557 | const MCInstrDesc &MCID = TII.get(CallOp); | |||
558 | MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL)); | |||
559 | ||||
560 | // See if we must truncate the function pointer. | |||
561 | // CALL_INDIRECT takes an i32, but in wasm64 we represent function pointers | |||
562 | // as 64-bit for uniformity with other pointer types. | |||
563 | // See also: WebAssemblyFastISel::selectCall | |||
564 | if (IsIndirect && MF.getSubtarget<WebAssemblySubtarget>().hasAddr64()) { | |||
565 | Register Reg32 = | |||
566 | MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass); | |||
567 | auto &FnPtr = CallParams.getOperand(0); | |||
568 | BuildMI(*BB, CallResults.getIterator(), DL, | |||
569 | TII.get(WebAssembly::I32_WRAP_I64), Reg32) | |||
570 | .addReg(FnPtr.getReg()); | |||
571 | FnPtr.setReg(Reg32); | |||
572 | } | |||
573 | ||||
574 | // Move the function pointer to the end of the arguments for indirect calls | |||
575 | if (IsIndirect) { | |||
576 | auto FnPtr = CallParams.getOperand(0); | |||
577 | CallParams.removeOperand(0); | |||
578 | ||||
579 | // For funcrefs, call_indirect is done through __funcref_call_table and the | |||
580 | // funcref is always installed in slot 0 of the table, therefore instead of having | |||
581 | // the function pointer added at the end of the params list, a zero (the index in | |||
582 | // __funcref_call_table is added). | |||
583 | if (IsFuncrefCall) { | |||
584 | Register RegZero = | |||
585 | MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass); | |||
586 | MachineInstrBuilder MIBC0 = | |||
587 | BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0); | |||
588 | ||||
589 | BB->insert(CallResults.getIterator(), MIBC0); | |||
590 | MachineInstrBuilder(MF, CallParams).addReg(RegZero); | |||
591 | } else | |||
592 | CallParams.addOperand(FnPtr); | |||
593 | } | |||
594 | ||||
595 | for (auto Def : CallResults.defs()) | |||
596 | MIB.add(Def); | |||
597 | ||||
598 | if (IsIndirect) { | |||
599 | // Placeholder for the type index. | |||
600 | MIB.addImm(0); | |||
601 | // The table into which this call_indirect indexes. | |||
602 | MCSymbolWasm *Table = IsFuncrefCall | |||
603 | ? WebAssembly::getOrCreateFuncrefCallTableSymbol( | |||
604 | MF.getContext(), Subtarget) | |||
605 | : WebAssembly::getOrCreateFunctionTableSymbol( | |||
606 | MF.getContext(), Subtarget); | |||
607 | if (Subtarget->hasReferenceTypes()) { | |||
608 | MIB.addSym(Table); | |||
609 | } else { | |||
610 | // For the MVP there is at most one table whose number is 0, but we can't | |||
611 | // write a table symbol or issue relocations. Instead we just ensure the | |||
612 | // table is live and write a zero. | |||
613 | Table->setNoStrip(); | |||
614 | MIB.addImm(0); | |||
615 | } | |||
616 | } | |||
617 | ||||
618 | for (auto Use : CallParams.uses()) | |||
619 | MIB.add(Use); | |||
620 | ||||
621 | BB->insert(CallResults.getIterator(), MIB); | |||
622 | CallParams.eraseFromParent(); | |||
623 | CallResults.eraseFromParent(); | |||
624 | ||||
625 | // If this is a funcref call, to avoid hidden GC roots, we need to clear the | |||
626 | // table slot with ref.null upon call_indirect return. | |||
627 | // | |||
628 | // This generates the following code, which comes right after a call_indirect | |||
629 | // of a funcref: | |||
630 | // | |||
631 | // i32.const 0 | |||
632 | // ref.null func | |||
633 | // table.set __funcref_call_table | |||
634 | if (IsIndirect && IsFuncrefCall) { | |||
635 | MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol( | |||
636 | MF.getContext(), Subtarget); | |||
637 | Register RegZero = | |||
638 | MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass); | |||
639 | MachineInstr *Const0 = | |||
640 | BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0); | |||
641 | BB->insertAfter(MIB.getInstr()->getIterator(), Const0); | |||
642 | ||||
643 | Register RegFuncref = | |||
644 | MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass); | |||
645 | MachineInstr *RefNull = | |||
646 | BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref); | |||
647 | BB->insertAfter(Const0->getIterator(), RefNull); | |||
648 | ||||
649 | MachineInstr *TableSet = | |||
650 | BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF)) | |||
651 | .addSym(Table) | |||
652 | .addReg(RegZero) | |||
653 | .addReg(RegFuncref); | |||
654 | BB->insertAfter(RefNull->getIterator(), TableSet); | |||
655 | } | |||
656 | ||||
657 | return BB; | |||
658 | } | |||
659 | ||||
660 | MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter( | |||
661 | MachineInstr &MI, MachineBasicBlock *BB) const { | |||
662 | const TargetInstrInfo &TII = *Subtarget->getInstrInfo(); | |||
663 | DebugLoc DL = MI.getDebugLoc(); | |||
664 | ||||
665 | switch (MI.getOpcode()) { | |||
666 | default: | |||
667 | llvm_unreachable("Unexpected instr type to insert")::llvm::llvm_unreachable_internal("Unexpected instr type to insert" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 667 ); | |||
668 | case WebAssembly::FP_TO_SINT_I32_F32: | |||
669 | return LowerFPToInt(MI, DL, BB, TII, false, false, false, | |||
670 | WebAssembly::I32_TRUNC_S_F32); | |||
671 | case WebAssembly::FP_TO_UINT_I32_F32: | |||
672 | return LowerFPToInt(MI, DL, BB, TII, true, false, false, | |||
673 | WebAssembly::I32_TRUNC_U_F32); | |||
674 | case WebAssembly::FP_TO_SINT_I64_F32: | |||
675 | return LowerFPToInt(MI, DL, BB, TII, false, true, false, | |||
676 | WebAssembly::I64_TRUNC_S_F32); | |||
677 | case WebAssembly::FP_TO_UINT_I64_F32: | |||
678 | return LowerFPToInt(MI, DL, BB, TII, true, true, false, | |||
679 | WebAssembly::I64_TRUNC_U_F32); | |||
680 | case WebAssembly::FP_TO_SINT_I32_F64: | |||
681 | return LowerFPToInt(MI, DL, BB, TII, false, false, true, | |||
682 | WebAssembly::I32_TRUNC_S_F64); | |||
683 | case WebAssembly::FP_TO_UINT_I32_F64: | |||
684 | return LowerFPToInt(MI, DL, BB, TII, true, false, true, | |||
685 | WebAssembly::I32_TRUNC_U_F64); | |||
686 | case WebAssembly::FP_TO_SINT_I64_F64: | |||
687 | return LowerFPToInt(MI, DL, BB, TII, false, true, true, | |||
688 | WebAssembly::I64_TRUNC_S_F64); | |||
689 | case WebAssembly::FP_TO_UINT_I64_F64: | |||
690 | return LowerFPToInt(MI, DL, BB, TII, true, true, true, | |||
691 | WebAssembly::I64_TRUNC_U_F64); | |||
692 | case WebAssembly::CALL_RESULTS: | |||
693 | case WebAssembly::RET_CALL_RESULTS: | |||
694 | return LowerCallResults(MI, DL, BB, Subtarget, TII); | |||
695 | } | |||
696 | } | |||
697 | ||||
698 | const char * | |||
699 | WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const { | |||
700 | switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) { | |||
701 | case WebAssemblyISD::FIRST_NUMBER: | |||
702 | case WebAssemblyISD::FIRST_MEM_OPCODE: | |||
703 | break; | |||
704 | #define HANDLE_NODETYPE(NODE) \ | |||
705 | case WebAssemblyISD::NODE: \ | |||
706 | return "WebAssemblyISD::" #NODE; | |||
707 | #define HANDLE_MEM_NODETYPE(NODE) HANDLE_NODETYPE(NODE) | |||
708 | #include "WebAssemblyISD.def" | |||
709 | #undef HANDLE_MEM_NODETYPE | |||
710 | #undef HANDLE_NODETYPE | |||
711 | } | |||
712 | return nullptr; | |||
713 | } | |||
714 | ||||
715 | std::pair<unsigned, const TargetRegisterClass *> | |||
716 | WebAssemblyTargetLowering::getRegForInlineAsmConstraint( | |||
717 | const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const { | |||
718 | // First, see if this is a constraint that directly corresponds to a | |||
719 | // WebAssembly register class. | |||
720 | if (Constraint.size() == 1) { | |||
721 | switch (Constraint[0]) { | |||
722 | case 'r': | |||
723 | assert(VT != MVT::iPTR && "Pointer MVT not expected here")(static_cast <bool> (VT != MVT::iPTR && "Pointer MVT not expected here" ) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Pointer MVT not expected here\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 723 , __extension__ __PRETTY_FUNCTION__)); | |||
724 | if (Subtarget->hasSIMD128() && VT.isVector()) { | |||
725 | if (VT.getSizeInBits() == 128) | |||
726 | return std::make_pair(0U, &WebAssembly::V128RegClass); | |||
727 | } | |||
728 | if (VT.isInteger() && !VT.isVector()) { | |||
729 | if (VT.getSizeInBits() <= 32) | |||
730 | return std::make_pair(0U, &WebAssembly::I32RegClass); | |||
731 | if (VT.getSizeInBits() <= 64) | |||
732 | return std::make_pair(0U, &WebAssembly::I64RegClass); | |||
733 | } | |||
734 | if (VT.isFloatingPoint() && !VT.isVector()) { | |||
735 | switch (VT.getSizeInBits()) { | |||
736 | case 32: | |||
737 | return std::make_pair(0U, &WebAssembly::F32RegClass); | |||
738 | case 64: | |||
739 | return std::make_pair(0U, &WebAssembly::F64RegClass); | |||
740 | default: | |||
741 | break; | |||
742 | } | |||
743 | } | |||
744 | break; | |||
745 | default: | |||
746 | break; | |||
747 | } | |||
748 | } | |||
749 | ||||
750 | return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT); | |||
751 | } | |||
752 | ||||
753 | bool WebAssemblyTargetLowering::isCheapToSpeculateCttz() const { | |||
754 | // Assume ctz is a relatively cheap operation. | |||
755 | return true; | |||
756 | } | |||
757 | ||||
758 | bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz() const { | |||
759 | // Assume clz is a relatively cheap operation. | |||
760 | return true; | |||
761 | } | |||
762 | ||||
763 | bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL, | |||
764 | const AddrMode &AM, | |||
765 | Type *Ty, unsigned AS, | |||
766 | Instruction *I) const { | |||
767 | // WebAssembly offsets are added as unsigned without wrapping. The | |||
768 | // isLegalAddressingMode gives us no way to determine if wrapping could be | |||
769 | // happening, so we approximate this by accepting only non-negative offsets. | |||
770 | if (AM.BaseOffs < 0) | |||
771 | return false; | |||
772 | ||||
773 | // WebAssembly has no scale register operands. | |||
774 | if (AM.Scale != 0) | |||
775 | return false; | |||
776 | ||||
777 | // Everything else is legal. | |||
778 | return true; | |||
779 | } | |||
780 | ||||
781 | bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses( | |||
782 | EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/, | |||
783 | MachineMemOperand::Flags /*Flags*/, bool *Fast) const { | |||
784 | // WebAssembly supports unaligned accesses, though it should be declared | |||
785 | // with the p2align attribute on loads and stores which do so, and there | |||
786 | // may be a performance impact. We tell LLVM they're "fast" because | |||
787 | // for the kinds of things that LLVM uses this for (merging adjacent stores | |||
788 | // of constants, etc.), WebAssembly implementations will either want the | |||
789 | // unaligned access or they'll split anyway. | |||
790 | if (Fast) | |||
791 | *Fast = true; | |||
792 | return true; | |||
793 | } | |||
794 | ||||
795 | bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT, | |||
796 | AttributeList Attr) const { | |||
797 | // The current thinking is that wasm engines will perform this optimization, | |||
798 | // so we can save on code size. | |||
799 | return true; | |||
800 | } | |||
801 | ||||
802 | bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const { | |||
803 | EVT ExtT = ExtVal.getValueType(); | |||
804 | EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0); | |||
805 | return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) || | |||
806 | (ExtT == MVT::v4i32 && MemT == MVT::v4i16) || | |||
807 | (ExtT == MVT::v2i64 && MemT == MVT::v2i32); | |||
808 | } | |||
809 | ||||
810 | bool WebAssemblyTargetLowering::isOffsetFoldingLegal( | |||
811 | const GlobalAddressSDNode *GA) const { | |||
812 | // Wasm doesn't support function addresses with offsets | |||
813 | const GlobalValue *GV = GA->getGlobal(); | |||
814 | return isa<Function>(GV) ? false : TargetLowering::isOffsetFoldingLegal(GA); | |||
815 | } | |||
816 | ||||
817 | EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL, | |||
818 | LLVMContext &C, | |||
819 | EVT VT) const { | |||
820 | if (VT.isVector()) | |||
821 | return VT.changeVectorElementTypeToInteger(); | |||
822 | ||||
823 | // So far, all branch instructions in Wasm take an I32 condition. | |||
824 | // The default TargetLowering::getSetCCResultType returns the pointer size, | |||
825 | // which would be useful to reduce instruction counts when testing | |||
826 | // against 64-bit pointers/values if at some point Wasm supports that. | |||
827 | return EVT::getIntegerVT(C, 32); | |||
828 | } | |||
829 | ||||
830 | bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, | |||
831 | const CallInst &I, | |||
832 | MachineFunction &MF, | |||
833 | unsigned Intrinsic) const { | |||
834 | switch (Intrinsic) { | |||
835 | case Intrinsic::wasm_memory_atomic_notify: | |||
836 | Info.opc = ISD::INTRINSIC_W_CHAIN; | |||
837 | Info.memVT = MVT::i32; | |||
838 | Info.ptrVal = I.getArgOperand(0); | |||
839 | Info.offset = 0; | |||
840 | Info.align = Align(4); | |||
841 | // atomic.notify instruction does not really load the memory specified with | |||
842 | // this argument, but MachineMemOperand should either be load or store, so | |||
843 | // we set this to a load. | |||
844 | // FIXME Volatile isn't really correct, but currently all LLVM atomic | |||
845 | // instructions are treated as volatiles in the backend, so we should be | |||
846 | // consistent. The same applies for wasm_atomic_wait intrinsics too. | |||
847 | Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; | |||
848 | return true; | |||
849 | case Intrinsic::wasm_memory_atomic_wait32: | |||
850 | Info.opc = ISD::INTRINSIC_W_CHAIN; | |||
851 | Info.memVT = MVT::i32; | |||
852 | Info.ptrVal = I.getArgOperand(0); | |||
853 | Info.offset = 0; | |||
854 | Info.align = Align(4); | |||
855 | Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; | |||
856 | return true; | |||
857 | case Intrinsic::wasm_memory_atomic_wait64: | |||
858 | Info.opc = ISD::INTRINSIC_W_CHAIN; | |||
859 | Info.memVT = MVT::i64; | |||
860 | Info.ptrVal = I.getArgOperand(0); | |||
861 | Info.offset = 0; | |||
862 | Info.align = Align(8); | |||
863 | Info.flags = MachineMemOperand::MOVolatile | MachineMemOperand::MOLoad; | |||
864 | return true; | |||
865 | default: | |||
866 | return false; | |||
867 | } | |||
868 | } | |||
869 | ||||
870 | void WebAssemblyTargetLowering::computeKnownBitsForTargetNode( | |||
871 | const SDValue Op, KnownBits &Known, const APInt &DemandedElts, | |||
872 | const SelectionDAG &DAG, unsigned Depth) const { | |||
873 | switch (Op.getOpcode()) { | |||
874 | default: | |||
875 | break; | |||
876 | case ISD::INTRINSIC_WO_CHAIN: { | |||
877 | unsigned IntNo = Op.getConstantOperandVal(0); | |||
878 | switch (IntNo) { | |||
879 | default: | |||
880 | break; | |||
881 | case Intrinsic::wasm_bitmask: { | |||
882 | unsigned BitWidth = Known.getBitWidth(); | |||
883 | EVT VT = Op.getOperand(1).getSimpleValueType(); | |||
884 | unsigned PossibleBits = VT.getVectorNumElements(); | |||
885 | APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits); | |||
886 | Known.Zero |= ZeroMask; | |||
887 | break; | |||
888 | } | |||
889 | } | |||
890 | } | |||
891 | } | |||
892 | } | |||
893 | ||||
894 | TargetLoweringBase::LegalizeTypeAction | |||
895 | WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const { | |||
896 | if (VT.isFixedLengthVector()) { | |||
897 | MVT EltVT = VT.getVectorElementType(); | |||
898 | // We have legal vector types with these lane types, so widening the | |||
899 | // vector would let us use some of the lanes directly without having to | |||
900 | // extend or truncate values. | |||
901 | if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 || | |||
902 | EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64) | |||
903 | return TypeWidenVector; | |||
904 | } | |||
905 | ||||
906 | return TargetLoweringBase::getPreferredVectorAction(VT); | |||
907 | } | |||
908 | ||||
909 | bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts( | |||
910 | SDValue Op, const TargetLoweringOpt &TLO) const { | |||
911 | // ISel process runs DAGCombiner after legalization; this step is called | |||
912 | // SelectionDAG optimization phase. This post-legalization combining process | |||
913 | // runs DAGCombiner on each node, and if there was a change to be made, | |||
914 | // re-runs legalization again on it and its user nodes to make sure | |||
915 | // everythiing is in a legalized state. | |||
916 | // | |||
917 | // The legalization calls lowering routines, and we do our custom lowering for | |||
918 | // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements | |||
919 | // into zeros. But there is a set of routines in DAGCombiner that turns unused | |||
920 | // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts | |||
921 | // turns unused vector elements into undefs. But this routine does not work | |||
922 | // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This | |||
923 | // combination can result in a infinite loop, in which undefs are converted to | |||
924 | // zeros in legalization and back to undefs in combining. | |||
925 | // | |||
926 | // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from | |||
927 | // running for build_vectors. | |||
928 | if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys) | |||
929 | return false; | |||
930 | return true; | |||
931 | } | |||
932 | ||||
933 | //===----------------------------------------------------------------------===// | |||
934 | // WebAssembly Lowering private implementation. | |||
935 | //===----------------------------------------------------------------------===// | |||
936 | ||||
937 | //===----------------------------------------------------------------------===// | |||
938 | // Lowering Code | |||
939 | //===----------------------------------------------------------------------===// | |||
940 | ||||
941 | static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) { | |||
942 | MachineFunction &MF = DAG.getMachineFunction(); | |||
943 | DAG.getContext()->diagnose( | |||
944 | DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc())); | |||
945 | } | |||
946 | ||||
947 | // Test whether the given calling convention is supported. | |||
948 | static bool callingConvSupported(CallingConv::ID CallConv) { | |||
949 | // We currently support the language-independent target-independent | |||
950 | // conventions. We don't yet have a way to annotate calls with properties like | |||
951 | // "cold", and we don't have any call-clobbered registers, so these are mostly | |||
952 | // all handled the same. | |||
953 | return CallConv == CallingConv::C || CallConv == CallingConv::Fast || | |||
954 | CallConv == CallingConv::Cold || | |||
955 | CallConv == CallingConv::PreserveMost || | |||
956 | CallConv == CallingConv::PreserveAll || | |||
957 | CallConv == CallingConv::CXX_FAST_TLS || | |||
958 | CallConv == CallingConv::WASM_EmscriptenInvoke || | |||
959 | CallConv == CallingConv::Swift; | |||
960 | } | |||
961 | ||||
962 | SDValue | |||
963 | WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI, | |||
964 | SmallVectorImpl<SDValue> &InVals) const { | |||
965 | SelectionDAG &DAG = CLI.DAG; | |||
966 | SDLoc DL = CLI.DL; | |||
967 | SDValue Chain = CLI.Chain; | |||
968 | SDValue Callee = CLI.Callee; | |||
969 | MachineFunction &MF = DAG.getMachineFunction(); | |||
970 | auto Layout = MF.getDataLayout(); | |||
971 | ||||
972 | CallingConv::ID CallConv = CLI.CallConv; | |||
973 | if (!callingConvSupported(CallConv)) | |||
974 | fail(DL, DAG, | |||
975 | "WebAssembly doesn't support language-specific or target-specific " | |||
976 | "calling conventions yet"); | |||
977 | if (CLI.IsPatchPoint) | |||
978 | fail(DL, DAG, "WebAssembly doesn't support patch point yet"); | |||
979 | ||||
980 | if (CLI.IsTailCall) { | |||
981 | auto NoTail = [&](const char *Msg) { | |||
982 | if (CLI.CB && CLI.CB->isMustTailCall()) | |||
983 | fail(DL, DAG, Msg); | |||
984 | CLI.IsTailCall = false; | |||
985 | }; | |||
986 | ||||
987 | if (!Subtarget->hasTailCall()) | |||
988 | NoTail("WebAssembly 'tail-call' feature not enabled"); | |||
989 | ||||
990 | // Varargs calls cannot be tail calls because the buffer is on the stack | |||
991 | if (CLI.IsVarArg) | |||
992 | NoTail("WebAssembly does not support varargs tail calls"); | |||
993 | ||||
994 | // Do not tail call unless caller and callee return types match | |||
995 | const Function &F = MF.getFunction(); | |||
996 | const TargetMachine &TM = getTargetMachine(); | |||
997 | Type *RetTy = F.getReturnType(); | |||
998 | SmallVector<MVT, 4> CallerRetTys; | |||
999 | SmallVector<MVT, 4> CalleeRetTys; | |||
1000 | computeLegalValueVTs(F, TM, RetTy, CallerRetTys); | |||
1001 | computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys); | |||
1002 | bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() && | |||
1003 | std::equal(CallerRetTys.begin(), CallerRetTys.end(), | |||
1004 | CalleeRetTys.begin()); | |||
1005 | if (!TypesMatch) | |||
1006 | NoTail("WebAssembly tail call requires caller and callee return types to " | |||
1007 | "match"); | |||
1008 | ||||
1009 | // If pointers to local stack values are passed, we cannot tail call | |||
1010 | if (CLI.CB) { | |||
1011 | for (auto &Arg : CLI.CB->args()) { | |||
1012 | Value *Val = Arg.get(); | |||
1013 | // Trace the value back through pointer operations | |||
1014 | while (true) { | |||
1015 | Value *Src = Val->stripPointerCastsAndAliases(); | |||
1016 | if (auto *GEP = dyn_cast<GetElementPtrInst>(Src)) | |||
1017 | Src = GEP->getPointerOperand(); | |||
1018 | if (Val == Src) | |||
1019 | break; | |||
1020 | Val = Src; | |||
1021 | } | |||
1022 | if (isa<AllocaInst>(Val)) { | |||
1023 | NoTail( | |||
1024 | "WebAssembly does not support tail calling with stack arguments"); | |||
1025 | break; | |||
1026 | } | |||
1027 | } | |||
1028 | } | |||
1029 | } | |||
1030 | ||||
1031 | SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins; | |||
1032 | SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs; | |||
1033 | SmallVectorImpl<SDValue> &OutVals = CLI.OutVals; | |||
1034 | ||||
1035 | // The generic code may have added an sret argument. If we're lowering an | |||
1036 | // invoke function, the ABI requires that the function pointer be the first | |||
1037 | // argument, so we may have to swap the arguments. | |||
1038 | if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 && | |||
1039 | Outs[0].Flags.isSRet()) { | |||
1040 | std::swap(Outs[0], Outs[1]); | |||
1041 | std::swap(OutVals[0], OutVals[1]); | |||
1042 | } | |||
1043 | ||||
1044 | bool HasSwiftSelfArg = false; | |||
1045 | bool HasSwiftErrorArg = false; | |||
1046 | unsigned NumFixedArgs = 0; | |||
1047 | for (unsigned I = 0; I < Outs.size(); ++I) { | |||
1048 | const ISD::OutputArg &Out = Outs[I]; | |||
1049 | SDValue &OutVal = OutVals[I]; | |||
1050 | HasSwiftSelfArg |= Out.Flags.isSwiftSelf(); | |||
1051 | HasSwiftErrorArg |= Out.Flags.isSwiftError(); | |||
1052 | if (Out.Flags.isNest()) | |||
1053 | fail(DL, DAG, "WebAssembly hasn't implemented nest arguments"); | |||
1054 | if (Out.Flags.isInAlloca()) | |||
1055 | fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments"); | |||
1056 | if (Out.Flags.isInConsecutiveRegs()) | |||
1057 | fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments"); | |||
1058 | if (Out.Flags.isInConsecutiveRegsLast()) | |||
1059 | fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments"); | |||
1060 | if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) { | |||
1061 | auto &MFI = MF.getFrameInfo(); | |||
1062 | int FI = MFI.CreateStackObject(Out.Flags.getByValSize(), | |||
1063 | Out.Flags.getNonZeroByValAlign(), | |||
1064 | /*isSS=*/false); | |||
1065 | SDValue SizeNode = | |||
1066 | DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32); | |||
1067 | SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout)); | |||
1068 | Chain = DAG.getMemcpy( | |||
1069 | Chain, DL, FINode, OutVal, SizeNode, Out.Flags.getNonZeroByValAlign(), | |||
1070 | /*isVolatile*/ false, /*AlwaysInline=*/false, | |||
1071 | /*isTailCall*/ false, MachinePointerInfo(), MachinePointerInfo()); | |||
1072 | OutVal = FINode; | |||
1073 | } | |||
1074 | // Count the number of fixed args *after* legalization. | |||
1075 | NumFixedArgs += Out.IsFixed; | |||
1076 | } | |||
1077 | ||||
1078 | bool IsVarArg = CLI.IsVarArg; | |||
1079 | auto PtrVT = getPointerTy(Layout); | |||
1080 | ||||
1081 | // For swiftcc, emit additional swiftself and swifterror arguments | |||
1082 | // if there aren't. These additional arguments are also added for callee | |||
1083 | // signature They are necessary to match callee and caller signature for | |||
1084 | // indirect call. | |||
1085 | if (CallConv == CallingConv::Swift) { | |||
1086 | if (!HasSwiftSelfArg) { | |||
1087 | NumFixedArgs++; | |||
1088 | ISD::OutputArg Arg; | |||
1089 | Arg.Flags.setSwiftSelf(); | |||
1090 | CLI.Outs.push_back(Arg); | |||
1091 | SDValue ArgVal = DAG.getUNDEF(PtrVT); | |||
1092 | CLI.OutVals.push_back(ArgVal); | |||
1093 | } | |||
1094 | if (!HasSwiftErrorArg) { | |||
1095 | NumFixedArgs++; | |||
1096 | ISD::OutputArg Arg; | |||
1097 | Arg.Flags.setSwiftError(); | |||
1098 | CLI.Outs.push_back(Arg); | |||
1099 | SDValue ArgVal = DAG.getUNDEF(PtrVT); | |||
1100 | CLI.OutVals.push_back(ArgVal); | |||
1101 | } | |||
1102 | } | |||
1103 | ||||
1104 | // Analyze operands of the call, assigning locations to each operand. | |||
1105 | SmallVector<CCValAssign, 16> ArgLocs; | |||
1106 | CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext()); | |||
1107 | ||||
1108 | if (IsVarArg) { | |||
1109 | // Outgoing non-fixed arguments are placed in a buffer. First | |||
1110 | // compute their offsets and the total amount of buffer space needed. | |||
1111 | for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) { | |||
1112 | const ISD::OutputArg &Out = Outs[I]; | |||
1113 | SDValue &Arg = OutVals[I]; | |||
1114 | EVT VT = Arg.getValueType(); | |||
1115 | assert(VT != MVT::iPTR && "Legalized args should be concrete")(static_cast <bool> (VT != MVT::iPTR && "Legalized args should be concrete" ) ? void (0) : __assert_fail ("VT != MVT::iPTR && \"Legalized args should be concrete\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1115 , __extension__ __PRETTY_FUNCTION__)); | |||
1116 | Type *Ty = VT.getTypeForEVT(*DAG.getContext()); | |||
1117 | Align Alignment = | |||
1118 | std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty)); | |||
1119 | unsigned Offset = | |||
1120 | CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment); | |||
1121 | CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(), | |||
1122 | Offset, VT.getSimpleVT(), | |||
1123 | CCValAssign::Full)); | |||
1124 | } | |||
1125 | } | |||
1126 | ||||
1127 | unsigned NumBytes = CCInfo.getAlignedCallFrameSize(); | |||
1128 | ||||
1129 | SDValue FINode; | |||
1130 | if (IsVarArg && NumBytes) { | |||
1131 | // For non-fixed arguments, next emit stores to store the argument values | |||
1132 | // to the stack buffer at the offsets computed above. | |||
1133 | int FI = MF.getFrameInfo().CreateStackObject(NumBytes, | |||
1134 | Layout.getStackAlignment(), | |||
1135 | /*isSS=*/false); | |||
1136 | unsigned ValNo = 0; | |||
1137 | SmallVector<SDValue, 8> Chains; | |||
1138 | for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) { | |||
1139 | assert(ArgLocs[ValNo].getValNo() == ValNo &&(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo && "ArgLocs should remain in order and only hold varargs args" ) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1140 , __extension__ __PRETTY_FUNCTION__)) | |||
1140 | "ArgLocs should remain in order and only hold varargs args")(static_cast <bool> (ArgLocs[ValNo].getValNo() == ValNo && "ArgLocs should remain in order and only hold varargs args" ) ? void (0) : __assert_fail ("ArgLocs[ValNo].getValNo() == ValNo && \"ArgLocs should remain in order and only hold varargs args\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1140 , __extension__ __PRETTY_FUNCTION__)); | |||
1141 | unsigned Offset = ArgLocs[ValNo++].getLocMemOffset(); | |||
1142 | FINode = DAG.getFrameIndex(FI, getPointerTy(Layout)); | |||
1143 | SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode, | |||
1144 | DAG.getConstant(Offset, DL, PtrVT)); | |||
1145 | Chains.push_back( | |||
1146 | DAG.getStore(Chain, DL, Arg, Add, | |||
1147 | MachinePointerInfo::getFixedStack(MF, FI, Offset))); | |||
1148 | } | |||
1149 | if (!Chains.empty()) | |||
1150 | Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains); | |||
1151 | } else if (IsVarArg) { | |||
1152 | FINode = DAG.getIntPtrConstant(0, DL); | |||
1153 | } | |||
1154 | ||||
1155 | if (Callee->getOpcode() == ISD::GlobalAddress) { | |||
1156 | // If the callee is a GlobalAddress node (quite common, every direct call | |||
1157 | // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress | |||
1158 | // doesn't at MO_GOT which is not needed for direct calls. | |||
1159 | GlobalAddressSDNode* GA = cast<GlobalAddressSDNode>(Callee); | |||
1160 | Callee = DAG.getTargetGlobalAddress(GA->getGlobal(), DL, | |||
1161 | getPointerTy(DAG.getDataLayout()), | |||
1162 | GA->getOffset()); | |||
1163 | Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL, | |||
1164 | getPointerTy(DAG.getDataLayout()), Callee); | |||
1165 | } | |||
1166 | ||||
1167 | // Compute the operands for the CALLn node. | |||
1168 | SmallVector<SDValue, 16> Ops; | |||
1169 | Ops.push_back(Chain); | |||
1170 | Ops.push_back(Callee); | |||
1171 | ||||
1172 | // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs | |||
1173 | // isn't reliable. | |||
1174 | Ops.append(OutVals.begin(), | |||
1175 | IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end()); | |||
1176 | // Add a pointer to the vararg buffer. | |||
1177 | if (IsVarArg) | |||
1178 | Ops.push_back(FINode); | |||
1179 | ||||
1180 | SmallVector<EVT, 8> InTys; | |||
1181 | for (const auto &In : Ins) { | |||
1182 | assert(!In.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!In.Flags.isByVal() && "byval is not valid for return values" ) ? void (0) : __assert_fail ("!In.Flags.isByVal() && \"byval is not valid for return values\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1182 , __extension__ __PRETTY_FUNCTION__)); | |||
1183 | assert(!In.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!In.Flags.isNest() && "nest is not valid for return values" ) ? void (0) : __assert_fail ("!In.Flags.isNest() && \"nest is not valid for return values\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1183 , __extension__ __PRETTY_FUNCTION__)); | |||
1184 | if (In.Flags.isInAlloca()) | |||
1185 | fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values"); | |||
1186 | if (In.Flags.isInConsecutiveRegs()) | |||
1187 | fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values"); | |||
1188 | if (In.Flags.isInConsecutiveRegsLast()) | |||
1189 | fail(DL, DAG, | |||
1190 | "WebAssembly hasn't implemented cons regs last return values"); | |||
1191 | // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in | |||
1192 | // registers. | |||
1193 | InTys.push_back(In.VT); | |||
1194 | } | |||
1195 | ||||
1196 | // Lastly, if this is a call to a funcref we need to add an instruction | |||
1197 | // table.set to the chain and transform the call. | |||
1198 | if (CLI.CB && | |||
1199 | WebAssembly::isFuncrefType(CLI.CB->getCalledOperand()->getType())) { | |||
1200 | // In the absence of function references proposal where a funcref call is | |||
1201 | // lowered to call_ref, using reference types we generate a table.set to set | |||
1202 | // the funcref to a special table used solely for this purpose, followed by | |||
1203 | // a call_indirect. Here we just generate the table set, and return the | |||
1204 | // SDValue of the table.set so that LowerCall can finalize the lowering by | |||
1205 | // generating the call_indirect. | |||
1206 | SDValue Chain = Ops[0]; | |||
1207 | ||||
1208 | MCSymbolWasm *Table = WebAssembly::getOrCreateFuncrefCallTableSymbol( | |||
1209 | MF.getContext(), Subtarget); | |||
1210 | SDValue Sym = DAG.getMCSymbol(Table, PtrVT); | |||
1211 | SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32); | |||
1212 | SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee}; | |||
1213 | SDValue TableSet = DAG.getMemIntrinsicNode( | |||
1214 | WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps, | |||
1215 | MVT::funcref, | |||
1216 | // Machine Mem Operand args | |||
1217 | MachinePointerInfo( | |||
1218 | WebAssembly::WasmAddressSpace::WASM_ADDRESS_SPACE_FUNCREF), | |||
1219 | CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()), | |||
1220 | MachineMemOperand::MOStore); | |||
1221 | ||||
1222 | Ops[0] = TableSet; // The new chain is the TableSet itself | |||
1223 | } | |||
1224 | ||||
1225 | if (CLI.IsTailCall) { | |||
1226 | // ret_calls do not return values to the current frame | |||
1227 | SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue); | |||
1228 | return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops); | |||
1229 | } | |||
1230 | ||||
1231 | InTys.push_back(MVT::Other); | |||
1232 | SDVTList InTyList = DAG.getVTList(InTys); | |||
1233 | SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops); | |||
1234 | ||||
1235 | for (size_t I = 0; I < Ins.size(); ++I) | |||
1236 | InVals.push_back(Res.getValue(I)); | |||
1237 | ||||
1238 | // Return the chain | |||
1239 | return Res.getValue(Ins.size()); | |||
1240 | } | |||
1241 | ||||
1242 | bool WebAssemblyTargetLowering::CanLowerReturn( | |||
1243 | CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/, | |||
1244 | const SmallVectorImpl<ISD::OutputArg> &Outs, | |||
1245 | LLVMContext & /*Context*/) const { | |||
1246 | // WebAssembly can only handle returning tuples with multivalue enabled | |||
1247 | return Subtarget->hasMultivalue() || Outs.size() <= 1; | |||
1248 | } | |||
1249 | ||||
1250 | SDValue WebAssemblyTargetLowering::LowerReturn( | |||
1251 | SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/, | |||
1252 | const SmallVectorImpl<ISD::OutputArg> &Outs, | |||
1253 | const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL, | |||
1254 | SelectionDAG &DAG) const { | |||
1255 | assert((Subtarget->hasMultivalue() || Outs.size() <= 1) &&(static_cast <bool> ((Subtarget->hasMultivalue() || Outs .size() <= 1) && "MVP WebAssembly can only return up to one value" ) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1256 , __extension__ __PRETTY_FUNCTION__)) | |||
1256 | "MVP WebAssembly can only return up to one value")(static_cast <bool> ((Subtarget->hasMultivalue() || Outs .size() <= 1) && "MVP WebAssembly can only return up to one value" ) ? void (0) : __assert_fail ("(Subtarget->hasMultivalue() || Outs.size() <= 1) && \"MVP WebAssembly can only return up to one value\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1256 , __extension__ __PRETTY_FUNCTION__)); | |||
1257 | if (!callingConvSupported(CallConv)) | |||
1258 | fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions"); | |||
1259 | ||||
1260 | SmallVector<SDValue, 4> RetOps(1, Chain); | |||
1261 | RetOps.append(OutVals.begin(), OutVals.end()); | |||
1262 | Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps); | |||
1263 | ||||
1264 | // Record the number and types of the return values. | |||
1265 | for (const ISD::OutputArg &Out : Outs) { | |||
1266 | assert(!Out.Flags.isByVal() && "byval is not valid for return values")(static_cast <bool> (!Out.Flags.isByVal() && "byval is not valid for return values" ) ? void (0) : __assert_fail ("!Out.Flags.isByVal() && \"byval is not valid for return values\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1266 , __extension__ __PRETTY_FUNCTION__)); | |||
1267 | assert(!Out.Flags.isNest() && "nest is not valid for return values")(static_cast <bool> (!Out.Flags.isNest() && "nest is not valid for return values" ) ? void (0) : __assert_fail ("!Out.Flags.isNest() && \"nest is not valid for return values\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1267 , __extension__ __PRETTY_FUNCTION__)); | |||
1268 | assert(Out.IsFixed && "non-fixed return value is not valid")(static_cast <bool> (Out.IsFixed && "non-fixed return value is not valid" ) ? void (0) : __assert_fail ("Out.IsFixed && \"non-fixed return value is not valid\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1268 , __extension__ __PRETTY_FUNCTION__)); | |||
1269 | if (Out.Flags.isInAlloca()) | |||
1270 | fail(DL, DAG, "WebAssembly hasn't implemented inalloca results"); | |||
1271 | if (Out.Flags.isInConsecutiveRegs()) | |||
1272 | fail(DL, DAG, "WebAssembly hasn't implemented cons regs results"); | |||
1273 | if (Out.Flags.isInConsecutiveRegsLast()) | |||
1274 | fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results"); | |||
1275 | } | |||
1276 | ||||
1277 | return Chain; | |||
1278 | } | |||
1279 | ||||
1280 | SDValue WebAssemblyTargetLowering::LowerFormalArguments( | |||
1281 | SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, | |||
1282 | const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL, | |||
1283 | SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const { | |||
1284 | if (!callingConvSupported(CallConv)) | |||
1285 | fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions"); | |||
1286 | ||||
1287 | MachineFunction &MF = DAG.getMachineFunction(); | |||
1288 | auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>(); | |||
1289 | ||||
1290 | // Set up the incoming ARGUMENTS value, which serves to represent the liveness | |||
1291 | // of the incoming values before they're represented by virtual registers. | |||
1292 | MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS); | |||
1293 | ||||
1294 | bool HasSwiftErrorArg = false; | |||
1295 | bool HasSwiftSelfArg = false; | |||
1296 | for (const ISD::InputArg &In : Ins) { | |||
1297 | HasSwiftSelfArg |= In.Flags.isSwiftSelf(); | |||
1298 | HasSwiftErrorArg |= In.Flags.isSwiftError(); | |||
1299 | if (In.Flags.isInAlloca()) | |||
1300 | fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments"); | |||
1301 | if (In.Flags.isNest()) | |||
1302 | fail(DL, DAG, "WebAssembly hasn't implemented nest arguments"); | |||
1303 | if (In.Flags.isInConsecutiveRegs()) | |||
1304 | fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments"); | |||
1305 | if (In.Flags.isInConsecutiveRegsLast()) | |||
1306 | fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments"); | |||
1307 | // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in | |||
1308 | // registers. | |||
1309 | InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT, | |||
1310 | DAG.getTargetConstant(InVals.size(), | |||
1311 | DL, MVT::i32)) | |||
1312 | : DAG.getUNDEF(In.VT)); | |||
1313 | ||||
1314 | // Record the number and types of arguments. | |||
1315 | MFI->addParam(In.VT); | |||
1316 | } | |||
1317 | ||||
1318 | // For swiftcc, emit additional swiftself and swifterror arguments | |||
1319 | // if there aren't. These additional arguments are also added for callee | |||
1320 | // signature They are necessary to match callee and caller signature for | |||
1321 | // indirect call. | |||
1322 | auto PtrVT = getPointerTy(MF.getDataLayout()); | |||
1323 | if (CallConv == CallingConv::Swift) { | |||
1324 | if (!HasSwiftSelfArg) { | |||
1325 | MFI->addParam(PtrVT); | |||
1326 | } | |||
1327 | if (!HasSwiftErrorArg) { | |||
1328 | MFI->addParam(PtrVT); | |||
1329 | } | |||
1330 | } | |||
1331 | // Varargs are copied into a buffer allocated by the caller, and a pointer to | |||
1332 | // the buffer is passed as an argument. | |||
1333 | if (IsVarArg) { | |||
1334 | MVT PtrVT = getPointerTy(MF.getDataLayout()); | |||
1335 | Register VarargVreg = | |||
1336 | MF.getRegInfo().createVirtualRegister(getRegClassFor(PtrVT)); | |||
1337 | MFI->setVarargBufferVreg(VarargVreg); | |||
1338 | Chain = DAG.getCopyToReg( | |||
1339 | Chain, DL, VarargVreg, | |||
1340 | DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT, | |||
1341 | DAG.getTargetConstant(Ins.size(), DL, MVT::i32))); | |||
1342 | MFI->addParam(PtrVT); | |||
1343 | } | |||
1344 | ||||
1345 | // Record the number and types of arguments and results. | |||
1346 | SmallVector<MVT, 4> Params; | |||
1347 | SmallVector<MVT, 4> Results; | |||
1348 | computeSignatureVTs(MF.getFunction().getFunctionType(), &MF.getFunction(), | |||
1349 | MF.getFunction(), DAG.getTarget(), Params, Results); | |||
1350 | for (MVT VT : Results) | |||
1351 | MFI->addResult(VT); | |||
1352 | // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify | |||
1353 | // the param logic here with ComputeSignatureVTs | |||
1354 | assert(MFI->getParams().size() == Params.size() &&(static_cast <bool> (MFI->getParams().size() == Params .size() && std::equal(MFI->getParams().begin(), MFI ->getParams().end(), Params.begin())) ? void (0) : __assert_fail ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1356 , __extension__ __PRETTY_FUNCTION__)) | |||
1355 | std::equal(MFI->getParams().begin(), MFI->getParams().end(),(static_cast <bool> (MFI->getParams().size() == Params .size() && std::equal(MFI->getParams().begin(), MFI ->getParams().end(), Params.begin())) ? void (0) : __assert_fail ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1356 , __extension__ __PRETTY_FUNCTION__)) | |||
1356 | Params.begin()))(static_cast <bool> (MFI->getParams().size() == Params .size() && std::equal(MFI->getParams().begin(), MFI ->getParams().end(), Params.begin())) ? void (0) : __assert_fail ("MFI->getParams().size() == Params.size() && std::equal(MFI->getParams().begin(), MFI->getParams().end(), Params.begin())" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1356 , __extension__ __PRETTY_FUNCTION__)); | |||
1357 | ||||
1358 | return Chain; | |||
1359 | } | |||
1360 | ||||
1361 | void WebAssemblyTargetLowering::ReplaceNodeResults( | |||
1362 | SDNode *N, SmallVectorImpl<SDValue> &Results, SelectionDAG &DAG) const { | |||
1363 | switch (N->getOpcode()) { | |||
1364 | case ISD::SIGN_EXTEND_INREG: | |||
1365 | // Do not add any results, signifying that N should not be custom lowered | |||
1366 | // after all. This happens because simd128 turns on custom lowering for | |||
1367 | // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an | |||
1368 | // illegal type. | |||
1369 | break; | |||
1370 | default: | |||
1371 | llvm_unreachable(::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1372 ) | |||
1372 | "ReplaceNodeResults not implemented for this op for WebAssembly!")::llvm::llvm_unreachable_internal("ReplaceNodeResults not implemented for this op for WebAssembly!" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1372 ); | |||
1373 | } | |||
1374 | } | |||
1375 | ||||
1376 | //===----------------------------------------------------------------------===// | |||
1377 | // Custom lowering hooks. | |||
1378 | //===----------------------------------------------------------------------===// | |||
1379 | ||||
1380 | SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op, | |||
1381 | SelectionDAG &DAG) const { | |||
1382 | SDLoc DL(Op); | |||
1383 | switch (Op.getOpcode()) { | |||
| ||||
1384 | default: | |||
1385 | llvm_unreachable("unimplemented operation lowering")::llvm::llvm_unreachable_internal("unimplemented operation lowering" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1385 ); | |||
1386 | return SDValue(); | |||
1387 | case ISD::FrameIndex: | |||
1388 | return LowerFrameIndex(Op, DAG); | |||
1389 | case ISD::GlobalAddress: | |||
1390 | return LowerGlobalAddress(Op, DAG); | |||
1391 | case ISD::GlobalTLSAddress: | |||
1392 | return LowerGlobalTLSAddress(Op, DAG); | |||
1393 | case ISD::ExternalSymbol: | |||
1394 | return LowerExternalSymbol(Op, DAG); | |||
1395 | case ISD::JumpTable: | |||
1396 | return LowerJumpTable(Op, DAG); | |||
1397 | case ISD::BR_JT: | |||
1398 | return LowerBR_JT(Op, DAG); | |||
1399 | case ISD::VASTART: | |||
1400 | return LowerVASTART(Op, DAG); | |||
1401 | case ISD::BlockAddress: | |||
1402 | case ISD::BRIND: | |||
1403 | fail(DL, DAG, "WebAssembly hasn't implemented computed gotos"); | |||
1404 | return SDValue(); | |||
1405 | case ISD::RETURNADDR: | |||
1406 | return LowerRETURNADDR(Op, DAG); | |||
1407 | case ISD::FRAMEADDR: | |||
1408 | return LowerFRAMEADDR(Op, DAG); | |||
1409 | case ISD::CopyToReg: | |||
1410 | return LowerCopyToReg(Op, DAG); | |||
1411 | case ISD::EXTRACT_VECTOR_ELT: | |||
1412 | case ISD::INSERT_VECTOR_ELT: | |||
1413 | return LowerAccessVectorElement(Op, DAG); | |||
1414 | case ISD::INTRINSIC_VOID: | |||
1415 | case ISD::INTRINSIC_WO_CHAIN: | |||
1416 | case ISD::INTRINSIC_W_CHAIN: | |||
1417 | return LowerIntrinsic(Op, DAG); | |||
1418 | case ISD::SIGN_EXTEND_INREG: | |||
1419 | return LowerSIGN_EXTEND_INREG(Op, DAG); | |||
1420 | case ISD::BUILD_VECTOR: | |||
1421 | return LowerBUILD_VECTOR(Op, DAG); | |||
1422 | case ISD::VECTOR_SHUFFLE: | |||
1423 | return LowerVECTOR_SHUFFLE(Op, DAG); | |||
1424 | case ISD::SETCC: | |||
1425 | return LowerSETCC(Op, DAG); | |||
1426 | case ISD::SHL: | |||
1427 | case ISD::SRA: | |||
1428 | case ISD::SRL: | |||
1429 | return LowerShift(Op, DAG); | |||
1430 | case ISD::FP_TO_SINT_SAT: | |||
1431 | case ISD::FP_TO_UINT_SAT: | |||
1432 | return LowerFP_TO_INT_SAT(Op, DAG); | |||
1433 | case ISD::LOAD: | |||
1434 | return LowerLoad(Op, DAG); | |||
1435 | case ISD::STORE: | |||
1436 | return LowerStore(Op, DAG); | |||
1437 | case ISD::CTPOP: | |||
1438 | case ISD::CTLZ: | |||
1439 | case ISD::CTTZ: | |||
1440 | return DAG.UnrollVectorOp(Op.getNode()); | |||
1441 | } | |||
1442 | } | |||
1443 | ||||
1444 | static bool IsWebAssemblyGlobal(SDValue Op) { | |||
1445 | if (const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op)) | |||
1446 | return WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace()); | |||
1447 | ||||
1448 | return false; | |||
1449 | } | |||
1450 | ||||
1451 | static Optional<unsigned> IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG) { | |||
1452 | const FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Op); | |||
1453 | if (!FI) | |||
1454 | return None; | |||
1455 | ||||
1456 | auto &MF = DAG.getMachineFunction(); | |||
1457 | return WebAssemblyFrameLowering::getLocalForStackObject(MF, FI->getIndex()); | |||
1458 | } | |||
1459 | ||||
1460 | static bool IsWebAssemblyTable(SDValue Op) { | |||
1461 | const GlobalAddressSDNode *GA = dyn_cast<GlobalAddressSDNode>(Op); | |||
1462 | if (GA && WebAssembly::isWasmVarAddressSpace(GA->getAddressSpace())) { | |||
1463 | const GlobalValue *Value = GA->getGlobal(); | |||
1464 | const Type *Ty = Value->getValueType(); | |||
1465 | ||||
1466 | if (Ty->isArrayTy() && WebAssembly::isRefType(Ty->getArrayElementType())) | |||
1467 | return true; | |||
1468 | } | |||
1469 | return false; | |||
1470 | } | |||
1471 | ||||
1472 | // This function will accept as Op any access to a table, so Op can | |||
1473 | // be the actual table or an offset into the table. | |||
1474 | static bool IsWebAssemblyTableWithOffset(SDValue Op) { | |||
1475 | if (Op->getOpcode() == ISD::ADD && Op->getNumOperands() == 2) | |||
1476 | return (Op->getOperand(1).getSimpleValueType() == MVT::i32 && | |||
1477 | IsWebAssemblyTableWithOffset(Op->getOperand(0))) || | |||
1478 | (Op->getOperand(0).getSimpleValueType() == MVT::i32 && | |||
1479 | IsWebAssemblyTableWithOffset(Op->getOperand(1))); | |||
1480 | ||||
1481 | return IsWebAssemblyTable(Op); | |||
1482 | } | |||
1483 | ||||
1484 | // Helper for table pattern matching used in LowerStore and LowerLoad | |||
1485 | bool WebAssemblyTargetLowering::MatchTableForLowering(SelectionDAG &DAG, | |||
1486 | const SDLoc &DL, | |||
1487 | const SDValue &Base, | |||
1488 | GlobalAddressSDNode *&GA, | |||
1489 | SDValue &Idx) const { | |||
1490 | // We expect the following graph for a load of the form: | |||
1491 | // table[<var> + <constant offset>] | |||
1492 | // | |||
1493 | // Case 1: | |||
1494 | // externref = load t1 | |||
1495 | // t1: i32 = add t2, i32:<constant offset> | |||
1496 | // t2: i32 = add tX, table | |||
1497 | // | |||
1498 | // This is in some cases simplified to just: | |||
1499 | // Case 2: | |||
1500 | // externref = load t1 | |||
1501 | // t1: i32 = add t2, i32:tX | |||
1502 | // | |||
1503 | // So, unfortunately we need to check for both cases and if we are in the | |||
1504 | // first case extract the table GlobalAddressNode and build a new node tY | |||
1505 | // that's tY: i32 = add i32:<constant offset>, i32:tX | |||
1506 | // | |||
1507 | if (IsWebAssemblyTable(Base)) { | |||
1508 | GA = cast<GlobalAddressSDNode>(Base); | |||
1509 | Idx = DAG.getConstant(0, DL, MVT::i32); | |||
1510 | } else { | |||
1511 | GA = dyn_cast<GlobalAddressSDNode>(Base->getOperand(0)); | |||
1512 | if (GA) { | |||
1513 | // We are in Case 2 above. | |||
1514 | Idx = Base->getOperand(1); | |||
1515 | assert(GA->getNumValues() == 1)(static_cast <bool> (GA->getNumValues() == 1) ? void (0) : __assert_fail ("GA->getNumValues() == 1", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 1515, __extension__ __PRETTY_FUNCTION__)); | |||
1516 | } else { | |||
1517 | // This might be Case 1 above (or an error) | |||
1518 | SDValue V = Base->getOperand(0); | |||
1519 | GA = dyn_cast<GlobalAddressSDNode>(V->getOperand(1)); | |||
1520 | ||||
1521 | if (V->getOpcode() != ISD::ADD || V->getNumOperands() != 2 || !GA) | |||
1522 | return false; | |||
1523 | ||||
1524 | SDValue IdxV = DAG.getNode(ISD::ADD, DL, MVT::i32, Base->getOperand(1), | |||
1525 | V->getOperand(0)); | |||
1526 | Idx = IdxV; | |||
1527 | } | |||
1528 | } | |||
1529 | ||||
1530 | return true; | |||
1531 | } | |||
1532 | ||||
1533 | SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op, | |||
1534 | SelectionDAG &DAG) const { | |||
1535 | SDLoc DL(Op); | |||
1536 | StoreSDNode *SN = cast<StoreSDNode>(Op.getNode()); | |||
1537 | const SDValue &Value = SN->getValue(); | |||
1538 | const SDValue &Base = SN->getBasePtr(); | |||
1539 | const SDValue &Offset = SN->getOffset(); | |||
1540 | ||||
1541 | if (IsWebAssemblyTableWithOffset(Base)) { | |||
1542 | if (!Offset->isUndef()) | |||
1543 | report_fatal_error( | |||
1544 | "unexpected offset when loading from webassembly table", false); | |||
1545 | ||||
1546 | SDValue Idx; | |||
1547 | GlobalAddressSDNode *GA; | |||
1548 | ||||
1549 | if (!MatchTableForLowering(DAG, DL, Base, GA, Idx)) | |||
1550 | report_fatal_error("failed pattern matching for lowering table store", | |||
1551 | false); | |||
1552 | ||||
1553 | SDVTList Tys = DAG.getVTList(MVT::Other); | |||
1554 | SDValue TableSetOps[] = {SN->getChain(), SDValue(GA, 0), Idx, Value}; | |||
1555 | SDValue TableSet = | |||
1556 | DAG.getMemIntrinsicNode(WebAssemblyISD::TABLE_SET, DL, Tys, TableSetOps, | |||
1557 | SN->getMemoryVT(), SN->getMemOperand()); | |||
1558 | return TableSet; | |||
1559 | } | |||
1560 | ||||
1561 | if (IsWebAssemblyGlobal(Base)) { | |||
1562 | if (!Offset->isUndef()) | |||
1563 | report_fatal_error("unexpected offset when storing to webassembly global", | |||
1564 | false); | |||
1565 | ||||
1566 | SDVTList Tys = DAG.getVTList(MVT::Other); | |||
1567 | SDValue Ops[] = {SN->getChain(), Value, Base}; | |||
1568 | return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops, | |||
1569 | SN->getMemoryVT(), SN->getMemOperand()); | |||
1570 | } | |||
1571 | ||||
1572 | if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) { | |||
1573 | if (!Offset->isUndef()) | |||
1574 | report_fatal_error("unexpected offset when storing to webassembly local", | |||
1575 | false); | |||
1576 | ||||
1577 | SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32); | |||
1578 | SDVTList Tys = DAG.getVTList(MVT::Other); // The chain. | |||
1579 | SDValue Ops[] = {SN->getChain(), Idx, Value}; | |||
1580 | return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops); | |||
1581 | } | |||
1582 | ||||
1583 | return Op; | |||
1584 | } | |||
1585 | ||||
1586 | SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op, | |||
1587 | SelectionDAG &DAG) const { | |||
1588 | SDLoc DL(Op); | |||
1589 | LoadSDNode *LN = cast<LoadSDNode>(Op.getNode()); | |||
1590 | const SDValue &Base = LN->getBasePtr(); | |||
1591 | const SDValue &Offset = LN->getOffset(); | |||
1592 | ||||
1593 | if (IsWebAssemblyTableWithOffset(Base)) { | |||
1594 | if (!Offset->isUndef()) | |||
1595 | report_fatal_error( | |||
1596 | "unexpected offset when loading from webassembly table", false); | |||
1597 | ||||
1598 | GlobalAddressSDNode *GA; | |||
1599 | SDValue Idx; | |||
1600 | ||||
1601 | if (!MatchTableForLowering(DAG, DL, Base, GA, Idx)) | |||
1602 | report_fatal_error("failed pattern matching for lowering table load", | |||
1603 | false); | |||
1604 | ||||
1605 | SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other); | |||
1606 | SDValue TableGetOps[] = {LN->getChain(), SDValue(GA, 0), Idx}; | |||
1607 | SDValue TableGet = | |||
1608 | DAG.getMemIntrinsicNode(WebAssemblyISD::TABLE_GET, DL, Tys, TableGetOps, | |||
1609 | LN->getMemoryVT(), LN->getMemOperand()); | |||
1610 | return TableGet; | |||
1611 | } | |||
1612 | ||||
1613 | if (IsWebAssemblyGlobal(Base)) { | |||
1614 | if (!Offset->isUndef()) | |||
1615 | report_fatal_error( | |||
1616 | "unexpected offset when loading from webassembly global", false); | |||
1617 | ||||
1618 | SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other); | |||
1619 | SDValue Ops[] = {LN->getChain(), Base}; | |||
1620 | return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops, | |||
1621 | LN->getMemoryVT(), LN->getMemOperand()); | |||
1622 | } | |||
1623 | ||||
1624 | if (Optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) { | |||
1625 | if (!Offset->isUndef()) | |||
1626 | report_fatal_error( | |||
1627 | "unexpected offset when loading from webassembly local", false); | |||
1628 | ||||
1629 | SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32); | |||
1630 | EVT LocalVT = LN->getValueType(0); | |||
1631 | SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT, | |||
1632 | {LN->getChain(), Idx}); | |||
1633 | SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL); | |||
1634 | assert(Result->getNumValues() == 2 && "Loads must carry a chain!")(static_cast <bool> (Result->getNumValues() == 2 && "Loads must carry a chain!") ? void (0) : __assert_fail ("Result->getNumValues() == 2 && \"Loads must carry a chain!\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1634 , __extension__ __PRETTY_FUNCTION__)); | |||
1635 | return Result; | |||
1636 | } | |||
1637 | ||||
1638 | return Op; | |||
1639 | } | |||
1640 | ||||
1641 | SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op, | |||
1642 | SelectionDAG &DAG) const { | |||
1643 | SDValue Src = Op.getOperand(2); | |||
1644 | if (isa<FrameIndexSDNode>(Src.getNode())) { | |||
1645 | // CopyToReg nodes don't support FrameIndex operands. Other targets select | |||
1646 | // the FI to some LEA-like instruction, but since we don't have that, we | |||
1647 | // need to insert some kind of instruction that can take an FI operand and | |||
1648 | // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy | |||
1649 | // local.copy between Op and its FI operand. | |||
1650 | SDValue Chain = Op.getOperand(0); | |||
1651 | SDLoc DL(Op); | |||
1652 | Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg(); | |||
1653 | EVT VT = Src.getValueType(); | |||
1654 | SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32 | |||
1655 | : WebAssembly::COPY_I64, | |||
1656 | DL, VT, Src), | |||
1657 | 0); | |||
1658 | return Op.getNode()->getNumValues() == 1 | |||
1659 | ? DAG.getCopyToReg(Chain, DL, Reg, Copy) | |||
1660 | : DAG.getCopyToReg(Chain, DL, Reg, Copy, | |||
1661 | Op.getNumOperands() == 4 ? Op.getOperand(3) | |||
1662 | : SDValue()); | |||
1663 | } | |||
1664 | return SDValue(); | |||
1665 | } | |||
1666 | ||||
1667 | SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op, | |||
1668 | SelectionDAG &DAG) const { | |||
1669 | int FI = cast<FrameIndexSDNode>(Op)->getIndex(); | |||
1670 | return DAG.getTargetFrameIndex(FI, Op.getValueType()); | |||
1671 | } | |||
1672 | ||||
1673 | SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op, | |||
1674 | SelectionDAG &DAG) const { | |||
1675 | SDLoc DL(Op); | |||
1676 | ||||
1677 | if (!Subtarget->getTargetTriple().isOSEmscripten()) { | |||
1678 | fail(DL, DAG, | |||
1679 | "Non-Emscripten WebAssembly hasn't implemented " | |||
1680 | "__builtin_return_address"); | |||
1681 | return SDValue(); | |||
1682 | } | |||
1683 | ||||
1684 | if (verifyReturnAddressArgumentIsConstant(Op, DAG)) | |||
1685 | return SDValue(); | |||
1686 | ||||
1687 | unsigned Depth = Op.getConstantOperandVal(0); | |||
1688 | MakeLibCallOptions CallOptions; | |||
1689 | return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(), | |||
1690 | {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL) | |||
1691 | .first; | |||
1692 | } | |||
1693 | ||||
1694 | SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op, | |||
1695 | SelectionDAG &DAG) const { | |||
1696 | // Non-zero depths are not supported by WebAssembly currently. Use the | |||
1697 | // legalizer's default expansion, which is to return 0 (what this function is | |||
1698 | // documented to do). | |||
1699 | if (Op.getConstantOperandVal(0) > 0) | |||
1700 | return SDValue(); | |||
1701 | ||||
1702 | DAG.getMachineFunction().getFrameInfo().setFrameAddressIsTaken(true); | |||
1703 | EVT VT = Op.getValueType(); | |||
1704 | Register FP = | |||
1705 | Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction()); | |||
1706 | return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT); | |||
1707 | } | |||
1708 | ||||
1709 | SDValue | |||
1710 | WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op, | |||
1711 | SelectionDAG &DAG) const { | |||
1712 | SDLoc DL(Op); | |||
1713 | const auto *GA = cast<GlobalAddressSDNode>(Op); | |||
1714 | ||||
1715 | MachineFunction &MF = DAG.getMachineFunction(); | |||
1716 | if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory()) | |||
1717 | report_fatal_error("cannot use thread-local storage without bulk memory", | |||
1718 | false); | |||
1719 | ||||
1720 | const GlobalValue *GV = GA->getGlobal(); | |||
1721 | ||||
1722 | // Currently Emscripten does not support dynamic linking with threads. | |||
1723 | // Therefore, if we have thread-local storage, only the local-exec model | |||
1724 | // is possible. | |||
1725 | // TODO: remove this and implement proper TLS models once Emscripten | |||
1726 | // supports dynamic linking with threads. | |||
1727 | if (GV->getThreadLocalMode() != GlobalValue::LocalExecTLSModel && | |||
1728 | !Subtarget->getTargetTriple().isOSEmscripten()) { | |||
1729 | report_fatal_error("only -ftls-model=local-exec is supported for now on " | |||
1730 | "non-Emscripten OSes: variable " + | |||
1731 | GV->getName(), | |||
1732 | false); | |||
1733 | } | |||
1734 | ||||
1735 | auto model = GV->getThreadLocalMode(); | |||
1736 | ||||
1737 | // Unsupported TLS modes | |||
1738 | assert(model != GlobalValue::NotThreadLocal)(static_cast <bool> (model != GlobalValue::NotThreadLocal ) ? void (0) : __assert_fail ("model != GlobalValue::NotThreadLocal" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1738 , __extension__ __PRETTY_FUNCTION__)); | |||
1739 | assert(model != GlobalValue::InitialExecTLSModel)(static_cast <bool> (model != GlobalValue::InitialExecTLSModel ) ? void (0) : __assert_fail ("model != GlobalValue::InitialExecTLSModel" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1739 , __extension__ __PRETTY_FUNCTION__)); | |||
1740 | ||||
1741 | if (model == GlobalValue::LocalExecTLSModel || | |||
1742 | model == GlobalValue::LocalDynamicTLSModel || | |||
1743 | (model == GlobalValue::GeneralDynamicTLSModel && | |||
1744 | getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV))) { | |||
1745 | // For DSO-local TLS variables we use offset from __tls_base | |||
1746 | ||||
1747 | MVT PtrVT = getPointerTy(DAG.getDataLayout()); | |||
1748 | auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64 | |||
1749 | : WebAssembly::GLOBAL_GET_I32; | |||
1750 | const char *BaseName = MF.createExternalSymbolName("__tls_base"); | |||
1751 | ||||
1752 | SDValue BaseAddr( | |||
1753 | DAG.getMachineNode(GlobalGet, DL, PtrVT, | |||
1754 | DAG.getTargetExternalSymbol(BaseName, PtrVT)), | |||
1755 | 0); | |||
1756 | ||||
1757 | SDValue TLSOffset = DAG.getTargetGlobalAddress( | |||
1758 | GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL); | |||
1759 | SDValue SymOffset = | |||
1760 | DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset); | |||
1761 | ||||
1762 | return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset); | |||
1763 | } | |||
1764 | ||||
1765 | assert(model == GlobalValue::GeneralDynamicTLSModel)(static_cast <bool> (model == GlobalValue::GeneralDynamicTLSModel ) ? void (0) : __assert_fail ("model == GlobalValue::GeneralDynamicTLSModel" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1765 , __extension__ __PRETTY_FUNCTION__)); | |||
1766 | ||||
1767 | EVT VT = Op.getValueType(); | |||
1768 | return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, | |||
1769 | DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, | |||
1770 | GA->getOffset(), | |||
1771 | WebAssemblyII::MO_GOT_TLS)); | |||
1772 | } | |||
1773 | ||||
1774 | SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op, | |||
1775 | SelectionDAG &DAG) const { | |||
1776 | SDLoc DL(Op); | |||
1777 | const auto *GA = cast<GlobalAddressSDNode>(Op); | |||
1778 | EVT VT = Op.getValueType(); | |||
1779 | assert(GA->getTargetFlags() == 0 &&(static_cast <bool> (GA->getTargetFlags() == 0 && "Unexpected target flags on generic GlobalAddressSDNode") ? void (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1780 , __extension__ __PRETTY_FUNCTION__)) | |||
1780 | "Unexpected target flags on generic GlobalAddressSDNode")(static_cast <bool> (GA->getTargetFlags() == 0 && "Unexpected target flags on generic GlobalAddressSDNode") ? void (0) : __assert_fail ("GA->getTargetFlags() == 0 && \"Unexpected target flags on generic GlobalAddressSDNode\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1780 , __extension__ __PRETTY_FUNCTION__)); | |||
1781 | if (!WebAssembly::isValidAddressSpace(GA->getAddressSpace())) | |||
1782 | fail(DL, DAG, "Invalid address space for WebAssembly target"); | |||
1783 | ||||
1784 | unsigned OperandFlags = 0; | |||
1785 | if (isPositionIndependent()) { | |||
1786 | const GlobalValue *GV = GA->getGlobal(); | |||
1787 | if (getTargetMachine().shouldAssumeDSOLocal(*GV->getParent(), GV)) { | |||
1788 | MachineFunction &MF = DAG.getMachineFunction(); | |||
1789 | MVT PtrVT = getPointerTy(MF.getDataLayout()); | |||
1790 | const char *BaseName; | |||
1791 | if (GV->getValueType()->isFunctionTy()) { | |||
1792 | BaseName = MF.createExternalSymbolName("__table_base"); | |||
1793 | OperandFlags = WebAssemblyII::MO_TABLE_BASE_REL; | |||
1794 | } | |||
1795 | else { | |||
1796 | BaseName = MF.createExternalSymbolName("__memory_base"); | |||
1797 | OperandFlags = WebAssemblyII::MO_MEMORY_BASE_REL; | |||
1798 | } | |||
1799 | SDValue BaseAddr = | |||
1800 | DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, | |||
1801 | DAG.getTargetExternalSymbol(BaseName, PtrVT)); | |||
1802 | ||||
1803 | SDValue SymAddr = DAG.getNode( | |||
1804 | WebAssemblyISD::WrapperREL, DL, VT, | |||
1805 | DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(), | |||
1806 | OperandFlags)); | |||
1807 | ||||
1808 | return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr); | |||
1809 | } | |||
1810 | OperandFlags = WebAssemblyII::MO_GOT; | |||
1811 | } | |||
1812 | ||||
1813 | return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, | |||
1814 | DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, | |||
1815 | GA->getOffset(), OperandFlags)); | |||
1816 | } | |||
1817 | ||||
1818 | SDValue | |||
1819 | WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op, | |||
1820 | SelectionDAG &DAG) const { | |||
1821 | SDLoc DL(Op); | |||
1822 | const auto *ES = cast<ExternalSymbolSDNode>(Op); | |||
1823 | EVT VT = Op.getValueType(); | |||
1824 | assert(ES->getTargetFlags() == 0 &&(static_cast <bool> (ES->getTargetFlags() == 0 && "Unexpected target flags on generic ExternalSymbolSDNode") ? void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1825 , __extension__ __PRETTY_FUNCTION__)) | |||
1825 | "Unexpected target flags on generic ExternalSymbolSDNode")(static_cast <bool> (ES->getTargetFlags() == 0 && "Unexpected target flags on generic ExternalSymbolSDNode") ? void (0) : __assert_fail ("ES->getTargetFlags() == 0 && \"Unexpected target flags on generic ExternalSymbolSDNode\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1825 , __extension__ __PRETTY_FUNCTION__)); | |||
1826 | return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT, | |||
1827 | DAG.getTargetExternalSymbol(ES->getSymbol(), VT)); | |||
1828 | } | |||
1829 | ||||
1830 | SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op, | |||
1831 | SelectionDAG &DAG) const { | |||
1832 | // There's no need for a Wrapper node because we always incorporate a jump | |||
1833 | // table operand into a BR_TABLE instruction, rather than ever | |||
1834 | // materializing it in a register. | |||
1835 | const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op); | |||
1836 | return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(), | |||
1837 | JT->getTargetFlags()); | |||
1838 | } | |||
1839 | ||||
1840 | SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op, | |||
1841 | SelectionDAG &DAG) const { | |||
1842 | SDLoc DL(Op); | |||
1843 | SDValue Chain = Op.getOperand(0); | |||
1844 | const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1)); | |||
1845 | SDValue Index = Op.getOperand(2); | |||
1846 | assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags")(static_cast <bool> (JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags") ? void (0) : __assert_fail ("JT->getTargetFlags() == 0 && \"WebAssembly doesn't set target flags\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1846 , __extension__ __PRETTY_FUNCTION__)); | |||
1847 | ||||
1848 | SmallVector<SDValue, 8> Ops; | |||
1849 | Ops.push_back(Chain); | |||
1850 | Ops.push_back(Index); | |||
1851 | ||||
1852 | MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo(); | |||
1853 | const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs; | |||
1854 | ||||
1855 | // Add an operand for each case. | |||
1856 | for (auto MBB : MBBs) | |||
1857 | Ops.push_back(DAG.getBasicBlock(MBB)); | |||
1858 | ||||
1859 | // Add the first MBB as a dummy default target for now. This will be replaced | |||
1860 | // with the proper default target (and the preceding range check eliminated) | |||
1861 | // if possible by WebAssemblyFixBrTableDefaults. | |||
1862 | Ops.push_back(DAG.getBasicBlock(*MBBs.begin())); | |||
1863 | return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops); | |||
1864 | } | |||
1865 | ||||
1866 | SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op, | |||
1867 | SelectionDAG &DAG) const { | |||
1868 | SDLoc DL(Op); | |||
1869 | EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout()); | |||
1870 | ||||
1871 | auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>(); | |||
1872 | const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue(); | |||
1873 | ||||
1874 | SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL, | |||
1875 | MFI->getVarargBufferVreg(), PtrVT); | |||
1876 | return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1), | |||
1877 | MachinePointerInfo(SV)); | |||
1878 | } | |||
1879 | ||||
1880 | SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op, | |||
1881 | SelectionDAG &DAG) const { | |||
1882 | MachineFunction &MF = DAG.getMachineFunction(); | |||
1883 | unsigned IntNo; | |||
1884 | switch (Op.getOpcode()) { | |||
1885 | case ISD::INTRINSIC_VOID: | |||
1886 | case ISD::INTRINSIC_W_CHAIN: | |||
1887 | IntNo = Op.getConstantOperandVal(1); | |||
1888 | break; | |||
1889 | case ISD::INTRINSIC_WO_CHAIN: | |||
1890 | IntNo = Op.getConstantOperandVal(0); | |||
1891 | break; | |||
1892 | default: | |||
1893 | llvm_unreachable("Invalid intrinsic")::llvm::llvm_unreachable_internal("Invalid intrinsic", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 1893); | |||
1894 | } | |||
1895 | SDLoc DL(Op); | |||
1896 | ||||
1897 | switch (IntNo) { | |||
1898 | default: | |||
1899 | return SDValue(); // Don't custom lower most intrinsics. | |||
1900 | ||||
1901 | case Intrinsic::wasm_lsda: { | |||
1902 | auto PtrVT = getPointerTy(MF.getDataLayout()); | |||
1903 | const char *SymName = MF.createExternalSymbolName( | |||
1904 | "GCC_except_table" + std::to_string(MF.getFunctionNumber())); | |||
1905 | if (isPositionIndependent()) { | |||
1906 | SDValue Node = DAG.getTargetExternalSymbol( | |||
1907 | SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL); | |||
1908 | const char *BaseName = MF.createExternalSymbolName("__memory_base"); | |||
1909 | SDValue BaseAddr = | |||
1910 | DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, | |||
1911 | DAG.getTargetExternalSymbol(BaseName, PtrVT)); | |||
1912 | SDValue SymAddr = | |||
1913 | DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node); | |||
1914 | return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr); | |||
1915 | } | |||
1916 | SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT); | |||
1917 | return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node); | |||
1918 | } | |||
1919 | ||||
1920 | case Intrinsic::wasm_shuffle: { | |||
1921 | // Drop in-chain and replace undefs, but otherwise pass through unchanged | |||
1922 | SDValue Ops[18]; | |||
1923 | size_t OpIdx = 0; | |||
1924 | Ops[OpIdx++] = Op.getOperand(1); | |||
1925 | Ops[OpIdx++] = Op.getOperand(2); | |||
1926 | while (OpIdx < 18) { | |||
1927 | const SDValue &MaskIdx = Op.getOperand(OpIdx + 1); | |||
1928 | if (MaskIdx.isUndef() || | |||
1929 | cast<ConstantSDNode>(MaskIdx.getNode())->getZExtValue() >= 32) { | |||
1930 | Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32); | |||
1931 | } else { | |||
1932 | Ops[OpIdx++] = MaskIdx; | |||
1933 | } | |||
1934 | } | |||
1935 | return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops); | |||
1936 | } | |||
1937 | } | |||
1938 | } | |||
1939 | ||||
1940 | SDValue | |||
1941 | WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op, | |||
1942 | SelectionDAG &DAG) const { | |||
1943 | SDLoc DL(Op); | |||
1944 | // If sign extension operations are disabled, allow sext_inreg only if operand | |||
1945 | // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign | |||
1946 | // extension operations, but allowing sext_inreg in this context lets us have | |||
1947 | // simple patterns to select extract_lane_s instructions. Expanding sext_inreg | |||
1948 | // everywhere would be simpler in this file, but would necessitate large and | |||
1949 | // brittle patterns to undo the expansion and select extract_lane_s | |||
1950 | // instructions. | |||
1951 | assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128())(static_cast <bool> (!Subtarget->hasSignExt() && Subtarget->hasSIMD128()) ? void (0) : __assert_fail ("!Subtarget->hasSignExt() && Subtarget->hasSIMD128()" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 1951 , __extension__ __PRETTY_FUNCTION__)); | |||
1952 | if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT) | |||
1953 | return SDValue(); | |||
1954 | ||||
1955 | const SDValue &Extract = Op.getOperand(0); | |||
1956 | MVT VecT = Extract.getOperand(0).getSimpleValueType(); | |||
1957 | if (VecT.getVectorElementType().getSizeInBits() > 32) | |||
1958 | return SDValue(); | |||
1959 | MVT ExtractedLaneT = | |||
1960 | cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT(); | |||
1961 | MVT ExtractedVecT = | |||
1962 | MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits()); | |||
1963 | if (ExtractedVecT == VecT) | |||
1964 | return Op; | |||
1965 | ||||
1966 | // Bitcast vector to appropriate type to ensure ISel pattern coverage | |||
1967 | const SDNode *Index = Extract.getOperand(1).getNode(); | |||
1968 | if (!isa<ConstantSDNode>(Index)) | |||
1969 | return SDValue(); | |||
1970 | unsigned IndexVal = cast<ConstantSDNode>(Index)->getZExtValue(); | |||
1971 | unsigned Scale = | |||
1972 | ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements(); | |||
1973 | assert(Scale > 1)(static_cast <bool> (Scale > 1) ? void (0) : __assert_fail ("Scale > 1", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 1973, __extension__ __PRETTY_FUNCTION__)); | |||
1974 | SDValue NewIndex = | |||
1975 | DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0)); | |||
1976 | SDValue NewExtract = DAG.getNode( | |||
1977 | ISD::EXTRACT_VECTOR_ELT, DL, Extract.getValueType(), | |||
1978 | DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex); | |||
1979 | return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract, | |||
1980 | Op.getOperand(1)); | |||
1981 | } | |||
1982 | ||||
1983 | static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG) { | |||
1984 | SDLoc DL(Op); | |||
1985 | if (Op.getValueType() != MVT::v2f64) | |||
1986 | return SDValue(); | |||
1987 | ||||
1988 | auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec, | |||
1989 | unsigned &Index) -> bool { | |||
1990 | switch (Op.getOpcode()) { | |||
1991 | case ISD::SINT_TO_FP: | |||
1992 | Opcode = WebAssemblyISD::CONVERT_LOW_S; | |||
1993 | break; | |||
1994 | case ISD::UINT_TO_FP: | |||
1995 | Opcode = WebAssemblyISD::CONVERT_LOW_U; | |||
1996 | break; | |||
1997 | case ISD::FP_EXTEND: | |||
1998 | Opcode = WebAssemblyISD::PROMOTE_LOW; | |||
1999 | break; | |||
2000 | default: | |||
2001 | return false; | |||
2002 | } | |||
2003 | ||||
2004 | auto ExtractVector = Op.getOperand(0); | |||
2005 | if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT) | |||
2006 | return false; | |||
2007 | ||||
2008 | if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode())) | |||
2009 | return false; | |||
2010 | ||||
2011 | SrcVec = ExtractVector.getOperand(0); | |||
2012 | Index = ExtractVector.getConstantOperandVal(1); | |||
2013 | return true; | |||
2014 | }; | |||
2015 | ||||
2016 | unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex; | |||
2017 | SDValue LHSSrcVec, RHSSrcVec; | |||
2018 | if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) || | |||
2019 | !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex)) | |||
2020 | return SDValue(); | |||
2021 | ||||
2022 | if (LHSOpcode != RHSOpcode) | |||
2023 | return SDValue(); | |||
2024 | ||||
2025 | MVT ExpectedSrcVT; | |||
2026 | switch (LHSOpcode) { | |||
2027 | case WebAssemblyISD::CONVERT_LOW_S: | |||
2028 | case WebAssemblyISD::CONVERT_LOW_U: | |||
2029 | ExpectedSrcVT = MVT::v4i32; | |||
2030 | break; | |||
2031 | case WebAssemblyISD::PROMOTE_LOW: | |||
2032 | ExpectedSrcVT = MVT::v4f32; | |||
2033 | break; | |||
2034 | } | |||
2035 | if (LHSSrcVec.getValueType() != ExpectedSrcVT) | |||
2036 | return SDValue(); | |||
2037 | ||||
2038 | auto Src = LHSSrcVec; | |||
2039 | if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) { | |||
2040 | // Shuffle the source vector so that the converted lanes are the low lanes. | |||
2041 | Src = DAG.getVectorShuffle( | |||
2042 | ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec, | |||
2043 | {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1}); | |||
2044 | } | |||
2045 | return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src); | |||
2046 | } | |||
2047 | ||||
2048 | SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op, | |||
2049 | SelectionDAG &DAG) const { | |||
2050 | if (auto ConvertLow = LowerConvertLow(Op, DAG)) | |||
2051 | return ConvertLow; | |||
2052 | ||||
2053 | SDLoc DL(Op); | |||
2054 | const EVT VecT = Op.getValueType(); | |||
2055 | const EVT LaneT = Op.getOperand(0).getValueType(); | |||
2056 | const size_t Lanes = Op.getNumOperands(); | |||
2057 | bool CanSwizzle = VecT == MVT::v16i8; | |||
2058 | ||||
2059 | // BUILD_VECTORs are lowered to the instruction that initializes the highest | |||
2060 | // possible number of lanes at once followed by a sequence of replace_lane | |||
2061 | // instructions to individually initialize any remaining lanes. | |||
2062 | ||||
2063 | // TODO: Tune this. For example, lanewise swizzling is very expensive, so | |||
2064 | // swizzled lanes should be given greater weight. | |||
2065 | ||||
2066 | // TODO: Investigate looping rather than always extracting/replacing specific | |||
2067 | // lanes to fill gaps. | |||
2068 | ||||
2069 | auto IsConstant = [](const SDValue &V) { | |||
2070 | return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP; | |||
2071 | }; | |||
2072 | ||||
2073 | // Returns the source vector and index vector pair if they exist. Checks for: | |||
2074 | // (extract_vector_elt | |||
2075 | // $src, | |||
2076 | // (sign_extend_inreg (extract_vector_elt $indices, $i)) | |||
2077 | // ) | |||
2078 | auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) { | |||
2079 | auto Bail = std::make_pair(SDValue(), SDValue()); | |||
2080 | if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT) | |||
2081 | return Bail; | |||
2082 | const SDValue &SwizzleSrc = Lane->getOperand(0); | |||
2083 | const SDValue &IndexExt = Lane->getOperand(1); | |||
2084 | if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG) | |||
2085 | return Bail; | |||
2086 | const SDValue &Index = IndexExt->getOperand(0); | |||
2087 | if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT) | |||
2088 | return Bail; | |||
2089 | const SDValue &SwizzleIndices = Index->getOperand(0); | |||
2090 | if (SwizzleSrc.getValueType() != MVT::v16i8 || | |||
2091 | SwizzleIndices.getValueType() != MVT::v16i8 || | |||
2092 | Index->getOperand(1)->getOpcode() != ISD::Constant || | |||
2093 | Index->getConstantOperandVal(1) != I) | |||
2094 | return Bail; | |||
2095 | return std::make_pair(SwizzleSrc, SwizzleIndices); | |||
2096 | }; | |||
2097 | ||||
2098 | // If the lane is extracted from another vector at a constant index, return | |||
2099 | // that vector. The source vector must not have more lanes than the dest | |||
2100 | // because the shufflevector indices are in terms of the destination lanes and | |||
2101 | // would not be able to address the smaller individual source lanes. | |||
2102 | auto GetShuffleSrc = [&](const SDValue &Lane) { | |||
2103 | if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT) | |||
2104 | return SDValue(); | |||
2105 | if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode())) | |||
2106 | return SDValue(); | |||
2107 | if (Lane->getOperand(0).getValueType().getVectorNumElements() > | |||
2108 | VecT.getVectorNumElements()) | |||
2109 | return SDValue(); | |||
2110 | return Lane->getOperand(0); | |||
2111 | }; | |||
2112 | ||||
2113 | using ValueEntry = std::pair<SDValue, size_t>; | |||
2114 | SmallVector<ValueEntry, 16> SplatValueCounts; | |||
2115 | ||||
2116 | using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>; | |||
2117 | SmallVector<SwizzleEntry, 16> SwizzleCounts; | |||
2118 | ||||
2119 | using ShuffleEntry = std::pair<SDValue, size_t>; | |||
2120 | SmallVector<ShuffleEntry, 16> ShuffleCounts; | |||
2121 | ||||
2122 | auto AddCount = [](auto &Counts, const auto &Val) { | |||
2123 | auto CountIt = | |||
2124 | llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; }); | |||
2125 | if (CountIt == Counts.end()) { | |||
2126 | Counts.emplace_back(Val, 1); | |||
2127 | } else { | |||
2128 | CountIt->second++; | |||
2129 | } | |||
2130 | }; | |||
2131 | ||||
2132 | auto GetMostCommon = [](auto &Counts) { | |||
2133 | auto CommonIt = | |||
2134 | std::max_element(Counts.begin(), Counts.end(), | |||
2135 | [](auto A, auto B) { return A.second < B.second; }); | |||
2136 | assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector")(static_cast <bool> (CommonIt != Counts.end() && "Unexpected all-undef build_vector") ? void (0) : __assert_fail ("CommonIt != Counts.end() && \"Unexpected all-undef build_vector\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2136 , __extension__ __PRETTY_FUNCTION__)); | |||
2137 | return *CommonIt; | |||
2138 | }; | |||
2139 | ||||
2140 | size_t NumConstantLanes = 0; | |||
2141 | ||||
2142 | // Count eligible lanes for each type of vector creation op | |||
2143 | for (size_t I = 0; I
| |||
2144 | const SDValue &Lane = Op->getOperand(I); | |||
2145 | if (Lane.isUndef()) | |||
2146 | continue; | |||
2147 | ||||
2148 | AddCount(SplatValueCounts, Lane); | |||
2149 | ||||
2150 | if (IsConstant(Lane)) | |||
2151 | NumConstantLanes++; | |||
2152 | if (auto ShuffleSrc = GetShuffleSrc(Lane)) | |||
2153 | AddCount(ShuffleCounts, ShuffleSrc); | |||
2154 | if (CanSwizzle
| |||
2155 | auto SwizzleSrcs = GetSwizzleSrcs(I, Lane); | |||
2156 | if (SwizzleSrcs.first) | |||
2157 | AddCount(SwizzleCounts, SwizzleSrcs); | |||
2158 | } | |||
2159 | } | |||
2160 | ||||
2161 | SDValue SplatValue; | |||
2162 | size_t NumSplatLanes; | |||
2163 | std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts); | |||
2164 | ||||
2165 | SDValue SwizzleSrc; | |||
2166 | SDValue SwizzleIndices; | |||
2167 | size_t NumSwizzleLanes = 0; | |||
2168 | if (SwizzleCounts.size()) | |||
2169 | std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices), | |||
2170 | NumSwizzleLanes) = GetMostCommon(SwizzleCounts); | |||
2171 | ||||
2172 | // Shuffles can draw from up to two vectors, so find the two most common | |||
2173 | // sources. | |||
2174 | SDValue ShuffleSrc1, ShuffleSrc2; | |||
2175 | size_t NumShuffleLanes = 0; | |||
2176 | if (ShuffleCounts.size()) { | |||
2177 | std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts); | |||
2178 | llvm::erase_if(ShuffleCounts, | |||
2179 | [&](const auto &Pair) { return Pair.first == ShuffleSrc1; }); | |||
2180 | } | |||
2181 | if (ShuffleCounts.size()) { | |||
2182 | size_t AdditionalShuffleLanes; | |||
2183 | std::tie(ShuffleSrc2, AdditionalShuffleLanes) = | |||
2184 | GetMostCommon(ShuffleCounts); | |||
2185 | NumShuffleLanes += AdditionalShuffleLanes; | |||
2186 | } | |||
2187 | ||||
2188 | // Predicate returning true if the lane is properly initialized by the | |||
2189 | // original instruction | |||
2190 | std::function<bool(size_t, const SDValue &)> IsLaneConstructed; | |||
2191 | SDValue Result; | |||
2192 | // Prefer swizzles over shuffles over vector consts over splats | |||
2193 | if (NumSwizzleLanes >= NumShuffleLanes && | |||
2194 | NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) { | |||
2195 | Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc, | |||
2196 | SwizzleIndices); | |||
2197 | auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices); | |||
2198 | IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) { | |||
2199 | return Swizzled == GetSwizzleSrcs(I, Lane); | |||
2200 | }; | |||
2201 | } else if (NumShuffleLanes
| |||
2202 | NumShuffleLanes >= NumSplatLanes) { | |||
2203 | size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8; | |||
2204 | size_t DestLaneCount = VecT.getVectorNumElements(); | |||
2205 | size_t Scale1 = 1; | |||
2206 | size_t Scale2 = 1; | |||
2207 | SDValue Src1 = ShuffleSrc1; | |||
2208 | SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT); | |||
2209 | if (Src1.getValueType() != VecT) { | |||
2210 | size_t LaneSize = | |||
2211 | Src1.getValueType().getVectorElementType().getFixedSizeInBits() / 8; | |||
2212 | assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void (0) : __assert_fail ("LaneSize > DestLaneSize", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2212, __extension__ __PRETTY_FUNCTION__)); | |||
2213 | Scale1 = LaneSize / DestLaneSize; | |||
2214 | Src1 = DAG.getBitcast(VecT, Src1); | |||
2215 | } | |||
2216 | if (Src2.getValueType() != VecT) { | |||
2217 | size_t LaneSize = | |||
2218 | Src2.getValueType().getVectorElementType().getFixedSizeInBits() / 8; | |||
2219 | assert(LaneSize > DestLaneSize)(static_cast <bool> (LaneSize > DestLaneSize) ? void (0) : __assert_fail ("LaneSize > DestLaneSize", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2219, __extension__ __PRETTY_FUNCTION__)); | |||
2220 | Scale2 = LaneSize / DestLaneSize; | |||
2221 | Src2 = DAG.getBitcast(VecT, Src2); | |||
2222 | } | |||
2223 | ||||
2224 | int Mask[16]; | |||
2225 | assert(DestLaneCount <= 16)(static_cast <bool> (DestLaneCount <= 16) ? void (0) : __assert_fail ("DestLaneCount <= 16", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2225, __extension__ __PRETTY_FUNCTION__)); | |||
2226 | for (size_t I = 0; I < DestLaneCount; ++I) { | |||
2227 | const SDValue &Lane = Op->getOperand(I); | |||
2228 | SDValue Src = GetShuffleSrc(Lane); | |||
2229 | if (Src == ShuffleSrc1) { | |||
2230 | Mask[I] = Lane->getConstantOperandVal(1) * Scale1; | |||
2231 | } else if (Src && Src == ShuffleSrc2) { | |||
2232 | Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2; | |||
2233 | } else { | |||
2234 | Mask[I] = -1; | |||
2235 | } | |||
2236 | } | |||
2237 | ArrayRef<int> MaskRef(Mask, DestLaneCount); | |||
2238 | Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef); | |||
2239 | IsLaneConstructed = [&](size_t, const SDValue &Lane) { | |||
2240 | auto Src = GetShuffleSrc(Lane); | |||
2241 | return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2); | |||
2242 | }; | |||
2243 | } else if (NumConstantLanes >= NumSplatLanes) { | |||
2244 | SmallVector<SDValue, 16> ConstLanes; | |||
2245 | for (const SDValue &Lane : Op->op_values()) { | |||
2246 | if (IsConstant(Lane)) { | |||
2247 | // Values may need to be fixed so that they will sign extend to be | |||
2248 | // within the expected range during ISel. Check whether the value is in | |||
2249 | // bounds based on the lane bit width and if it is out of bounds, lop | |||
2250 | // off the extra bits and subtract 2^n to reflect giving the high bit | |||
2251 | // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it | |||
2252 | // cannot possibly be out of range. | |||
2253 | auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode()); | |||
2254 | int64_t Val = Const
| |||
2255 | uint64_t LaneBits = 128 / Lanes; | |||
2256 | assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && "Unexpected out of bounds negative value" ) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2257 , __extension__ __PRETTY_FUNCTION__)) | |||
| ||||
2257 | "Unexpected out of bounds negative value")(static_cast <bool> ((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && "Unexpected out of bounds negative value" ) ? void (0) : __assert_fail ("(LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) && \"Unexpected out of bounds negative value\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2257 , __extension__ __PRETTY_FUNCTION__)); | |||
2258 | if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) { | |||
2259 | auto NewVal = ((uint64_t)Val % (1ll << LaneBits)) - (1ll << LaneBits); | |||
2260 | ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT)); | |||
2261 | } else { | |||
2262 | ConstLanes.push_back(Lane); | |||
2263 | } | |||
2264 | } else if (LaneT.isFloatingPoint()) { | |||
2265 | ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT)); | |||
2266 | } else { | |||
2267 | ConstLanes.push_back(DAG.getConstant(0, DL, LaneT)); | |||
2268 | } | |||
2269 | } | |||
2270 | Result = DAG.getBuildVector(VecT, DL, ConstLanes); | |||
2271 | IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) { | |||
2272 | return IsConstant(Lane); | |||
2273 | }; | |||
2274 | } else { | |||
2275 | // Use a splat, but possibly a load_splat | |||
2276 | LoadSDNode *SplattedLoad; | |||
2277 | if ((SplattedLoad = dyn_cast<LoadSDNode>(SplatValue)) && | |||
2278 | SplattedLoad->getMemoryVT() == VecT.getVectorElementType()) { | |||
2279 | Result = DAG.getMemIntrinsicNode( | |||
2280 | WebAssemblyISD::LOAD_SPLAT, DL, DAG.getVTList(VecT), | |||
2281 | {SplattedLoad->getChain(), SplattedLoad->getBasePtr(), | |||
2282 | SplattedLoad->getOffset()}, | |||
2283 | SplattedLoad->getMemoryVT(), SplattedLoad->getMemOperand()); | |||
2284 | } else { | |||
2285 | Result = DAG.getSplatBuildVector(VecT, DL, SplatValue); | |||
2286 | } | |||
2287 | IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) { | |||
2288 | return Lane == SplatValue; | |||
2289 | }; | |||
2290 | } | |||
2291 | ||||
2292 | assert(Result)(static_cast <bool> (Result) ? void (0) : __assert_fail ("Result", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2292, __extension__ __PRETTY_FUNCTION__)); | |||
2293 | assert(IsLaneConstructed)(static_cast <bool> (IsLaneConstructed) ? void (0) : __assert_fail ("IsLaneConstructed", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2293, __extension__ __PRETTY_FUNCTION__)); | |||
2294 | ||||
2295 | // Add replace_lane instructions for any unhandled values | |||
2296 | for (size_t I = 0; I < Lanes; ++I) { | |||
2297 | const SDValue &Lane = Op->getOperand(I); | |||
2298 | if (!Lane.isUndef() && !IsLaneConstructed(I, Lane)) | |||
2299 | Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane, | |||
2300 | DAG.getConstant(I, DL, MVT::i32)); | |||
2301 | } | |||
2302 | ||||
2303 | return Result; | |||
2304 | } | |||
2305 | ||||
2306 | SDValue | |||
2307 | WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op, | |||
2308 | SelectionDAG &DAG) const { | |||
2309 | SDLoc DL(Op); | |||
2310 | ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask(); | |||
2311 | MVT VecType = Op.getOperand(0).getSimpleValueType(); | |||
2312 | assert(VecType.is128BitVector() && "Unexpected shuffle vector type")(static_cast <bool> (VecType.is128BitVector() && "Unexpected shuffle vector type") ? void (0) : __assert_fail ("VecType.is128BitVector() && \"Unexpected shuffle vector type\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2312 , __extension__ __PRETTY_FUNCTION__)); | |||
2313 | size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8; | |||
2314 | ||||
2315 | // Space for two vector args and sixteen mask indices | |||
2316 | SDValue Ops[18]; | |||
2317 | size_t OpIdx = 0; | |||
2318 | Ops[OpIdx++] = Op.getOperand(0); | |||
2319 | Ops[OpIdx++] = Op.getOperand(1); | |||
2320 | ||||
2321 | // Expand mask indices to byte indices and materialize them as operands | |||
2322 | for (int M : Mask) { | |||
2323 | for (size_t J = 0; J < LaneBytes; ++J) { | |||
2324 | // Lower undefs (represented by -1 in mask) to zero | |||
2325 | uint64_t ByteIndex = M == -1 ? 0 : (uint64_t)M * LaneBytes + J; | |||
2326 | Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32); | |||
2327 | } | |||
2328 | } | |||
2329 | ||||
2330 | return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops); | |||
2331 | } | |||
2332 | ||||
2333 | SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op, | |||
2334 | SelectionDAG &DAG) const { | |||
2335 | SDLoc DL(Op); | |||
2336 | // The legalizer does not know how to expand the unsupported comparison modes | |||
2337 | // of i64x2 vectors, so we manually unroll them here. | |||
2338 | assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64)(static_cast <bool> (Op->getOperand(0)->getSimpleValueType (0) == MVT::v2i64) ? void (0) : __assert_fail ("Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2338 , __extension__ __PRETTY_FUNCTION__)); | |||
2339 | SmallVector<SDValue, 2> LHS, RHS; | |||
2340 | DAG.ExtractVectorElements(Op->getOperand(0), LHS); | |||
2341 | DAG.ExtractVectorElements(Op->getOperand(1), RHS); | |||
2342 | const SDValue &CC = Op->getOperand(2); | |||
2343 | auto MakeLane = [&](unsigned I) { | |||
2344 | return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I], | |||
2345 | DAG.getConstant(uint64_t(-1), DL, MVT::i64), | |||
2346 | DAG.getConstant(uint64_t(0), DL, MVT::i64), CC); | |||
2347 | }; | |||
2348 | return DAG.getBuildVector(Op->getValueType(0), DL, | |||
2349 | {MakeLane(0), MakeLane(1)}); | |||
2350 | } | |||
2351 | ||||
2352 | SDValue | |||
2353 | WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op, | |||
2354 | SelectionDAG &DAG) const { | |||
2355 | // Allow constant lane indices, expand variable lane indices | |||
2356 | SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode(); | |||
2357 | if (isa<ConstantSDNode>(IdxNode) || IdxNode->isUndef()) | |||
2358 | return Op; | |||
2359 | else | |||
2360 | // Perform default expansion | |||
2361 | return SDValue(); | |||
2362 | } | |||
2363 | ||||
2364 | static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG) { | |||
2365 | EVT LaneT = Op.getSimpleValueType().getVectorElementType(); | |||
2366 | // 32-bit and 64-bit unrolled shifts will have proper semantics | |||
2367 | if (LaneT.bitsGE(MVT::i32)) | |||
2368 | return DAG.UnrollVectorOp(Op.getNode()); | |||
2369 | // Otherwise mask the shift value to get proper semantics from 32-bit shift | |||
2370 | SDLoc DL(Op); | |||
2371 | size_t NumLanes = Op.getSimpleValueType().getVectorNumElements(); | |||
2372 | SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32); | |||
2373 | unsigned ShiftOpcode = Op.getOpcode(); | |||
2374 | SmallVector<SDValue, 16> ShiftedElements; | |||
2375 | DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32); | |||
2376 | SmallVector<SDValue, 16> ShiftElements; | |||
2377 | DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32); | |||
2378 | SmallVector<SDValue, 16> UnrolledOps; | |||
2379 | for (size_t i = 0; i < NumLanes; ++i) { | |||
2380 | SDValue MaskedShiftValue = | |||
2381 | DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask); | |||
2382 | SDValue ShiftedValue = ShiftedElements[i]; | |||
2383 | if (ShiftOpcode == ISD::SRA) | |||
2384 | ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32, | |||
2385 | ShiftedValue, DAG.getValueType(LaneT)); | |||
2386 | UnrolledOps.push_back( | |||
2387 | DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue)); | |||
2388 | } | |||
2389 | return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps); | |||
2390 | } | |||
2391 | ||||
2392 | SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op, | |||
2393 | SelectionDAG &DAG) const { | |||
2394 | SDLoc DL(Op); | |||
2395 | ||||
2396 | // Only manually lower vector shifts | |||
2397 | assert(Op.getSimpleValueType().isVector())(static_cast <bool> (Op.getSimpleValueType().isVector() ) ? void (0) : __assert_fail ("Op.getSimpleValueType().isVector()" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2397 , __extension__ __PRETTY_FUNCTION__)); | |||
2398 | ||||
2399 | auto ShiftVal = DAG.getSplatValue(Op.getOperand(1)); | |||
2400 | if (!ShiftVal) | |||
2401 | return unrollVectorShift(Op, DAG); | |||
2402 | ||||
2403 | // Use anyext because none of the high bits can affect the shift | |||
2404 | ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32); | |||
2405 | ||||
2406 | unsigned Opcode; | |||
2407 | switch (Op.getOpcode()) { | |||
2408 | case ISD::SHL: | |||
2409 | Opcode = WebAssemblyISD::VEC_SHL; | |||
2410 | break; | |||
2411 | case ISD::SRA: | |||
2412 | Opcode = WebAssemblyISD::VEC_SHR_S; | |||
2413 | break; | |||
2414 | case ISD::SRL: | |||
2415 | Opcode = WebAssemblyISD::VEC_SHR_U; | |||
2416 | break; | |||
2417 | default: | |||
2418 | llvm_unreachable("unexpected opcode")::llvm::llvm_unreachable_internal("unexpected opcode", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2418); | |||
2419 | } | |||
2420 | ||||
2421 | return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal); | |||
2422 | } | |||
2423 | ||||
2424 | SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op, | |||
2425 | SelectionDAG &DAG) const { | |||
2426 | SDLoc DL(Op); | |||
2427 | EVT ResT = Op.getValueType(); | |||
2428 | EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT(); | |||
2429 | ||||
2430 | if ((ResT == MVT::i32 || ResT == MVT::i64) && | |||
2431 | (SatVT == MVT::i32 || SatVT == MVT::i64)) | |||
2432 | return Op; | |||
2433 | ||||
2434 | if (ResT == MVT::v4i32 && SatVT == MVT::i32) | |||
2435 | return Op; | |||
2436 | ||||
2437 | return SDValue(); | |||
2438 | } | |||
2439 | ||||
2440 | //===----------------------------------------------------------------------===// | |||
2441 | // Custom DAG combine hooks | |||
2442 | //===----------------------------------------------------------------------===// | |||
2443 | static SDValue | |||
2444 | performVECTOR_SHUFFLECombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { | |||
2445 | auto &DAG = DCI.DAG; | |||
2446 | auto Shuffle = cast<ShuffleVectorSDNode>(N); | |||
2447 | ||||
2448 | // Hoist vector bitcasts that don't change the number of lanes out of unary | |||
2449 | // shuffles, where they are less likely to get in the way of other combines. | |||
2450 | // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) -> | |||
2451 | // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask)))) | |||
2452 | SDValue Bitcast = N->getOperand(0); | |||
2453 | if (Bitcast.getOpcode() != ISD::BITCAST) | |||
2454 | return SDValue(); | |||
2455 | if (!N->getOperand(1).isUndef()) | |||
2456 | return SDValue(); | |||
2457 | SDValue CastOp = Bitcast.getOperand(0); | |||
2458 | MVT SrcType = CastOp.getSimpleValueType(); | |||
2459 | MVT DstType = Bitcast.getSimpleValueType(); | |||
2460 | if (!SrcType.is128BitVector() || | |||
2461 | SrcType.getVectorNumElements() != DstType.getVectorNumElements()) | |||
2462 | return SDValue(); | |||
2463 | SDValue NewShuffle = DAG.getVectorShuffle( | |||
2464 | SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask()); | |||
2465 | return DAG.getBitcast(DstType, NewShuffle); | |||
2466 | } | |||
2467 | ||||
2468 | static SDValue | |||
2469 | performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { | |||
2470 | auto &DAG = DCI.DAG; | |||
2471 | assert(N->getOpcode() == ISD::SIGN_EXTEND ||(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2472 , __extension__ __PRETTY_FUNCTION__)) | |||
2472 | N->getOpcode() == ISD::ZERO_EXTEND)(static_cast <bool> (N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND) ? void (0) : __assert_fail ("N->getOpcode() == ISD::SIGN_EXTEND || N->getOpcode() == ISD::ZERO_EXTEND" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2472 , __extension__ __PRETTY_FUNCTION__)); | |||
2473 | ||||
2474 | // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if | |||
2475 | // possible before the extract_subvector can be expanded. | |||
2476 | auto Extract = N->getOperand(0); | |||
2477 | if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR) | |||
2478 | return SDValue(); | |||
2479 | auto Source = Extract.getOperand(0); | |||
2480 | auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1)); | |||
2481 | if (IndexNode == nullptr) | |||
2482 | return SDValue(); | |||
2483 | auto Index = IndexNode->getZExtValue(); | |||
2484 | ||||
2485 | // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the | |||
2486 | // extracted subvector is the low or high half of its source. | |||
2487 | EVT ResVT = N->getValueType(0); | |||
2488 | if (ResVT == MVT::v8i16) { | |||
2489 | if (Extract.getValueType() != MVT::v8i8 || | |||
2490 | Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8)) | |||
2491 | return SDValue(); | |||
2492 | } else if (ResVT == MVT::v4i32) { | |||
2493 | if (Extract.getValueType() != MVT::v4i16 || | |||
2494 | Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4)) | |||
2495 | return SDValue(); | |||
2496 | } else if (ResVT == MVT::v2i64) { | |||
2497 | if (Extract.getValueType() != MVT::v2i32 || | |||
2498 | Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2)) | |||
2499 | return SDValue(); | |||
2500 | } else { | |||
2501 | return SDValue(); | |||
2502 | } | |||
2503 | ||||
2504 | bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND; | |||
2505 | bool IsLow = Index == 0; | |||
2506 | ||||
2507 | unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S | |||
2508 | : WebAssemblyISD::EXTEND_HIGH_S) | |||
2509 | : (IsLow ? WebAssemblyISD::EXTEND_LOW_U | |||
2510 | : WebAssemblyISD::EXTEND_HIGH_U); | |||
2511 | ||||
2512 | return DAG.getNode(Op, SDLoc(N), ResVT, Source); | |||
2513 | } | |||
2514 | ||||
2515 | static SDValue | |||
2516 | performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI) { | |||
2517 | auto &DAG = DCI.DAG; | |||
2518 | ||||
2519 | auto GetWasmConversionOp = [](unsigned Op) { | |||
2520 | switch (Op) { | |||
2521 | case ISD::FP_TO_SINT_SAT: | |||
2522 | return WebAssemblyISD::TRUNC_SAT_ZERO_S; | |||
2523 | case ISD::FP_TO_UINT_SAT: | |||
2524 | return WebAssemblyISD::TRUNC_SAT_ZERO_U; | |||
2525 | case ISD::FP_ROUND: | |||
2526 | return WebAssemblyISD::DEMOTE_ZERO; | |||
2527 | } | |||
2528 | llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2528); | |||
2529 | }; | |||
2530 | ||||
2531 | auto IsZeroSplat = [](SDValue SplatVal) { | |||
2532 | auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode()); | |||
2533 | APInt SplatValue, SplatUndef; | |||
2534 | unsigned SplatBitSize; | |||
2535 | bool HasAnyUndefs; | |||
2536 | return Splat && | |||
2537 | Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, | |||
2538 | HasAnyUndefs) && | |||
2539 | SplatValue == 0; | |||
2540 | }; | |||
2541 | ||||
2542 | if (N->getOpcode() == ISD::CONCAT_VECTORS) { | |||
2543 | // Combine this: | |||
2544 | // | |||
2545 | // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0))) | |||
2546 | // | |||
2547 | // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x). | |||
2548 | // | |||
2549 | // Or this: | |||
2550 | // | |||
2551 | // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0))) | |||
2552 | // | |||
2553 | // into (f32x4.demote_zero_f64x2 $x). | |||
2554 | EVT ResVT; | |||
2555 | EVT ExpectedConversionType; | |||
2556 | auto Conversion = N->getOperand(0); | |||
2557 | auto ConversionOp = Conversion.getOpcode(); | |||
2558 | switch (ConversionOp) { | |||
2559 | case ISD::FP_TO_SINT_SAT: | |||
2560 | case ISD::FP_TO_UINT_SAT: | |||
2561 | ResVT = MVT::v4i32; | |||
2562 | ExpectedConversionType = MVT::v2i32; | |||
2563 | break; | |||
2564 | case ISD::FP_ROUND: | |||
2565 | ResVT = MVT::v4f32; | |||
2566 | ExpectedConversionType = MVT::v2f32; | |||
2567 | break; | |||
2568 | default: | |||
2569 | return SDValue(); | |||
2570 | } | |||
2571 | ||||
2572 | if (N->getValueType(0) != ResVT) | |||
2573 | return SDValue(); | |||
2574 | ||||
2575 | if (Conversion.getValueType() != ExpectedConversionType) | |||
2576 | return SDValue(); | |||
2577 | ||||
2578 | auto Source = Conversion.getOperand(0); | |||
2579 | if (Source.getValueType() != MVT::v2f64) | |||
2580 | return SDValue(); | |||
2581 | ||||
2582 | if (!IsZeroSplat(N->getOperand(1)) || | |||
2583 | N->getOperand(1).getValueType() != ExpectedConversionType) | |||
2584 | return SDValue(); | |||
2585 | ||||
2586 | unsigned Op = GetWasmConversionOp(ConversionOp); | |||
2587 | return DAG.getNode(Op, SDLoc(N), ResVT, Source); | |||
2588 | } | |||
2589 | ||||
2590 | // Combine this: | |||
2591 | // | |||
2592 | // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32) | |||
2593 | // | |||
2594 | // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x). | |||
2595 | // | |||
2596 | // Or this: | |||
2597 | // | |||
2598 | // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0))))) | |||
2599 | // | |||
2600 | // into (f32x4.demote_zero_f64x2 $x). | |||
2601 | EVT ResVT; | |||
2602 | auto ConversionOp = N->getOpcode(); | |||
2603 | switch (ConversionOp) { | |||
2604 | case ISD::FP_TO_SINT_SAT: | |||
2605 | case ISD::FP_TO_UINT_SAT: | |||
2606 | ResVT = MVT::v4i32; | |||
2607 | break; | |||
2608 | case ISD::FP_ROUND: | |||
2609 | ResVT = MVT::v4f32; | |||
2610 | break; | |||
2611 | default: | |||
2612 | llvm_unreachable("unexpected op")::llvm::llvm_unreachable_internal("unexpected op", "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp" , 2612); | |||
2613 | } | |||
2614 | ||||
2615 | if (N->getValueType(0) != ResVT) | |||
2616 | return SDValue(); | |||
2617 | ||||
2618 | auto Concat = N->getOperand(0); | |||
2619 | if (Concat.getValueType() != MVT::v4f64) | |||
2620 | return SDValue(); | |||
2621 | ||||
2622 | auto Source = Concat.getOperand(0); | |||
2623 | if (Source.getValueType() != MVT::v2f64) | |||
2624 | return SDValue(); | |||
2625 | ||||
2626 | if (!IsZeroSplat(Concat.getOperand(1)) || | |||
2627 | Concat.getOperand(1).getValueType() != MVT::v2f64) | |||
2628 | return SDValue(); | |||
2629 | ||||
2630 | unsigned Op = GetWasmConversionOp(ConversionOp); | |||
2631 | return DAG.getNode(Op, SDLoc(N), ResVT, Source); | |||
2632 | } | |||
2633 | ||||
2634 | // Helper to extract VectorWidth bits from Vec, starting from IdxVal. | |||
2635 | static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, | |||
2636 | const SDLoc &DL, unsigned VectorWidth) { | |||
2637 | EVT VT = Vec.getValueType(); | |||
2638 | EVT ElVT = VT.getVectorElementType(); | |||
2639 | unsigned Factor = VT.getSizeInBits() / VectorWidth; | |||
2640 | EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT, | |||
2641 | VT.getVectorNumElements() / Factor); | |||
2642 | ||||
2643 | // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR | |||
2644 | unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits(); | |||
2645 | assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2")(static_cast <bool> (isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2") ? void (0) : __assert_fail ("isPowerOf2_32(ElemsPerChunk) && \"Elements per chunk not power of 2\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2645 , __extension__ __PRETTY_FUNCTION__)); | |||
2646 | ||||
2647 | // This is the index of the first element of the VectorWidth-bit chunk | |||
2648 | // we want. Since ElemsPerChunk is a power of 2 just need to clear bits. | |||
2649 | IdxVal &= ~(ElemsPerChunk - 1); | |||
2650 | ||||
2651 | // If the input is a buildvector just emit a smaller one. | |||
2652 | if (Vec.getOpcode() == ISD::BUILD_VECTOR) | |||
2653 | return DAG.getBuildVector(ResultVT, DL, | |||
2654 | Vec->ops().slice(IdxVal, ElemsPerChunk)); | |||
2655 | ||||
2656 | SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL); | |||
2657 | return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx); | |||
2658 | } | |||
2659 | ||||
2660 | // Helper to recursively truncate vector elements in half with NARROW_U. DstVT | |||
2661 | // is the expected destination value type after recursion. In is the initial | |||
2662 | // input. Note that the input should have enough leading zero bits to prevent | |||
2663 | // NARROW_U from saturating results. | |||
2664 | static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL, | |||
2665 | SelectionDAG &DAG) { | |||
2666 | EVT SrcVT = In.getValueType(); | |||
2667 | ||||
2668 | // No truncation required, we might get here due to recursive calls. | |||
2669 | if (SrcVT == DstVT) | |||
2670 | return In; | |||
2671 | ||||
2672 | unsigned SrcSizeInBits = SrcVT.getSizeInBits(); | |||
2673 | unsigned NumElems = SrcVT.getVectorNumElements(); | |||
2674 | if (!isPowerOf2_32(NumElems)) | |||
2675 | return SDValue(); | |||
2676 | assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation")(static_cast <bool> (DstVT.getVectorNumElements() == NumElems && "Illegal truncation") ? void (0) : __assert_fail ( "DstVT.getVectorNumElements() == NumElems && \"Illegal truncation\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2676 , __extension__ __PRETTY_FUNCTION__)); | |||
2677 | assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation")(static_cast <bool> (SrcSizeInBits > DstVT.getSizeInBits () && "Illegal truncation") ? void (0) : __assert_fail ("SrcSizeInBits > DstVT.getSizeInBits() && \"Illegal truncation\"" , "llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp", 2677 , __extension__ __PRETTY_FUNCTION__)); | |||
2678 | ||||
2679 | LLVMContext &Ctx = *DAG.getContext(); | |||
2680 | EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2); | |||
2681 | ||||
2682 | // Narrow to the largest type possible: | |||
2683 | // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u. | |||
2684 | EVT InVT = MVT::i16, OutVT = MVT::i8; | |||
2685 | if (SrcVT.getScalarSizeInBits() > 16) { | |||
2686 | InVT = MVT::i32; | |||
2687 | OutVT = MVT::i16; | |||
2688 | } | |||
2689 | unsigned SubSizeInBits = SrcSizeInBits / 2; | |||
2690 | InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits()); | |||
2691 | OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits()); | |||
2692 | ||||
2693 | // Split lower/upper subvectors. | |||
2694 | SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits); | |||
2695 | SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits); | |||
2696 | ||||
2697 | // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors. | |||
2698 | if (SrcVT.is256BitVector() && DstVT.is128BitVector()) { | |||
2699 | Lo = DAG.getBitcast(InVT, Lo); | |||
2700 | Hi = DAG.getBitcast(InVT, Hi); | |||
2701 | SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi); | |||
2702 | return DAG.getBitcast(DstVT, Res); | |||
2703 | } | |||
2704 | ||||
2705 | // Recursively narrow lower/upper subvectors, concat result and narrow again. | |||
2706 | EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2); | |||
2707 | Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG); | |||
2708 | Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG); | |||
2709 | ||||
2710 | PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems); | |||
2711 | SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi); | |||
2712 | return truncateVectorWithNARROW(DstVT, Res, DL, DAG); | |||
2713 | } | |||
2714 | ||||
2715 | static SDValue performTruncateCombine(SDNode *N, | |||
2716 | TargetLowering::DAGCombinerInfo &DCI) { | |||
2717 | auto &DAG = DCI.DAG; | |||
2718 | ||||
2719 | SDValue In = N->getOperand(0); | |||
2720 | EVT InVT = In.getValueType(); | |||
2721 | if (!InVT.isSimple()) | |||
2722 | return SDValue(); | |||
2723 | ||||
2724 | EVT OutVT = N->getValueType(0); | |||
2725 | if (!OutVT.isVector()) | |||
2726 | return SDValue(); | |||
2727 | ||||
2728 | EVT OutSVT = OutVT.getVectorElementType(); | |||
2729 | EVT InSVT = InVT.getVectorElementType(); | |||
2730 | // Currently only cover truncate to v16i8 or v8i16. | |||
2731 | if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) && | |||
2732 | (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector())) | |||
2733 | return SDValue(); | |||
2734 | ||||
2735 | SDLoc DL(N); | |||
2736 | APInt Mask = APInt::getLowBitsSet(InVT.getScalarSizeInBits(), | |||
2737 | OutVT.getScalarSizeInBits()); | |||
2738 | In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT)); | |||
2739 | return truncateVectorWithNARROW(OutVT, In, DL, DAG); | |||
2740 | } | |||
2741 | ||||
2742 | SDValue | |||
2743 | WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N, | |||
2744 | DAGCombinerInfo &DCI) const { | |||
2745 | switch (N->getOpcode()) { | |||
2746 | default: | |||
2747 | return SDValue(); | |||
2748 | case ISD::VECTOR_SHUFFLE: | |||
2749 | return performVECTOR_SHUFFLECombine(N, DCI); | |||
2750 | case ISD::SIGN_EXTEND: | |||
2751 | case ISD::ZERO_EXTEND: | |||
2752 | return performVectorExtendCombine(N, DCI); | |||
2753 | case ISD::FP_TO_SINT_SAT: | |||
2754 | case ISD::FP_TO_UINT_SAT: | |||
2755 | case ISD::FP_ROUND: | |||
2756 | case ISD::CONCAT_VECTORS: | |||
2757 | return performVectorTruncZeroCombine(N, DCI); | |||
2758 | case ISD::TRUNCATE: | |||
2759 | return performTruncateCombine(N, DCI); | |||
2760 | } | |||
2761 | } |