LLVM 22.0.0git
WebAssemblyISelLowering.cpp
Go to the documentation of this file.
1//=- WebAssemblyISelLowering.cpp - WebAssembly DAG Lowering Implementation -==//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8///
9/// \file
10/// This file implements the WebAssemblyTargetLowering class.
11///
12//===----------------------------------------------------------------------===//
13
32#include "llvm/IR/Function.h"
34#include "llvm/IR/Intrinsics.h"
35#include "llvm/IR/IntrinsicsWebAssembly.h"
40using namespace llvm;
41
42#define DEBUG_TYPE "wasm-lower"
43
45 const TargetMachine &TM, const WebAssemblySubtarget &STI)
46 : TargetLowering(TM), Subtarget(&STI) {
47 auto MVTPtr = Subtarget->hasAddr64() ? MVT::i64 : MVT::i32;
48
49 // Set the load count for memcmp expand optimization
52
53 // Booleans always contain 0 or 1.
55 // Except in SIMD vectors
57 // We don't know the microarchitecture here, so just reduce register pressure.
59 // Tell ISel that we have a stack pointer.
61 Subtarget->hasAddr64() ? WebAssembly::SP64 : WebAssembly::SP32);
62 // Set up the register classes.
63 addRegisterClass(MVT::i32, &WebAssembly::I32RegClass);
64 addRegisterClass(MVT::i64, &WebAssembly::I64RegClass);
65 addRegisterClass(MVT::f32, &WebAssembly::F32RegClass);
66 addRegisterClass(MVT::f64, &WebAssembly::F64RegClass);
67 if (Subtarget->hasSIMD128()) {
68 addRegisterClass(MVT::v16i8, &WebAssembly::V128RegClass);
69 addRegisterClass(MVT::v8i16, &WebAssembly::V128RegClass);
70 addRegisterClass(MVT::v4i32, &WebAssembly::V128RegClass);
71 addRegisterClass(MVT::v4f32, &WebAssembly::V128RegClass);
72 addRegisterClass(MVT::v2i64, &WebAssembly::V128RegClass);
73 addRegisterClass(MVT::v2f64, &WebAssembly::V128RegClass);
74 }
75 if (Subtarget->hasFP16()) {
76 addRegisterClass(MVT::v8f16, &WebAssembly::V128RegClass);
77 }
78 if (Subtarget->hasReferenceTypes()) {
79 addRegisterClass(MVT::externref, &WebAssembly::EXTERNREFRegClass);
80 addRegisterClass(MVT::funcref, &WebAssembly::FUNCREFRegClass);
81 if (Subtarget->hasExceptionHandling()) {
82 addRegisterClass(MVT::exnref, &WebAssembly::EXNREFRegClass);
83 }
84 }
85 // Compute derived properties from the register classes.
86 computeRegisterProperties(Subtarget->getRegisterInfo());
87
88 // Transform loads and stores to pointers in address space 1 to loads and
89 // stores to WebAssembly global variables, outside linear memory.
90 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64}) {
91 setOperationAction(ISD::LOAD, T, Custom);
92 setOperationAction(ISD::STORE, T, Custom);
93 }
94 if (Subtarget->hasSIMD128()) {
95 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
96 MVT::v2f64}) {
97 setOperationAction(ISD::LOAD, T, Custom);
98 setOperationAction(ISD::STORE, T, Custom);
99 }
100 }
101 if (Subtarget->hasFP16()) {
102 setOperationAction(ISD::LOAD, MVT::v8f16, Custom);
103 setOperationAction(ISD::STORE, MVT::v8f16, Custom);
104 }
105 if (Subtarget->hasReferenceTypes()) {
106 // We need custom load and store lowering for both externref, funcref and
107 // Other. The MVT::Other here represents tables of reference types.
108 for (auto T : {MVT::externref, MVT::funcref, MVT::Other}) {
109 setOperationAction(ISD::LOAD, T, Custom);
110 setOperationAction(ISD::STORE, T, Custom);
111 }
112 }
113
119 setOperationAction(ISD::BRIND, MVT::Other, Custom);
121
122 // Take the default expansion for va_arg, va_copy, and va_end. There is no
123 // default action for va_start, so we do that custom.
124 setOperationAction(ISD::VASTART, MVT::Other, Custom);
125 setOperationAction(ISD::VAARG, MVT::Other, Expand);
126 setOperationAction(ISD::VACOPY, MVT::Other, Expand);
127 setOperationAction(ISD::VAEND, MVT::Other, Expand);
128
129 for (auto T : {MVT::f32, MVT::f64, MVT::v4f32, MVT::v2f64, MVT::v8f16}) {
130 if (!Subtarget->hasFP16() && T == MVT::v8f16) {
131 continue;
132 }
133 // Don't expand the floating-point types to constant pools.
135 // Expand floating-point comparisons.
136 for (auto CC : {ISD::SETO, ISD::SETUO, ISD::SETUEQ, ISD::SETONE,
139 // Expand floating-point library function operators.
140 for (auto Op :
141 {ISD::FSIN, ISD::FCOS, ISD::FSINCOS, ISD::FPOW, ISD::FREM, ISD::FMA})
143 // Note supported floating-point library function operators that otherwise
144 // default to expand.
145 for (auto Op : {ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FNEARBYINT,
146 ISD::FRINT, ISD::FROUNDEVEN})
148 // Support minimum and maximum, which otherwise default to expand.
149 setOperationAction(ISD::FMINIMUM, T, Legal);
150 setOperationAction(ISD::FMAXIMUM, T, Legal);
151 // When experimental v8f16 support is enabled these instructions don't need
152 // to be expanded.
153 if (T != MVT::v8f16) {
154 setOperationAction(ISD::FP16_TO_FP, T, Expand);
155 setOperationAction(ISD::FP_TO_FP16, T, Expand);
156 }
158 setTruncStoreAction(T, MVT::f16, Expand);
159 }
160
161 // Expand unavailable integer operations.
162 for (auto Op :
166 for (auto T : {MVT::i32, MVT::i64})
168 if (Subtarget->hasSIMD128())
169 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
171 }
172
173 if (Subtarget->hasWideArithmetic()) {
179 }
180
181 if (Subtarget->hasNontrappingFPToInt())
183 for (auto T : {MVT::i32, MVT::i64})
185
186 // SIMD-specific configuration
187 if (Subtarget->hasSIMD128()) {
188
190
191 // Combine wide-vector muls, with extend inputs, to extmul_half.
193
194 // Combine vector mask reductions into alltrue/anytrue
196
197 // Convert vector to integer bitcasts to bitmask
198 setTargetDAGCombine(ISD::BITCAST);
199
200 // Hoist bitcasts out of shuffles
202
203 // Combine extends of extract_subvectors into widening ops
205
206 // Combine int_to_fp or fp_extend of extract_vectors and vice versa into
207 // conversions ops
210
211 // Combine fp_to_{s,u}int_sat or fp_round of concat_vectors or vice versa
212 // into conversion ops
215
217
218 // Support saturating add/sub for i8x16 and i16x8
220 for (auto T : {MVT::v16i8, MVT::v8i16})
222
223 // Support integer abs
224 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
226
227 // Custom lower BUILD_VECTORs to minimize number of replace_lanes
228 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
229 MVT::v2f64})
231
232 if (Subtarget->hasFP16())
234
235 // We have custom shuffle lowering to expose the shuffle mask
236 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
237 MVT::v2f64})
239
240 if (Subtarget->hasFP16())
242
243 // Support splatting
244 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
245 MVT::v2f64})
247
248 setOperationAction(ISD::AVGCEILU, {MVT::v8i16, MVT::v16i8}, Legal);
249
250 // Custom lowering since wasm shifts must have a scalar shift amount
251 for (auto Op : {ISD::SHL, ISD::SRA, ISD::SRL})
252 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
254
255 // Custom lower lane accesses to expand out variable indices
257 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
258 MVT::v2f64})
260
261 // There is no i8x16.mul instruction
262 setOperationAction(ISD::MUL, MVT::v16i8, Expand);
263
264 // There is no vector conditional select instruction
265 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v4f32, MVT::v2i64,
266 MVT::v2f64})
268
269 // Expand integer operations supported for scalars but not SIMD
270 for (auto Op :
272 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64})
274
275 // But we do have integer min and max operations
276 for (auto Op : {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX})
277 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
279
280 // And we have popcnt for i8x16. It can be used to expand ctlz/cttz.
281 setOperationAction(ISD::CTPOP, MVT::v16i8, Legal);
282 setOperationAction(ISD::CTLZ, MVT::v16i8, Expand);
283 setOperationAction(ISD::CTTZ, MVT::v16i8, Expand);
284
285 // Custom lower bit counting operations for other types to scalarize them.
286 for (auto Op : {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP})
287 for (auto T : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
289
290 // Expand float operations supported for scalars but not SIMD
291 for (auto Op : {ISD::FCOPYSIGN, ISD::FLOG, ISD::FLOG2, ISD::FLOG10,
292 ISD::FEXP, ISD::FEXP2, ISD::FEXP10})
293 for (auto T : {MVT::v4f32, MVT::v2f64})
295
296 // Unsigned comparison operations are unavailable for i64x2 vectors.
298 setCondCodeAction(CC, MVT::v2i64, Custom);
299
300 // 64x2 conversions are not in the spec
301 for (auto Op :
303 for (auto T : {MVT::v2i64, MVT::v2f64})
305
306 // But saturating fp_to_int converstions are
308 setOperationAction(Op, MVT::v4i32, Custom);
309 if (Subtarget->hasFP16()) {
310 setOperationAction(Op, MVT::v8i16, Custom);
311 }
312 }
313
314 // Support vector extending
318 }
319
320 // Partial MLA reductions.
321 for (auto Op : {ISD::PARTIAL_REDUCE_SMLA, ISD::PARTIAL_REDUCE_UMLA}) {
322 setPartialReduceMLAAction(Op, MVT::v4i32, MVT::v16i8, Legal);
323 setPartialReduceMLAAction(Op, MVT::v4i32, MVT::v8i16, Legal);
324 }
325 }
326
327 // As a special case, these operators use the type to mean the type to
328 // sign-extend from.
330 if (!Subtarget->hasSignExt()) {
331 // Sign extends are legal only when extending a vector extract
332 auto Action = Subtarget->hasSIMD128() ? Custom : Expand;
333 for (auto T : {MVT::i8, MVT::i16, MVT::i32})
335 }
338
339 // Dynamic stack allocation: use the default expansion.
340 setOperationAction(ISD::STACKSAVE, MVT::Other, Expand);
341 setOperationAction(ISD::STACKRESTORE, MVT::Other, Expand);
342 setOperationAction(ISD::DYNAMIC_STACKALLOC, MVTPtr, Expand);
343
347
348 // Expand these forms; we pattern-match the forms that we can handle in isel.
349 for (auto T : {MVT::i32, MVT::i64, MVT::f32, MVT::f64})
350 for (auto Op : {ISD::BR_CC, ISD::SELECT_CC})
352
353 // We have custom switch handling.
354 setOperationAction(ISD::BR_JT, MVT::Other, Custom);
355
356 // WebAssembly doesn't have:
357 // - Floating-point extending loads.
358 // - Floating-point truncating stores.
359 // - i1 extending loads.
360 // - truncating SIMD stores and most extending loads
361 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
362 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
363 for (auto T : MVT::integer_valuetypes())
364 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
365 setLoadExtAction(Ext, T, MVT::i1, Promote);
366 if (Subtarget->hasSIMD128()) {
367 for (auto T : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32,
368 MVT::v2f64}) {
369 for (auto MemT : MVT::fixedlen_vector_valuetypes()) {
370 if (MVT(T) != MemT) {
372 for (auto Ext : {ISD::EXTLOAD, ISD::ZEXTLOAD, ISD::SEXTLOAD})
373 setLoadExtAction(Ext, T, MemT, Expand);
374 }
375 }
376 }
377 // But some vector extending loads are legal
378 for (auto Ext : {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}) {
379 setLoadExtAction(Ext, MVT::v8i16, MVT::v8i8, Legal);
380 setLoadExtAction(Ext, MVT::v4i32, MVT::v4i16, Legal);
381 setLoadExtAction(Ext, MVT::v2i64, MVT::v2i32, Legal);
382 }
383 setLoadExtAction(ISD::EXTLOAD, MVT::v2f64, MVT::v2f32, Legal);
384 }
385
386 // Don't do anything clever with build_pairs
388
389 // Trap lowers to wasm unreachable
390 setOperationAction(ISD::TRAP, MVT::Other, Legal);
391 setOperationAction(ISD::DEBUGTRAP, MVT::Other, Legal);
392
393 // Exception handling intrinsics
397
399
400 // Always convert switches to br_tables unless there is only one case, which
401 // is equivalent to a simple branch. This reduces code size for wasm, and we
402 // defer possible jump table optimizations to the VM.
404}
405
414
423
425WebAssemblyTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const {
426 // We have wasm instructions for these
427 switch (AI->getOperation()) {
435 default:
436 break;
437 }
439}
440
441bool WebAssemblyTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
442 // Implementation copied from X86TargetLowering.
443 unsigned Opc = VecOp.getOpcode();
444
445 // Assume target opcodes can't be scalarized.
446 // TODO - do we have any exceptions?
448 return false;
449
450 // If the vector op is not supported, try to convert to scalar.
451 EVT VecVT = VecOp.getValueType();
453 return true;
454
455 // If the vector op is supported, but the scalar op is not, the transform may
456 // not be worthwhile.
457 EVT ScalarVT = VecVT.getScalarType();
458 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
459}
460
461FastISel *WebAssemblyTargetLowering::createFastISel(
462 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo) const {
463 return WebAssembly::createFastISel(FuncInfo, LibInfo);
464}
465
466MVT WebAssemblyTargetLowering::getScalarShiftAmountTy(const DataLayout & /*DL*/,
467 EVT VT) const {
468 unsigned BitWidth = NextPowerOf2(VT.getSizeInBits() - 1);
469 if (BitWidth > 1 && BitWidth < 8)
470 BitWidth = 8;
471
472 if (BitWidth > 64) {
473 // The shift will be lowered to a libcall, and compiler-rt libcalls expect
474 // the count to be an i32.
475 BitWidth = 32;
477 "32-bit shift counts ought to be enough for anyone");
478 }
479
482 "Unable to represent scalar shift amount type");
483 return Result;
484}
485
486// Lower an fp-to-int conversion operator from the LLVM opcode, which has an
487// undefined result on invalid/overflow, to the WebAssembly opcode, which
488// traps on invalid/overflow.
491 const TargetInstrInfo &TII,
492 bool IsUnsigned, bool Int64,
493 bool Float64, unsigned LoweredOpcode) {
495
496 Register OutReg = MI.getOperand(0).getReg();
497 Register InReg = MI.getOperand(1).getReg();
498
499 unsigned Abs = Float64 ? WebAssembly::ABS_F64 : WebAssembly::ABS_F32;
500 unsigned FConst = Float64 ? WebAssembly::CONST_F64 : WebAssembly::CONST_F32;
501 unsigned LT = Float64 ? WebAssembly::LT_F64 : WebAssembly::LT_F32;
502 unsigned GE = Float64 ? WebAssembly::GE_F64 : WebAssembly::GE_F32;
503 unsigned IConst = Int64 ? WebAssembly::CONST_I64 : WebAssembly::CONST_I32;
504 unsigned Eqz = WebAssembly::EQZ_I32;
505 unsigned And = WebAssembly::AND_I32;
506 int64_t Limit = Int64 ? INT64_MIN : INT32_MIN;
507 int64_t Substitute = IsUnsigned ? 0 : Limit;
508 double CmpVal = IsUnsigned ? -(double)Limit * 2.0 : -(double)Limit;
509 auto &Context = BB->getParent()->getFunction().getContext();
510 Type *Ty = Float64 ? Type::getDoubleTy(Context) : Type::getFloatTy(Context);
511
512 const BasicBlock *LLVMBB = BB->getBasicBlock();
513 MachineFunction *F = BB->getParent();
514 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
515 MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(LLVMBB);
516 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
517
519 F->insert(It, FalseMBB);
520 F->insert(It, TrueMBB);
521 F->insert(It, DoneMBB);
522
523 // Transfer the remainder of BB and its successor edges to DoneMBB.
524 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
526
527 BB->addSuccessor(TrueMBB);
528 BB->addSuccessor(FalseMBB);
529 TrueMBB->addSuccessor(DoneMBB);
530 FalseMBB->addSuccessor(DoneMBB);
531
532 unsigned Tmp0, Tmp1, CmpReg, EqzReg, FalseReg, TrueReg;
533 Tmp0 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
534 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
535 CmpReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
536 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
537 FalseReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
538 TrueReg = MRI.createVirtualRegister(MRI.getRegClass(OutReg));
539
540 MI.eraseFromParent();
541 // For signed numbers, we can do a single comparison to determine whether
542 // fabs(x) is within range.
543 if (IsUnsigned) {
544 Tmp0 = InReg;
545 } else {
546 BuildMI(BB, DL, TII.get(Abs), Tmp0).addReg(InReg);
547 }
548 BuildMI(BB, DL, TII.get(FConst), Tmp1)
549 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, CmpVal)));
550 BuildMI(BB, DL, TII.get(LT), CmpReg).addReg(Tmp0).addReg(Tmp1);
551
552 // For unsigned numbers, we have to do a separate comparison with zero.
553 if (IsUnsigned) {
554 Tmp1 = MRI.createVirtualRegister(MRI.getRegClass(InReg));
555 Register SecondCmpReg =
556 MRI.createVirtualRegister(&WebAssembly::I32RegClass);
557 Register AndReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
558 BuildMI(BB, DL, TII.get(FConst), Tmp1)
559 .addFPImm(cast<ConstantFP>(ConstantFP::get(Ty, 0.0)));
560 BuildMI(BB, DL, TII.get(GE), SecondCmpReg).addReg(Tmp0).addReg(Tmp1);
561 BuildMI(BB, DL, TII.get(And), AndReg).addReg(CmpReg).addReg(SecondCmpReg);
562 CmpReg = AndReg;
563 }
564
565 BuildMI(BB, DL, TII.get(Eqz), EqzReg).addReg(CmpReg);
566
567 // Create the CFG diamond to select between doing the conversion or using
568 // the substitute value.
569 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(TrueMBB).addReg(EqzReg);
570 BuildMI(FalseMBB, DL, TII.get(LoweredOpcode), FalseReg).addReg(InReg);
571 BuildMI(FalseMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
572 BuildMI(TrueMBB, DL, TII.get(IConst), TrueReg).addImm(Substitute);
573 BuildMI(*DoneMBB, DoneMBB->begin(), DL, TII.get(TargetOpcode::PHI), OutReg)
574 .addReg(FalseReg)
575 .addMBB(FalseMBB)
576 .addReg(TrueReg)
577 .addMBB(TrueMBB);
578
579 return DoneMBB;
580}
581
582// Lower a `MEMCPY` instruction into a CFG triangle around a `MEMORY_COPY`
583// instuction to handle the zero-length case.
586 const TargetInstrInfo &TII, bool Int64) {
588
589 MachineOperand DstMem = MI.getOperand(0);
590 MachineOperand SrcMem = MI.getOperand(1);
591 MachineOperand Dst = MI.getOperand(2);
592 MachineOperand Src = MI.getOperand(3);
593 MachineOperand Len = MI.getOperand(4);
594
595 // We're going to add an extra use to `Len` to test if it's zero; that
596 // use shouldn't be a kill, even if the original use is.
597 MachineOperand NoKillLen = Len;
598 NoKillLen.setIsKill(false);
599
600 // Decide on which `MachineInstr` opcode we're going to use.
601 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
602 unsigned MemoryCopy =
603 Int64 ? WebAssembly::MEMORY_COPY_A64 : WebAssembly::MEMORY_COPY_A32;
604
605 // Create two new basic blocks; one for the new `memory.fill` that we can
606 // branch over, and one for the rest of the instructions after the original
607 // `memory.fill`.
608 const BasicBlock *LLVMBB = BB->getBasicBlock();
609 MachineFunction *F = BB->getParent();
610 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
611 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
612
614 F->insert(It, TrueMBB);
615 F->insert(It, DoneMBB);
616
617 // Transfer the remainder of BB and its successor edges to DoneMBB.
618 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
620
621 // Connect the CFG edges.
622 BB->addSuccessor(TrueMBB);
623 BB->addSuccessor(DoneMBB);
624 TrueMBB->addSuccessor(DoneMBB);
625
626 // Create a virtual register for the `Eqz` result.
627 unsigned EqzReg;
628 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
629
630 // Erase the original `memory.copy`.
631 MI.eraseFromParent();
632
633 // Test if `Len` is zero.
634 BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);
635
636 // Insert a new `memory.copy`.
637 BuildMI(TrueMBB, DL, TII.get(MemoryCopy))
638 .add(DstMem)
639 .add(SrcMem)
640 .add(Dst)
641 .add(Src)
642 .add(Len);
643
644 // Create the CFG triangle.
645 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);
646 BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
647
648 return DoneMBB;
649}
650
651// Lower a `MEMSET` instruction into a CFG triangle around a `MEMORY_FILL`
652// instuction to handle the zero-length case.
655 const TargetInstrInfo &TII, bool Int64) {
657
658 MachineOperand Mem = MI.getOperand(0);
659 MachineOperand Dst = MI.getOperand(1);
660 MachineOperand Val = MI.getOperand(2);
661 MachineOperand Len = MI.getOperand(3);
662
663 // We're going to add an extra use to `Len` to test if it's zero; that
664 // use shouldn't be a kill, even if the original use is.
665 MachineOperand NoKillLen = Len;
666 NoKillLen.setIsKill(false);
667
668 // Decide on which `MachineInstr` opcode we're going to use.
669 unsigned Eqz = Int64 ? WebAssembly::EQZ_I64 : WebAssembly::EQZ_I32;
670 unsigned MemoryFill =
671 Int64 ? WebAssembly::MEMORY_FILL_A64 : WebAssembly::MEMORY_FILL_A32;
672
673 // Create two new basic blocks; one for the new `memory.fill` that we can
674 // branch over, and one for the rest of the instructions after the original
675 // `memory.fill`.
676 const BasicBlock *LLVMBB = BB->getBasicBlock();
677 MachineFunction *F = BB->getParent();
678 MachineBasicBlock *TrueMBB = F->CreateMachineBasicBlock(LLVMBB);
679 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(LLVMBB);
680
682 F->insert(It, TrueMBB);
683 F->insert(It, DoneMBB);
684
685 // Transfer the remainder of BB and its successor edges to DoneMBB.
686 DoneMBB->splice(DoneMBB->begin(), BB, std::next(MI.getIterator()), BB->end());
688
689 // Connect the CFG edges.
690 BB->addSuccessor(TrueMBB);
691 BB->addSuccessor(DoneMBB);
692 TrueMBB->addSuccessor(DoneMBB);
693
694 // Create a virtual register for the `Eqz` result.
695 unsigned EqzReg;
696 EqzReg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
697
698 // Erase the original `memory.fill`.
699 MI.eraseFromParent();
700
701 // Test if `Len` is zero.
702 BuildMI(BB, DL, TII.get(Eqz), EqzReg).add(NoKillLen);
703
704 // Insert a new `memory.copy`.
705 BuildMI(TrueMBB, DL, TII.get(MemoryFill)).add(Mem).add(Dst).add(Val).add(Len);
706
707 // Create the CFG triangle.
708 BuildMI(BB, DL, TII.get(WebAssembly::BR_IF)).addMBB(DoneMBB).addReg(EqzReg);
709 BuildMI(TrueMBB, DL, TII.get(WebAssembly::BR)).addMBB(DoneMBB);
710
711 return DoneMBB;
712}
713
714static MachineBasicBlock *
716 const WebAssemblySubtarget *Subtarget,
717 const TargetInstrInfo &TII) {
718 MachineInstr &CallParams = *CallResults.getPrevNode();
719 assert(CallParams.getOpcode() == WebAssembly::CALL_PARAMS);
720 assert(CallResults.getOpcode() == WebAssembly::CALL_RESULTS ||
721 CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS);
722
723 bool IsIndirect =
724 CallParams.getOperand(0).isReg() || CallParams.getOperand(0).isFI();
725 bool IsRetCall = CallResults.getOpcode() == WebAssembly::RET_CALL_RESULTS;
726
727 bool IsFuncrefCall = false;
728 if (IsIndirect && CallParams.getOperand(0).isReg()) {
729 Register Reg = CallParams.getOperand(0).getReg();
730 const MachineFunction *MF = BB->getParent();
731 const MachineRegisterInfo &MRI = MF->getRegInfo();
732 const TargetRegisterClass *TRC = MRI.getRegClass(Reg);
733 IsFuncrefCall = (TRC == &WebAssembly::FUNCREFRegClass);
734 assert(!IsFuncrefCall || Subtarget->hasReferenceTypes());
735 }
736
737 unsigned CallOp;
738 if (IsIndirect && IsRetCall) {
739 CallOp = WebAssembly::RET_CALL_INDIRECT;
740 } else if (IsIndirect) {
741 CallOp = WebAssembly::CALL_INDIRECT;
742 } else if (IsRetCall) {
743 CallOp = WebAssembly::RET_CALL;
744 } else {
745 CallOp = WebAssembly::CALL;
746 }
747
748 MachineFunction &MF = *BB->getParent();
749 const MCInstrDesc &MCID = TII.get(CallOp);
750 MachineInstrBuilder MIB(MF, MF.CreateMachineInstr(MCID, DL));
751
752 // Move the function pointer to the end of the arguments for indirect calls
753 if (IsIndirect) {
754 auto FnPtr = CallParams.getOperand(0);
755 CallParams.removeOperand(0);
756
757 // For funcrefs, call_indirect is done through __funcref_call_table and the
758 // funcref is always installed in slot 0 of the table, therefore instead of
759 // having the function pointer added at the end of the params list, a zero
760 // (the index in
761 // __funcref_call_table is added).
762 if (IsFuncrefCall) {
763 Register RegZero =
764 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
765 MachineInstrBuilder MIBC0 =
766 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
767
768 BB->insert(CallResults.getIterator(), MIBC0);
769 MachineInstrBuilder(MF, CallParams).addReg(RegZero);
770 } else
771 CallParams.addOperand(FnPtr);
772 }
773
774 for (auto Def : CallResults.defs())
775 MIB.add(Def);
776
777 if (IsIndirect) {
778 // Placeholder for the type index.
779 // This gets replaced with the correct value in WebAssemblyMCInstLower.cpp
780 MIB.addImm(0);
781 // The table into which this call_indirect indexes.
782 MCSymbolWasm *Table = IsFuncrefCall
784 MF.getContext(), Subtarget)
786 MF.getContext(), Subtarget);
787 if (Subtarget->hasCallIndirectOverlong()) {
788 MIB.addSym(Table);
789 } else {
790 // For the MVP there is at most one table whose number is 0, but we can't
791 // write a table symbol or issue relocations. Instead we just ensure the
792 // table is live and write a zero.
793 Table->setNoStrip();
794 MIB.addImm(0);
795 }
796 }
797
798 for (auto Use : CallParams.uses())
799 MIB.add(Use);
800
801 BB->insert(CallResults.getIterator(), MIB);
802 CallParams.eraseFromParent();
803 CallResults.eraseFromParent();
804
805 // If this is a funcref call, to avoid hidden GC roots, we need to clear the
806 // table slot with ref.null upon call_indirect return.
807 //
808 // This generates the following code, which comes right after a call_indirect
809 // of a funcref:
810 //
811 // i32.const 0
812 // ref.null func
813 // table.set __funcref_call_table
814 if (IsIndirect && IsFuncrefCall) {
816 MF.getContext(), Subtarget);
817 Register RegZero =
818 MF.getRegInfo().createVirtualRegister(&WebAssembly::I32RegClass);
819 MachineInstr *Const0 =
820 BuildMI(MF, DL, TII.get(WebAssembly::CONST_I32), RegZero).addImm(0);
821 BB->insertAfter(MIB.getInstr()->getIterator(), Const0);
822
823 Register RegFuncref =
824 MF.getRegInfo().createVirtualRegister(&WebAssembly::FUNCREFRegClass);
825 MachineInstr *RefNull =
826 BuildMI(MF, DL, TII.get(WebAssembly::REF_NULL_FUNCREF), RegFuncref);
827 BB->insertAfter(Const0->getIterator(), RefNull);
828
829 MachineInstr *TableSet =
830 BuildMI(MF, DL, TII.get(WebAssembly::TABLE_SET_FUNCREF))
831 .addSym(Table)
832 .addReg(RegZero)
833 .addReg(RegFuncref);
834 BB->insertAfter(RefNull->getIterator(), TableSet);
835 }
836
837 return BB;
838}
839
840MachineBasicBlock *WebAssemblyTargetLowering::EmitInstrWithCustomInserter(
841 MachineInstr &MI, MachineBasicBlock *BB) const {
842 const TargetInstrInfo &TII = *Subtarget->getInstrInfo();
843 DebugLoc DL = MI.getDebugLoc();
844
845 switch (MI.getOpcode()) {
846 default:
847 llvm_unreachable("Unexpected instr type to insert");
848 case WebAssembly::FP_TO_SINT_I32_F32:
849 return LowerFPToInt(MI, DL, BB, TII, false, false, false,
850 WebAssembly::I32_TRUNC_S_F32);
851 case WebAssembly::FP_TO_UINT_I32_F32:
852 return LowerFPToInt(MI, DL, BB, TII, true, false, false,
853 WebAssembly::I32_TRUNC_U_F32);
854 case WebAssembly::FP_TO_SINT_I64_F32:
855 return LowerFPToInt(MI, DL, BB, TII, false, true, false,
856 WebAssembly::I64_TRUNC_S_F32);
857 case WebAssembly::FP_TO_UINT_I64_F32:
858 return LowerFPToInt(MI, DL, BB, TII, true, true, false,
859 WebAssembly::I64_TRUNC_U_F32);
860 case WebAssembly::FP_TO_SINT_I32_F64:
861 return LowerFPToInt(MI, DL, BB, TII, false, false, true,
862 WebAssembly::I32_TRUNC_S_F64);
863 case WebAssembly::FP_TO_UINT_I32_F64:
864 return LowerFPToInt(MI, DL, BB, TII, true, false, true,
865 WebAssembly::I32_TRUNC_U_F64);
866 case WebAssembly::FP_TO_SINT_I64_F64:
867 return LowerFPToInt(MI, DL, BB, TII, false, true, true,
868 WebAssembly::I64_TRUNC_S_F64);
869 case WebAssembly::FP_TO_UINT_I64_F64:
870 return LowerFPToInt(MI, DL, BB, TII, true, true, true,
871 WebAssembly::I64_TRUNC_U_F64);
872 case WebAssembly::MEMCPY_A32:
873 return LowerMemcpy(MI, DL, BB, TII, false);
874 case WebAssembly::MEMCPY_A64:
875 return LowerMemcpy(MI, DL, BB, TII, true);
876 case WebAssembly::MEMSET_A32:
877 return LowerMemset(MI, DL, BB, TII, false);
878 case WebAssembly::MEMSET_A64:
879 return LowerMemset(MI, DL, BB, TII, true);
880 case WebAssembly::CALL_RESULTS:
881 case WebAssembly::RET_CALL_RESULTS:
882 return LowerCallResults(MI, DL, BB, Subtarget, TII);
883 }
884}
885
886const char *
887WebAssemblyTargetLowering::getTargetNodeName(unsigned Opcode) const {
888 switch (static_cast<WebAssemblyISD::NodeType>(Opcode)) {
890 break;
891#define HANDLE_NODETYPE(NODE) \
892 case WebAssemblyISD::NODE: \
893 return "WebAssemblyISD::" #NODE;
894#include "WebAssemblyISD.def"
895#undef HANDLE_NODETYPE
896 }
897 return nullptr;
898}
899
900std::pair<unsigned, const TargetRegisterClass *>
901WebAssemblyTargetLowering::getRegForInlineAsmConstraint(
902 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
903 // First, see if this is a constraint that directly corresponds to a
904 // WebAssembly register class.
905 if (Constraint.size() == 1) {
906 switch (Constraint[0]) {
907 case 'r':
908 assert(VT != MVT::iPTR && "Pointer MVT not expected here");
909 if (Subtarget->hasSIMD128() && VT.isVector()) {
910 if (VT.getSizeInBits() == 128)
911 return std::make_pair(0U, &WebAssembly::V128RegClass);
912 }
913 if (VT.isInteger() && !VT.isVector()) {
914 if (VT.getSizeInBits() <= 32)
915 return std::make_pair(0U, &WebAssembly::I32RegClass);
916 if (VT.getSizeInBits() <= 64)
917 return std::make_pair(0U, &WebAssembly::I64RegClass);
918 }
919 if (VT.isFloatingPoint() && !VT.isVector()) {
920 switch (VT.getSizeInBits()) {
921 case 32:
922 return std::make_pair(0U, &WebAssembly::F32RegClass);
923 case 64:
924 return std::make_pair(0U, &WebAssembly::F64RegClass);
925 default:
926 break;
927 }
928 }
929 break;
930 default:
931 break;
932 }
933 }
934
936}
937
938bool WebAssemblyTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
939 // Assume ctz is a relatively cheap operation.
940 return true;
941}
942
943bool WebAssemblyTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
944 // Assume clz is a relatively cheap operation.
945 return true;
946}
947
948bool WebAssemblyTargetLowering::isLegalAddressingMode(const DataLayout &DL,
949 const AddrMode &AM,
950 Type *Ty, unsigned AS,
951 Instruction *I) const {
952 // WebAssembly offsets are added as unsigned without wrapping. The
953 // isLegalAddressingMode gives us no way to determine if wrapping could be
954 // happening, so we approximate this by accepting only non-negative offsets.
955 if (AM.BaseOffs < 0)
956 return false;
957
958 // WebAssembly has no scale register operands.
959 if (AM.Scale != 0)
960 return false;
961
962 // Everything else is legal.
963 return true;
964}
965
966bool WebAssemblyTargetLowering::allowsMisalignedMemoryAccesses(
967 EVT /*VT*/, unsigned /*AddrSpace*/, Align /*Align*/,
968 MachineMemOperand::Flags /*Flags*/, unsigned *Fast) const {
969 // WebAssembly supports unaligned accesses, though it should be declared
970 // with the p2align attribute on loads and stores which do so, and there
971 // may be a performance impact. We tell LLVM they're "fast" because
972 // for the kinds of things that LLVM uses this for (merging adjacent stores
973 // of constants, etc.), WebAssembly implementations will either want the
974 // unaligned access or they'll split anyway.
975 if (Fast)
976 *Fast = 1;
977 return true;
978}
979
980bool WebAssemblyTargetLowering::isIntDivCheap(EVT VT,
981 AttributeList Attr) const {
982 // The current thinking is that wasm engines will perform this optimization,
983 // so we can save on code size.
984 return true;
985}
986
987bool WebAssemblyTargetLowering::isVectorLoadExtDesirable(SDValue ExtVal) const {
988 EVT ExtT = ExtVal.getValueType();
989 EVT MemT = cast<LoadSDNode>(ExtVal->getOperand(0))->getValueType(0);
990 return (ExtT == MVT::v8i16 && MemT == MVT::v8i8) ||
991 (ExtT == MVT::v4i32 && MemT == MVT::v4i16) ||
992 (ExtT == MVT::v2i64 && MemT == MVT::v2i32);
993}
994
995bool WebAssemblyTargetLowering::isOffsetFoldingLegal(
996 const GlobalAddressSDNode *GA) const {
997 // Wasm doesn't support function addresses with offsets
998 const GlobalValue *GV = GA->getGlobal();
1000}
1001
1002EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL,
1003 LLVMContext &C,
1004 EVT VT) const {
1005 if (VT.isVector())
1007
1008 // So far, all branch instructions in Wasm take an I32 condition.
1009 // The default TargetLowering::getSetCCResultType returns the pointer size,
1010 // which would be useful to reduce instruction counts when testing
1011 // against 64-bit pointers/values if at some point Wasm supports that.
1012 return EVT::getIntegerVT(C, 32);
1013}
1014
1015bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info,
1016 const CallInst &I,
1017 MachineFunction &MF,
1018 unsigned Intrinsic) const {
1019 switch (Intrinsic) {
1020 case Intrinsic::wasm_memory_atomic_notify:
1022 Info.memVT = MVT::i32;
1023 Info.ptrVal = I.getArgOperand(0);
1024 Info.offset = 0;
1025 Info.align = Align(4);
1026 // atomic.notify instruction does not really load the memory specified with
1027 // this argument, but MachineMemOperand should either be load or store, so
1028 // we set this to a load.
1029 // FIXME Volatile isn't really correct, but currently all LLVM atomic
1030 // instructions are treated as volatiles in the backend, so we should be
1031 // consistent. The same applies for wasm_atomic_wait intrinsics too.
1033 return true;
1034 case Intrinsic::wasm_memory_atomic_wait32:
1036 Info.memVT = MVT::i32;
1037 Info.ptrVal = I.getArgOperand(0);
1038 Info.offset = 0;
1039 Info.align = Align(4);
1041 return true;
1042 case Intrinsic::wasm_memory_atomic_wait64:
1044 Info.memVT = MVT::i64;
1045 Info.ptrVal = I.getArgOperand(0);
1046 Info.offset = 0;
1047 Info.align = Align(8);
1049 return true;
1050 case Intrinsic::wasm_loadf16_f32:
1052 Info.memVT = MVT::f16;
1053 Info.ptrVal = I.getArgOperand(0);
1054 Info.offset = 0;
1055 Info.align = Align(2);
1057 return true;
1058 case Intrinsic::wasm_storef16_f32:
1060 Info.memVT = MVT::f16;
1061 Info.ptrVal = I.getArgOperand(1);
1062 Info.offset = 0;
1063 Info.align = Align(2);
1065 return true;
1066 default:
1067 return false;
1068 }
1069}
1070
1071void WebAssemblyTargetLowering::computeKnownBitsForTargetNode(
1072 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
1073 const SelectionDAG &DAG, unsigned Depth) const {
1074 switch (Op.getOpcode()) {
1075 default:
1076 break;
1078 unsigned IntNo = Op.getConstantOperandVal(0);
1079 switch (IntNo) {
1080 default:
1081 break;
1082 case Intrinsic::wasm_bitmask: {
1083 unsigned BitWidth = Known.getBitWidth();
1084 EVT VT = Op.getOperand(1).getSimpleValueType();
1085 unsigned PossibleBits = VT.getVectorNumElements();
1086 APInt ZeroMask = APInt::getHighBitsSet(BitWidth, BitWidth - PossibleBits);
1087 Known.Zero |= ZeroMask;
1088 break;
1089 }
1090 }
1091 break;
1092 }
1093
1094 // For 128-bit addition if the upper bits are all zero then it's known that
1095 // the upper bits of the result will have all bits guaranteed zero except the
1096 // first.
1097 case WebAssemblyISD::I64_ADD128:
1098 if (Op.getResNo() == 1) {
1099 SDValue LHS_HI = Op.getOperand(1);
1100 SDValue RHS_HI = Op.getOperand(3);
1101 if (isNullConstant(LHS_HI) && isNullConstant(RHS_HI))
1102 Known.Zero.setBitsFrom(1);
1103 }
1104 break;
1105 }
1106}
1107
1109WebAssemblyTargetLowering::getPreferredVectorAction(MVT VT) const {
1110 if (VT.isFixedLengthVector()) {
1111 MVT EltVT = VT.getVectorElementType();
1112 // We have legal vector types with these lane types, so widening the
1113 // vector would let us use some of the lanes directly without having to
1114 // extend or truncate values.
1115 if (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
1116 EltVT == MVT::i64 || EltVT == MVT::f32 || EltVT == MVT::f64)
1117 return TypeWidenVector;
1118 }
1119
1121}
1122
1123bool WebAssemblyTargetLowering::shouldSimplifyDemandedVectorElts(
1124 SDValue Op, const TargetLoweringOpt &TLO) const {
1125 // ISel process runs DAGCombiner after legalization; this step is called
1126 // SelectionDAG optimization phase. This post-legalization combining process
1127 // runs DAGCombiner on each node, and if there was a change to be made,
1128 // re-runs legalization again on it and its user nodes to make sure
1129 // everythiing is in a legalized state.
1130 //
1131 // The legalization calls lowering routines, and we do our custom lowering for
1132 // build_vectors (LowerBUILD_VECTOR), which converts undef vector elements
1133 // into zeros. But there is a set of routines in DAGCombiner that turns unused
1134 // (= not demanded) nodes into undef, among which SimplifyDemandedVectorElts
1135 // turns unused vector elements into undefs. But this routine does not work
1136 // with our custom LowerBUILD_VECTOR, which turns undefs into zeros. This
1137 // combination can result in a infinite loop, in which undefs are converted to
1138 // zeros in legalization and back to undefs in combining.
1139 //
1140 // So after DAG is legalized, we prevent SimplifyDemandedVectorElts from
1141 // running for build_vectors.
1142 if (Op.getOpcode() == ISD::BUILD_VECTOR && TLO.LegalOps && TLO.LegalTys)
1143 return false;
1144 return true;
1145}
1146
1147//===----------------------------------------------------------------------===//
1148// WebAssembly Lowering private implementation.
1149//===----------------------------------------------------------------------===//
1150
1151//===----------------------------------------------------------------------===//
1152// Lowering Code
1153//===----------------------------------------------------------------------===//
1154
1155static void fail(const SDLoc &DL, SelectionDAG &DAG, const char *Msg) {
1157 DAG.getContext()->diagnose(
1158 DiagnosticInfoUnsupported(MF.getFunction(), Msg, DL.getDebugLoc()));
1159}
1160
1161// Test whether the given calling convention is supported.
1163 // We currently support the language-independent target-independent
1164 // conventions. We don't yet have a way to annotate calls with properties like
1165 // "cold", and we don't have any call-clobbered registers, so these are mostly
1166 // all handled the same.
1167 return CallConv == CallingConv::C || CallConv == CallingConv::Fast ||
1168 CallConv == CallingConv::Cold ||
1169 CallConv == CallingConv::PreserveMost ||
1170 CallConv == CallingConv::PreserveAll ||
1171 CallConv == CallingConv::CXX_FAST_TLS ||
1173 CallConv == CallingConv::Swift;
1174}
1175
1176SDValue
1177WebAssemblyTargetLowering::LowerCall(CallLoweringInfo &CLI,
1178 SmallVectorImpl<SDValue> &InVals) const {
1179 SelectionDAG &DAG = CLI.DAG;
1180 SDLoc DL = CLI.DL;
1181 SDValue Chain = CLI.Chain;
1182 SDValue Callee = CLI.Callee;
1183 MachineFunction &MF = DAG.getMachineFunction();
1184 auto Layout = MF.getDataLayout();
1185
1186 CallingConv::ID CallConv = CLI.CallConv;
1187 if (!callingConvSupported(CallConv))
1188 fail(DL, DAG,
1189 "WebAssembly doesn't support language-specific or target-specific "
1190 "calling conventions yet");
1191 if (CLI.IsPatchPoint)
1192 fail(DL, DAG, "WebAssembly doesn't support patch point yet");
1193
1194 if (CLI.IsTailCall) {
1195 auto NoTail = [&](const char *Msg) {
1196 if (CLI.CB && CLI.CB->isMustTailCall())
1197 fail(DL, DAG, Msg);
1198 CLI.IsTailCall = false;
1199 };
1200
1201 if (!Subtarget->hasTailCall())
1202 NoTail("WebAssembly 'tail-call' feature not enabled");
1203
1204 // Varargs calls cannot be tail calls because the buffer is on the stack
1205 if (CLI.IsVarArg)
1206 NoTail("WebAssembly does not support varargs tail calls");
1207
1208 // Do not tail call unless caller and callee return types match
1209 const Function &F = MF.getFunction();
1210 const TargetMachine &TM = getTargetMachine();
1211 Type *RetTy = F.getReturnType();
1212 SmallVector<MVT, 4> CallerRetTys;
1213 SmallVector<MVT, 4> CalleeRetTys;
1214 computeLegalValueVTs(F, TM, RetTy, CallerRetTys);
1215 computeLegalValueVTs(F, TM, CLI.RetTy, CalleeRetTys);
1216 bool TypesMatch = CallerRetTys.size() == CalleeRetTys.size() &&
1217 std::equal(CallerRetTys.begin(), CallerRetTys.end(),
1218 CalleeRetTys.begin());
1219 if (!TypesMatch)
1220 NoTail("WebAssembly tail call requires caller and callee return types to "
1221 "match");
1222
1223 // If pointers to local stack values are passed, we cannot tail call
1224 if (CLI.CB) {
1225 for (auto &Arg : CLI.CB->args()) {
1226 Value *Val = Arg.get();
1227 // Trace the value back through pointer operations
1228 while (true) {
1229 Value *Src = Val->stripPointerCastsAndAliases();
1230 if (auto *GEP = dyn_cast<GetElementPtrInst>(Src))
1231 Src = GEP->getPointerOperand();
1232 if (Val == Src)
1233 break;
1234 Val = Src;
1235 }
1236 if (isa<AllocaInst>(Val)) {
1237 NoTail(
1238 "WebAssembly does not support tail calling with stack arguments");
1239 break;
1240 }
1241 }
1242 }
1243 }
1244
1245 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
1246 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
1247 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
1248
1249 // The generic code may have added an sret argument. If we're lowering an
1250 // invoke function, the ABI requires that the function pointer be the first
1251 // argument, so we may have to swap the arguments.
1252 if (CallConv == CallingConv::WASM_EmscriptenInvoke && Outs.size() >= 2 &&
1253 Outs[0].Flags.isSRet()) {
1254 std::swap(Outs[0], Outs[1]);
1255 std::swap(OutVals[0], OutVals[1]);
1256 }
1257
1258 bool HasSwiftSelfArg = false;
1259 bool HasSwiftErrorArg = false;
1260 unsigned NumFixedArgs = 0;
1261 for (unsigned I = 0; I < Outs.size(); ++I) {
1262 const ISD::OutputArg &Out = Outs[I];
1263 SDValue &OutVal = OutVals[I];
1264 HasSwiftSelfArg |= Out.Flags.isSwiftSelf();
1265 HasSwiftErrorArg |= Out.Flags.isSwiftError();
1266 if (Out.Flags.isNest())
1267 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1268 if (Out.Flags.isInAlloca())
1269 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1270 if (Out.Flags.isInConsecutiveRegs())
1271 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1273 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1274 if (Out.Flags.isByVal() && Out.Flags.getByValSize() != 0) {
1275 auto &MFI = MF.getFrameInfo();
1276 int FI = MFI.CreateStackObject(Out.Flags.getByValSize(),
1278 /*isSS=*/false);
1279 SDValue SizeNode =
1280 DAG.getConstant(Out.Flags.getByValSize(), DL, MVT::i32);
1281 SDValue FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1282 Chain = DAG.getMemcpy(Chain, DL, FINode, OutVal, SizeNode,
1284 /*isVolatile*/ false, /*AlwaysInline=*/false,
1285 /*CI=*/nullptr, std::nullopt, MachinePointerInfo(),
1286 MachinePointerInfo());
1287 OutVal = FINode;
1288 }
1289 // Count the number of fixed args *after* legalization.
1290 NumFixedArgs += !Out.Flags.isVarArg();
1291 }
1292
1293 bool IsVarArg = CLI.IsVarArg;
1294 auto PtrVT = getPointerTy(Layout);
1295
1296 // For swiftcc, emit additional swiftself and swifterror arguments
1297 // if there aren't. These additional arguments are also added for callee
1298 // signature They are necessary to match callee and caller signature for
1299 // indirect call.
1300 if (CallConv == CallingConv::Swift) {
1301 Type *PtrTy = PointerType::getUnqual(*DAG.getContext());
1302 if (!HasSwiftSelfArg) {
1303 NumFixedArgs++;
1304 ISD::ArgFlagsTy Flags;
1305 Flags.setSwiftSelf();
1306 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1307 CLI.Outs.push_back(Arg);
1308 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1309 CLI.OutVals.push_back(ArgVal);
1310 }
1311 if (!HasSwiftErrorArg) {
1312 NumFixedArgs++;
1313 ISD::ArgFlagsTy Flags;
1314 Flags.setSwiftError();
1315 ISD::OutputArg Arg(Flags, PtrVT, EVT(PtrVT), PtrTy, 0, 0);
1316 CLI.Outs.push_back(Arg);
1317 SDValue ArgVal = DAG.getUNDEF(PtrVT);
1318 CLI.OutVals.push_back(ArgVal);
1319 }
1320 }
1321
1322 // Analyze operands of the call, assigning locations to each operand.
1324 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
1325
1326 if (IsVarArg) {
1327 // Outgoing non-fixed arguments are placed in a buffer. First
1328 // compute their offsets and the total amount of buffer space needed.
1329 for (unsigned I = NumFixedArgs; I < Outs.size(); ++I) {
1330 const ISD::OutputArg &Out = Outs[I];
1331 SDValue &Arg = OutVals[I];
1332 EVT VT = Arg.getValueType();
1333 assert(VT != MVT::iPTR && "Legalized args should be concrete");
1334 Type *Ty = VT.getTypeForEVT(*DAG.getContext());
1335 Align Alignment =
1336 std::max(Out.Flags.getNonZeroOrigAlign(), Layout.getABITypeAlign(Ty));
1337 unsigned Offset =
1338 CCInfo.AllocateStack(Layout.getTypeAllocSize(Ty), Alignment);
1339 CCInfo.addLoc(CCValAssign::getMem(ArgLocs.size(), VT.getSimpleVT(),
1340 Offset, VT.getSimpleVT(),
1342 }
1343 }
1344
1345 unsigned NumBytes = CCInfo.getAlignedCallFrameSize();
1346
1347 SDValue FINode;
1348 if (IsVarArg && NumBytes) {
1349 // For non-fixed arguments, next emit stores to store the argument values
1350 // to the stack buffer at the offsets computed above.
1351 MaybeAlign StackAlign = Layout.getStackAlignment();
1352 assert(StackAlign && "data layout string is missing stack alignment");
1353 int FI = MF.getFrameInfo().CreateStackObject(NumBytes, *StackAlign,
1354 /*isSS=*/false);
1355 unsigned ValNo = 0;
1357 for (SDValue Arg : drop_begin(OutVals, NumFixedArgs)) {
1358 assert(ArgLocs[ValNo].getValNo() == ValNo &&
1359 "ArgLocs should remain in order and only hold varargs args");
1360 unsigned Offset = ArgLocs[ValNo++].getLocMemOffset();
1361 FINode = DAG.getFrameIndex(FI, getPointerTy(Layout));
1362 SDValue Add = DAG.getNode(ISD::ADD, DL, PtrVT, FINode,
1363 DAG.getConstant(Offset, DL, PtrVT));
1364 Chains.push_back(
1365 DAG.getStore(Chain, DL, Arg, Add,
1367 }
1368 if (!Chains.empty())
1369 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, Chains);
1370 } else if (IsVarArg) {
1371 FINode = DAG.getIntPtrConstant(0, DL);
1372 }
1373
1374 if (Callee->getOpcode() == ISD::GlobalAddress) {
1375 // If the callee is a GlobalAddress node (quite common, every direct call
1376 // is) turn it into a TargetGlobalAddress node so that LowerGlobalAddress
1377 // doesn't at MO_GOT which is not needed for direct calls.
1378 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Callee);
1381 GA->getOffset());
1382 Callee = DAG.getNode(WebAssemblyISD::Wrapper, DL,
1383 getPointerTy(DAG.getDataLayout()), Callee);
1384 }
1385
1386 // Compute the operands for the CALLn node.
1388 Ops.push_back(Chain);
1389 Ops.push_back(Callee);
1390
1391 // Add all fixed arguments. Note that for non-varargs calls, NumFixedArgs
1392 // isn't reliable.
1393 Ops.append(OutVals.begin(),
1394 IsVarArg ? OutVals.begin() + NumFixedArgs : OutVals.end());
1395 // Add a pointer to the vararg buffer.
1396 if (IsVarArg)
1397 Ops.push_back(FINode);
1398
1399 SmallVector<EVT, 8> InTys;
1400 for (const auto &In : Ins) {
1401 assert(!In.Flags.isByVal() && "byval is not valid for return values");
1402 assert(!In.Flags.isNest() && "nest is not valid for return values");
1403 if (In.Flags.isInAlloca())
1404 fail(DL, DAG, "WebAssembly hasn't implemented inalloca return values");
1405 if (In.Flags.isInConsecutiveRegs())
1406 fail(DL, DAG, "WebAssembly hasn't implemented cons regs return values");
1407 if (In.Flags.isInConsecutiveRegsLast())
1408 fail(DL, DAG,
1409 "WebAssembly hasn't implemented cons regs last return values");
1410 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1411 // registers.
1412 InTys.push_back(In.VT);
1413 }
1414
1415 // Lastly, if this is a call to a funcref we need to add an instruction
1416 // table.set to the chain and transform the call.
1418 CLI.CB->getCalledOperand()->getType())) {
1419 // In the absence of function references proposal where a funcref call is
1420 // lowered to call_ref, using reference types we generate a table.set to set
1421 // the funcref to a special table used solely for this purpose, followed by
1422 // a call_indirect. Here we just generate the table set, and return the
1423 // SDValue of the table.set so that LowerCall can finalize the lowering by
1424 // generating the call_indirect.
1425 SDValue Chain = Ops[0];
1426
1428 MF.getContext(), Subtarget);
1429 SDValue Sym = DAG.getMCSymbol(Table, PtrVT);
1430 SDValue TableSlot = DAG.getConstant(0, DL, MVT::i32);
1431 SDValue TableSetOps[] = {Chain, Sym, TableSlot, Callee};
1432 SDValue TableSet = DAG.getMemIntrinsicNode(
1433 WebAssemblyISD::TABLE_SET, DL, DAG.getVTList(MVT::Other), TableSetOps,
1434 MVT::funcref,
1435 // Machine Mem Operand args
1436 MachinePointerInfo(
1438 CLI.CB->getCalledOperand()->getPointerAlignment(DAG.getDataLayout()),
1440
1441 Ops[0] = TableSet; // The new chain is the TableSet itself
1442 }
1443
1444 if (CLI.IsTailCall) {
1445 // ret_calls do not return values to the current frame
1446 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
1447 return DAG.getNode(WebAssemblyISD::RET_CALL, DL, NodeTys, Ops);
1448 }
1449
1450 InTys.push_back(MVT::Other);
1451 SDVTList InTyList = DAG.getVTList(InTys);
1452 SDValue Res = DAG.getNode(WebAssemblyISD::CALL, DL, InTyList, Ops);
1453
1454 for (size_t I = 0; I < Ins.size(); ++I)
1455 InVals.push_back(Res.getValue(I));
1456
1457 // Return the chain
1458 return Res.getValue(Ins.size());
1459}
1460
1461bool WebAssemblyTargetLowering::CanLowerReturn(
1462 CallingConv::ID /*CallConv*/, MachineFunction & /*MF*/, bool /*IsVarArg*/,
1463 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext & /*Context*/,
1464 const Type *RetTy) const {
1465 // WebAssembly can only handle returning tuples with multivalue enabled
1466 return WebAssembly::canLowerReturn(Outs.size(), Subtarget);
1467}
1468
1469SDValue WebAssemblyTargetLowering::LowerReturn(
1470 SDValue Chain, CallingConv::ID CallConv, bool /*IsVarArg*/,
1472 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
1473 SelectionDAG &DAG) const {
1474 assert(WebAssembly::canLowerReturn(Outs.size(), Subtarget) &&
1475 "MVP WebAssembly can only return up to one value");
1476 if (!callingConvSupported(CallConv))
1477 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1478
1479 SmallVector<SDValue, 4> RetOps(1, Chain);
1480 RetOps.append(OutVals.begin(), OutVals.end());
1481 Chain = DAG.getNode(WebAssemblyISD::RETURN, DL, MVT::Other, RetOps);
1482
1483 // Record the number and types of the return values.
1484 for (const ISD::OutputArg &Out : Outs) {
1485 assert(!Out.Flags.isByVal() && "byval is not valid for return values");
1486 assert(!Out.Flags.isNest() && "nest is not valid for return values");
1487 assert(!Out.Flags.isVarArg() && "non-fixed return value is not valid");
1488 if (Out.Flags.isInAlloca())
1489 fail(DL, DAG, "WebAssembly hasn't implemented inalloca results");
1490 if (Out.Flags.isInConsecutiveRegs())
1491 fail(DL, DAG, "WebAssembly hasn't implemented cons regs results");
1493 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last results");
1494 }
1495
1496 return Chain;
1497}
1498
1499SDValue WebAssemblyTargetLowering::LowerFormalArguments(
1500 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
1501 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
1502 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
1503 if (!callingConvSupported(CallConv))
1504 fail(DL, DAG, "WebAssembly doesn't support non-C calling conventions");
1505
1506 MachineFunction &MF = DAG.getMachineFunction();
1507 auto *MFI = MF.getInfo<WebAssemblyFunctionInfo>();
1508
1509 // Set up the incoming ARGUMENTS value, which serves to represent the liveness
1510 // of the incoming values before they're represented by virtual registers.
1511 MF.getRegInfo().addLiveIn(WebAssembly::ARGUMENTS);
1512
1513 bool HasSwiftErrorArg = false;
1514 bool HasSwiftSelfArg = false;
1515 for (const ISD::InputArg &In : Ins) {
1516 HasSwiftSelfArg |= In.Flags.isSwiftSelf();
1517 HasSwiftErrorArg |= In.Flags.isSwiftError();
1518 if (In.Flags.isInAlloca())
1519 fail(DL, DAG, "WebAssembly hasn't implemented inalloca arguments");
1520 if (In.Flags.isNest())
1521 fail(DL, DAG, "WebAssembly hasn't implemented nest arguments");
1522 if (In.Flags.isInConsecutiveRegs())
1523 fail(DL, DAG, "WebAssembly hasn't implemented cons regs arguments");
1524 if (In.Flags.isInConsecutiveRegsLast())
1525 fail(DL, DAG, "WebAssembly hasn't implemented cons regs last arguments");
1526 // Ignore In.getNonZeroOrigAlign() because all our arguments are passed in
1527 // registers.
1528 InVals.push_back(In.Used ? DAG.getNode(WebAssemblyISD::ARGUMENT, DL, In.VT,
1529 DAG.getTargetConstant(InVals.size(),
1530 DL, MVT::i32))
1531 : DAG.getUNDEF(In.VT));
1532
1533 // Record the number and types of arguments.
1534 MFI->addParam(In.VT);
1535 }
1536
1537 // For swiftcc, emit additional swiftself and swifterror arguments
1538 // if there aren't. These additional arguments are also added for callee
1539 // signature They are necessary to match callee and caller signature for
1540 // indirect call.
1541 auto PtrVT = getPointerTy(MF.getDataLayout());
1542 if (CallConv == CallingConv::Swift) {
1543 if (!HasSwiftSelfArg) {
1544 MFI->addParam(PtrVT);
1545 }
1546 if (!HasSwiftErrorArg) {
1547 MFI->addParam(PtrVT);
1548 }
1549 }
1550 // Varargs are copied into a buffer allocated by the caller, and a pointer to
1551 // the buffer is passed as an argument.
1552 if (IsVarArg) {
1553 MVT PtrVT = getPointerTy(MF.getDataLayout());
1554 Register VarargVreg =
1556 MFI->setVarargBufferVreg(VarargVreg);
1557 Chain = DAG.getCopyToReg(
1558 Chain, DL, VarargVreg,
1559 DAG.getNode(WebAssemblyISD::ARGUMENT, DL, PtrVT,
1560 DAG.getTargetConstant(Ins.size(), DL, MVT::i32)));
1561 MFI->addParam(PtrVT);
1562 }
1563
1564 // Record the number and types of arguments and results.
1565 SmallVector<MVT, 4> Params;
1568 MF.getFunction(), DAG.getTarget(), Params, Results);
1569 for (MVT VT : Results)
1570 MFI->addResult(VT);
1571 // TODO: Use signatures in WebAssemblyMachineFunctionInfo too and unify
1572 // the param logic here with ComputeSignatureVTs
1573 assert(MFI->getParams().size() == Params.size() &&
1574 std::equal(MFI->getParams().begin(), MFI->getParams().end(),
1575 Params.begin()));
1576
1577 return Chain;
1578}
1579
1580void WebAssemblyTargetLowering::ReplaceNodeResults(
1582 switch (N->getOpcode()) {
1584 // Do not add any results, signifying that N should not be custom lowered
1585 // after all. This happens because simd128 turns on custom lowering for
1586 // SIGN_EXTEND_INREG, but for non-vector sign extends the result might be an
1587 // illegal type.
1588 break;
1591 // Do not add any results, signifying that N should not be custom lowered.
1592 // EXTEND_VECTOR_INREG is implemented for some vectors, but not all.
1593 break;
1594 case ISD::ADD:
1595 case ISD::SUB:
1596 Results.push_back(Replace128Op(N, DAG));
1597 break;
1598 default:
1600 "ReplaceNodeResults not implemented for this op for WebAssembly!");
1601 }
1602}
1603
1604//===----------------------------------------------------------------------===//
1605// Custom lowering hooks.
1606//===----------------------------------------------------------------------===//
1607
1608SDValue WebAssemblyTargetLowering::LowerOperation(SDValue Op,
1609 SelectionDAG &DAG) const {
1610 SDLoc DL(Op);
1611 switch (Op.getOpcode()) {
1612 default:
1613 llvm_unreachable("unimplemented operation lowering");
1614 return SDValue();
1615 case ISD::FrameIndex:
1616 return LowerFrameIndex(Op, DAG);
1617 case ISD::GlobalAddress:
1618 return LowerGlobalAddress(Op, DAG);
1620 return LowerGlobalTLSAddress(Op, DAG);
1622 return LowerExternalSymbol(Op, DAG);
1623 case ISD::JumpTable:
1624 return LowerJumpTable(Op, DAG);
1625 case ISD::BR_JT:
1626 return LowerBR_JT(Op, DAG);
1627 case ISD::VASTART:
1628 return LowerVASTART(Op, DAG);
1629 case ISD::BlockAddress:
1630 case ISD::BRIND:
1631 fail(DL, DAG, "WebAssembly hasn't implemented computed gotos");
1632 return SDValue();
1633 case ISD::RETURNADDR:
1634 return LowerRETURNADDR(Op, DAG);
1635 case ISD::FRAMEADDR:
1636 return LowerFRAMEADDR(Op, DAG);
1637 case ISD::CopyToReg:
1638 return LowerCopyToReg(Op, DAG);
1641 return LowerAccessVectorElement(Op, DAG);
1645 return LowerIntrinsic(Op, DAG);
1647 return LowerSIGN_EXTEND_INREG(Op, DAG);
1650 return LowerEXTEND_VECTOR_INREG(Op, DAG);
1651 case ISD::BUILD_VECTOR:
1652 return LowerBUILD_VECTOR(Op, DAG);
1654 return LowerVECTOR_SHUFFLE(Op, DAG);
1655 case ISD::SETCC:
1656 return LowerSETCC(Op, DAG);
1657 case ISD::SHL:
1658 case ISD::SRA:
1659 case ISD::SRL:
1660 return LowerShift(Op, DAG);
1663 return LowerFP_TO_INT_SAT(Op, DAG);
1664 case ISD::LOAD:
1665 return LowerLoad(Op, DAG);
1666 case ISD::STORE:
1667 return LowerStore(Op, DAG);
1668 case ISD::CTPOP:
1669 case ISD::CTLZ:
1670 case ISD::CTTZ:
1671 return DAG.UnrollVectorOp(Op.getNode());
1672 case ISD::CLEAR_CACHE:
1673 report_fatal_error("llvm.clear_cache is not supported on wasm");
1674 case ISD::SMUL_LOHI:
1675 case ISD::UMUL_LOHI:
1676 return LowerMUL_LOHI(Op, DAG);
1677 case ISD::UADDO:
1678 return LowerUADDO(Op, DAG);
1679 }
1680}
1681
1685
1686 return false;
1687}
1688
1689static std::optional<unsigned> IsWebAssemblyLocal(SDValue Op,
1690 SelectionDAG &DAG) {
1692 if (!FI)
1693 return std::nullopt;
1694
1695 auto &MF = DAG.getMachineFunction();
1697}
1698
1699SDValue WebAssemblyTargetLowering::LowerStore(SDValue Op,
1700 SelectionDAG &DAG) const {
1701 SDLoc DL(Op);
1702 StoreSDNode *SN = cast<StoreSDNode>(Op.getNode());
1703 const SDValue &Value = SN->getValue();
1704 const SDValue &Base = SN->getBasePtr();
1705 const SDValue &Offset = SN->getOffset();
1706
1708 if (!Offset->isUndef())
1709 report_fatal_error("unexpected offset when storing to webassembly global",
1710 false);
1711
1712 SDVTList Tys = DAG.getVTList(MVT::Other);
1713 SDValue Ops[] = {SN->getChain(), Value, Base};
1714 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_SET, DL, Tys, Ops,
1715 SN->getMemoryVT(), SN->getMemOperand());
1716 }
1717
1718 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1719 if (!Offset->isUndef())
1720 report_fatal_error("unexpected offset when storing to webassembly local",
1721 false);
1722
1723 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1724 SDVTList Tys = DAG.getVTList(MVT::Other); // The chain.
1725 SDValue Ops[] = {SN->getChain(), Idx, Value};
1726 return DAG.getNode(WebAssemblyISD::LOCAL_SET, DL, Tys, Ops);
1727 }
1728
1731 "Encountered an unlowerable store to the wasm_var address space",
1732 false);
1733
1734 return Op;
1735}
1736
1737SDValue WebAssemblyTargetLowering::LowerLoad(SDValue Op,
1738 SelectionDAG &DAG) const {
1739 SDLoc DL(Op);
1740 LoadSDNode *LN = cast<LoadSDNode>(Op.getNode());
1741 const SDValue &Base = LN->getBasePtr();
1742 const SDValue &Offset = LN->getOffset();
1743
1745 if (!Offset->isUndef())
1747 "unexpected offset when loading from webassembly global", false);
1748
1749 SDVTList Tys = DAG.getVTList(LN->getValueType(0), MVT::Other);
1750 SDValue Ops[] = {LN->getChain(), Base};
1751 return DAG.getMemIntrinsicNode(WebAssemblyISD::GLOBAL_GET, DL, Tys, Ops,
1752 LN->getMemoryVT(), LN->getMemOperand());
1753 }
1754
1755 if (std::optional<unsigned> Local = IsWebAssemblyLocal(Base, DAG)) {
1756 if (!Offset->isUndef())
1758 "unexpected offset when loading from webassembly local", false);
1759
1760 SDValue Idx = DAG.getTargetConstant(*Local, Base, MVT::i32);
1761 EVT LocalVT = LN->getValueType(0);
1762 SDValue LocalGet = DAG.getNode(WebAssemblyISD::LOCAL_GET, DL, LocalVT,
1763 {LN->getChain(), Idx});
1764 SDValue Result = DAG.getMergeValues({LocalGet, LN->getChain()}, DL);
1765 assert(Result->getNumValues() == 2 && "Loads must carry a chain!");
1766 return Result;
1767 }
1768
1771 "Encountered an unlowerable load from the wasm_var address space",
1772 false);
1773
1774 return Op;
1775}
1776
1777SDValue WebAssemblyTargetLowering::LowerMUL_LOHI(SDValue Op,
1778 SelectionDAG &DAG) const {
1779 assert(Subtarget->hasWideArithmetic());
1780 assert(Op.getValueType() == MVT::i64);
1781 SDLoc DL(Op);
1782 unsigned Opcode;
1783 switch (Op.getOpcode()) {
1784 case ISD::UMUL_LOHI:
1785 Opcode = WebAssemblyISD::I64_MUL_WIDE_U;
1786 break;
1787 case ISD::SMUL_LOHI:
1788 Opcode = WebAssemblyISD::I64_MUL_WIDE_S;
1789 break;
1790 default:
1791 llvm_unreachable("unexpected opcode");
1792 }
1793 SDValue LHS = Op.getOperand(0);
1794 SDValue RHS = Op.getOperand(1);
1795 SDValue Lo =
1796 DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64), LHS, RHS);
1797 SDValue Hi(Lo.getNode(), 1);
1798 SDValue Ops[] = {Lo, Hi};
1799 return DAG.getMergeValues(Ops, DL);
1800}
1801
1802// Lowers `UADDO` intrinsics to an `i64.add128` instruction when it's enabled.
1803//
1804// This enables generating a single wasm instruction for this operation where
1805// the upper half of both operands are constant zeros. The upper half of the
1806// result is then whether the overflow happened.
1807SDValue WebAssemblyTargetLowering::LowerUADDO(SDValue Op,
1808 SelectionDAG &DAG) const {
1809 assert(Subtarget->hasWideArithmetic());
1810 assert(Op.getValueType() == MVT::i64);
1811 assert(Op.getOpcode() == ISD::UADDO);
1812 SDLoc DL(Op);
1813 SDValue LHS = Op.getOperand(0);
1814 SDValue RHS = Op.getOperand(1);
1815 SDValue Zero = DAG.getConstant(0, DL, MVT::i64);
1816 SDValue Result =
1817 DAG.getNode(WebAssemblyISD::I64_ADD128, DL,
1818 DAG.getVTList(MVT::i64, MVT::i64), LHS, Zero, RHS, Zero);
1819 SDValue CarryI64(Result.getNode(), 1);
1820 SDValue CarryI32 = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, CarryI64);
1821 SDValue Ops[] = {Result, CarryI32};
1822 return DAG.getMergeValues(Ops, DL);
1823}
1824
1825SDValue WebAssemblyTargetLowering::Replace128Op(SDNode *N,
1826 SelectionDAG &DAG) const {
1827 assert(Subtarget->hasWideArithmetic());
1828 assert(N->getValueType(0) == MVT::i128);
1829 SDLoc DL(N);
1830 unsigned Opcode;
1831 switch (N->getOpcode()) {
1832 case ISD::ADD:
1833 Opcode = WebAssemblyISD::I64_ADD128;
1834 break;
1835 case ISD::SUB:
1836 Opcode = WebAssemblyISD::I64_SUB128;
1837 break;
1838 default:
1839 llvm_unreachable("unexpected opcode");
1840 }
1841 SDValue LHS = N->getOperand(0);
1842 SDValue RHS = N->getOperand(1);
1843
1844 SDValue C0 = DAG.getConstant(0, DL, MVT::i64);
1845 SDValue C1 = DAG.getConstant(1, DL, MVT::i64);
1846 SDValue LHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C0);
1847 SDValue LHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, LHS, C1);
1848 SDValue RHS_0 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C0);
1849 SDValue RHS_1 = DAG.getNode(ISD::EXTRACT_ELEMENT, DL, MVT::i64, RHS, C1);
1850 SDValue Result_LO = DAG.getNode(Opcode, DL, DAG.getVTList(MVT::i64, MVT::i64),
1851 LHS_0, LHS_1, RHS_0, RHS_1);
1852 SDValue Result_HI(Result_LO.getNode(), 1);
1853 return DAG.getNode(ISD::BUILD_PAIR, DL, N->getVTList(), Result_LO, Result_HI);
1854}
1855
1856SDValue WebAssemblyTargetLowering::LowerCopyToReg(SDValue Op,
1857 SelectionDAG &DAG) const {
1858 SDValue Src = Op.getOperand(2);
1859 if (isa<FrameIndexSDNode>(Src.getNode())) {
1860 // CopyToReg nodes don't support FrameIndex operands. Other targets select
1861 // the FI to some LEA-like instruction, but since we don't have that, we
1862 // need to insert some kind of instruction that can take an FI operand and
1863 // produces a value usable by CopyToReg (i.e. in a vreg). So insert a dummy
1864 // local.copy between Op and its FI operand.
1865 SDValue Chain = Op.getOperand(0);
1866 SDLoc DL(Op);
1867 Register Reg = cast<RegisterSDNode>(Op.getOperand(1))->getReg();
1868 EVT VT = Src.getValueType();
1869 SDValue Copy(DAG.getMachineNode(VT == MVT::i32 ? WebAssembly::COPY_I32
1870 : WebAssembly::COPY_I64,
1871 DL, VT, Src),
1872 0);
1873 return Op.getNode()->getNumValues() == 1
1874 ? DAG.getCopyToReg(Chain, DL, Reg, Copy)
1875 : DAG.getCopyToReg(Chain, DL, Reg, Copy,
1876 Op.getNumOperands() == 4 ? Op.getOperand(3)
1877 : SDValue());
1878 }
1879 return SDValue();
1880}
1881
1882SDValue WebAssemblyTargetLowering::LowerFrameIndex(SDValue Op,
1883 SelectionDAG &DAG) const {
1884 int FI = cast<FrameIndexSDNode>(Op)->getIndex();
1885 return DAG.getTargetFrameIndex(FI, Op.getValueType());
1886}
1887
1888SDValue WebAssemblyTargetLowering::LowerRETURNADDR(SDValue Op,
1889 SelectionDAG &DAG) const {
1890 SDLoc DL(Op);
1891
1892 if (!Subtarget->getTargetTriple().isOSEmscripten()) {
1893 fail(DL, DAG,
1894 "Non-Emscripten WebAssembly hasn't implemented "
1895 "__builtin_return_address");
1896 return SDValue();
1897 }
1898
1899 unsigned Depth = Op.getConstantOperandVal(0);
1900 MakeLibCallOptions CallOptions;
1901 return makeLibCall(DAG, RTLIB::RETURN_ADDRESS, Op.getValueType(),
1902 {DAG.getConstant(Depth, DL, MVT::i32)}, CallOptions, DL)
1903 .first;
1904}
1905
1906SDValue WebAssemblyTargetLowering::LowerFRAMEADDR(SDValue Op,
1907 SelectionDAG &DAG) const {
1908 // Non-zero depths are not supported by WebAssembly currently. Use the
1909 // legalizer's default expansion, which is to return 0 (what this function is
1910 // documented to do).
1911 if (Op.getConstantOperandVal(0) > 0)
1912 return SDValue();
1913
1915 EVT VT = Op.getValueType();
1916 Register FP =
1917 Subtarget->getRegisterInfo()->getFrameRegister(DAG.getMachineFunction());
1918 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), FP, VT);
1919}
1920
1921SDValue
1922WebAssemblyTargetLowering::LowerGlobalTLSAddress(SDValue Op,
1923 SelectionDAG &DAG) const {
1924 SDLoc DL(Op);
1925 const auto *GA = cast<GlobalAddressSDNode>(Op);
1926
1927 MachineFunction &MF = DAG.getMachineFunction();
1928 if (!MF.getSubtarget<WebAssemblySubtarget>().hasBulkMemory())
1929 report_fatal_error("cannot use thread-local storage without bulk memory",
1930 false);
1931
1932 const GlobalValue *GV = GA->getGlobal();
1933
1934 // Currently only Emscripten supports dynamic linking with threads. Therefore,
1935 // on other targets, if we have thread-local storage, only the local-exec
1936 // model is possible.
1937 auto model = Subtarget->getTargetTriple().isOSEmscripten()
1938 ? GV->getThreadLocalMode()
1940
1941 // Unsupported TLS modes
1944
1945 if (model == GlobalValue::LocalExecTLSModel ||
1948 getTargetMachine().shouldAssumeDSOLocal(GV))) {
1949 // For DSO-local TLS variables we use offset from __tls_base
1950
1951 MVT PtrVT = getPointerTy(DAG.getDataLayout());
1952 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
1953 : WebAssembly::GLOBAL_GET_I32;
1954 const char *BaseName = MF.createExternalSymbolName("__tls_base");
1955
1957 DAG.getMachineNode(GlobalGet, DL, PtrVT,
1958 DAG.getTargetExternalSymbol(BaseName, PtrVT)),
1959 0);
1960
1961 SDValue TLSOffset = DAG.getTargetGlobalAddress(
1962 GV, DL, PtrVT, GA->getOffset(), WebAssemblyII::MO_TLS_BASE_REL);
1963 SDValue SymOffset =
1964 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, TLSOffset);
1965
1966 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymOffset);
1967 }
1968
1970
1971 EVT VT = Op.getValueType();
1972 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
1973 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
1974 GA->getOffset(),
1976}
1977
1978SDValue WebAssemblyTargetLowering::LowerGlobalAddress(SDValue Op,
1979 SelectionDAG &DAG) const {
1980 SDLoc DL(Op);
1981 const auto *GA = cast<GlobalAddressSDNode>(Op);
1982 EVT VT = Op.getValueType();
1983 assert(GA->getTargetFlags() == 0 &&
1984 "Unexpected target flags on generic GlobalAddressSDNode");
1986 fail(DL, DAG, "Invalid address space for WebAssembly target");
1987
1988 unsigned OperandFlags = 0;
1989 const GlobalValue *GV = GA->getGlobal();
1990 // Since WebAssembly tables cannot yet be shared accross modules, we don't
1991 // need special treatment for tables in PIC mode.
1992 if (isPositionIndependent() &&
1994 if (getTargetMachine().shouldAssumeDSOLocal(GV)) {
1995 MachineFunction &MF = DAG.getMachineFunction();
1996 MVT PtrVT = getPointerTy(MF.getDataLayout());
1997 const char *BaseName;
1998 if (GV->getValueType()->isFunctionTy()) {
1999 BaseName = MF.createExternalSymbolName("__table_base");
2001 } else {
2002 BaseName = MF.createExternalSymbolName("__memory_base");
2004 }
2006 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
2007 DAG.getTargetExternalSymbol(BaseName, PtrVT));
2008
2009 SDValue SymAddr = DAG.getNode(
2010 WebAssemblyISD::WrapperREL, DL, VT,
2011 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT, GA->getOffset(),
2012 OperandFlags));
2013
2014 return DAG.getNode(ISD::ADD, DL, VT, BaseAddr, SymAddr);
2015 }
2017 }
2018
2019 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
2020 DAG.getTargetGlobalAddress(GA->getGlobal(), DL, VT,
2021 GA->getOffset(), OperandFlags));
2022}
2023
2024SDValue
2025WebAssemblyTargetLowering::LowerExternalSymbol(SDValue Op,
2026 SelectionDAG &DAG) const {
2027 SDLoc DL(Op);
2028 const auto *ES = cast<ExternalSymbolSDNode>(Op);
2029 EVT VT = Op.getValueType();
2030 assert(ES->getTargetFlags() == 0 &&
2031 "Unexpected target flags on generic ExternalSymbolSDNode");
2032 return DAG.getNode(WebAssemblyISD::Wrapper, DL, VT,
2033 DAG.getTargetExternalSymbol(ES->getSymbol(), VT));
2034}
2035
2036SDValue WebAssemblyTargetLowering::LowerJumpTable(SDValue Op,
2037 SelectionDAG &DAG) const {
2038 // There's no need for a Wrapper node because we always incorporate a jump
2039 // table operand into a BR_TABLE instruction, rather than ever
2040 // materializing it in a register.
2041 const JumpTableSDNode *JT = cast<JumpTableSDNode>(Op);
2042 return DAG.getTargetJumpTable(JT->getIndex(), Op.getValueType(),
2043 JT->getTargetFlags());
2044}
2045
2046SDValue WebAssemblyTargetLowering::LowerBR_JT(SDValue Op,
2047 SelectionDAG &DAG) const {
2048 SDLoc DL(Op);
2049 SDValue Chain = Op.getOperand(0);
2050 const auto *JT = cast<JumpTableSDNode>(Op.getOperand(1));
2051 SDValue Index = Op.getOperand(2);
2052 assert(JT->getTargetFlags() == 0 && "WebAssembly doesn't set target flags");
2053
2055 Ops.push_back(Chain);
2056 Ops.push_back(Index);
2057
2058 MachineJumpTableInfo *MJTI = DAG.getMachineFunction().getJumpTableInfo();
2059 const auto &MBBs = MJTI->getJumpTables()[JT->getIndex()].MBBs;
2060
2061 // Add an operand for each case.
2062 for (auto *MBB : MBBs)
2063 Ops.push_back(DAG.getBasicBlock(MBB));
2064
2065 // Add the first MBB as a dummy default target for now. This will be replaced
2066 // with the proper default target (and the preceding range check eliminated)
2067 // if possible by WebAssemblyFixBrTableDefaults.
2068 Ops.push_back(DAG.getBasicBlock(*MBBs.begin()));
2069 return DAG.getNode(WebAssemblyISD::BR_TABLE, DL, MVT::Other, Ops);
2070}
2071
2072SDValue WebAssemblyTargetLowering::LowerVASTART(SDValue Op,
2073 SelectionDAG &DAG) const {
2074 SDLoc DL(Op);
2075 EVT PtrVT = getPointerTy(DAG.getMachineFunction().getDataLayout());
2076
2077 auto *MFI = DAG.getMachineFunction().getInfo<WebAssemblyFunctionInfo>();
2078 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
2079
2080 SDValue ArgN = DAG.getCopyFromReg(DAG.getEntryNode(), DL,
2081 MFI->getVarargBufferVreg(), PtrVT);
2082 return DAG.getStore(Op.getOperand(0), DL, ArgN, Op.getOperand(1),
2083 MachinePointerInfo(SV));
2084}
2085
2086SDValue WebAssemblyTargetLowering::LowerIntrinsic(SDValue Op,
2087 SelectionDAG &DAG) const {
2088 MachineFunction &MF = DAG.getMachineFunction();
2089 unsigned IntNo;
2090 switch (Op.getOpcode()) {
2093 IntNo = Op.getConstantOperandVal(1);
2094 break;
2096 IntNo = Op.getConstantOperandVal(0);
2097 break;
2098 default:
2099 llvm_unreachable("Invalid intrinsic");
2100 }
2101 SDLoc DL(Op);
2102
2103 switch (IntNo) {
2104 default:
2105 return SDValue(); // Don't custom lower most intrinsics.
2106
2107 case Intrinsic::wasm_lsda: {
2108 auto PtrVT = getPointerTy(MF.getDataLayout());
2109 const char *SymName = MF.createExternalSymbolName(
2110 "GCC_except_table" + std::to_string(MF.getFunctionNumber()));
2111 if (isPositionIndependent()) {
2113 SymName, PtrVT, WebAssemblyII::MO_MEMORY_BASE_REL);
2114 const char *BaseName = MF.createExternalSymbolName("__memory_base");
2116 DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT,
2117 DAG.getTargetExternalSymbol(BaseName, PtrVT));
2118 SDValue SymAddr =
2119 DAG.getNode(WebAssemblyISD::WrapperREL, DL, PtrVT, Node);
2120 return DAG.getNode(ISD::ADD, DL, PtrVT, BaseAddr, SymAddr);
2121 }
2122 SDValue Node = DAG.getTargetExternalSymbol(SymName, PtrVT);
2123 return DAG.getNode(WebAssemblyISD::Wrapper, DL, PtrVT, Node);
2124 }
2125
2126 case Intrinsic::wasm_shuffle: {
2127 // Drop in-chain and replace undefs, but otherwise pass through unchanged
2128 SDValue Ops[18];
2129 size_t OpIdx = 0;
2130 Ops[OpIdx++] = Op.getOperand(1);
2131 Ops[OpIdx++] = Op.getOperand(2);
2132 while (OpIdx < 18) {
2133 const SDValue &MaskIdx = Op.getOperand(OpIdx + 1);
2134 if (MaskIdx.isUndef() || MaskIdx.getNode()->getAsZExtVal() >= 32) {
2135 bool isTarget = MaskIdx.getNode()->getOpcode() == ISD::TargetConstant;
2136 Ops[OpIdx++] = DAG.getConstant(0, DL, MVT::i32, isTarget);
2137 } else {
2138 Ops[OpIdx++] = MaskIdx;
2139 }
2140 }
2141 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2142 }
2143
2144 case Intrinsic::thread_pointer: {
2145 MVT PtrVT = getPointerTy(DAG.getDataLayout());
2146 auto GlobalGet = PtrVT == MVT::i64 ? WebAssembly::GLOBAL_GET_I64
2147 : WebAssembly::GLOBAL_GET_I32;
2148 const char *TlsBase = MF.createExternalSymbolName("__tls_base");
2149 return SDValue(
2150 DAG.getMachineNode(GlobalGet, DL, PtrVT,
2151 DAG.getTargetExternalSymbol(TlsBase, PtrVT)),
2152 0);
2153 }
2154 }
2155}
2156
2157SDValue
2158WebAssemblyTargetLowering::LowerSIGN_EXTEND_INREG(SDValue Op,
2159 SelectionDAG &DAG) const {
2160 SDLoc DL(Op);
2161 // If sign extension operations are disabled, allow sext_inreg only if operand
2162 // is a vector extract of an i8 or i16 lane. SIMD does not depend on sign
2163 // extension operations, but allowing sext_inreg in this context lets us have
2164 // simple patterns to select extract_lane_s instructions. Expanding sext_inreg
2165 // everywhere would be simpler in this file, but would necessitate large and
2166 // brittle patterns to undo the expansion and select extract_lane_s
2167 // instructions.
2168 assert(!Subtarget->hasSignExt() && Subtarget->hasSIMD128());
2169 if (Op.getOperand(0).getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2170 return SDValue();
2171
2172 const SDValue &Extract = Op.getOperand(0);
2173 MVT VecT = Extract.getOperand(0).getSimpleValueType();
2174 if (VecT.getVectorElementType().getSizeInBits() > 32)
2175 return SDValue();
2176 MVT ExtractedLaneT =
2177 cast<VTSDNode>(Op.getOperand(1).getNode())->getVT().getSimpleVT();
2178 MVT ExtractedVecT =
2179 MVT::getVectorVT(ExtractedLaneT, 128 / ExtractedLaneT.getSizeInBits());
2180 if (ExtractedVecT == VecT)
2181 return Op;
2182
2183 // Bitcast vector to appropriate type to ensure ISel pattern coverage
2184 const SDNode *Index = Extract.getOperand(1).getNode();
2185 if (!isa<ConstantSDNode>(Index))
2186 return SDValue();
2187 unsigned IndexVal = Index->getAsZExtVal();
2188 unsigned Scale =
2189 ExtractedVecT.getVectorNumElements() / VecT.getVectorNumElements();
2190 assert(Scale > 1);
2191 SDValue NewIndex =
2192 DAG.getConstant(IndexVal * Scale, DL, Index->getValueType(0));
2193 SDValue NewExtract = DAG.getNode(
2195 DAG.getBitcast(ExtractedVecT, Extract.getOperand(0)), NewIndex);
2196 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, Op.getValueType(), NewExtract,
2197 Op.getOperand(1));
2198}
2199
2200static SDValue GetExtendHigh(SDValue Op, unsigned UserOpc, EVT VT,
2201 SelectionDAG &DAG) {
2202 if (Op.getOpcode() != ISD::VECTOR_SHUFFLE)
2203 return SDValue();
2204
2205 assert((UserOpc == WebAssemblyISD::EXTEND_LOW_U ||
2206 UserOpc == WebAssemblyISD::EXTEND_LOW_S) &&
2207 "expected extend_low");
2208 auto *Shuffle = cast<ShuffleVectorSDNode>(Op.getNode());
2209
2210 ArrayRef<int> Mask = Shuffle->getMask();
2211 // Look for a shuffle which moves from the high half to the low half.
2212 size_t FirstIdx = Mask.size() / 2;
2213 for (size_t i = 0; i < Mask.size() / 2; ++i) {
2214 if (Mask[i] != static_cast<int>(FirstIdx + i)) {
2215 return SDValue();
2216 }
2217 }
2218
2219 SDLoc DL(Op);
2220 unsigned Opc = UserOpc == WebAssemblyISD::EXTEND_LOW_S
2221 ? WebAssemblyISD::EXTEND_HIGH_S
2222 : WebAssemblyISD::EXTEND_HIGH_U;
2223 return DAG.getNode(Opc, DL, VT, Shuffle->getOperand(0));
2224}
2225
2226SDValue
2227WebAssemblyTargetLowering::LowerEXTEND_VECTOR_INREG(SDValue Op,
2228 SelectionDAG &DAG) const {
2229 SDLoc DL(Op);
2230 EVT VT = Op.getValueType();
2231 SDValue Src = Op.getOperand(0);
2232 EVT SrcVT = Src.getValueType();
2233
2234 if (SrcVT.getVectorElementType() == MVT::i1 ||
2235 SrcVT.getVectorElementType() == MVT::i64)
2236 return SDValue();
2237
2238 assert(VT.getScalarSizeInBits() % SrcVT.getScalarSizeInBits() == 0 &&
2239 "Unexpected extension factor.");
2240 unsigned Scale = VT.getScalarSizeInBits() / SrcVT.getScalarSizeInBits();
2241
2242 if (Scale != 2 && Scale != 4 && Scale != 8)
2243 return SDValue();
2244
2245 unsigned Ext;
2246 switch (Op.getOpcode()) {
2248 Ext = WebAssemblyISD::EXTEND_LOW_U;
2249 break;
2251 Ext = WebAssemblyISD::EXTEND_LOW_S;
2252 break;
2253 }
2254
2255 if (Scale == 2) {
2256 // See if we can use EXTEND_HIGH.
2257 if (auto ExtendHigh = GetExtendHigh(Op.getOperand(0), Ext, VT, DAG))
2258 return ExtendHigh;
2259 }
2260
2261 SDValue Ret = Src;
2262 while (Scale != 1) {
2263 Ret = DAG.getNode(Ext, DL,
2264 Ret.getValueType()
2265 .widenIntegerVectorElementType(*DAG.getContext())
2266 .getHalfNumVectorElementsVT(*DAG.getContext()),
2267 Ret);
2268 Scale /= 2;
2269 }
2270 assert(Ret.getValueType() == VT);
2271 return Ret;
2272}
2273
2275 SDLoc DL(Op);
2276 if (Op.getValueType() != MVT::v2f64)
2277 return SDValue();
2278
2279 auto GetConvertedLane = [](SDValue Op, unsigned &Opcode, SDValue &SrcVec,
2280 unsigned &Index) -> bool {
2281 switch (Op.getOpcode()) {
2282 case ISD::SINT_TO_FP:
2283 Opcode = WebAssemblyISD::CONVERT_LOW_S;
2284 break;
2285 case ISD::UINT_TO_FP:
2286 Opcode = WebAssemblyISD::CONVERT_LOW_U;
2287 break;
2288 case ISD::FP_EXTEND:
2289 Opcode = WebAssemblyISD::PROMOTE_LOW;
2290 break;
2291 default:
2292 return false;
2293 }
2294
2295 auto ExtractVector = Op.getOperand(0);
2296 if (ExtractVector.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2297 return false;
2298
2299 if (!isa<ConstantSDNode>(ExtractVector.getOperand(1).getNode()))
2300 return false;
2301
2302 SrcVec = ExtractVector.getOperand(0);
2303 Index = ExtractVector.getConstantOperandVal(1);
2304 return true;
2305 };
2306
2307 unsigned LHSOpcode, RHSOpcode, LHSIndex, RHSIndex;
2308 SDValue LHSSrcVec, RHSSrcVec;
2309 if (!GetConvertedLane(Op.getOperand(0), LHSOpcode, LHSSrcVec, LHSIndex) ||
2310 !GetConvertedLane(Op.getOperand(1), RHSOpcode, RHSSrcVec, RHSIndex))
2311 return SDValue();
2312
2313 if (LHSOpcode != RHSOpcode)
2314 return SDValue();
2315
2316 MVT ExpectedSrcVT;
2317 switch (LHSOpcode) {
2318 case WebAssemblyISD::CONVERT_LOW_S:
2319 case WebAssemblyISD::CONVERT_LOW_U:
2320 ExpectedSrcVT = MVT::v4i32;
2321 break;
2322 case WebAssemblyISD::PROMOTE_LOW:
2323 ExpectedSrcVT = MVT::v4f32;
2324 break;
2325 }
2326 if (LHSSrcVec.getValueType() != ExpectedSrcVT)
2327 return SDValue();
2328
2329 auto Src = LHSSrcVec;
2330 if (LHSIndex != 0 || RHSIndex != 1 || LHSSrcVec != RHSSrcVec) {
2331 // Shuffle the source vector so that the converted lanes are the low lanes.
2332 Src = DAG.getVectorShuffle(
2333 ExpectedSrcVT, DL, LHSSrcVec, RHSSrcVec,
2334 {static_cast<int>(LHSIndex), static_cast<int>(RHSIndex) + 4, -1, -1});
2335 }
2336 return DAG.getNode(LHSOpcode, DL, MVT::v2f64, Src);
2337}
2338
2339SDValue WebAssemblyTargetLowering::LowerBUILD_VECTOR(SDValue Op,
2340 SelectionDAG &DAG) const {
2341 MVT VT = Op.getSimpleValueType();
2342 if (VT == MVT::v8f16) {
2343 // BUILD_VECTOR can't handle FP16 operands since Wasm doesn't have a scaler
2344 // FP16 type, so cast them to I16s.
2345 MVT IVT = VT.changeVectorElementType(MVT::i16);
2347 for (unsigned I = 0, E = Op.getNumOperands(); I < E; ++I)
2348 NewOps.push_back(DAG.getBitcast(MVT::i16, Op.getOperand(I)));
2349 SDValue Res = DAG.getNode(ISD::BUILD_VECTOR, SDLoc(), IVT, NewOps);
2350 return DAG.getBitcast(VT, Res);
2351 }
2352
2353 if (auto ConvertLow = LowerConvertLow(Op, DAG))
2354 return ConvertLow;
2355
2356 SDLoc DL(Op);
2357 const EVT VecT = Op.getValueType();
2358 const EVT LaneT = Op.getOperand(0).getValueType();
2359 const size_t Lanes = Op.getNumOperands();
2360 bool CanSwizzle = VecT == MVT::v16i8;
2361
2362 // BUILD_VECTORs are lowered to the instruction that initializes the highest
2363 // possible number of lanes at once followed by a sequence of replace_lane
2364 // instructions to individually initialize any remaining lanes.
2365
2366 // TODO: Tune this. For example, lanewise swizzling is very expensive, so
2367 // swizzled lanes should be given greater weight.
2368
2369 // TODO: Investigate looping rather than always extracting/replacing specific
2370 // lanes to fill gaps.
2371
2372 auto IsConstant = [](const SDValue &V) {
2373 return V.getOpcode() == ISD::Constant || V.getOpcode() == ISD::ConstantFP;
2374 };
2375
2376 // Returns the source vector and index vector pair if they exist. Checks for:
2377 // (extract_vector_elt
2378 // $src,
2379 // (sign_extend_inreg (extract_vector_elt $indices, $i))
2380 // )
2381 auto GetSwizzleSrcs = [](size_t I, const SDValue &Lane) {
2382 auto Bail = std::make_pair(SDValue(), SDValue());
2383 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2384 return Bail;
2385 const SDValue &SwizzleSrc = Lane->getOperand(0);
2386 const SDValue &IndexExt = Lane->getOperand(1);
2387 if (IndexExt->getOpcode() != ISD::SIGN_EXTEND_INREG)
2388 return Bail;
2389 const SDValue &Index = IndexExt->getOperand(0);
2390 if (Index->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2391 return Bail;
2392 const SDValue &SwizzleIndices = Index->getOperand(0);
2393 if (SwizzleSrc.getValueType() != MVT::v16i8 ||
2394 SwizzleIndices.getValueType() != MVT::v16i8 ||
2395 Index->getOperand(1)->getOpcode() != ISD::Constant ||
2396 Index->getConstantOperandVal(1) != I)
2397 return Bail;
2398 return std::make_pair(SwizzleSrc, SwizzleIndices);
2399 };
2400
2401 // If the lane is extracted from another vector at a constant index, return
2402 // that vector. The source vector must not have more lanes than the dest
2403 // because the shufflevector indices are in terms of the destination lanes and
2404 // would not be able to address the smaller individual source lanes.
2405 auto GetShuffleSrc = [&](const SDValue &Lane) {
2406 if (Lane->getOpcode() != ISD::EXTRACT_VECTOR_ELT)
2407 return SDValue();
2408 if (!isa<ConstantSDNode>(Lane->getOperand(1).getNode()))
2409 return SDValue();
2410 if (Lane->getOperand(0).getValueType().getVectorNumElements() >
2411 VecT.getVectorNumElements())
2412 return SDValue();
2413 return Lane->getOperand(0);
2414 };
2415
2416 using ValueEntry = std::pair<SDValue, size_t>;
2417 SmallVector<ValueEntry, 16> SplatValueCounts;
2418
2419 using SwizzleEntry = std::pair<std::pair<SDValue, SDValue>, size_t>;
2420 SmallVector<SwizzleEntry, 16> SwizzleCounts;
2421
2422 using ShuffleEntry = std::pair<SDValue, size_t>;
2423 SmallVector<ShuffleEntry, 16> ShuffleCounts;
2424
2425 auto AddCount = [](auto &Counts, const auto &Val) {
2426 auto CountIt =
2427 llvm::find_if(Counts, [&Val](auto E) { return E.first == Val; });
2428 if (CountIt == Counts.end()) {
2429 Counts.emplace_back(Val, 1);
2430 } else {
2431 CountIt->second++;
2432 }
2433 };
2434
2435 auto GetMostCommon = [](auto &Counts) {
2436 auto CommonIt = llvm::max_element(Counts, llvm::less_second());
2437 assert(CommonIt != Counts.end() && "Unexpected all-undef build_vector");
2438 return *CommonIt;
2439 };
2440
2441 size_t NumConstantLanes = 0;
2442
2443 // Count eligible lanes for each type of vector creation op
2444 for (size_t I = 0; I < Lanes; ++I) {
2445 const SDValue &Lane = Op->getOperand(I);
2446 if (Lane.isUndef())
2447 continue;
2448
2449 AddCount(SplatValueCounts, Lane);
2450
2451 if (IsConstant(Lane))
2452 NumConstantLanes++;
2453 if (auto ShuffleSrc = GetShuffleSrc(Lane))
2454 AddCount(ShuffleCounts, ShuffleSrc);
2455 if (CanSwizzle) {
2456 auto SwizzleSrcs = GetSwizzleSrcs(I, Lane);
2457 if (SwizzleSrcs.first)
2458 AddCount(SwizzleCounts, SwizzleSrcs);
2459 }
2460 }
2461
2462 SDValue SplatValue;
2463 size_t NumSplatLanes;
2464 std::tie(SplatValue, NumSplatLanes) = GetMostCommon(SplatValueCounts);
2465
2466 SDValue SwizzleSrc;
2467 SDValue SwizzleIndices;
2468 size_t NumSwizzleLanes = 0;
2469 if (SwizzleCounts.size())
2470 std::forward_as_tuple(std::tie(SwizzleSrc, SwizzleIndices),
2471 NumSwizzleLanes) = GetMostCommon(SwizzleCounts);
2472
2473 // Shuffles can draw from up to two vectors, so find the two most common
2474 // sources.
2475 SDValue ShuffleSrc1, ShuffleSrc2;
2476 size_t NumShuffleLanes = 0;
2477 if (ShuffleCounts.size()) {
2478 std::tie(ShuffleSrc1, NumShuffleLanes) = GetMostCommon(ShuffleCounts);
2479 llvm::erase_if(ShuffleCounts,
2480 [&](const auto &Pair) { return Pair.first == ShuffleSrc1; });
2481 }
2482 if (ShuffleCounts.size()) {
2483 size_t AdditionalShuffleLanes;
2484 std::tie(ShuffleSrc2, AdditionalShuffleLanes) =
2485 GetMostCommon(ShuffleCounts);
2486 NumShuffleLanes += AdditionalShuffleLanes;
2487 }
2488
2489 // Predicate returning true if the lane is properly initialized by the
2490 // original instruction
2491 std::function<bool(size_t, const SDValue &)> IsLaneConstructed;
2493 // Prefer swizzles over shuffles over vector consts over splats
2494 if (NumSwizzleLanes >= NumShuffleLanes &&
2495 NumSwizzleLanes >= NumConstantLanes && NumSwizzleLanes >= NumSplatLanes) {
2496 Result = DAG.getNode(WebAssemblyISD::SWIZZLE, DL, VecT, SwizzleSrc,
2497 SwizzleIndices);
2498 auto Swizzled = std::make_pair(SwizzleSrc, SwizzleIndices);
2499 IsLaneConstructed = [&, Swizzled](size_t I, const SDValue &Lane) {
2500 return Swizzled == GetSwizzleSrcs(I, Lane);
2501 };
2502 } else if (NumShuffleLanes >= NumConstantLanes &&
2503 NumShuffleLanes >= NumSplatLanes) {
2504 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits() / 8;
2505 size_t DestLaneCount = VecT.getVectorNumElements();
2506 size_t Scale1 = 1;
2507 size_t Scale2 = 1;
2508 SDValue Src1 = ShuffleSrc1;
2509 SDValue Src2 = ShuffleSrc2 ? ShuffleSrc2 : DAG.getUNDEF(VecT);
2510 if (Src1.getValueType() != VecT) {
2511 size_t LaneSize =
2513 assert(LaneSize > DestLaneSize);
2514 Scale1 = LaneSize / DestLaneSize;
2515 Src1 = DAG.getBitcast(VecT, Src1);
2516 }
2517 if (Src2.getValueType() != VecT) {
2518 size_t LaneSize =
2520 assert(LaneSize > DestLaneSize);
2521 Scale2 = LaneSize / DestLaneSize;
2522 Src2 = DAG.getBitcast(VecT, Src2);
2523 }
2524
2525 int Mask[16];
2526 assert(DestLaneCount <= 16);
2527 for (size_t I = 0; I < DestLaneCount; ++I) {
2528 const SDValue &Lane = Op->getOperand(I);
2529 SDValue Src = GetShuffleSrc(Lane);
2530 if (Src == ShuffleSrc1) {
2531 Mask[I] = Lane->getConstantOperandVal(1) * Scale1;
2532 } else if (Src && Src == ShuffleSrc2) {
2533 Mask[I] = DestLaneCount + Lane->getConstantOperandVal(1) * Scale2;
2534 } else {
2535 Mask[I] = -1;
2536 }
2537 }
2538 ArrayRef<int> MaskRef(Mask, DestLaneCount);
2539 Result = DAG.getVectorShuffle(VecT, DL, Src1, Src2, MaskRef);
2540 IsLaneConstructed = [&](size_t, const SDValue &Lane) {
2541 auto Src = GetShuffleSrc(Lane);
2542 return Src == ShuffleSrc1 || (Src && Src == ShuffleSrc2);
2543 };
2544 } else if (NumConstantLanes >= NumSplatLanes) {
2545 SmallVector<SDValue, 16> ConstLanes;
2546 for (const SDValue &Lane : Op->op_values()) {
2547 if (IsConstant(Lane)) {
2548 // Values may need to be fixed so that they will sign extend to be
2549 // within the expected range during ISel. Check whether the value is in
2550 // bounds based on the lane bit width and if it is out of bounds, lop
2551 // off the extra bits and subtract 2^n to reflect giving the high bit
2552 // value -2^(n-1) rather than +2^(n-1). Skip the i64 case because it
2553 // cannot possibly be out of range.
2554 auto *Const = dyn_cast<ConstantSDNode>(Lane.getNode());
2555 int64_t Val = Const ? Const->getSExtValue() : 0;
2556 uint64_t LaneBits = 128 / Lanes;
2557 assert((LaneBits == 64 || Val >= -(1ll << (LaneBits - 1))) &&
2558 "Unexpected out of bounds negative value");
2559 if (Const && LaneBits != 64 && Val > (1ll << (LaneBits - 1)) - 1) {
2560 uint64_t Mask = (1ll << LaneBits) - 1;
2561 auto NewVal = (((uint64_t)Val & Mask) - (1ll << LaneBits)) & Mask;
2562 ConstLanes.push_back(DAG.getConstant(NewVal, SDLoc(Lane), LaneT));
2563 } else {
2564 ConstLanes.push_back(Lane);
2565 }
2566 } else if (LaneT.isFloatingPoint()) {
2567 ConstLanes.push_back(DAG.getConstantFP(0, DL, LaneT));
2568 } else {
2569 ConstLanes.push_back(DAG.getConstant(0, DL, LaneT));
2570 }
2571 }
2572 Result = DAG.getBuildVector(VecT, DL, ConstLanes);
2573 IsLaneConstructed = [&IsConstant](size_t _, const SDValue &Lane) {
2574 return IsConstant(Lane);
2575 };
2576 } else {
2577 size_t DestLaneSize = VecT.getVectorElementType().getFixedSizeInBits();
2578 if (NumSplatLanes == 1 && Op->getOperand(0) == SplatValue &&
2579 (DestLaneSize == 32 || DestLaneSize == 64)) {
2580 // Could be selected to load_zero.
2581 Result = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecT, SplatValue);
2582 } else {
2583 // Use a splat (which might be selected as a load splat)
2584 Result = DAG.getSplatBuildVector(VecT, DL, SplatValue);
2585 }
2586 IsLaneConstructed = [&SplatValue](size_t _, const SDValue &Lane) {
2587 return Lane == SplatValue;
2588 };
2589 }
2590
2591 assert(Result);
2592 assert(IsLaneConstructed);
2593
2594 // Add replace_lane instructions for any unhandled values
2595 for (size_t I = 0; I < Lanes; ++I) {
2596 const SDValue &Lane = Op->getOperand(I);
2597 if (!Lane.isUndef() && !IsLaneConstructed(I, Lane))
2598 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VecT, Result, Lane,
2599 DAG.getConstant(I, DL, MVT::i32));
2600 }
2601
2602 return Result;
2603}
2604
2605SDValue
2606WebAssemblyTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
2607 SelectionDAG &DAG) const {
2608 SDLoc DL(Op);
2609 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Op.getNode())->getMask();
2610 MVT VecType = Op.getOperand(0).getSimpleValueType();
2611 assert(VecType.is128BitVector() && "Unexpected shuffle vector type");
2612 size_t LaneBytes = VecType.getVectorElementType().getSizeInBits() / 8;
2613
2614 // Space for two vector args and sixteen mask indices
2615 SDValue Ops[18];
2616 size_t OpIdx = 0;
2617 Ops[OpIdx++] = Op.getOperand(0);
2618 Ops[OpIdx++] = Op.getOperand(1);
2619
2620 // Expand mask indices to byte indices and materialize them as operands
2621 for (int M : Mask) {
2622 for (size_t J = 0; J < LaneBytes; ++J) {
2623 // Lower undefs (represented by -1 in mask) to {0..J}, which use a
2624 // whole lane of vector input, to allow further reduction at VM. E.g.
2625 // match an 8x16 byte shuffle to an equivalent cheaper 32x4 shuffle.
2626 uint64_t ByteIndex = M == -1 ? J : (uint64_t)M * LaneBytes + J;
2627 Ops[OpIdx++] = DAG.getConstant(ByteIndex, DL, MVT::i32);
2628 }
2629 }
2630
2631 return DAG.getNode(WebAssemblyISD::SHUFFLE, DL, Op.getValueType(), Ops);
2632}
2633
2634SDValue WebAssemblyTargetLowering::LowerSETCC(SDValue Op,
2635 SelectionDAG &DAG) const {
2636 SDLoc DL(Op);
2637 // The legalizer does not know how to expand the unsupported comparison modes
2638 // of i64x2 vectors, so we manually unroll them here.
2639 assert(Op->getOperand(0)->getSimpleValueType(0) == MVT::v2i64);
2641 DAG.ExtractVectorElements(Op->getOperand(0), LHS);
2642 DAG.ExtractVectorElements(Op->getOperand(1), RHS);
2643 const SDValue &CC = Op->getOperand(2);
2644 auto MakeLane = [&](unsigned I) {
2645 return DAG.getNode(ISD::SELECT_CC, DL, MVT::i64, LHS[I], RHS[I],
2646 DAG.getConstant(uint64_t(-1), DL, MVT::i64),
2647 DAG.getConstant(uint64_t(0), DL, MVT::i64), CC);
2648 };
2649 return DAG.getBuildVector(Op->getValueType(0), DL,
2650 {MakeLane(0), MakeLane(1)});
2651}
2652
2653SDValue
2654WebAssemblyTargetLowering::LowerAccessVectorElement(SDValue Op,
2655 SelectionDAG &DAG) const {
2656 // Allow constant lane indices, expand variable lane indices
2657 SDNode *IdxNode = Op.getOperand(Op.getNumOperands() - 1).getNode();
2658 if (isa<ConstantSDNode>(IdxNode)) {
2659 // Ensure the index type is i32 to match the tablegen patterns
2660 uint64_t Idx = IdxNode->getAsZExtVal();
2661 SmallVector<SDValue, 3> Ops(Op.getNode()->ops());
2662 Ops[Op.getNumOperands() - 1] =
2663 DAG.getConstant(Idx, SDLoc(IdxNode), MVT::i32);
2664 return DAG.getNode(Op.getOpcode(), SDLoc(Op), Op.getValueType(), Ops);
2665 }
2666 // Perform default expansion
2667 return SDValue();
2668}
2669
2671 EVT LaneT = Op.getSimpleValueType().getVectorElementType();
2672 // 32-bit and 64-bit unrolled shifts will have proper semantics
2673 if (LaneT.bitsGE(MVT::i32))
2674 return DAG.UnrollVectorOp(Op.getNode());
2675 // Otherwise mask the shift value to get proper semantics from 32-bit shift
2676 SDLoc DL(Op);
2677 size_t NumLanes = Op.getSimpleValueType().getVectorNumElements();
2678 SDValue Mask = DAG.getConstant(LaneT.getSizeInBits() - 1, DL, MVT::i32);
2679 unsigned ShiftOpcode = Op.getOpcode();
2680 SmallVector<SDValue, 16> ShiftedElements;
2681 DAG.ExtractVectorElements(Op.getOperand(0), ShiftedElements, 0, 0, MVT::i32);
2682 SmallVector<SDValue, 16> ShiftElements;
2683 DAG.ExtractVectorElements(Op.getOperand(1), ShiftElements, 0, 0, MVT::i32);
2684 SmallVector<SDValue, 16> UnrolledOps;
2685 for (size_t i = 0; i < NumLanes; ++i) {
2686 SDValue MaskedShiftValue =
2687 DAG.getNode(ISD::AND, DL, MVT::i32, ShiftElements[i], Mask);
2688 SDValue ShiftedValue = ShiftedElements[i];
2689 if (ShiftOpcode == ISD::SRA)
2690 ShiftedValue = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i32,
2691 ShiftedValue, DAG.getValueType(LaneT));
2692 UnrolledOps.push_back(
2693 DAG.getNode(ShiftOpcode, DL, MVT::i32, ShiftedValue, MaskedShiftValue));
2694 }
2695 return DAG.getBuildVector(Op.getValueType(), DL, UnrolledOps);
2696}
2697
2698SDValue WebAssemblyTargetLowering::LowerShift(SDValue Op,
2699 SelectionDAG &DAG) const {
2700 SDLoc DL(Op);
2701
2702 // Only manually lower vector shifts
2703 assert(Op.getSimpleValueType().isVector());
2704
2705 uint64_t LaneBits = Op.getValueType().getScalarSizeInBits();
2706 auto ShiftVal = Op.getOperand(1);
2707
2708 // Try to skip bitmask operation since it is implied inside shift instruction
2709 auto SkipImpliedMask = [](SDValue MaskOp, uint64_t MaskBits) {
2710 if (MaskOp.getOpcode() != ISD::AND)
2711 return MaskOp;
2712 SDValue LHS = MaskOp.getOperand(0);
2713 SDValue RHS = MaskOp.getOperand(1);
2714 if (MaskOp.getValueType().isVector()) {
2715 APInt MaskVal;
2716 if (!ISD::isConstantSplatVector(RHS.getNode(), MaskVal))
2717 std::swap(LHS, RHS);
2718
2719 if (ISD::isConstantSplatVector(RHS.getNode(), MaskVal) &&
2720 MaskVal == MaskBits)
2721 MaskOp = LHS;
2722 } else {
2723 if (!isa<ConstantSDNode>(RHS.getNode()))
2724 std::swap(LHS, RHS);
2725
2726 auto ConstantRHS = dyn_cast<ConstantSDNode>(RHS.getNode());
2727 if (ConstantRHS && ConstantRHS->getAPIntValue() == MaskBits)
2728 MaskOp = LHS;
2729 }
2730
2731 return MaskOp;
2732 };
2733
2734 // Skip vector and operation
2735 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2736 ShiftVal = DAG.getSplatValue(ShiftVal);
2737 if (!ShiftVal)
2738 return unrollVectorShift(Op, DAG);
2739
2740 // Skip scalar and operation
2741 ShiftVal = SkipImpliedMask(ShiftVal, LaneBits - 1);
2742 // Use anyext because none of the high bits can affect the shift
2743 ShiftVal = DAG.getAnyExtOrTrunc(ShiftVal, DL, MVT::i32);
2744
2745 unsigned Opcode;
2746 switch (Op.getOpcode()) {
2747 case ISD::SHL:
2748 Opcode = WebAssemblyISD::VEC_SHL;
2749 break;
2750 case ISD::SRA:
2751 Opcode = WebAssemblyISD::VEC_SHR_S;
2752 break;
2753 case ISD::SRL:
2754 Opcode = WebAssemblyISD::VEC_SHR_U;
2755 break;
2756 default:
2757 llvm_unreachable("unexpected opcode");
2758 }
2759
2760 return DAG.getNode(Opcode, DL, Op.getValueType(), Op.getOperand(0), ShiftVal);
2761}
2762
2763SDValue WebAssemblyTargetLowering::LowerFP_TO_INT_SAT(SDValue Op,
2764 SelectionDAG &DAG) const {
2765 EVT ResT = Op.getValueType();
2766 EVT SatVT = cast<VTSDNode>(Op.getOperand(1))->getVT();
2767
2768 if ((ResT == MVT::i32 || ResT == MVT::i64) &&
2769 (SatVT == MVT::i32 || SatVT == MVT::i64))
2770 return Op;
2771
2772 if (ResT == MVT::v4i32 && SatVT == MVT::i32)
2773 return Op;
2774
2775 if (ResT == MVT::v8i16 && SatVT == MVT::i16)
2776 return Op;
2777
2778 return SDValue();
2779}
2780
2781//===----------------------------------------------------------------------===//
2782// Custom DAG combine hooks
2783//===----------------------------------------------------------------------===//
2784static SDValue
2786 auto &DAG = DCI.DAG;
2787 auto Shuffle = cast<ShuffleVectorSDNode>(N);
2788
2789 // Hoist vector bitcasts that don't change the number of lanes out of unary
2790 // shuffles, where they are less likely to get in the way of other combines.
2791 // (shuffle (vNxT1 (bitcast (vNxT0 x))), undef, mask) ->
2792 // (vNxT1 (bitcast (vNxT0 (shuffle x, undef, mask))))
2793 SDValue Bitcast = N->getOperand(0);
2794 if (Bitcast.getOpcode() != ISD::BITCAST)
2795 return SDValue();
2796 if (!N->getOperand(1).isUndef())
2797 return SDValue();
2798 SDValue CastOp = Bitcast.getOperand(0);
2799 EVT SrcType = CastOp.getValueType();
2800 EVT DstType = Bitcast.getValueType();
2801 if (!SrcType.is128BitVector() ||
2802 SrcType.getVectorNumElements() != DstType.getVectorNumElements())
2803 return SDValue();
2804 SDValue NewShuffle = DAG.getVectorShuffle(
2805 SrcType, SDLoc(N), CastOp, DAG.getUNDEF(SrcType), Shuffle->getMask());
2806 return DAG.getBitcast(DstType, NewShuffle);
2807}
2808
2809/// Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get
2810/// split up into scalar instructions during legalization, and the vector
2811/// extending instructions are selected in performVectorExtendCombine below.
2812static SDValue
2815 auto &DAG = DCI.DAG;
2816 assert(N->getOpcode() == ISD::UINT_TO_FP ||
2817 N->getOpcode() == ISD::SINT_TO_FP);
2818
2819 EVT InVT = N->getOperand(0)->getValueType(0);
2820 EVT ResVT = N->getValueType(0);
2821 MVT ExtVT;
2822 if (ResVT == MVT::v4f32 && (InVT == MVT::v4i16 || InVT == MVT::v4i8))
2823 ExtVT = MVT::v4i32;
2824 else if (ResVT == MVT::v2f64 && (InVT == MVT::v2i16 || InVT == MVT::v2i8))
2825 ExtVT = MVT::v2i32;
2826 else
2827 return SDValue();
2828
2829 unsigned Op =
2831 SDValue Conv = DAG.getNode(Op, SDLoc(N), ExtVT, N->getOperand(0));
2832 return DAG.getNode(N->getOpcode(), SDLoc(N), ResVT, Conv);
2833}
2834
2835static SDValue
2838 auto &DAG = DCI.DAG;
2839
2840 SDNodeFlags Flags = N->getFlags();
2841 SDValue Op0 = N->getOperand(0);
2842 EVT VT = N->getValueType(0);
2843
2844 // Optimize uitofp to sitofp when the sign bit is known to be zero.
2845 // Depending on the target (runtime) backend, this might be performance
2846 // neutral (e.g. AArch64) or a significant improvement (e.g. x86_64).
2847 if (VT.isVector() && (Flags.hasNonNeg() || DAG.SignBitIsZero(Op0))) {
2848 return DAG.getNode(ISD::SINT_TO_FP, SDLoc(N), VT, Op0);
2849 }
2850
2851 return SDValue();
2852}
2853
2854static SDValue
2856 auto &DAG = DCI.DAG;
2857 assert(N->getOpcode() == ISD::SIGN_EXTEND ||
2858 N->getOpcode() == ISD::ZERO_EXTEND);
2859
2860 // Combine ({s,z}ext (extract_subvector src, i)) into a widening operation if
2861 // possible before the extract_subvector can be expanded.
2862 auto Extract = N->getOperand(0);
2863 if (Extract.getOpcode() != ISD::EXTRACT_SUBVECTOR)
2864 return SDValue();
2865 auto Source = Extract.getOperand(0);
2866 auto *IndexNode = dyn_cast<ConstantSDNode>(Extract.getOperand(1));
2867 if (IndexNode == nullptr)
2868 return SDValue();
2869 auto Index = IndexNode->getZExtValue();
2870
2871 // Only v8i8, v4i16, and v2i32 extracts can be widened, and only if the
2872 // extracted subvector is the low or high half of its source.
2873 EVT ResVT = N->getValueType(0);
2874 if (ResVT == MVT::v8i16) {
2875 if (Extract.getValueType() != MVT::v8i8 ||
2876 Source.getValueType() != MVT::v16i8 || (Index != 0 && Index != 8))
2877 return SDValue();
2878 } else if (ResVT == MVT::v4i32) {
2879 if (Extract.getValueType() != MVT::v4i16 ||
2880 Source.getValueType() != MVT::v8i16 || (Index != 0 && Index != 4))
2881 return SDValue();
2882 } else if (ResVT == MVT::v2i64) {
2883 if (Extract.getValueType() != MVT::v2i32 ||
2884 Source.getValueType() != MVT::v4i32 || (Index != 0 && Index != 2))
2885 return SDValue();
2886 } else {
2887 return SDValue();
2888 }
2889
2890 bool IsSext = N->getOpcode() == ISD::SIGN_EXTEND;
2891 bool IsLow = Index == 0;
2892
2893 unsigned Op = IsSext ? (IsLow ? WebAssemblyISD::EXTEND_LOW_S
2894 : WebAssemblyISD::EXTEND_HIGH_S)
2895 : (IsLow ? WebAssemblyISD::EXTEND_LOW_U
2896 : WebAssemblyISD::EXTEND_HIGH_U);
2897
2898 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2899}
2900
2901static SDValue
2903 auto &DAG = DCI.DAG;
2904
2905 auto GetWasmConversionOp = [](unsigned Op) {
2906 switch (Op) {
2908 return WebAssemblyISD::TRUNC_SAT_ZERO_S;
2910 return WebAssemblyISD::TRUNC_SAT_ZERO_U;
2911 case ISD::FP_ROUND:
2912 return WebAssemblyISD::DEMOTE_ZERO;
2913 }
2914 llvm_unreachable("unexpected op");
2915 };
2916
2917 auto IsZeroSplat = [](SDValue SplatVal) {
2918 auto *Splat = dyn_cast<BuildVectorSDNode>(SplatVal.getNode());
2919 APInt SplatValue, SplatUndef;
2920 unsigned SplatBitSize;
2921 bool HasAnyUndefs;
2922 // Endianness doesn't matter in this context because we are looking for
2923 // an all-zero value.
2924 return Splat &&
2925 Splat->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
2926 HasAnyUndefs) &&
2927 SplatValue == 0;
2928 };
2929
2930 if (N->getOpcode() == ISD::CONCAT_VECTORS) {
2931 // Combine this:
2932 //
2933 // (concat_vectors (v2i32 (fp_to_{s,u}int_sat $x, 32)), (v2i32 (splat 0)))
2934 //
2935 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2936 //
2937 // Or this:
2938 //
2939 // (concat_vectors (v2f32 (fp_round (v2f64 $x))), (v2f32 (splat 0)))
2940 //
2941 // into (f32x4.demote_zero_f64x2 $x).
2942 EVT ResVT;
2943 EVT ExpectedConversionType;
2944 auto Conversion = N->getOperand(0);
2945 auto ConversionOp = Conversion.getOpcode();
2946 switch (ConversionOp) {
2949 ResVT = MVT::v4i32;
2950 ExpectedConversionType = MVT::v2i32;
2951 break;
2952 case ISD::FP_ROUND:
2953 ResVT = MVT::v4f32;
2954 ExpectedConversionType = MVT::v2f32;
2955 break;
2956 default:
2957 return SDValue();
2958 }
2959
2960 if (N->getValueType(0) != ResVT)
2961 return SDValue();
2962
2963 if (Conversion.getValueType() != ExpectedConversionType)
2964 return SDValue();
2965
2966 auto Source = Conversion.getOperand(0);
2967 if (Source.getValueType() != MVT::v2f64)
2968 return SDValue();
2969
2970 if (!IsZeroSplat(N->getOperand(1)) ||
2971 N->getOperand(1).getValueType() != ExpectedConversionType)
2972 return SDValue();
2973
2974 unsigned Op = GetWasmConversionOp(ConversionOp);
2975 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
2976 }
2977
2978 // Combine this:
2979 //
2980 // (fp_to_{s,u}int_sat (concat_vectors $x, (v2f64 (splat 0))), 32)
2981 //
2982 // into (i32x4.trunc_sat_f64x2_zero_{s,u} $x).
2983 //
2984 // Or this:
2985 //
2986 // (v4f32 (fp_round (concat_vectors $x, (v2f64 (splat 0)))))
2987 //
2988 // into (f32x4.demote_zero_f64x2 $x).
2989 EVT ResVT;
2990 auto ConversionOp = N->getOpcode();
2991 switch (ConversionOp) {
2994 ResVT = MVT::v4i32;
2995 break;
2996 case ISD::FP_ROUND:
2997 ResVT = MVT::v4f32;
2998 break;
2999 default:
3000 llvm_unreachable("unexpected op");
3001 }
3002
3003 if (N->getValueType(0) != ResVT)
3004 return SDValue();
3005
3006 auto Concat = N->getOperand(0);
3007 if (Concat.getValueType() != MVT::v4f64)
3008 return SDValue();
3009
3010 auto Source = Concat.getOperand(0);
3011 if (Source.getValueType() != MVT::v2f64)
3012 return SDValue();
3013
3014 if (!IsZeroSplat(Concat.getOperand(1)) ||
3015 Concat.getOperand(1).getValueType() != MVT::v2f64)
3016 return SDValue();
3017
3018 unsigned Op = GetWasmConversionOp(ConversionOp);
3019 return DAG.getNode(Op, SDLoc(N), ResVT, Source);
3020}
3021
3022// Helper to extract VectorWidth bits from Vec, starting from IdxVal.
3023static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG,
3024 const SDLoc &DL, unsigned VectorWidth) {
3025 EVT VT = Vec.getValueType();
3026 EVT ElVT = VT.getVectorElementType();
3027 unsigned Factor = VT.getSizeInBits() / VectorWidth;
3028 EVT ResultVT = EVT::getVectorVT(*DAG.getContext(), ElVT,
3029 VT.getVectorNumElements() / Factor);
3030
3031 // Extract the relevant VectorWidth bits. Generate an EXTRACT_SUBVECTOR
3032 unsigned ElemsPerChunk = VectorWidth / ElVT.getSizeInBits();
3033 assert(isPowerOf2_32(ElemsPerChunk) && "Elements per chunk not power of 2");
3034
3035 // This is the index of the first element of the VectorWidth-bit chunk
3036 // we want. Since ElemsPerChunk is a power of 2 just need to clear bits.
3037 IdxVal &= ~(ElemsPerChunk - 1);
3038
3039 // If the input is a buildvector just emit a smaller one.
3040 if (Vec.getOpcode() == ISD::BUILD_VECTOR)
3041 return DAG.getBuildVector(ResultVT, DL,
3042 Vec->ops().slice(IdxVal, ElemsPerChunk));
3043
3044 SDValue VecIdx = DAG.getIntPtrConstant(IdxVal, DL);
3045 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, ResultVT, Vec, VecIdx);
3046}
3047
3048// Helper to recursively truncate vector elements in half with NARROW_U. DstVT
3049// is the expected destination value type after recursion. In is the initial
3050// input. Note that the input should have enough leading zero bits to prevent
3051// NARROW_U from saturating results.
3053 SelectionDAG &DAG) {
3054 EVT SrcVT = In.getValueType();
3055
3056 // No truncation required, we might get here due to recursive calls.
3057 if (SrcVT == DstVT)
3058 return In;
3059
3060 unsigned SrcSizeInBits = SrcVT.getSizeInBits();
3061 unsigned NumElems = SrcVT.getVectorNumElements();
3062 if (!isPowerOf2_32(NumElems))
3063 return SDValue();
3064 assert(DstVT.getVectorNumElements() == NumElems && "Illegal truncation");
3065 assert(SrcSizeInBits > DstVT.getSizeInBits() && "Illegal truncation");
3066
3067 LLVMContext &Ctx = *DAG.getContext();
3068 EVT PackedSVT = EVT::getIntegerVT(Ctx, SrcVT.getScalarSizeInBits() / 2);
3069
3070 // Narrow to the largest type possible:
3071 // vXi64/vXi32 -> i16x8.narrow_i32x4_u and vXi16 -> i8x16.narrow_i16x8_u.
3072 EVT InVT = MVT::i16, OutVT = MVT::i8;
3073 if (SrcVT.getScalarSizeInBits() > 16) {
3074 InVT = MVT::i32;
3075 OutVT = MVT::i16;
3076 }
3077 unsigned SubSizeInBits = SrcSizeInBits / 2;
3078 InVT = EVT::getVectorVT(Ctx, InVT, SubSizeInBits / InVT.getSizeInBits());
3079 OutVT = EVT::getVectorVT(Ctx, OutVT, SubSizeInBits / OutVT.getSizeInBits());
3080
3081 // Split lower/upper subvectors.
3082 SDValue Lo = extractSubVector(In, 0, DAG, DL, SubSizeInBits);
3083 SDValue Hi = extractSubVector(In, NumElems / 2, DAG, DL, SubSizeInBits);
3084
3085 // 256bit -> 128bit truncate - Narrow lower/upper 128-bit subvectors.
3086 if (SrcVT.is256BitVector() && DstVT.is128BitVector()) {
3087 Lo = DAG.getBitcast(InVT, Lo);
3088 Hi = DAG.getBitcast(InVT, Hi);
3089 SDValue Res = DAG.getNode(WebAssemblyISD::NARROW_U, DL, OutVT, Lo, Hi);
3090 return DAG.getBitcast(DstVT, Res);
3091 }
3092
3093 // Recursively narrow lower/upper subvectors, concat result and narrow again.
3094 EVT PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems / 2);
3095 Lo = truncateVectorWithNARROW(PackedVT, Lo, DL, DAG);
3096 Hi = truncateVectorWithNARROW(PackedVT, Hi, DL, DAG);
3097
3098 PackedVT = EVT::getVectorVT(Ctx, PackedSVT, NumElems);
3099 SDValue Res = DAG.getNode(ISD::CONCAT_VECTORS, DL, PackedVT, Lo, Hi);
3100 return truncateVectorWithNARROW(DstVT, Res, DL, DAG);
3101}
3102
3105 auto &DAG = DCI.DAG;
3106
3107 SDValue In = N->getOperand(0);
3108 EVT InVT = In.getValueType();
3109 if (!InVT.isSimple())
3110 return SDValue();
3111
3112 EVT OutVT = N->getValueType(0);
3113 if (!OutVT.isVector())
3114 return SDValue();
3115
3116 EVT OutSVT = OutVT.getVectorElementType();
3117 EVT InSVT = InVT.getVectorElementType();
3118 // Currently only cover truncate to v16i8 or v8i16.
3119 if (!((InSVT == MVT::i16 || InSVT == MVT::i32 || InSVT == MVT::i64) &&
3120 (OutSVT == MVT::i8 || OutSVT == MVT::i16) && OutVT.is128BitVector()))
3121 return SDValue();
3122
3123 SDLoc DL(N);
3125 OutVT.getScalarSizeInBits());
3126 In = DAG.getNode(ISD::AND, DL, InVT, In, DAG.getConstant(Mask, DL, InVT));
3127 return truncateVectorWithNARROW(OutVT, In, DL, DAG);
3128}
3129
3132 using namespace llvm::SDPatternMatch;
3133 auto &DAG = DCI.DAG;
3134 SDLoc DL(N);
3135 SDValue Src = N->getOperand(0);
3136 EVT VT = N->getValueType(0);
3137 EVT SrcVT = Src.getValueType();
3138
3139 if (!(DCI.isBeforeLegalize() && VT.isScalarInteger() &&
3140 SrcVT.isFixedLengthVector() && SrcVT.getScalarType() == MVT::i1))
3141 return SDValue();
3142
3143 unsigned NumElts = SrcVT.getVectorNumElements();
3144 EVT Width = MVT::getIntegerVT(128 / NumElts);
3145
3146 // bitcast <N x i1> to iN, where N = 2, 4, 8, 16 (legal)
3147 // ==> bitmask
3148 if (NumElts == 2 || NumElts == 4 || NumElts == 8 || NumElts == 16) {
3149 return DAG.getZExtOrTrunc(
3150 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
3151 {DAG.getConstant(Intrinsic::wasm_bitmask, DL, MVT::i32),
3152 DAG.getSExtOrTrunc(N->getOperand(0), DL,
3153 SrcVT.changeVectorElementType(Width))}),
3154 DL, VT);
3155 }
3156
3157 // bitcast <N x i1>(setcc ...) to concat iN, where N = 32 and 64 (illegal)
3158 if (NumElts == 32 || NumElts == 64) {
3159 // Strategy: We will setcc them seperately in v16i8 -> v16i1
3160 // Bitcast them to i16, extend them to either i32 or i64.
3161 // Add them together, shifting left 1 by 1.
3162 SDValue Concat, SetCCVector;
3163 ISD::CondCode SetCond;
3164
3165 if (!sd_match(N, m_BitCast(m_c_SetCC(m_Value(Concat), m_Value(SetCCVector),
3166 m_CondCode(SetCond)))))
3167 return SDValue();
3168 if (Concat.getOpcode() != ISD::CONCAT_VECTORS)
3169 return SDValue();
3170
3171 uint64_t ElementWidth =
3173
3174 SmallVector<SDValue> VectorsToShuffle;
3175 for (size_t I = 0; I < Concat->ops().size(); I++) {
3176 VectorsToShuffle.push_back(DAG.getBitcast(
3177 MVT::i16,
3178 DAG.getSetCC(DL, MVT::v16i1, Concat->ops()[I],
3179 extractSubVector(SetCCVector, I * (128 / ElementWidth),
3180 DAG, DL, 128),
3181 SetCond)));
3182 }
3183
3184 MVT ReturnType = VectorsToShuffle.size() == 2 ? MVT::i32 : MVT::i64;
3185 SDValue ReturningInteger = DAG.getConstant(0, DL, ReturnType);
3186
3187 for (SDValue V : VectorsToShuffle) {
3188 ReturningInteger = DAG.getNode(
3189 ISD::SHL, DL, ReturnType,
3190 {DAG.getShiftAmountConstant(16, ReturnType, DL), ReturningInteger});
3191
3192 SDValue ExtendedV = DAG.getZExtOrTrunc(V, DL, ReturnType);
3193 ReturningInteger =
3194 DAG.getNode(ISD::ADD, DL, ReturnType, {ReturningInteger, ExtendedV});
3195 }
3196
3197 return ReturningInteger;
3198 }
3199
3200 return SDValue();
3201}
3202
3204 // any_true (setcc <X>, 0, eq) => (not (all_true X))
3205 // all_true (setcc <X>, 0, eq) => (not (any_true X))
3206 // any_true (setcc <X>, 0, ne) => (any_true X)
3207 // all_true (setcc <X>, 0, ne) => (all_true X)
3208 assert(N->getOpcode() == ISD::INTRINSIC_WO_CHAIN);
3209 using namespace llvm::SDPatternMatch;
3210
3211 SDValue LHS;
3212 if (!sd_match(N->getOperand(1),
3214 return SDValue();
3215 EVT LT = LHS.getValueType();
3216 if (LT.getScalarSizeInBits() > 128 / LT.getVectorNumElements())
3217 return SDValue();
3218
3219 auto CombineSetCC = [&N, &DAG](Intrinsic::WASMIntrinsics InPre,
3220 ISD::CondCode SetType,
3221 Intrinsic::WASMIntrinsics InPost) {
3222 if (N->getConstantOperandVal(0) != InPre)
3223 return SDValue();
3224
3225 SDValue LHS;
3226 if (!sd_match(N->getOperand(1), m_c_SetCC(m_Value(LHS), m_Zero(),
3227 m_SpecificCondCode(SetType))))
3228 return SDValue();
3229
3230 SDLoc DL(N);
3231 SDValue Ret = DAG.getZExtOrTrunc(
3232 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
3233 {DAG.getConstant(InPost, DL, MVT::i32), LHS}),
3234 DL, MVT::i1);
3235 if (SetType == ISD::SETEQ)
3236 Ret = DAG.getNOT(DL, Ret, MVT::i1);
3237 return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
3238 };
3239
3240 if (SDValue AnyTrueEQ = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETEQ,
3241 Intrinsic::wasm_alltrue))
3242 return AnyTrueEQ;
3243 if (SDValue AllTrueEQ = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETEQ,
3244 Intrinsic::wasm_anytrue))
3245 return AllTrueEQ;
3246 if (SDValue AnyTrueNE = CombineSetCC(Intrinsic::wasm_anytrue, ISD::SETNE,
3247 Intrinsic::wasm_anytrue))
3248 return AnyTrueNE;
3249 if (SDValue AllTrueNE = CombineSetCC(Intrinsic::wasm_alltrue, ISD::SETNE,
3250 Intrinsic::wasm_alltrue))
3251 return AllTrueNE;
3252
3253 return SDValue();
3254}
3255
3256template <int MatchRHS, ISD::CondCode MatchCond, bool RequiresNegate,
3257 Intrinsic::ID Intrin>
3259 SDValue LHS = N->getOperand(0);
3260 SDValue RHS = N->getOperand(1);
3261 SDValue Cond = N->getOperand(2);
3262 if (MatchCond != cast<CondCodeSDNode>(Cond)->get())
3263 return SDValue();
3264
3265 if (MatchRHS != cast<ConstantSDNode>(RHS)->getSExtValue())
3266 return SDValue();
3267
3268 SDLoc DL(N);
3269 SDValue Ret = DAG.getZExtOrTrunc(
3270 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
3271 {DAG.getConstant(Intrin, DL, MVT::i32),
3272 DAG.getSExtOrTrunc(LHS->getOperand(0), DL, VecVT)}),
3273 DL, MVT::i1);
3274 if (RequiresNegate)
3275 Ret = DAG.getNOT(DL, Ret, MVT::i1);
3276 return DAG.getZExtOrTrunc(Ret, DL, N->getValueType(0));
3277}
3278
3279/// Try to convert a i128 comparison to a v16i8 comparison before type
3280/// legalization splits it up into chunks
3281static SDValue
3283 const WebAssemblySubtarget *Subtarget) {
3284
3285 SDLoc DL(N);
3286 SDValue X = N->getOperand(0);
3287 SDValue Y = N->getOperand(1);
3288 EVT VT = N->getValueType(0);
3289 EVT OpVT = X.getValueType();
3290
3291 SelectionDAG &DAG = DCI.DAG;
3293 Attribute::NoImplicitFloat))
3294 return SDValue();
3295
3296 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
3297 // We're looking for an oversized integer equality comparison with SIMD
3298 if (!OpVT.isScalarInteger() || !OpVT.isByteSized() || OpVT != MVT::i128 ||
3299 !Subtarget->hasSIMD128() || !isIntEqualitySetCC(CC))
3300 return SDValue();
3301
3302 // Don't perform this combine if constructing the vector will be expensive.
3303 auto IsVectorBitCastCheap = [](SDValue X) {
3305 return isa<ConstantSDNode>(X) || X.getOpcode() == ISD::LOAD;
3306 };
3307
3308 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
3309 return SDValue();
3310
3311 SDValue VecX = DAG.getBitcast(MVT::v16i8, X);
3312 SDValue VecY = DAG.getBitcast(MVT::v16i8, Y);
3313 SDValue Cmp = DAG.getSetCC(DL, MVT::v16i8, VecX, VecY, CC);
3314
3315 SDValue Intr =
3316 DAG.getNode(ISD::INTRINSIC_WO_CHAIN, DL, MVT::i32,
3317 {DAG.getConstant(CC == ISD::SETEQ ? Intrinsic::wasm_alltrue
3318 : Intrinsic::wasm_anytrue,
3319 DL, MVT::i32),
3320 Cmp});
3321
3322 return DAG.getSetCC(DL, VT, Intr, DAG.getConstant(0, DL, MVT::i32),
3323 ISD::SETNE);
3324}
3325
3328 const WebAssemblySubtarget *Subtarget) {
3329 if (!DCI.isBeforeLegalize())
3330 return SDValue();
3331
3332 EVT VT = N->getValueType(0);
3333 if (!VT.isScalarInteger())
3334 return SDValue();
3335
3336 if (SDValue V = combineVectorSizedSetCCEquality(N, DCI, Subtarget))
3337 return V;
3338
3339 SDValue LHS = N->getOperand(0);
3340 if (LHS->getOpcode() != ISD::BITCAST)
3341 return SDValue();
3342
3343 EVT FromVT = LHS->getOperand(0).getValueType();
3344 if (!FromVT.isFixedLengthVector() || FromVT.getVectorElementType() != MVT::i1)
3345 return SDValue();
3346
3347 unsigned NumElts = FromVT.getVectorNumElements();
3348 if (NumElts != 2 && NumElts != 4 && NumElts != 8 && NumElts != 16)
3349 return SDValue();
3350
3351 if (!cast<ConstantSDNode>(N->getOperand(1)))
3352 return SDValue();
3353
3354 EVT VecVT = FromVT.changeVectorElementType(MVT::getIntegerVT(128 / NumElts));
3355 auto &DAG = DCI.DAG;
3356 // setcc (iN (bitcast (vNi1 X))), 0, ne
3357 // ==> any_true (vNi1 X)
3359 N, VecVT, DAG)) {
3360 return Match;
3361 }
3362 // setcc (iN (bitcast (vNi1 X))), 0, eq
3363 // ==> xor (any_true (vNi1 X)), -1
3365 N, VecVT, DAG)) {
3366 return Match;
3367 }
3368 // setcc (iN (bitcast (vNi1 X))), -1, eq
3369 // ==> all_true (vNi1 X)
3371 N, VecVT, DAG)) {
3372 return Match;
3373 }
3374 // setcc (iN (bitcast (vNi1 X))), -1, ne
3375 // ==> xor (all_true (vNi1 X)), -1
3377 N, VecVT, DAG)) {
3378 return Match;
3379 }
3380 return SDValue();
3381}
3382
3384 EVT VT = N->getValueType(0);
3385 if (VT != MVT::v8i32 && VT != MVT::v16i32)
3386 return SDValue();
3387
3388 // Mul with extending inputs.
3389 SDValue LHS = N->getOperand(0);
3390 SDValue RHS = N->getOperand(1);
3391 if (LHS.getOpcode() != RHS.getOpcode())
3392 return SDValue();
3393
3394 if (LHS.getOpcode() != ISD::SIGN_EXTEND &&
3395 LHS.getOpcode() != ISD::ZERO_EXTEND)
3396 return SDValue();
3397
3398 if (LHS->getOperand(0).getValueType() != RHS->getOperand(0).getValueType())
3399 return SDValue();
3400
3401 EVT FromVT = LHS->getOperand(0).getValueType();
3402 EVT EltTy = FromVT.getVectorElementType();
3403 if (EltTy != MVT::i8)
3404 return SDValue();
3405
3406 // For an input DAG that looks like this
3407 // %a = input_type
3408 // %b = input_type
3409 // %lhs = extend %a to output_type
3410 // %rhs = extend %b to output_type
3411 // %mul = mul %lhs, %rhs
3412
3413 // input_type | output_type | instructions
3414 // v16i8 | v16i32 | %low = i16x8.extmul_low_i8x16_ %a, %b
3415 // | | %high = i16x8.extmul_high_i8x16_, %a, %b
3416 // | | %low_low = i32x4.ext_low_i16x8_ %low
3417 // | | %low_high = i32x4.ext_high_i16x8_ %low
3418 // | | %high_low = i32x4.ext_low_i16x8_ %high
3419 // | | %high_high = i32x4.ext_high_i16x8_ %high
3420 // | | %res = concat_vector(...)
3421 // v8i8 | v8i32 | %low = i16x8.extmul_low_i8x16_ %a, %b
3422 // | | %low_low = i32x4.ext_low_i16x8_ %low
3423 // | | %low_high = i32x4.ext_high_i16x8_ %low
3424 // | | %res = concat_vector(%low_low, %low_high)
3425
3426 SDLoc DL(N);
3427 unsigned NumElts = VT.getVectorNumElements();
3428 SDValue ExtendInLHS = LHS->getOperand(0);
3429 SDValue ExtendInRHS = RHS->getOperand(0);
3430 bool IsSigned = LHS->getOpcode() == ISD::SIGN_EXTEND;
3431 unsigned ExtendLowOpc =
3432 IsSigned ? WebAssemblyISD::EXTEND_LOW_S : WebAssemblyISD::EXTEND_LOW_U;
3433 unsigned ExtendHighOpc =
3434 IsSigned ? WebAssemblyISD::EXTEND_HIGH_S : WebAssemblyISD::EXTEND_HIGH_U;
3435
3436 auto GetExtendLow = [&DAG, &DL, &ExtendLowOpc](EVT VT, SDValue Op) {
3437 return DAG.getNode(ExtendLowOpc, DL, VT, Op);
3438 };
3439 auto GetExtendHigh = [&DAG, &DL, &ExtendHighOpc](EVT VT, SDValue Op) {
3440 return DAG.getNode(ExtendHighOpc, DL, VT, Op);
3441 };
3442
3443 if (NumElts == 16) {
3444 SDValue LowLHS = GetExtendLow(MVT::v8i16, ExtendInLHS);
3445 SDValue LowRHS = GetExtendLow(MVT::v8i16, ExtendInRHS);
3446 SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);
3447 SDValue HighLHS = GetExtendHigh(MVT::v8i16, ExtendInLHS);
3448 SDValue HighRHS = GetExtendHigh(MVT::v8i16, ExtendInRHS);
3449 SDValue MulHigh = DAG.getNode(ISD::MUL, DL, MVT::v8i16, HighLHS, HighRHS);
3450 SDValue SubVectors[] = {
3451 GetExtendLow(MVT::v4i32, MulLow),
3452 GetExtendHigh(MVT::v4i32, MulLow),
3453 GetExtendLow(MVT::v4i32, MulHigh),
3454 GetExtendHigh(MVT::v4i32, MulHigh),
3455 };
3456 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, SubVectors);
3457 } else {
3458 assert(NumElts == 8);
3459 SDValue LowLHS = DAG.getNode(LHS->getOpcode(), DL, MVT::v8i16, ExtendInLHS);
3460 SDValue LowRHS = DAG.getNode(RHS->getOpcode(), DL, MVT::v8i16, ExtendInRHS);
3461 SDValue MulLow = DAG.getNode(ISD::MUL, DL, MVT::v8i16, LowLHS, LowRHS);
3462 SDValue Lo = GetExtendLow(MVT::v4i32, MulLow);
3463 SDValue Hi = GetExtendHigh(MVT::v4i32, MulLow);
3464 return DAG.getNode(ISD::CONCAT_VECTORS, DL, VT, Lo, Hi);
3465 }
3466 return SDValue();
3467}
3468
3471 assert(N->getOpcode() == ISD::MUL);
3472 EVT VT = N->getValueType(0);
3473 if (!VT.isVector())
3474 return SDValue();
3475
3476 if (auto Res = TryWideExtMulCombine(N, DCI.DAG))
3477 return Res;
3478
3479 // We don't natively support v16i8 or v8i8 mul, but we do support v8i16. So,
3480 // extend them to v8i16. Only do this before legalization in case a narrow
3481 // vector is widened and may be simplified later.
3482 if (!DCI.isBeforeLegalize() || (VT != MVT::v8i8 && VT != MVT::v16i8))
3483 return SDValue();
3484
3485 SDLoc DL(N);
3486 SelectionDAG &DAG = DCI.DAG;
3487 SDValue LHS = N->getOperand(0);
3488 SDValue RHS = N->getOperand(1);
3489 EVT MulVT = MVT::v8i16;
3490
3491 if (VT == MVT::v8i8) {
3492 SDValue PromotedLHS = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, LHS,
3493 DAG.getUNDEF(MVT::v8i8));
3494 SDValue PromotedRHS = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, RHS,
3495 DAG.getUNDEF(MVT::v8i8));
3496 SDValue LowLHS =
3497 DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, PromotedLHS);
3498 SDValue LowRHS =
3499 DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, PromotedRHS);
3500 SDValue MulLow = DAG.getBitcast(
3501 MVT::v16i8, DAG.getNode(ISD::MUL, DL, MulVT, LowLHS, LowRHS));
3502 // Take the low byte of each lane.
3503 SDValue Shuffle = DAG.getVectorShuffle(
3504 MVT::v16i8, DL, MulLow, DAG.getUNDEF(MVT::v16i8),
3505 {0, 2, 4, 6, 8, 10, 12, 14, -1, -1, -1, -1, -1, -1, -1, -1});
3506 return extractSubVector(Shuffle, 0, DAG, DL, 64);
3507 } else {
3508 assert(VT == MVT::v16i8 && "Expected v16i8");
3509 SDValue LowLHS = DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, LHS);
3510 SDValue LowRHS = DAG.getNode(WebAssemblyISD::EXTEND_LOW_U, DL, MulVT, RHS);
3511 SDValue HighLHS =
3512 DAG.getNode(WebAssemblyISD::EXTEND_HIGH_U, DL, MulVT, LHS);
3513 SDValue HighRHS =
3514 DAG.getNode(WebAssemblyISD::EXTEND_HIGH_U, DL, MulVT, RHS);
3515
3516 SDValue MulLow =
3517 DAG.getBitcast(VT, DAG.getNode(ISD::MUL, DL, MulVT, LowLHS, LowRHS));
3518 SDValue MulHigh =
3519 DAG.getBitcast(VT, DAG.getNode(ISD::MUL, DL, MulVT, HighLHS, HighRHS));
3520
3521 // Take the low byte of each lane.
3522 return DAG.getVectorShuffle(
3523 VT, DL, MulLow, MulHigh,
3524 {0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20, 22, 24, 26, 28, 30});
3525 }
3526}
3527
3528SDValue
3529WebAssemblyTargetLowering::PerformDAGCombine(SDNode *N,
3530 DAGCombinerInfo &DCI) const {
3531 switch (N->getOpcode()) {
3532 default:
3533 return SDValue();
3534 case ISD::BITCAST:
3535 return performBitcastCombine(N, DCI);
3536 case ISD::SETCC:
3537 return performSETCCCombine(N, DCI, Subtarget);
3539 return performVECTOR_SHUFFLECombine(N, DCI);
3540 case ISD::SIGN_EXTEND:
3541 case ISD::ZERO_EXTEND:
3542 return performVectorExtendCombine(N, DCI);
3543 case ISD::UINT_TO_FP:
3544 if (auto ExtCombine = performVectorExtendToFPCombine(N, DCI))
3545 return ExtCombine;
3546 return performVectorNonNegToFPCombine(N, DCI);
3547 case ISD::SINT_TO_FP:
3548 return performVectorExtendToFPCombine(N, DCI);
3551 case ISD::FP_ROUND:
3553 return performVectorTruncZeroCombine(N, DCI);
3554 case ISD::TRUNCATE:
3555 return performTruncateCombine(N, DCI);
3557 return performAnyAllCombine(N, DCI.DAG);
3558 case ISD::MUL:
3559 return performMulCombine(N, DCI);
3560 }
3561}
unsigned const MachineRegisterInfo * MRI
static SDValue performMulCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const AArch64Subtarget *Subtarget)
static SDValue performTruncateCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
return SDValue()
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
Function Alias Analysis false
Function Alias Analysis Results
static void fail(const SDLoc &DL, SelectionDAG &DAG, const Twine &Msg, SDValue Val={})
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
Analysis containing CSE Info
Definition CSEInfo.cpp:27
Hexagon Common GEP
const HexagonInstrInfo * TII
#define _
IRTranslator LLVM IR MI
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define F(x, y, z)
Definition MD5.cpp:55
#define I(x, y, z)
Definition MD5.cpp:58
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
MachineInstr unsigned OpIdx
static SDValue performVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const RISCVTargetLowering &TLI)
static SDValue combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const RISCVSubtarget &Subtarget)
Try to map an integer comparison with size > XLEN to vector instructions before type legalization spl...
const SmallVectorImpl< MachineOperand > & Cond
Contains matchers for matching SelectionDAG nodes and values.
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static TableGen::Emitter::OptClass< SkeletonEmitter > X("gen-skeleton-class", "Generate example skeleton class")
static MachineBasicBlock * LowerFPToInt(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool IsUnsigned, bool Int64, bool Float64, unsigned LoweredOpcode)
static bool callingConvSupported(CallingConv::ID CallConv)
static SDValue TryWideExtMulCombine(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * LowerMemcpy(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool Int64)
static std::optional< unsigned > IsWebAssemblyLocal(SDValue Op, SelectionDAG &DAG)
static SDValue performVectorExtendCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performVectorNonNegToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue unrollVectorShift(SDValue Op, SelectionDAG &DAG)
static SDValue performAnyAllCombine(SDNode *N, SelectionDAG &DAG)
static MachineBasicBlock * LowerCallResults(MachineInstr &CallResults, DebugLoc DL, MachineBasicBlock *BB, const WebAssemblySubtarget *Subtarget, const TargetInstrInfo &TII)
static SDValue TryMatchTrue(SDNode *N, EVT VecVT, SelectionDAG &DAG)
static SDValue GetExtendHigh(SDValue Op, unsigned UserOpc, EVT VT, SelectionDAG &DAG)
static SDValue performVectorTruncZeroCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static bool IsWebAssemblyGlobal(SDValue Op)
static MachineBasicBlock * LowerMemset(MachineInstr &MI, DebugLoc DL, MachineBasicBlock *BB, const TargetInstrInfo &TII, bool Int64)
static SDValue performVectorExtendToFPCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
Convert ({u,s}itofp vec) --> ({u,s}itofp ({s,z}ext vec)) so it doesn't get split up into scalar instr...
static SDValue LowerConvertLow(SDValue Op, SelectionDAG &DAG)
static SDValue extractSubVector(SDValue Vec, unsigned IdxVal, SelectionDAG &DAG, const SDLoc &DL, unsigned VectorWidth)
static SDValue performBitcastCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue truncateVectorWithNARROW(EVT DstVT, SDValue In, const SDLoc &DL, SelectionDAG &DAG)
This file defines the interfaces that WebAssembly uses to lower LLVM code into a selection DAG.
This file provides WebAssembly-specific target descriptions.
This file declares WebAssembly-specific per-machine-function information.
This file declares the WebAssembly-specific subclass of TargetSubtarget.
This file declares the WebAssembly-specific subclass of TargetMachine.
This file contains the declaration of the WebAssembly-specific type parsing utility functions.
This file contains the declaration of the WebAssembly-specific utility functions.
X86 cmov Conversion
static constexpr int Concat[]
Value * RHS
Value * LHS
Class for arbitrary precision integers.
Definition APInt.h:78
void setBitsFrom(unsigned loBit)
Set the top bits starting from loBit.
Definition APInt.h:1385
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:306
static APInt getHighBitsSet(unsigned numBits, unsigned hiBitsSet)
Constructs an APInt value that has the top hiBitsSet bits set.
Definition APInt.h:296
ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:41
an instruction that atomically reads a memory location, combines it with another value,...
@ Add
*p = old + v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
BinOp getOperation() const
LLVM Basic Block Representation.
Definition BasicBlock.h:62
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
This class represents a function call, abstracting a target machine's calling convention.
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:63
A debug info location.
Definition DebugLoc.h:124
Diagnostic information for unsupported feature in backend.
This is a fast-path instruction selection class that generates poor code and doesn't support illegal ...
Definition FastISel.h:66
FunctionLoweringInfo - This contains information that is global to a function that is used when lower...
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:209
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:359
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:727
LLVM_ABI unsigned getAddressSpace() const
const GlobalValue * getGlobal() const
ThreadLocalMode getThreadLocalMode() const
Type * getValueType() const
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:68
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
Describe properties that are true of each instruction in the target description file.
void setNoStrip() const
Machine Value Type.
bool is128BitVector() const
Return true if this is a 128-bit vector type.
@ INVALID_SIMPLE_VALUE_TYPE
static auto integer_fixedlen_vector_valuetypes()
MVT changeVectorElementType(MVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
static auto integer_valuetypes()
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
bool isFixedLengthVector() const
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)
Insert MI into the instruction list before I, possibly inside a bundle.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
iterator insertAfter(iterator I, MachineInstr *MI)
Insert MI into the instruction list after I.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
unsigned getFunctionNumber() const
getFunctionNumber - Return a unique ID for the current function.
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
const char * createExternalSymbolName(StringRef Name)
Allocate a string and populate it with the given external symbol name.
MCContext & getContext() const
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
const MachineJumpTableInfo * getJumpTableInfo() const
getJumpTableInfo - Return the jump table info object for the current function.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & add(const MachineOperand &MO) const
const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const
const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addFPImm(const ConstantFP *Val) const
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
MachineInstr * getInstr() const
If conversion operators fail, use this method to get the MachineInstr explicitly.
Representation of each machine instruction.
mop_range defs()
Returns all explicit operands that are register definitions.
unsigned getOpcode() const
Returns the opcode of this MachineInstr.
LLVM_ABI void addOperand(MachineFunction &MF, const MachineOperand &Op)
Add the specified operand to the instruction.
mop_range uses()
Returns all operands which may be register uses.
LLVM_ABI void eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
LLVM_ABI void removeOperand(unsigned OpNo)
Erase an operand from an instruction, leaving it with one fewer operand than it started with.
const MachineOperand & getOperand(unsigned i) const
const std::vector< MachineJumpTableEntry > & getJumpTables() const
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MOLoad
The memory access reads data.
@ MOStore
The memory access writes data.
MachineOperand class - Representation of each machine instruction operand.
bool isReg() const
isReg - Tests if this is a MO_Register operand.
void setIsKill(bool Val=true)
Register getReg() const
getReg - Returns the register number.
bool isFI() const
isFI - Tests if this is a MO_FrameIndex operand.
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
void addLiveIn(MCRegister Reg, Register vreg=Register())
addLiveIn - Add the specified register as a live-in.
unsigned getAddressSpace() const
Return the address space for the associated pointer.
MachineMemOperand * getMemOperand() const
Return a MachineMemOperand object describing the memory reference performed by operation.
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
static PointerType * getUnqual(Type *ElementType)
This constructs a pointer to an object of the specified type in the default address space (address sp...
Wrapper class representing virtual and physical registers.
Definition Register.h:19
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
ArrayRef< SDUse > ops() const
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
const SDValue & getOperand(unsigned Num) const
uint64_t getConstantOperandVal(unsigned Num) const
Helper method returns the integer value of a ConstantSDNode operand.
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
const SDValue & getOperand(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getSplatValue(SDValue V, bool LegalTypes=false)
If V is a splat vector, return its scalar source operand by extracting that element from the source v...
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI void ExtractVectorElements(SDValue Op, SmallVectorImpl< SDValue > &Args, unsigned Start=0, unsigned Count=0, EVT EltVT=EVT())
Append the extracted elements from Start to Count out of the vector Op in Args.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false)
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
LLVM_ABI SDValue UnrollVectorOp(SDNode *N, unsigned ResNE=0)
Utility function used by legalize and lowering to "unroll" a vector operation by splitting out the sc...
LLVM_ABI SDValue getConstantFP(double Val, const SDLoc &DL, EVT VT, bool isTarget=false)
Create a ConstantFPSDNode wrapping a constant value.
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
const DataLayout & getDataLayout() const
SDValue getTargetFrameIndex(int FI, EVT VT)
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const AAMDNodes &AAInfo=AAMDNodes())
Helper function to build ISD::STORE nodes.
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI SDValue getBasicBlock(MachineBasicBlock *MBB)
const TargetMachine & getTarget() const
LLVM_ABI SDValue getAnyExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either any-extending or truncat...
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue getMCSymbol(MCSymbol *Sym, EVT VT)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
const SDValue & getBasePtr() const
const SDValue & getOffset() const
const SDValue & getValue() const
StringRef - Represent a constant reference to a string, i.e.
Definition StringRef.h:55
constexpr size_t size() const
size - Get the string size.
Definition StringRef.h:146
TargetInstrInfo - Interface to description of machine instruction set.
Provides information about what library functions are available for the current target.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
unsigned MaxLoadsPerMemcmp
Specify maximum number of load instructions per memcmp call.
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setMinimumJumpTableEntries(unsigned Val)
Indicate the minimum number of blocks to generate jump tables.
void setPartialReduceMLAAction(unsigned Opc, MVT AccVT, MVT InputVT, LegalizeAction Action)
Indicate how a PARTIAL_REDUCE_U/SMLA node with Acc type AccVT and Input type InputVT should be treate...
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
unsigned MaxLoadsPerMemcmpOptSize
Likewise for functions with the OptSize attribute.
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
virtual MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
void setSchedulingPreference(Sched::Preference Pref)
Specify the target scheduling preference.
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::Libcall LC, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
bool isPositionIndependent() const
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
TargetLowering(const TargetLowering &)=delete
virtual bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const
Return true if folding a constant offset with the given GlobalAddress is legal.
Primary interface to the complete machine description for the target machine.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:45
bool isFunctionTy() const
True if this is an instance of FunctionType.
Definition Type.h:258
static LLVM_ABI Type * getDoubleTy(LLVMContext &C)
Definition Type.cpp:286
static LLVM_ABI Type * getFloatTy(LLVMContext &C)
Definition Type.cpp:285
A Use represents the edge between a Value definition and its users.
Definition Use.h:35
LLVM_ABI const Value * stripPointerCastsAndAliases() const
Strip off pointer casts, all-zero GEPs, address space casts, and aliases.
Definition Value.cpp:705
static std::optional< unsigned > getLocalForStackObject(MachineFunction &MF, int FrameIndex)
WebAssemblyTargetLowering(const TargetMachine &TM, const WebAssemblySubtarget &STI)
MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const override
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
MVT getPointerMemTy(const DataLayout &DL, uint32_t AS=0) const override
Return the in-memory pointer type for the given address space, defaults to the pointer type from the ...
self_iterator getIterator()
Definition ilist_node.h:123
#define INT64_MIN
Definition DataTypes.h:74
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ Swift
Calling convention for Swift.
Definition CallingConv.h:69
@ PreserveMost
Used for runtime calls that preserves most registers.
Definition CallingConv.h:63
@ CXX_FAST_TLS
Used for access functions.
Definition CallingConv.h:72
@ WASM_EmscriptenInvoke
For emscripten __invoke_* functions.
@ Cold
Attempts to make code in the caller as efficient as possible under the assumption that the call is no...
Definition CallingConv.h:47
@ PreserveAll
Used for runtime calls that preserves (almost) all registers.
Definition CallingConv.h:66
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:801
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:270
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:765
@ ADDC
Carry-setting nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:289
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:259
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:511
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:215
@ GlobalAddress
Definition ISDOpcodes.h:88
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:862
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:571
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:738
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition ISDOpcodes.h:892
@ SDIVREM
SDIVREM/UDIVREM - Divide two integers and produce both a quotient and remainder result.
Definition ISDOpcodes.h:275
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:249
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ GlobalTLSAddress
Definition ISDOpcodes.h:89
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:826
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:656
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:369
@ EXTRACT_ELEMENT
EXTRACT_ELEMENT - This is used to get the lower or upper (determined by a Constant,...
Definition ISDOpcodes.h:242
@ SPLAT_VECTOR
SPLAT_VECTOR(VAL) - Returns a vector with the scalar value VAL duplicated in all lanes.
Definition ISDOpcodes.h:663
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:695
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:756
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:636
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:601
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:563
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition ISDOpcodes.h:219
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:832
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:793
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:870
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:718
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:110
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:908
@ TargetConstant
TargetConstant* - Like Constant*, but the DAG does not do any folding, simplification,...
Definition ISDOpcodes.h:174
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:730
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:200
@ ADDE
Carry-using nodes for multiple precision addition and subtraction.
Definition ISDOpcodes.h:299
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:552
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:53
@ ExternalSymbol
Definition ISDOpcodes.h:93
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:941
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition ISDOpcodes.h:903
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:927
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:838
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:815
@ FCOPYSIGN
FCOPYSIGN(X, Y) - Return the value of X with the sign of Y.
Definition ISDOpcodes.h:521
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:360
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:208
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:543
LLVM_ABI bool isConstantSplatVector(const SDNode *N, APInt &SplatValue)
Node predicates.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
OperandFlags
These are flags set on operands, but should be considered private, all access should go through the M...
Definition MCInstrDesc.h:51
CastOperator_match< OpTy, Instruction::BitCast > m_BitCast(const OpTy &Op)
Matches BitCast.
class_match< Value > m_Value()
Match an arbitrary value and ignore it.
is_zero m_Zero()
Match any null constant or a vector with all elements equal to 0.
bool sd_match(SDNode *N, const SelectionDAG *DAG, Pattern &&P)
CondCode_match m_SpecificCondCode(ISD::CondCode CC)
Match a conditional code SDNode with a specific ISD::CondCode.
CondCode_match m_CondCode()
Match any conditional code SDNode.
TernaryOpc_match< T0_P, T1_P, T2_P, true, false > m_c_SetCC(const T0_P &LHS, const T1_P &RHS, const T2_P &CC)
MCSymbolWasm * getOrCreateFunctionTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __indirect_function_table, for use in call_indirect and in function bitcasts.
bool isWebAssemblyFuncrefType(const Type *Ty)
Return true if this is a WebAssembly Funcref Type.
bool isWebAssemblyTableType(const Type *Ty)
Return true if the table represents a WebAssembly table type.
MCSymbolWasm * getOrCreateFuncrefCallTableSymbol(MCContext &Ctx, const WebAssemblySubtarget *Subtarget)
Returns the __funcref_call_table, for use in funcref calls when lowered to table.set + call_indirect.
FastISel * createFastISel(FunctionLoweringInfo &funcInfo, const TargetLibraryInfo *libInfo)
bool isValidAddressSpace(unsigned AS)
bool canLowerReturn(size_t ResultSize, const WebAssemblySubtarget *Subtarget)
Returns true if the function's return value(s) can be lowered directly, i.e., not indirectly via a po...
bool isWasmVarAddressSpace(unsigned AS)
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
This is an optimization pass for GlobalISel generic memory operations.
auto drop_begin(T &&RangeOrContainer, size_t N=1)
Return a range covering RangeOrContainer with the first N elements excluded.
Definition STLExtras.h:318
unsigned Log2_32_Ceil(uint32_t Value)
Return the ceil log base 2 of the specified value, 32 if the value is zero.
Definition MathExtras.h:355
@ Offset
Definition DWP.cpp:477
FunctionAddr VTableAddr Value
Definition InstrProf.h:137
void computeSignatureVTs(const FunctionType *Ty, const Function *TargetFunc, const Function &ContextFunc, const TargetMachine &TM, SmallVectorImpl< MVT > &Params, SmallVectorImpl< MVT > &Results)
auto size(R &&Range, std::enable_if_t< std::is_base_of< std::random_access_iterator_tag, typename std::iterator_traits< decltype(Range.begin())>::iterator_category >::value, void > *=nullptr)
Get the size of a range.
Definition STLExtras.h:1657
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:644
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:288
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:167
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:548
@ Add
Sum of integers.
DWARFExpression::Operation Op
auto max_element(R &&Range)
Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...
Definition STLExtras.h:2010
constexpr unsigned BitWidth
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:560
auto find_if(R &&Range, UnaryPredicate P)
Provide wrappers to std::find_if which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1738
void erase_if(Container &C, UnaryPredicate P)
Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...
Definition STLExtras.h:2100
void computeLegalValueVTs(const WebAssemblyTargetLowering &TLI, LLVMContext &Ctx, const DataLayout &DL, Type *Ty, SmallVectorImpl< MVT > &ValueVTs)
constexpr uint64_t NextPowerOf2(uint64_t A)
Returns the next power of two (in 64-bits) that is strictly greater than A.
Definition MathExtras.h:384
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:872
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
Extended Value Type.
Definition ValueTypes.h:35
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition ValueTypes.h:94
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:137
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:74
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:147
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:373
bool isByteSized() const
Return true if the bit size is a multiple of 8.
Definition ValueTypes.h:243
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:385
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:316
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition ValueTypes.h:207
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:65
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:381
bool isFixedLengthVector() const
Definition ValueTypes.h:181
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:168
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:323
bool bitsGE(EVT VT) const
Return true if this has no less bits than VT.
Definition ValueTypes.h:292
bool is256BitVector() const
Return true if this is a 256-bit vector type.
Definition ValueTypes.h:212
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:328
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:157
EVT changeVectorElementType(EVT EltVT) const
Return a VT for a vector type whose attributes match ourselves with the exception of the element type...
Definition ValueTypes.h:102
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:336
Align getNonZeroOrigAlign() const
unsigned getByValSize() const
bool isInConsecutiveRegsLast() const
Align getNonZeroByValAlign() const
unsigned getBitWidth() const
Get the bit width of this value.
Definition KnownBits.h:44
Matching combinators.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
These are IR-level optimization flags that may be propagated to SDNodes.
This structure is used to pass arguments to makeLibCall function.