LLVM 24.0.0git
LoongArchISelLowering.cpp
Go to the documentation of this file.
1//=- LoongArchISelLowering.cpp - LoongArch DAG Lowering Implementation ---===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that LoongArch uses to lower LLVM code into
10// a selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
15#include "LoongArch.h"
19#include "LoongArchSubtarget.h"
23#include "llvm/ADT/SmallSet.h"
24#include "llvm/ADT/Statistic.h"
30#include "llvm/IR/IRBuilder.h"
32#include "llvm/IR/IntrinsicsLoongArch.h"
34#include "llvm/Support/Debug.h"
39
40using namespace llvm;
41
42#define DEBUG_TYPE "loongarch-isel-lowering"
43
44STATISTIC(NumTailCalls, "Number of tail calls");
45
54
56 "loongarch-materialize-float-imm", cl::Hidden,
57 cl::desc("Maximum number of instructions used (including code sequence "
58 "to generate the value and moving the value to FPR) when "
59 "materializing floating-point immediates (default = 3)"),
61 cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"),
63 "Materialize FP immediate within 2 instructions"),
65 "Materialize FP immediate within 3 instructions"),
67 "Materialize FP immediate within 4 instructions"),
69 "Materialize FP immediate within 5 instructions"),
71 "Materialize FP immediate within 6 instructions "
72 "(behaves same as 5 on loongarch64)")));
73
74static cl::opt<bool> ZeroDivCheck("loongarch-check-zero-division", cl::Hidden,
75 cl::desc("Trap on integer division by zero."),
76 cl::init(false));
77
79 const LoongArchSubtarget &STI)
80 : TargetLowering(TM, STI), Subtarget(STI) {
81
82 MVT GRLenVT = Subtarget.getGRLenVT();
83
84 // Set up the register classes.
85
86 addRegisterClass(GRLenVT, &LoongArch::GPRRegClass);
87 if (Subtarget.hasBasicF())
88 addRegisterClass(MVT::f32, &LoongArch::FPR32RegClass);
89 if (Subtarget.hasBasicD())
90 addRegisterClass(MVT::f64, &LoongArch::FPR64RegClass);
91
92 static const MVT::SimpleValueType LSXVTs[] = {
93 MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::v4f32, MVT::v2f64};
94 static const MVT::SimpleValueType LASXVTs[] = {
95 MVT::v32i8, MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v8f32, MVT::v4f64};
96
97 if (Subtarget.hasExtLSX())
98 for (MVT VT : LSXVTs)
99 addRegisterClass(VT, &LoongArch::LSX128RegClass);
100
101 if (Subtarget.hasExtLASX())
102 for (MVT VT : LASXVTs)
103 addRegisterClass(VT, &LoongArch::LASX256RegClass);
104
105 // Set operations for LA32 and LA64.
106
108 MVT::i1, Promote);
109
114 if (Subtarget.hasBasicF())
118
121 GRLenVT, Custom);
122
124
129
131 setOperationAction(ISD::TRAP, MVT::Other, Legal);
132
136
138
139 // BITREV/REVB requires the 32S feature.
140 if (STI.has32S()) {
141 // Expand bitreverse.i16 with native-width bitrev and shift for now, before
142 // we get to know which of sll and revb.2h is faster.
145
146 // LA32 does not have REVB.2W and REVB.D due to the 64-bit operands, and
147 // the narrower REVB.W does not exist. But LA32 does have REVB.2H, so i16
148 // and i32 could still be byte-swapped relatively cheaply.
150 } else {
158 }
159
166
169
170 // Set operations for LA64 only.
171
172 if (Subtarget.is64Bit()) {
190
194 Custom);
196 }
197
198 // Set operations for LA32 only.
199
200 if (!Subtarget.is64Bit()) {
206 if (Subtarget.hasBasicD())
208 }
209
211
212 static const ISD::CondCode FPCCToExpand[] = {
215
216 // Set operations for 'F' feature.
217
218 if (Subtarget.hasBasicF()) {
219 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::f16, Expand);
220 setTruncStoreAction(MVT::f32, MVT::f16, Expand);
221 setLoadExtAction(ISD::EXTLOAD, MVT::f32, MVT::bf16, Expand);
222 setTruncStoreAction(MVT::f32, MVT::bf16, Expand);
223 setCondCodeAction(FPCCToExpand, MVT::f32, Expand);
224
243 Subtarget.isSoftFPABI() ? LibCall : Custom);
245 Subtarget.isSoftFPABI() ? LibCall : Custom);
248 Subtarget.isSoftFPABI() ? LibCall : Custom);
251
252 if (Subtarget.is64Bit())
254
255 if (!Subtarget.hasBasicD()) {
257 if (Subtarget.is64Bit()) {
260 }
261 }
262 }
263
264 // Set operations for 'D' feature.
265
266 if (Subtarget.hasBasicD()) {
267 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f16, Expand);
268 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::f32, Expand);
269 setLoadExtAction(ISD::EXTLOAD, MVT::f64, MVT::bf16, Expand);
270 setTruncStoreAction(MVT::f64, MVT::bf16, Expand);
271 setTruncStoreAction(MVT::f64, MVT::f16, Expand);
272 setTruncStoreAction(MVT::f64, MVT::f32, Expand);
273 setCondCodeAction(FPCCToExpand, MVT::f64, Expand);
274
294 Subtarget.isSoftFPABI() ? LibCall : Custom);
297 Subtarget.isSoftFPABI() ? LibCall : Custom);
298
299 if (Subtarget.is64Bit())
301 }
302
303 // Set operations for 'LSX' feature.
304
305 if (Subtarget.hasExtLSX()) {
307 // Expand all truncating stores and extending loads.
308 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
309 setTruncStoreAction(VT, InnerVT, Expand);
312 setLoadExtAction(ISD::EXTLOAD, VT, InnerVT, Expand);
313 }
314 // By default everything must be expanded. Then we will selectively turn
315 // on ones that can be effectively codegen'd.
316 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
318 }
319
320 for (MVT VT : LSXVTs) {
324
328
333 }
334 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) {
337 Legal);
339 VT, Legal);
346 Expand);
361 }
362 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
364 for (MVT VT : {MVT::v8i16, MVT::v4i32, MVT::v2i64})
366 for (MVT VT : {MVT::v4i32, MVT::v2i64}) {
370 }
372 for (MVT VT : {MVT::v4f32, MVT::v2f64}) {
380 VT, Expand);
388 }
390 setOperationAction(ISD::FCEIL, {MVT::f32, MVT::f64}, Legal);
391 setOperationAction(ISD::FFLOOR, {MVT::f32, MVT::f64}, Legal);
392 setOperationAction(ISD::FTRUNC, {MVT::f32, MVT::f64}, Legal);
393 setOperationAction(ISD::FROUNDEVEN, {MVT::f32, MVT::f64}, Legal);
394
395 for (MVT VT :
396 {MVT::v16i8, MVT::v8i8, MVT::v4i8, MVT::v2i8, MVT::v8i16, MVT::v4i16,
397 MVT::v2i16, MVT::v4i32, MVT::v2i32, MVT::v2i64}) {
407 }
410 // We want to legalize this to an f64 load rather than an i64 load.
411 setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
412 for (MVT VT : {MVT::v2i64, MVT::v4i32, MVT::v8i16})
414 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64, MVT::v16i32, MVT::v8i64,
415 MVT::v16i64})
417 }
418
419 // Set operations for 'LASX' feature.
420
421 if (Subtarget.hasExtLASX()) {
422 for (MVT VT : LASXVTs) {
426
432
436 }
437 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) {
440 Legal);
442 VT, Legal);
449 Expand);
465 }
466 for (MVT VT : {MVT::v32i8, MVT::v16i16, MVT::v8i32})
468 for (MVT VT : {MVT::v16i16, MVT::v8i32, MVT::v4i64})
470 for (MVT VT : {MVT::v8i32, MVT::v4i64})
472 for (MVT VT : {MVT::v8i32, MVT::v4i32, MVT::v4i64}) {
476 }
477 for (MVT VT : {MVT::v8f32, MVT::v4f64}) {
485 VT, Expand);
493 }
496 for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16}) {
500 }
501 for (MVT VT :
502 {MVT::v2i64, MVT::v4i32, MVT::v4i64, MVT::v8i16, MVT::v8i32}) {
505 }
506 for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32})
508 }
509
510 // Set DAG combine for LA32 and LA64.
511 if (Subtarget.hasBasicF()) {
513 }
514
522
523 // On targets with the 32S feature, `select` is expanded into
524 // maskeqz + masknez + or (3 instructions), which is more expensive than
525 // on most other architectures where a single cmov-like instruction
526 // suffices. Enable a combine that can turn
527 // select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
528 // select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
529 // for binop in {add, or, xor, sub}, replacing the 3-insn select (plus
530 // the original binop) with a single mask instruction plus the binop.
531 if (Subtarget.has32S())
533
534 // Set DAG combine for 'LSX' feature.
535
536 if (Subtarget.hasExtLSX()) {
549 }
550
551 // Set DAG combine for 'LASX' feature.
552 if (Subtarget.hasExtLASX()) {
555 }
556
557 // Compute derived properties from the register classes.
558 computeRegisterProperties(Subtarget.getRegisterInfo());
559
561
564
565 setMaxAtomicSizeInBitsSupported(Subtarget.getGRLen());
566
568
569 // Function alignments.
571 // Set preferred alignments.
572 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
573 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
574 setMaxBytesForAlignment(Subtarget.getMaxBytesForAlignment());
575
576 // cmpxchg sizes down to 8 bits become legal if LAMCAS is available.
577 if (Subtarget.hasLAMCAS())
579
580 if (Subtarget.hasSCQ()) {
583 }
584
585 // Disable strict node mutation.
586 IsStrictFPEnabled = true;
587}
588
590 const GlobalAddressSDNode *GA) const {
591 // In order to maximise the opportunity for common subexpression elimination,
592 // keep a separate ADD node for the global address offset instead of folding
593 // it in the global address node. Later peephole optimisations may choose to
594 // fold it back in when profitable.
595 return false;
596}
597
599 SelectionDAG &DAG) const {
600 switch (Op.getOpcode()) {
602 return lowerATOMIC_FENCE(Op, DAG);
604 return lowerEH_DWARF_CFA(Op, DAG);
606 return lowerGlobalAddress(Op, DAG);
608 return lowerGlobalTLSAddress(Op, DAG);
610 return lowerINTRINSIC_WO_CHAIN(Op, DAG);
612 return lowerINTRINSIC_W_CHAIN(Op, DAG);
614 return lowerINTRINSIC_VOID(Op, DAG);
616 return lowerBlockAddress(Op, DAG);
617 case ISD::JumpTable:
618 return lowerJumpTable(Op, DAG);
619 case ISD::SHL_PARTS:
620 return lowerShiftLeftParts(Op, DAG);
621 case ISD::SRA_PARTS:
622 return lowerShiftRightParts(Op, DAG, true);
623 case ISD::SRL_PARTS:
624 return lowerShiftRightParts(Op, DAG, false);
626 return lowerConstantPool(Op, DAG);
627 case ISD::FP_TO_SINT:
628 return lowerFP_TO_SINT(Op, DAG);
629 case ISD::FP_TO_UINT:
630 return lowerFP_TO_UINT(Op, DAG);
632 return lowerFP_TO_SINT_SAT(Op, DAG);
633 case ISD::BITCAST:
634 return lowerBITCAST(Op, DAG);
635 case ISD::UINT_TO_FP:
636 return lowerUINT_TO_FP(Op, DAG);
637 case ISD::SINT_TO_FP:
638 return lowerSINT_TO_FP(Op, DAG);
639 case ISD::VASTART:
640 return lowerVASTART(Op, DAG);
641 case ISD::FRAMEADDR:
642 return lowerFRAMEADDR(Op, DAG);
643 case ISD::RETURNADDR:
644 return lowerRETURNADDR(Op, DAG);
646 return lowerSET_ROUNDING(Op, DAG);
648 return lowerGET_ROUNDING(Op, DAG);
650 return lowerWRITE_REGISTER(Op, DAG);
652 return lowerINSERT_VECTOR_ELT(Op, DAG);
654 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
656 return lowerBUILD_VECTOR(Op, DAG);
658 return lowerCONCAT_VECTORS(Op, DAG);
660 return lowerVECTOR_SHUFFLE(Op, DAG);
661 case ISD::BITREVERSE:
662 return lowerBITREVERSE(Op, DAG);
664 return lowerSCALAR_TO_VECTOR(Op, DAG);
665 case ISD::PREFETCH:
666 return lowerPREFETCH(Op, DAG);
667 case ISD::SELECT:
668 return lowerSELECT(Op, DAG);
669 case ISD::BRCOND:
670 return lowerBRCOND(Op, DAG);
671 case ISD::FP_TO_FP16:
672 return lowerFP_TO_FP16(Op, DAG);
673 case ISD::FP16_TO_FP:
674 return lowerFP16_TO_FP(Op, DAG);
675 case ISD::FP_TO_BF16:
676 return lowerFP_TO_BF16(Op, DAG);
677 case ISD::BF16_TO_FP:
678 return lowerBF16_TO_FP(Op, DAG);
680 return lowerVECREDUCE_ADD(Op, DAG);
681 case ISD::ROTL:
682 case ISD::ROTR:
683 return lowerRotate(Op, DAG);
691 return lowerVECREDUCE(Op, DAG);
692 case ISD::ConstantFP:
693 return lowerConstantFP(Op, DAG);
694 case ISD::SETCC:
695 return lowerSETCC(Op, DAG);
696 case ISD::FP_ROUND:
697 return lowerFP_ROUND(Op, DAG);
698 case ISD::FP_EXTEND:
699 return lowerFP_EXTEND(Op, DAG);
701 return lowerSIGN_EXTEND_VECTOR_INREG(Op, DAG);
703 return lowerDYNAMIC_STACKALLOC(Op, DAG);
704 case ISD::ANY_EXTEND:
705 return lowerANY_EXTEND(Op, DAG);
706 case ISD::VSELECT:
707 return lowerVSELECT(Op, DAG);
708 }
709 return SDValue();
710}
711
712// Helper to attempt to return a cheaper, bit-inverted version of \p V.
714 // TODO: don't always ignore oneuse constraints.
716 EVT VT = V.getValueType();
717
718 // Match not(xor X, -1) -> X.
719 if (V.getOpcode() == ISD::XOR &&
720 (ISD::isBuildVectorAllOnes(V.getOperand(1)) ||
721 isAllOnesConstant(V.getOperand(1))))
722 return V.getOperand(0);
723
724 // Match not(extract_subvector(not(X)) -> extract_subvector(X).
725 if (V.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
726 (isNullConstant(V.getOperand(1)) || V.getOperand(0).hasOneUse())) {
727 if (SDValue Not = isNOT(V.getOperand(0), DAG)) {
728 Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
729 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, SDLoc(Not), VT, Not,
730 V.getOperand(1));
731 }
732 }
733
734 // Match not(SplatVector(not(X)) -> SplatVector(X).
735 if (V.getOpcode() == ISD::BUILD_VECTOR) {
736 if (SDValue SplatValue =
737 cast<BuildVectorSDNode>(V.getNode())->getSplatValue()) {
738 if (!V->isOnlyUserOf(SplatValue.getNode()))
739 return SDValue();
740
741 if (SDValue Not = isNOT(SplatValue, DAG)) {
742 Not = DAG.getBitcast(V.getOperand(0).getValueType(), Not);
743 return DAG.getSplat(VT, SDLoc(Not), Not);
744 }
745 }
746 }
747
748 // Match not(or(not(X),not(Y))) -> and(X, Y).
749 if (V.getOpcode() == ISD::OR && DAG.getTargetLoweringInfo().isTypeLegal(VT) &&
750 V.getOperand(0).hasOneUse() && V.getOperand(1).hasOneUse()) {
751 // TODO: Handle cases with single NOT operand -> VANDN
752 if (SDValue Op1 = isNOT(V.getOperand(1), DAG))
753 if (SDValue Op0 = isNOT(V.getOperand(0), DAG))
754 return DAG.getNode(ISD::AND, SDLoc(V), VT, DAG.getBitcast(VT, Op0),
755 DAG.getBitcast(VT, Op1));
756 }
757
758 // TODO: Add more matching patterns. Such as,
759 // not(concat_vectors(not(X), not(Y))) -> concat_vectors(X, Y).
760 // not(slt(C, X)) -> slt(X - 1, C)
761 return SDValue();
762}
763
764// Combine two ISD::FP_ROUND / LoongArchISD::VFCVT nodes with same type to
765// LoongArchISD::VFCVT. For example:
766// x1 = fp_round x, 0
767// y1 = fp_round y, 0
768// z = concat_vectors x1, y1
769// Or
770// x1 = LoongArch::VFCVT undef, x
771// y1 = LoongArch::VFCVT undef, y
772// z = LoongArchISD::VPACKEV y1, x1; or LoongArchISD::VPERMI y1, x1, 68
773// can be combined to:
774// z = LoongArch::VFCVT y, x
777 assert(((N->getOpcode() == ISD::CONCAT_VECTORS && N->getNumOperands() == 2) ||
778 (N->getOpcode() == LoongArchISD::VPACKEV) ||
779 (N->getOpcode() == LoongArchISD::VPERMI)) &&
780 "Invalid Node");
781
782 SDValue Op0 = peekThroughBitcasts(N->getOperand(0));
783 SDValue Op1 = peekThroughBitcasts(N->getOperand(1));
784 unsigned Opcode0 = Op0.getOpcode();
785 unsigned Opcode1 = Op1.getOpcode();
786 if (Opcode0 != Opcode1)
787 return SDValue();
788
789 if (Opcode0 != ISD::FP_ROUND && Opcode0 != LoongArchISD::VFCVT)
790 return SDValue();
791
792 // Check if two nodes have only one use.
793 if (!Op0.hasOneUse() || !Op1.hasOneUse())
794 return SDValue();
795
796 EVT VT = N.getValueType();
797 EVT SVT0 = Op0.getValueType();
798 EVT SVT1 = Op1.getValueType();
799 // Check if two nodes have the same result type.
800 if (SVT0 != SVT1)
801 return SDValue();
802
803 // Check if two nodes have the same operand type.
804 EVT SSVT0 = Op0.getOperand(0).getValueType();
805 EVT SSVT1 = Op1.getOperand(0).getValueType();
806 if (SSVT0 != SSVT1)
807 return SDValue();
808
809 if (N->getOpcode() == ISD::CONCAT_VECTORS && Opcode0 == ISD::FP_ROUND) {
810 if (Subtarget.hasExtLASX() && VT.is256BitVector() && SVT0 == MVT::v4f32 &&
811 SSVT0 == MVT::v4f64) {
812 // A vector_shuffle is required in the final step, as xvfcvt instruction
813 // operates on each 128-bit segament as a lane.
814 SDValue Res = DAG.getNode(LoongArchISD::VFCVT, DL, MVT::v8f32,
815 Op1.getOperand(0), Op0.getOperand(0));
816 SDValue Undef = DAG.getUNDEF(Res.getValueType());
817 // After VFCVT, the high part of Res comes from the high parts of Op0 and
818 // Op1, and the low part comes from the low parts of Op0 and Op1. However,
819 // the desired order requires Op0 to fully occupy the lower half and Op1
820 // the upper half of Res. The Mask reorders the elements of Res to achieve
821 // this:
822 // - The first four elements (0, 1, 4, 5) come from Op0.
823 // - The next four elements (2, 3, 6, 7) come from Op1.
824 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
825 Res = DAG.getVectorShuffle(Res.getValueType(), DL, Res, Undef, Mask);
826 return DAG.getBitcast(VT, Res);
827 }
828 }
829
830 if ((N->getOpcode() == LoongArchISD::VPACKEV ||
831 N->getOpcode() == LoongArchISD::VPERMI) &&
832 Opcode0 == LoongArchISD::VFCVT) {
833 // For VPACKEV or VPERMI, check if the first operation of VFCVT is undef.
834 if (!Op0.getOperand(0).isUndef() || !Op1.getOperand(0).isUndef())
835 return SDValue();
836
837 if (!Subtarget.hasExtLSX() || SVT0 != MVT::v4f32 || SSVT0 != MVT::v2f64)
838 return SDValue();
839
840 if (N->getOpcode() == LoongArchISD::VPACKEV &&
841 (VT == MVT::v2i64 || VT == MVT::v2f64)) {
842 SDValue Res = DAG.getNode(LoongArchISD::VFCVT, DL, MVT::v4f32,
843 Op0.getOperand(1), Op1.getOperand(1));
844 return DAG.getBitcast(VT, Res);
845 }
846
847 if (N->getOpcode() == LoongArchISD::VPERMI && VT == MVT::v4f32) {
848 int64_t Imm = cast<ConstantSDNode>(N->getOperand(2))->getSExtValue();
849 if (Imm != 68)
850 return SDValue();
851 return DAG.getNode(LoongArchISD::VFCVT, DL, MVT::v4f32, Op0.getOperand(1),
852 Op1.getOperand(1));
853 }
854 }
855
856 return SDValue();
857}
858
859SDValue LoongArchTargetLowering::lowerFP_ROUND(SDValue Op,
860 SelectionDAG &DAG) const {
861 SDLoc DL(Op);
862 SDValue In = Op.getOperand(0);
863 MVT VT = Op.getSimpleValueType();
864 MVT SVT = In.getSimpleValueType();
865
866 if (VT == MVT::v4f32 && SVT == MVT::v4f64) {
867 SDValue Lo, Hi;
868 std::tie(Lo, Hi) = DAG.SplitVector(In, DL);
869 return DAG.getNode(LoongArchISD::VFCVT, DL, VT, Hi, Lo);
870 }
871
872 return SDValue();
873}
874
875SDValue LoongArchTargetLowering::lowerFP_EXTEND(SDValue Op,
876 SelectionDAG &DAG) const {
877
878 SDLoc DL(Op);
879 EVT VT = Op.getValueType();
880 SDValue Src = Op->getOperand(0);
881 EVT SVT = Src.getValueType();
882
883 bool V2F32ToV2F64 =
884 VT == MVT::v2f64 && SVT == MVT::v2f32 && Subtarget.hasExtLSX();
885 bool V4F32ToV4F64 =
886 VT == MVT::v4f64 && SVT == MVT::v4f32 && Subtarget.hasExtLASX();
887 if (!V2F32ToV2F64 && !V4F32ToV4F64)
888 return SDValue();
889
890 // Check if Op is the high part of vector.
891 auto CheckVecHighPart = [](SDValue Op) {
893 if (Op.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
894 SDValue SOp = Op.getOperand(0);
895 EVT SVT = SOp.getValueType();
896 if (!SVT.isVector() || (SVT.getVectorNumElements() % 2 != 0))
897 return SDValue();
898
899 const uint64_t Imm = Op.getConstantOperandVal(1);
900 if (Imm == SVT.getVectorNumElements() / 2)
901 return SOp;
902 return SDValue();
903 }
904 return SDValue();
905 };
906
907 unsigned Opcode;
908 SDValue VFCVTOp;
909 EVT WideOpVT = SVT.getSimpleVT().getDoubleNumVectorElementsVT();
910 SDValue ZeroIdx = DAG.getVectorIdxConstant(0, DL);
911
912 // If the operand of ISD::FP_EXTEND comes from the high part of vector,
913 // generate LoongArchISD::VFCVTH, otherwise LoongArchISD::VFCVTL.
914 if (SDValue V = CheckVecHighPart(Src)) {
915 assert(V.getValueSizeInBits() == WideOpVT.getSizeInBits() &&
916 "Unexpected wide vector");
917 Opcode = LoongArchISD::VFCVTH;
918 VFCVTOp = DAG.getBitcast(WideOpVT, V);
919 } else {
920 Opcode = LoongArchISD::VFCVTL;
921 VFCVTOp = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WideOpVT,
922 DAG.getUNDEF(WideOpVT), Src, ZeroIdx);
923 }
924
925 // v2f64 = fp_extend v2f32
926 if (V2F32ToV2F64)
927 return DAG.getNode(Opcode, DL, VT, VFCVTOp);
928
929 // v4f64 = fp_extend v4f32
930 if (V4F32ToV4F64) {
931 // XVFCVT instruction operates on each 128-bit segment as a lane, so a
932 // vector_shuffle is required firstly.
933 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
934 SDValue Res = DAG.getVectorShuffle(WideOpVT, DL, VFCVTOp,
935 DAG.getUNDEF(WideOpVT), Mask);
936 Res = DAG.getNode(Opcode, DL, VT, Res);
937 return Res;
938 }
939
940 return SDValue();
941}
942
943SDValue LoongArchTargetLowering::lowerVSELECT(SDValue Op,
944 SelectionDAG &DAG) const {
945 SDValue Cond = Op.getOperand(0);
946 SDValue LHS = Op.getOperand(1);
947 SDValue RHS = Op.getOperand(2);
948 MVT VT = Op.getSimpleValueType();
949 SDLoc DL(Op);
950
951 // Try to lower vselect to vector_shuffle. All cases with constant
952 // build_vector condition will be handled. NOTE: On 32-bit platform, vselect
953 // with v2i64/v4i64 type condition will never enter this because of the extra
954 // bitcast. It can be considered separately later.
956 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Cond.getNode());
957 APInt SplatValue, SplatUndef;
958 unsigned SplatBitSize;
959 bool HasAnyUndefs;
960
961 // If the Cond is a BUILD_VECTOR with splat constants, using a vldi or
962 // constant broadcast instruction and a vbitsel is better.
963 if (BVN->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
964 /*MinSplatBits=*/8) &&
965 (SplatBitSize == 8 || SplatBitSize == 16 || SplatBitSize == 32 ||
966 SplatBitSize == 64))
967 return Op;
968
970 EVT CondVT = Cond.getValueType();
971 int NumElts = CondVT.getVectorNumElements();
972 Mask.resize(NumElts, -1);
973 for (int i = 0; i < NumElts; ++i) {
974 SDValue CondOp = Cond.getOperand(i);
975 if (CondOp.isUndef())
976 continue;
977 auto *CondElt = dyn_cast<ConstantSDNode>(CondOp);
978 if (CondElt->getZExtValue() == 0)
979 Mask[i] = i + NumElts;
980 else
981 Mask[i] = i;
982 }
983
984 return DAG.getVectorShuffle(VT, DL, LHS, RHS, Mask);
985 }
986
987 // Fallback to match patterns in tablegen.
988 return Op;
989}
990
991SDValue LoongArchTargetLowering::lowerConstantFP(SDValue Op,
992 SelectionDAG &DAG) const {
993 EVT VT = Op.getValueType();
994 ConstantFPSDNode *CFP = cast<ConstantFPSDNode>(Op);
995 const APFloat &FPVal = CFP->getValueAPF();
996 SDLoc DL(CFP);
997
998 assert((VT == MVT::f32 && Subtarget.hasBasicF()) ||
999 (VT == MVT::f64 && Subtarget.hasBasicD()));
1000
1001 // If value is 0.0 or -0.0, just ignore it.
1002 if (FPVal.isZero())
1003 return SDValue();
1004
1005 // If lsx enabled, use cheaper 'vldi' instruction if possible.
1006 if (isFPImmVLDILegal(FPVal, VT))
1007 return SDValue();
1008
1009 // The legalizer builds immediates the size of a GPR from their bits. Fall
1010 // back to a load from the constant pool for expensive immediates.
1011 if (VT.getSizeInBits() == Subtarget.getGRLen() ||
1012 !isFPImmCheapInGPR(FPVal, VT))
1013 return SDValue();
1014
1015 // Construct as integer, and move to float register.
1016 APInt INTVal = FPVal.bitcastToAPInt();
1017 if (VT == MVT::f32) {
1018 assert(Subtarget.is64Bit());
1019 SDValue NewVal = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64,
1020 DAG.getConstant(INTVal, DL, MVT::i32));
1021 return DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, VT, NewVal);
1022 }
1023
1024 assert(VT == MVT::f64 && !Subtarget.is64Bit());
1025 SDValue Lo = DAG.getConstant(INTVal.trunc(32), DL, MVT::i32);
1026 SDValue Hi = DAG.getConstant(INTVal.lshr(32).trunc(32), DL, MVT::i32);
1027 return DAG.getNode(LoongArchISD::MOVGR2FR_D_LO_HI, DL, VT, Lo, Hi);
1028}
1029
1030// Ensure SETCC result and operand have the same bit width; isel does not
1031// support mismatched widths.
1032SDValue LoongArchTargetLowering::lowerSETCC(SDValue Op,
1033 SelectionDAG &DAG) const {
1034 SDLoc DL(Op);
1035 EVT ResultVT = Op.getValueType();
1036 EVT OperandVT = Op.getOperand(0).getValueType();
1037
1038 EVT SetCCResultVT =
1039 getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), OperandVT);
1040
1041 if (ResultVT == SetCCResultVT)
1042 return Op;
1043
1044 assert(Op.getOperand(0).getValueType() == Op.getOperand(1).getValueType() &&
1045 "SETCC operands must have the same type!");
1046
1047 SDValue SetCCNode =
1048 DAG.getNode(ISD::SETCC, DL, SetCCResultVT, Op.getOperand(0),
1049 Op.getOperand(1), Op.getOperand(2));
1050
1051 if (ResultVT.bitsGT(SetCCResultVT))
1052 SetCCNode = DAG.getNode(ISD::SIGN_EXTEND, DL, ResultVT, SetCCNode);
1053 else if (ResultVT.bitsLT(SetCCResultVT))
1054 SetCCNode = DAG.getNode(ISD::TRUNCATE, DL, ResultVT, SetCCNode);
1055
1056 return SetCCNode;
1057}
1058
1059// Lower SIGN_EXTEND_VECTOR_INREG with VSLLWIL, which sign-extends the low half
1060// of the source elements (and shifts them left by 0).
1061// For example:
1062// %b = sext <8 x i8> %a to <8 x i16>
1063// can be lowered to:
1064// VSLLWIL.H.B vr1, vr1, 0
1065//
1066// VSLLWIL is slower than VILVL and VSRAI, so a 4x extension instead
1067// replicates each source element across its destination element by
1068// interleaving the source with itself twice, then shifts the copy in the top
1069// bits down arithmetically:
1070// %b = sext <4 x i8> %a to <4 x i32>
1071// can be lowered to:
1072// VILVL.B vr1, vr1, vr1
1073// VILVL.H vr1, vr1, vr1
1074// VSRAI.W vr1, vr1, 24
1075//
1076// If every extended source element is already all-ones or all-zeros, e.g. a
1077// comparison result, the interleaves alone produce the sign extension, so
1078// neither VSLLWIL nor the VSRAI is needed:
1079// %b = sext <8 x i8> %mask to <8 x i16>
1080// can be lowered to:
1081// VILVL.B vr1, vr1, vr1
1082SDValue LoongArchTargetLowering::lowerSIGN_EXTEND_VECTOR_INREG(
1083 SDValue Op, SelectionDAG &DAG) const {
1084 SDLoc DL(Op);
1085 SDValue Src = Op.getOperand(0);
1086 MVT SrcVT = Src.getSimpleValueType();
1087 MVT DstVT = Op.getSimpleValueType();
1088 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
1089 unsigned DstEltBits = DstVT.getScalarSizeInBits();
1090
1091 // Leave larger ratios to the generic shuffle + shift expansion, which only
1092 // needs a single VEXTRINS to move the second element into place.
1093 unsigned Ratio = DstEltBits / SrcEltBits;
1094 if (!SrcVT.is128BitVector() || Ratio > 4)
1095 return SDValue();
1096
1097 // Only the low elements of the source are extended.
1098 APInt DemandedElts = APInt::getLowBitsSet(SrcVT.getVectorNumElements(),
1099 DstVT.getVectorNumElements());
1100 bool IsSignSplat = DAG.ComputeNumSignBits(Src, DemandedElts) == SrcEltBits;
1101
1102 if (Ratio == 2 && !IsSignSplat)
1103 return DAG.getNode(LoongArchISD::VSLLWIL, DL, DstVT, Src,
1104 DAG.getConstant(0, DL, Subtarget.getGRLenVT()));
1105
1106 SDValue Res = Src;
1107 for (unsigned EltBits = SrcEltBits; EltBits < DstEltBits; EltBits *= 2) {
1108 MVT VT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), 128 / EltBits);
1109 Res = DAG.getBitcast(VT, Res);
1110 Res = DAG.getNode(LoongArchISD::VILVL, DL, VT, Res, Res);
1111 }
1112 Res = DAG.getBitcast(DstVT, Res);
1113 if (IsSignSplat)
1114 return Res;
1115 return DAG.getNode(ISD::SRA, DL, DstVT, Res,
1116 DAG.getConstant(DstEltBits - SrcEltBits, DL, DstVT));
1117}
1118
1119// ANY_EXTEND can be replaced by ZERO_EXTEND when LASX is enabled.
1120SDValue LoongArchTargetLowering::lowerANY_EXTEND(SDValue Op,
1121 SelectionDAG &DAG) const {
1122 assert(Subtarget.hasExtLASX());
1123 // We don't have corresponding instrunction for ANY_EXTEND, lowering it to
1124 // ZERO_EXTEND won't break its semantics, while avoid scalar extract/insert.
1125 return DAG.getNode(ISD::ZERO_EXTEND, SDLoc(Op), Op.getValueType(),
1126 Op.getOperand(0));
1127}
1128
1129// Lower vecreduce_add using vhaddw instructions.
1130// For Example:
1131// call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
1132// can be lowered to:
1133// VHADDW_D_W vr0, vr0, vr0
1134// VHADDW_Q_D vr0, vr0, vr0
1135// VPICKVE2GR_D a0, vr0, 0
1136// ADDI_W a0, a0, 0
1137SDValue LoongArchTargetLowering::lowerVECREDUCE_ADD(SDValue Op,
1138 SelectionDAG &DAG) const {
1139
1140 SDLoc DL(Op);
1141 MVT OpVT = Op.getSimpleValueType();
1142 SDValue Val = Op.getOperand(0);
1143
1144 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1145 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1146 unsigned ResBits = OpVT.getScalarSizeInBits();
1147
1148 unsigned LegalVecSize = 128;
1149 bool isLASX256Vector =
1150 Subtarget.hasExtLASX() && Val.getValueSizeInBits() == 256;
1151
1152 // Ensure operand type legal or enable it legal.
1153 while (!isTypeLegal(Val.getSimpleValueType())) {
1154 Val = DAG.WidenVector(Val, DL);
1155 }
1156
1157 // NumEles is designed for iterations count, v4i32 for LSX
1158 // and v8i32 for LASX should have the same count.
1159 if (isLASX256Vector) {
1160 NumEles /= 2;
1161 LegalVecSize = 256;
1162 }
1163
1164 EleBits *= 2;
1165 for (unsigned i = 1; i < NumEles; i *= 2, EleBits *= 2) {
1166 EleBits = std::min(EleBits, 64u);
1167 MVT IntTy = MVT::getIntegerVT(EleBits);
1168 MVT VecTy = MVT::getVectorVT(IntTy, LegalVecSize / EleBits);
1169 Val = DAG.getNode(LoongArchISD::VHADDW, DL, VecTy, Val, Val);
1170 }
1171
1172 if (isLASX256Vector) {
1173 SDValue Tmp = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, Val,
1174 DAG.getConstant(2, DL, Subtarget.getGRLenVT()));
1175 Val = DAG.getNode(ISD::ADD, DL, MVT::v4i64, Tmp, Val);
1176 }
1177
1178 Val = DAG.getBitcast(MVT::getVectorVT(OpVT, LegalVecSize / ResBits), Val);
1179 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Val,
1180 DAG.getConstant(0, DL, Subtarget.getGRLenVT()));
1181}
1182
1183// Lower vecreduce_and/or/xor/[s/u]max/[s/u]min.
1184// For Example:
1185// call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
1186// can be lowered to:
1187// VBSRL_V vr1, vr0, 8
1188// VMAX_W vr0, vr1, vr0
1189// VBSRL_V vr1, vr0, 4
1190// VMAX_W vr0, vr1, vr0
1191// VPICKVE2GR_W a0, vr0, 0
1192// For 256 bit vector, it is illegal and will be spilt into
1193// two 128 bit vector by default then processed by this.
1194SDValue LoongArchTargetLowering::lowerVECREDUCE(SDValue Op,
1195 SelectionDAG &DAG) const {
1196 SDLoc DL(Op);
1197
1198 MVT OpVT = Op.getSimpleValueType();
1199 SDValue Val = Op.getOperand(0);
1200
1201 unsigned NumEles = Val.getSimpleValueType().getVectorNumElements();
1202 unsigned EleBits = Val.getSimpleValueType().getScalarSizeInBits();
1203
1204 // Ensure operand type legal or enable it legal.
1205 while (!isTypeLegal(Val.getSimpleValueType())) {
1206 Val = DAG.WidenVector(Val, DL);
1207 }
1208
1209 unsigned Opcode = ISD::getVecReduceBaseOpcode(Op.getOpcode());
1210 MVT VecTy = Val.getSimpleValueType();
1211 MVT GRLenVT = Subtarget.getGRLenVT();
1212
1213 for (int i = NumEles; i > 1; i /= 2) {
1214 SDValue ShiftAmt = DAG.getConstant(i * EleBits / 16, DL, GRLenVT);
1215 SDValue Tmp = DAG.getNode(LoongArchISD::VBSRL, DL, VecTy, Val, ShiftAmt);
1216 Val = DAG.getNode(Opcode, DL, VecTy, Tmp, Val);
1217 }
1218
1219 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, OpVT, Val,
1220 DAG.getConstant(0, DL, GRLenVT));
1221}
1222
1223SDValue LoongArchTargetLowering::lowerPREFETCH(SDValue Op,
1224 SelectionDAG &DAG) const {
1225 unsigned IsData = Op.getConstantOperandVal(4);
1226
1227 // We don't support non-data prefetch.
1228 // Just preserve the chain.
1229 if (!IsData)
1230 return Op.getOperand(0);
1231
1232 return Op;
1233}
1234
1235SDValue LoongArchTargetLowering::lowerRotate(SDValue Op,
1236 SelectionDAG &DAG) const {
1237 MVT VT = Op.getSimpleValueType();
1238 assert(VT.isVector() && "Unexpected type");
1239
1240 SDLoc DL(Op);
1241 SDValue R = Op.getOperand(0);
1242 SDValue Amt = Op.getOperand(1);
1243 unsigned Opcode = Op.getOpcode();
1244 unsigned EltSizeInBits = VT.getScalarSizeInBits();
1245
1246 auto checkCstSplat = [](SDValue V, APInt &CstSplatValue) {
1247 if (V.getOpcode() != ISD::BUILD_VECTOR)
1248 return false;
1249 if (SDValue SplatValue =
1250 cast<BuildVectorSDNode>(V.getNode())->getSplatValue()) {
1251 if (auto *C = dyn_cast<ConstantSDNode>(SplatValue)) {
1252 CstSplatValue = C->getAPIntValue();
1253 return true;
1254 }
1255 }
1256 return false;
1257 };
1258
1259 // Check for constant splat rotation amount.
1260 APInt CstSplatValue;
1261 bool IsCstSplat = checkCstSplat(Amt, CstSplatValue);
1262 bool isROTL = Opcode == ISD::ROTL;
1263
1264 // Check for splat rotate by zero.
1265 if (IsCstSplat && CstSplatValue.urem(EltSizeInBits) == 0)
1266 return R;
1267
1268 // LoongArch targets always prefer ISD::ROTR.
1269 if (isROTL) {
1270 SDValue Zero = DAG.getConstant(0, DL, VT);
1271 return DAG.getNode(ISD::ROTR, DL, VT, R,
1272 DAG.getNode(ISD::SUB, DL, VT, Zero, Amt));
1273 }
1274
1275 // Rotate by a immediate.
1276 if (IsCstSplat) {
1277 // ISD::ROTR: Attemp to rotate by a positive immediate.
1278 SDValue Bits = DAG.getConstant(EltSizeInBits, DL, VT);
1279 if (SDValue Urem =
1280 DAG.FoldConstantArithmetic(ISD::UREM, DL, VT, {Amt, Bits}))
1281 return DAG.getNode(Opcode, DL, VT, R, Urem);
1282 }
1283
1284 return Op;
1285}
1286
1287// Return true if Val is equal to (setcc LHS, RHS, CC).
1288// Return false if Val is the inverse of (setcc LHS, RHS, CC).
1289// Otherwise, return std::nullopt.
1290static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
1291 ISD::CondCode CC, SDValue Val) {
1292 assert(Val->getOpcode() == ISD::SETCC);
1293 SDValue LHS2 = Val.getOperand(0);
1294 SDValue RHS2 = Val.getOperand(1);
1295 ISD::CondCode CC2 = Val.getOperandCondCode(2);
1296
1297 if (LHS == LHS2 && RHS == RHS2) {
1298 if (CC == CC2)
1299 return true;
1300 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
1301 return false;
1302 } else if (LHS == RHS2 && RHS == LHS2) {
1304 if (CC == CC2)
1305 return true;
1306 if (CC == ISD::getSetCCInverse(CC2, LHS2.getValueType()))
1307 return false;
1308 }
1309
1310 return std::nullopt;
1311}
1312
1315 SDValue CondV = N->getOperand(0);
1316 SDValue TrueV = N->getOperand(1);
1317 SDValue FalseV = N->getOperand(2);
1318 MVT VT = N->getSimpleValueType(0);
1319 SDLoc DL(N);
1320
1321 // (select c, -1, y) -> -c | y
1322 if (isAllOnesConstant(TrueV)) {
1323 SDValue Neg = DAG.getNegative(CondV, DL, VT);
1324 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(FalseV));
1325 }
1326 // (select c, y, -1) -> (c-1) | y
1327 if (isAllOnesConstant(FalseV)) {
1328 SDValue Neg =
1329 DAG.getNode(ISD::ADD, DL, VT, CondV, DAG.getAllOnesConstant(DL, VT));
1330 return DAG.getNode(ISD::OR, DL, VT, Neg, DAG.getFreeze(TrueV));
1331 }
1332
1333 // (select c, 0, y) -> (c-1) & y
1334 if (isNullConstant(TrueV)) {
1335 SDValue Neg =
1336 DAG.getNode(ISD::ADD, DL, VT, CondV, DAG.getAllOnesConstant(DL, VT));
1337 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(FalseV));
1338 }
1339 // (select c, y, 0) -> -c & y
1340 if (isNullConstant(FalseV)) {
1341 SDValue Neg = DAG.getNegative(CondV, DL, VT);
1342 return DAG.getNode(ISD::AND, DL, VT, Neg, DAG.getFreeze(TrueV));
1343 }
1344
1345 // select c, ~x, x --> xor -c, x
1346 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV)) {
1347 const APInt &TrueVal = TrueV->getAsAPIntVal();
1348 const APInt &FalseVal = FalseV->getAsAPIntVal();
1349 if (~TrueVal == FalseVal) {
1350 SDValue Neg = DAG.getNegative(CondV, DL, VT);
1351 return DAG.getNode(ISD::XOR, DL, VT, Neg, FalseV);
1352 }
1353 }
1354
1355 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
1356 // when both truev and falsev are also setcc.
1357 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
1358 FalseV.getOpcode() == ISD::SETCC) {
1359 SDValue LHS = CondV.getOperand(0);
1360 SDValue RHS = CondV.getOperand(1);
1362
1363 // (select x, x, y) -> x | y
1364 // (select !x, x, y) -> x & y
1365 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, TrueV)) {
1366 return DAG.getNode(*MatchResult ? ISD::OR : ISD::AND, DL, VT, TrueV,
1367 DAG.getFreeze(FalseV));
1368 }
1369 // (select x, y, x) -> x & y
1370 // (select !x, y, x) -> x | y
1371 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, FalseV)) {
1372 return DAG.getNode(*MatchResult ? ISD::AND : ISD::OR, DL, VT,
1373 DAG.getFreeze(TrueV), FalseV);
1374 }
1375 }
1376
1377 return SDValue();
1378}
1379
1380// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
1381// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
1382// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
1383// being `0` or `-1`. In such cases we can replace `select` with `and`.
1384// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
1385// than `c0`?
1386static SDValue
1389 unsigned SelOpNo = 0;
1390 SDValue Sel = BO->getOperand(0);
1391 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
1392 SelOpNo = 1;
1393 Sel = BO->getOperand(1);
1394 }
1395
1396 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
1397 return SDValue();
1398
1399 unsigned ConstSelOpNo = 1;
1400 unsigned OtherSelOpNo = 2;
1401 if (!isa<ConstantSDNode>(Sel->getOperand(ConstSelOpNo))) {
1402 ConstSelOpNo = 2;
1403 OtherSelOpNo = 1;
1404 }
1405 SDValue ConstSelOp = Sel->getOperand(ConstSelOpNo);
1406 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(ConstSelOp);
1407 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
1408 return SDValue();
1409
1410 SDValue ConstBinOp = BO->getOperand(SelOpNo ^ 1);
1411 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(ConstBinOp);
1412 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
1413 return SDValue();
1414
1415 SDLoc DL(Sel);
1416 EVT VT = BO->getValueType(0);
1417
1418 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
1419 if (SelOpNo == 1)
1420 std::swap(NewConstOps[0], NewConstOps[1]);
1421
1422 SDValue NewConstOp =
1423 DAG.FoldConstantArithmetic(BO->getOpcode(), DL, VT, NewConstOps);
1424 if (!NewConstOp)
1425 return SDValue();
1426
1427 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
1428 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
1429 return SDValue();
1430
1431 SDValue OtherSelOp = Sel->getOperand(OtherSelOpNo);
1432 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
1433 if (SelOpNo == 1)
1434 std::swap(NewNonConstOps[0], NewNonConstOps[1]);
1435 SDValue NewNonConstOp = DAG.getNode(BO->getOpcode(), DL, VT, NewNonConstOps);
1436
1437 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
1438 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
1439 return DAG.getSelect(DL, VT, Sel.getOperand(0), NewT, NewF);
1440}
1441
1442// Changes the condition code and swaps operands if necessary, so the SetCC
1443// operation matches one of the comparisons supported directly by branches
1444// in the LoongArch ISA. May adjust compares to favor compare with 0 over
1445// compare with 1/-1.
1447 ISD::CondCode &CC, SelectionDAG &DAG) {
1448 // If this is a single bit test that can't be handled by ANDI, shift the
1449 // bit to be tested to the MSB and perform a signed compare with 0.
1450 if (isIntEqualitySetCC(CC) && isNullConstant(RHS) &&
1451 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
1452 isa<ConstantSDNode>(LHS.getOperand(1))) {
1453 uint64_t Mask = LHS.getConstantOperandVal(1);
1454 if ((isPowerOf2_64(Mask) || isMask_64(Mask)) && !isInt<12>(Mask)) {
1455 unsigned ShAmt = 0;
1456 if (isPowerOf2_64(Mask)) {
1458 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Mask);
1459 } else {
1460 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Mask);
1461 }
1462
1463 LHS = LHS.getOperand(0);
1464 if (ShAmt != 0)
1465 LHS = DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS,
1466 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
1467 return;
1468 }
1469 }
1470
1471 if (auto *RHSC = dyn_cast<ConstantSDNode>(RHS)) {
1472 int64_t C = RHSC->getSExtValue();
1473 switch (CC) {
1474 default:
1475 break;
1476 case ISD::SETGT:
1477 // Convert X > -1 to X >= 0.
1478 if (C == -1) {
1479 RHS = DAG.getConstant(0, DL, RHS.getValueType());
1480 CC = ISD::SETGE;
1481 return;
1482 }
1483 break;
1484 case ISD::SETLT:
1485 // Convert X < 1 to 0 >= X.
1486 if (C == 1) {
1487 RHS = LHS;
1488 LHS = DAG.getConstant(0, DL, RHS.getValueType());
1489 CC = ISD::SETGE;
1490 return;
1491 }
1492 break;
1493 }
1494 }
1495
1496 switch (CC) {
1497 default:
1498 break;
1499 case ISD::SETGT:
1500 case ISD::SETLE:
1501 case ISD::SETUGT:
1502 case ISD::SETULE:
1504 std::swap(LHS, RHS);
1505 break;
1506 }
1507}
1508
1509SDValue LoongArchTargetLowering::lowerSELECT(SDValue Op,
1510 SelectionDAG &DAG) const {
1511 SDValue CondV = Op.getOperand(0);
1512 SDValue TrueV = Op.getOperand(1);
1513 SDValue FalseV = Op.getOperand(2);
1514 SDLoc DL(Op);
1515 MVT VT = Op.getSimpleValueType();
1516 MVT GRLenVT = Subtarget.getGRLenVT();
1517
1518 if (SDValue V = combineSelectToBinOp(Op.getNode(), DAG, Subtarget))
1519 return V;
1520
1521 if (Op.hasOneUse()) {
1522 unsigned UseOpc = Op->user_begin()->getOpcode();
1523 if (isBinOp(UseOpc) && DAG.isSafeToSpeculativelyExecute(UseOpc)) {
1524 SDNode *BinOp = *Op->user_begin();
1525 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(*Op->user_begin(),
1526 DAG, Subtarget)) {
1527 DAG.ReplaceAllUsesWith(BinOp, &NewSel);
1528 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
1529 // may return a constant node and cause crash in lowerSELECT.
1530 if (NewSel.getOpcode() == ISD::SELECT)
1531 return lowerSELECT(NewSel, DAG);
1532 return NewSel;
1533 }
1534 }
1535 }
1536
1537 // If the condition is not an integer SETCC which operates on GRLenVT, we need
1538 // to emit a LoongArchISD::SELECT_CC comparing the condition to zero. i.e.:
1539 // (select condv, truev, falsev)
1540 // -> (loongarchisd::select_cc condv, zero, setne, truev, falsev)
1541 if (CondV.getOpcode() != ISD::SETCC ||
1542 CondV.getOperand(0).getSimpleValueType() != GRLenVT) {
1543 SDValue Zero = DAG.getConstant(0, DL, GRLenVT);
1544 SDValue SetNE = DAG.getCondCode(ISD::SETNE);
1545
1546 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
1547
1548 return DAG.getNode(LoongArchISD::SELECT_CC, DL, VT, Ops);
1549 }
1550
1551 // If the CondV is the output of a SETCC node which operates on GRLenVT
1552 // inputs, then merge the SETCC node into the lowered LoongArchISD::SELECT_CC
1553 // to take advantage of the integer compare+branch instructions. i.e.: (select
1554 // (setcc lhs, rhs, cc), truev, falsev)
1555 // -> (loongarchisd::select_cc lhs, rhs, cc, truev, falsev)
1556 SDValue LHS = CondV.getOperand(0);
1557 SDValue RHS = CondV.getOperand(1);
1558 ISD::CondCode CCVal = CondV.getOperandCondCode(2);
1559
1560 // Special case for a select of 2 constants that have a difference of 1.
1561 // Normally this is done by DAGCombine, but if the select is introduced by
1562 // type legalization or op legalization, we miss it. Restricting to SETLT
1563 // case for now because that is what signed saturating add/sub need.
1564 // FIXME: We don't need the condition to be SETLT or even a SETCC,
1565 // but we would probably want to swap the true/false values if the condition
1566 // is SETGE/SETLE to avoid an XORI.
1567 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
1568 CCVal == ISD::SETLT) {
1569 const APInt &TrueVal = TrueV->getAsAPIntVal();
1570 const APInt &FalseVal = FalseV->getAsAPIntVal();
1571 if (TrueVal - 1 == FalseVal)
1572 return DAG.getNode(ISD::ADD, DL, VT, CondV, FalseV);
1573 if (TrueVal + 1 == FalseVal)
1574 return DAG.getNode(ISD::SUB, DL, VT, FalseV, CondV);
1575 }
1576
1577 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
1578 // 1 < x ? x : 1 -> 0 < x ? x : 1
1579 if (isOneConstant(LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
1580 RHS == TrueV && LHS == FalseV) {
1581 LHS = DAG.getConstant(0, DL, VT);
1582 // 0 <u x is the same as x != 0.
1583 if (CCVal == ISD::SETULT) {
1584 std::swap(LHS, RHS);
1585 CCVal = ISD::SETNE;
1586 }
1587 }
1588
1589 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
1590 if (isAllOnesConstant(RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
1591 RHS == FalseV) {
1592 RHS = DAG.getConstant(0, DL, VT);
1593 }
1594
1595 SDValue TargetCC = DAG.getCondCode(CCVal);
1596
1597 if (isa<ConstantSDNode>(TrueV) && !isa<ConstantSDNode>(FalseV)) {
1598 // (select (setcc lhs, rhs, CC), constant, falsev)
1599 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
1600 std::swap(TrueV, FalseV);
1601 TargetCC = DAG.getCondCode(ISD::getSetCCInverse(CCVal, LHS.getValueType()));
1602 }
1603
1604 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
1605 return DAG.getNode(LoongArchISD::SELECT_CC, DL, VT, Ops);
1606}
1607
1608SDValue LoongArchTargetLowering::lowerBRCOND(SDValue Op,
1609 SelectionDAG &DAG) const {
1610 SDValue CondV = Op.getOperand(1);
1611 SDLoc DL(Op);
1612 MVT GRLenVT = Subtarget.getGRLenVT();
1613
1614 if (CondV.getOpcode() == ISD::SETCC) {
1615 if (CondV.getOperand(0).getValueType() == GRLenVT) {
1616 SDValue LHS = CondV.getOperand(0);
1617 SDValue RHS = CondV.getOperand(1);
1618 ISD::CondCode CCVal = CondV.getOperandCondCode(2);
1619
1620 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
1621
1622 SDValue TargetCC = DAG.getCondCode(CCVal);
1623 return DAG.getNode(LoongArchISD::BR_CC, DL, Op.getValueType(),
1624 Op.getOperand(0), LHS, RHS, TargetCC,
1625 Op.getOperand(2));
1626 } else if (CondV.getOperand(0).getValueType().isFloatingPoint()) {
1627 return DAG.getNode(LoongArchISD::BRCOND, DL, Op.getValueType(),
1628 Op.getOperand(0), CondV, Op.getOperand(2));
1629 }
1630 }
1631
1632 return DAG.getNode(LoongArchISD::BR_CC, DL, Op.getValueType(),
1633 Op.getOperand(0), CondV, DAG.getConstant(0, DL, GRLenVT),
1634 DAG.getCondCode(ISD::SETNE), Op.getOperand(2));
1635}
1636
1637SDValue
1638LoongArchTargetLowering::lowerSCALAR_TO_VECTOR(SDValue Op,
1639 SelectionDAG &DAG) const {
1640 SDLoc DL(Op);
1641 MVT OpVT = Op.getSimpleValueType();
1642
1643 SDValue Vector = DAG.getUNDEF(OpVT);
1644 SDValue Val = Op.getOperand(0);
1645 SDValue Idx = DAG.getConstant(0, DL, Subtarget.getGRLenVT());
1646
1647 return DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, OpVT, Vector, Val, Idx);
1648}
1649
1650SDValue LoongArchTargetLowering::lowerBITREVERSE(SDValue Op,
1651 SelectionDAG &DAG) const {
1652 EVT ResTy = Op->getValueType(0);
1653 SDValue Src = Op->getOperand(0);
1654 SDLoc DL(Op);
1655
1656 // LoongArchISD::BITREV_8B is not supported on LA32.
1657 if (!Subtarget.is64Bit() && (ResTy == MVT::v16i8 || ResTy == MVT::v32i8))
1658 return SDValue();
1659
1660 EVT NewVT = ResTy.is128BitVector() ? MVT::v2i64 : MVT::v4i64;
1661 unsigned int OrigEltNum = ResTy.getVectorNumElements();
1662 unsigned int NewEltNum = NewVT.getVectorNumElements();
1663
1664 SDValue NewSrc = DAG.getNode(ISD::BITCAST, DL, NewVT, Src);
1665
1667 for (unsigned int i = 0; i < NewEltNum; i++) {
1668 SDValue Op = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::i64, NewSrc,
1669 DAG.getConstant(i, DL, Subtarget.getGRLenVT()));
1670 unsigned RevOp = (ResTy == MVT::v16i8 || ResTy == MVT::v32i8)
1671 ? (unsigned)LoongArchISD::BITREV_8B
1672 : (unsigned)ISD::BITREVERSE;
1673 Ops.push_back(DAG.getNode(RevOp, DL, MVT::i64, Op));
1674 }
1675 SDValue Res =
1676 DAG.getNode(ISD::BITCAST, DL, ResTy, DAG.getBuildVector(NewVT, DL, Ops));
1677
1678 switch (ResTy.getSimpleVT().SimpleTy) {
1679 default:
1680 return SDValue();
1681 case MVT::v16i8:
1682 case MVT::v32i8:
1683 return Res;
1684 case MVT::v8i16:
1685 case MVT::v16i16:
1686 case MVT::v4i32:
1687 case MVT::v8i32: {
1689 for (unsigned int i = 0; i < NewEltNum; i++)
1690 for (int j = OrigEltNum / NewEltNum - 1; j >= 0; j--)
1691 Mask.push_back(j + (OrigEltNum / NewEltNum) * i);
1692 return DAG.getVectorShuffle(ResTy, DL, Res, DAG.getUNDEF(ResTy), Mask);
1693 }
1694 }
1695}
1696
1697// Widen element type to get a new mask value (if possible).
1698// For example:
1699// shufflevector <4 x i32> %a, <4 x i32> %b,
1700// <4 x i32> <i32 6, i32 7, i32 2, i32 3>
1701// is equivalent to:
1702// shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> <i32 3, i32 1>
1703// can be lowered to:
1704// VPACKOD_D vr0, vr0, vr1
1706 SDValue V1, SDValue V2, SelectionDAG &DAG) {
1707 unsigned EltBits = VT.getScalarSizeInBits();
1708
1709 if (EltBits > 32 || EltBits == 1)
1710 return SDValue();
1711
1712 SmallVector<int, 8> NewMask;
1713 if (widenShuffleMaskElts(Mask, NewMask)) {
1714 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(EltBits * 2)
1715 : MVT::getIntegerVT(EltBits * 2);
1716 MVT NewVT = MVT::getVectorVT(NewEltVT, VT.getVectorNumElements() / 2);
1717 if (DAG.getTargetLoweringInfo().isTypeLegal(NewVT)) {
1718 SDValue NewV1 = DAG.getBitcast(NewVT, V1);
1719 SDValue NewV2 = DAG.getBitcast(NewVT, V2);
1720 return DAG.getBitcast(
1721 VT, DAG.getVectorShuffle(NewVT, DL, NewV1, NewV2, NewMask));
1722 }
1723 }
1724
1725 return SDValue();
1726}
1727
1728/// Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI
1729/// instruction.
1730// The funciton matches elements from one of the input vector shuffled to the
1731// left or right with zeroable elements 'shifted in'. It handles both the
1732// strictly bit-wise element shifts and the byte shfit across an entire 128-bit
1733// lane.
1734// Mostly copied from X86.
1735static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode,
1736 unsigned ScalarSizeInBits, ArrayRef<int> Mask,
1737 int MaskOffset, const APInt &Zeroable) {
1738 int Size = Mask.size();
1739 unsigned SizeInBits = Size * ScalarSizeInBits;
1740
1741 auto CheckZeros = [&](int Shift, int Scale, bool Left) {
1742 for (int i = 0; i < Size; i += Scale)
1743 for (int j = 0; j < Shift; ++j)
1744 if (!Zeroable[i + j + (Left ? 0 : (Scale - Shift))])
1745 return false;
1746
1747 return true;
1748 };
1749
1750 auto isSequentialOrUndefInRange = [&](unsigned Pos, unsigned Size, int Low,
1751 int Step = 1) {
1752 for (unsigned i = Pos, e = Pos + Size; i != e; ++i, Low += Step)
1753 if (!(Mask[i] == -1 || Mask[i] == Low))
1754 return false;
1755 return true;
1756 };
1757
1758 auto MatchShift = [&](int Shift, int Scale, bool Left) {
1759 for (int i = 0; i != Size; i += Scale) {
1760 unsigned Pos = Left ? i + Shift : i;
1761 unsigned Low = Left ? i : i + Shift;
1762 unsigned Len = Scale - Shift;
1763 if (!isSequentialOrUndefInRange(Pos, Len, Low + MaskOffset))
1764 return -1;
1765 }
1766
1767 int ShiftEltBits = ScalarSizeInBits * Scale;
1768 bool ByteShift = ShiftEltBits > 64;
1769 Opcode = Left ? (ByteShift ? LoongArchISD::VBSLL : LoongArchISD::VSLLI)
1770 : (ByteShift ? LoongArchISD::VBSRL : LoongArchISD::VSRLI);
1771 int ShiftAmt = Shift * ScalarSizeInBits / (ByteShift ? 8 : 1);
1772
1773 // Normalize the scale for byte shifts to still produce an i64 element
1774 // type.
1775 Scale = ByteShift ? Scale / 2 : Scale;
1776
1777 // We need to round trip through the appropriate type for the shift.
1778 MVT ShiftSVT = MVT::getIntegerVT(ScalarSizeInBits * Scale);
1779 ShiftVT = ByteShift ? MVT::getVectorVT(MVT::i8, SizeInBits / 8)
1780 : MVT::getVectorVT(ShiftSVT, Size / Scale);
1781 return (int)ShiftAmt;
1782 };
1783
1784 unsigned MaxWidth = 128;
1785 for (int Scale = 2; Scale * ScalarSizeInBits <= MaxWidth; Scale *= 2)
1786 for (int Shift = 1; Shift != Scale; ++Shift)
1787 for (bool Left : {true, false})
1788 if (CheckZeros(Shift, Scale, Left)) {
1789 int ShiftAmt = MatchShift(Shift, Scale, Left);
1790 if (0 < ShiftAmt)
1791 return ShiftAmt;
1792 }
1793
1794 // no match
1795 return -1;
1796}
1797
1798/// Lower VECTOR_SHUFFLE as shift (if possible).
1799///
1800/// For example:
1801/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1802/// <4 x i32> <i32 4, i32 0, i32 1, i32 2>
1803/// is lowered to:
1804/// (VBSLL_V $v0, $v0, 4)
1805///
1806/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
1807/// <4 x i32> <i32 4, i32 0, i32 4, i32 2>
1808/// is lowered to:
1809/// (VSLLI_D $v0, $v0, 32)
1811 MVT VT, SDValue V1, SDValue V2,
1812 SelectionDAG &DAG,
1814 const APInt &Zeroable) {
1815 int Size = Mask.size();
1816 assert(Size == (int)VT.getVectorNumElements() && "Unexpected mask size");
1817
1818 MVT ShiftVT;
1819 SDValue V = V1;
1820 unsigned Opcode;
1821
1822 // Try to match shuffle against V1 shift.
1823 int ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
1824 Mask, 0, Zeroable);
1825
1826 // If V1 failed, try to match shuffle against V2 shift.
1827 if (ShiftAmt < 0) {
1828 ShiftAmt = matchShuffleAsShift(ShiftVT, Opcode, VT.getScalarSizeInBits(),
1829 Mask, Size, Zeroable);
1830 V = V2;
1831 }
1832
1833 if (ShiftAmt < 0)
1834 return SDValue();
1835
1836 assert(DAG.getTargetLoweringInfo().isTypeLegal(ShiftVT) &&
1837 "Illegal integer vector type");
1838 V = DAG.getBitcast(ShiftVT, V);
1839 V = DAG.getNode(Opcode, DL, ShiftVT, V,
1840 DAG.getConstant(ShiftAmt, DL, Subtarget.getGRLenVT()));
1841 return DAG.getBitcast(VT, V);
1842}
1843
1844/// Determine whether a range fits a regular pattern of values.
1845/// This function accounts for the possibility of jumping over the End iterator.
1846template <typename ValType>
1847static bool
1849 unsigned CheckStride,
1851 ValType ExpectedIndex, unsigned ExpectedIndexStride) {
1852 auto &I = Begin;
1853
1854 while (I != End) {
1855 if (*I != -1 && *I != ExpectedIndex)
1856 return false;
1857 ExpectedIndex += ExpectedIndexStride;
1858
1859 // Incrementing past End is undefined behaviour so we must increment one
1860 // step at a time and check for End at each step.
1861 for (unsigned n = 0; n < CheckStride && I != End; ++n, ++I)
1862 ; // Empty loop body.
1863 }
1864 return true;
1865}
1866
1867/// Compute whether each element of a shuffle is zeroable.
1868///
1869/// A "zeroable" vector shuffle element is one which can be lowered to zero.
1871 SDValue V2, APInt &KnownUndef,
1872 APInt &KnownZero) {
1873 int Size = Mask.size();
1874 KnownUndef = KnownZero = APInt::getZero(Size);
1875
1877 V2 = peekThroughBitcasts(V2);
1878
1879 bool V1IsZero = ISD::isBuildVectorAllZeros(V1);
1880 bool V2IsZero = ISD::isBuildVectorAllZeros(V2);
1881
1882 int VectorSizeInBits = V1.getValueSizeInBits();
1883 int ScalarSizeInBits = VectorSizeInBits / Size;
1884 assert(!(VectorSizeInBits % ScalarSizeInBits) && "Illegal shuffle mask size");
1885 (void)ScalarSizeInBits;
1886
1887 for (int i = 0; i < Size; ++i) {
1888 int M = Mask[i];
1889 if (M < 0) {
1890 KnownUndef.setBit(i);
1891 continue;
1892 }
1893 if ((M >= 0 && M < Size && V1IsZero) || (M >= Size && V2IsZero)) {
1894 KnownZero.setBit(i);
1895 continue;
1896 }
1897 }
1898}
1899
1900/// Test whether a shuffle mask is equivalent within each sub-lane.
1901///
1902/// The specific repeated shuffle mask is populated in \p RepeatedMask, as it is
1903/// non-trivial to compute in the face of undef lanes. The representation is
1904/// suitable for use with existing 128-bit shuffles as entries from the second
1905/// vector have been remapped to [LaneSize, 2*LaneSize).
1906static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT,
1907 ArrayRef<int> Mask,
1908 SmallVectorImpl<int> &RepeatedMask) {
1909 auto LaneSize = LaneSizeInBits / VT.getScalarSizeInBits();
1910 RepeatedMask.assign(LaneSize, -1);
1911 int Size = Mask.size();
1912 for (int i = 0; i < Size; ++i) {
1913 assert(Mask[i] == -1 || Mask[i] >= 0);
1914 if (Mask[i] < 0)
1915 continue;
1916 if ((Mask[i] % Size) / LaneSize != i / LaneSize)
1917 // This entry crosses lanes, so there is no way to model this shuffle.
1918 return false;
1919
1920 // Ok, handle the in-lane shuffles by detecting if and when they repeat.
1921 // Adjust second vector indices to start at LaneSize instead of Size.
1922 int LocalM =
1923 Mask[i] < Size ? Mask[i] % LaneSize : Mask[i] % LaneSize + LaneSize;
1924 if (RepeatedMask[i % LaneSize] < 0)
1925 // This is the first non-undef entry in this slot of a 128-bit lane.
1926 RepeatedMask[i % LaneSize] = LocalM;
1927 else if (RepeatedMask[i % LaneSize] != LocalM)
1928 // Found a mismatch with the repeated mask.
1929 return false;
1930 }
1931 return true;
1932}
1933
1934/// Attempts to match vector shuffle as byte rotation.
1936 ArrayRef<int> Mask) {
1937
1938 SDValue Lo, Hi;
1939 SmallVector<int, 16> RepeatedMask;
1940
1941 if (!isRepeatedShuffleMask(128, VT, Mask, RepeatedMask))
1942 return -1;
1943
1944 int NumElts = RepeatedMask.size();
1945 int Rotation = 0;
1946 int Scale = 16 / NumElts;
1947
1948 for (int i = 0; i < NumElts; ++i) {
1949 int M = RepeatedMask[i];
1950 assert((M == -1 || (0 <= M && M < (2 * NumElts))) &&
1951 "Unexpected mask index.");
1952 if (M < 0)
1953 continue;
1954
1955 // Determine where a rotated vector would have started.
1956 int StartIdx = i - (M % NumElts);
1957 if (StartIdx == 0)
1958 return -1;
1959
1960 // If we found the tail of a vector the rotation must be the missing
1961 // front. If we found the head of a vector, it must be how much of the
1962 // head.
1963 int CandidateRotation = StartIdx < 0 ? -StartIdx : NumElts - StartIdx;
1964
1965 if (Rotation == 0)
1966 Rotation = CandidateRotation;
1967 else if (Rotation != CandidateRotation)
1968 return -1;
1969
1970 // Compute which value this mask is pointing at.
1971 SDValue MaskV = M < NumElts ? V1 : V2;
1972
1973 // Compute which of the two target values this index should be assigned
1974 // to. This reflects whether the high elements are remaining or the low
1975 // elements are remaining.
1976 SDValue &TargetV = StartIdx < 0 ? Hi : Lo;
1977
1978 // Either set up this value if we've not encountered it before, or check
1979 // that it remains consistent.
1980 if (!TargetV)
1981 TargetV = MaskV;
1982 else if (TargetV != MaskV)
1983 return -1;
1984 }
1985
1986 // Check that we successfully analyzed the mask, and normalize the results.
1987 assert(Rotation != 0 && "Failed to locate a viable rotation!");
1988 assert((Lo || Hi) && "Failed to find a rotated input vector!");
1989 if (!Lo)
1990 Lo = Hi;
1991 else if (!Hi)
1992 Hi = Lo;
1993
1994 V1 = Lo;
1995 V2 = Hi;
1996
1997 return Rotation * Scale;
1998}
1999
2000/// Lower VECTOR_SHUFFLE as byte rotate (if possible).
2001///
2002/// For example:
2003/// %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b,
2004/// <2 x i32> <i32 3, i32 0>
2005/// is lowered to:
2006/// (VBSRL_V $v1, $v1, 8)
2007/// (VBSLL_V $v0, $v0, 8)
2008/// (VOR_V $v0, $V0, $v1)
2009static SDValue
2011 SDValue V1, SDValue V2, SelectionDAG &DAG,
2013
2014 SDValue Lo = V1, Hi = V2;
2015 int ByteRotation = matchShuffleAsByteRotate(VT, Lo, Hi, Mask);
2016 if (ByteRotation <= 0)
2017 return SDValue();
2018
2019 MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8);
2020 Lo = DAG.getBitcast(ByteVT, Lo);
2021 Hi = DAG.getBitcast(ByteVT, Hi);
2022
2023 int LoByteShift = 16 - ByteRotation;
2024 int HiByteShift = ByteRotation;
2025 MVT GRLenVT = Subtarget.getGRLenVT();
2026
2027 SDValue LoShift = DAG.getNode(LoongArchISD::VBSLL, DL, ByteVT, Lo,
2028 DAG.getConstant(LoByteShift, DL, GRLenVT));
2029 SDValue HiShift = DAG.getNode(LoongArchISD::VBSRL, DL, ByteVT, Hi,
2030 DAG.getConstant(HiByteShift, DL, GRLenVT));
2031 return DAG.getBitcast(VT, DAG.getNode(ISD::OR, DL, ByteVT, LoShift, HiShift));
2032}
2033
2034/// Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
2035///
2036/// For example:
2037/// %2 = shufflevector <4 x i32> %0, <4 x i32> zeroinitializer,
2038/// <4 x i32> <i32 0, i32 4, i32 1, i32 4>
2039/// %3 = bitcast <4 x i32> %2 to <2 x i64>
2040/// is lowered to:
2041/// (VREPLI $v1, 0)
2042/// (VILVL $v0, $v1, $v0)
2044 ArrayRef<int> Mask, MVT VT,
2045 SDValue V1, SDValue V2,
2046 SelectionDAG &DAG,
2047 const APInt &Zeroable) {
2048 int Bits = VT.getSizeInBits();
2049 int EltBits = VT.getScalarSizeInBits();
2050 int NumElements = VT.getVectorNumElements();
2051
2052 if (Zeroable.isAllOnes())
2053 return DAG.getConstant(0, DL, VT);
2054
2055 // Define a helper function to check a particular ext-scale and lower to it if
2056 // valid.
2057 auto Lower = [&](int Scale) -> SDValue {
2058 SDValue InputV;
2059 bool AnyExt = true;
2060 int Offset = 0;
2061 for (int i = 0; i < NumElements; i++) {
2062 int M = Mask[i];
2063 if (M < 0)
2064 continue;
2065 if (i % Scale != 0) {
2066 // Each of the extended elements need to be zeroable.
2067 if (!Zeroable[i])
2068 return SDValue();
2069
2070 AnyExt = false;
2071 continue;
2072 }
2073
2074 // Each of the base elements needs to be consecutive indices into the
2075 // same input vector.
2076 SDValue V = M < NumElements ? V1 : V2;
2077 M = M % NumElements;
2078 if (!InputV) {
2079 InputV = V;
2080 Offset = M - (i / Scale);
2081
2082 // These offset can't be handled
2083 if (Offset % (NumElements / Scale))
2084 return SDValue();
2085 } else if (InputV != V)
2086 return SDValue();
2087
2088 if (M != (Offset + (i / Scale)))
2089 return SDValue(); // Non-consecutive strided elements.
2090 }
2091
2092 // If we fail to find an input, we have a zero-shuffle which should always
2093 // have already been handled.
2094 if (!InputV)
2095 return SDValue();
2096
2097 do {
2098 unsigned VilVLoHi = LoongArchISD::VILVL;
2099 if (Offset >= (NumElements / 2)) {
2100 VilVLoHi = LoongArchISD::VILVH;
2101 Offset -= (NumElements / 2);
2102 }
2103
2104 MVT InputVT = MVT::getVectorVT(MVT::getIntegerVT(EltBits), NumElements);
2105 SDValue Ext =
2106 AnyExt ? DAG.getFreeze(InputV) : DAG.getConstant(0, DL, InputVT);
2107 InputV = DAG.getBitcast(InputVT, InputV);
2108 Ext = DAG.getBitcast(InputVT, Ext);
2109 InputV = DAG.getNode(VilVLoHi, DL, InputVT, Ext, InputV);
2110 Scale /= 2;
2111 EltBits *= 2;
2112 NumElements /= 2;
2113 } while (Scale > 1);
2114 return DAG.getBitcast(VT, InputV);
2115 };
2116
2117 // Each iteration, try extending the elements half as much, but into twice as
2118 // many elements.
2119 for (int NumExtElements = Bits / 64; NumExtElements < NumElements;
2120 NumExtElements *= 2) {
2121 if (SDValue V = Lower(NumElements / NumExtElements))
2122 return V;
2123 }
2124 return SDValue();
2125}
2126
2127/// Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
2128///
2129/// VREPLVEI performs vector broadcast based on an element specified by an
2130/// integer immediate, with its mask being similar to:
2131/// <x, x, x, ...>
2132/// where x is any valid index.
2133///
2134/// When undef's appear in the mask they are treated as if they were whatever
2135/// value is necessary in order to fit the above form.
2136static SDValue
2138 SDValue V1, SelectionDAG &DAG,
2140 int SplatIndex = -1;
2141 for (const auto &M : Mask) {
2142 if (M != -1) {
2143 SplatIndex = M;
2144 break;
2145 }
2146 }
2147
2148 if (SplatIndex == -1)
2149 return DAG.getUNDEF(VT);
2150
2151 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2152 if (fitsRegularPattern<int>(Mask.begin(), 1, Mask.end(), SplatIndex, 0)) {
2153 return DAG.getNode(LoongArchISD::VREPLVEI, DL, VT, V1,
2154 DAG.getConstant(SplatIndex, DL, Subtarget.getGRLenVT()));
2155 }
2156
2157 return SDValue();
2158}
2159
2160/// Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
2161///
2162/// VSHUF4I splits the vector into blocks of four elements, then shuffles these
2163/// elements according to a <4 x i2> constant (encoded as an integer immediate).
2164///
2165/// It is therefore possible to lower into VSHUF4I when the mask takes the form:
2166/// <a, b, c, d, a+4, b+4, c+4, d+4, a+8, b+8, c+8, d+8, ...>
2167/// When undef's appear they are treated as if they were whatever value is
2168/// necessary in order to fit the above forms.
2169///
2170/// For example:
2171/// %2 = shufflevector <8 x i16> %0, <8 x i16> undef,
2172/// <8 x i32> <i32 3, i32 2, i32 1, i32 0,
2173/// i32 7, i32 6, i32 5, i32 4>
2174/// is lowered to:
2175/// (VSHUF4I_H $v0, $v1, 27)
2176/// where the 27 comes from:
2177/// 3 + (2 << 2) + (1 << 4) + (0 << 6)
2178static SDValue
2180 SDValue V1, SDValue V2, SelectionDAG &DAG,
2182
2183 unsigned SubVecSize = 4;
2184 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2185 SubVecSize = 2;
2186
2187 int SubMask[4] = {-1, -1, -1, -1};
2188 for (unsigned i = 0; i < SubVecSize; ++i) {
2189 for (unsigned j = i; j < Mask.size(); j += SubVecSize) {
2190 int M = Mask[j];
2191
2192 // Convert from vector index to 4-element subvector index
2193 // If an index refers to an element outside of the subvector then give up
2194 if (M != -1) {
2195 M -= 4 * (j / SubVecSize);
2196 if (M < 0 || M >= 4)
2197 return SDValue();
2198 }
2199
2200 // If the mask has an undef, replace it with the current index.
2201 // Note that it might still be undef if the current index is also undef
2202 if (SubMask[i] == -1)
2203 SubMask[i] = M;
2204 // Check that non-undef values are the same as in the mask. If they
2205 // aren't then give up
2206 else if (M != -1 && M != SubMask[i])
2207 return SDValue();
2208 }
2209 }
2210
2211 // Calculate the immediate. Replace any remaining undefs with zero
2212 int Imm = 0;
2213 for (int i = SubVecSize - 1; i >= 0; --i) {
2214 int M = SubMask[i];
2215
2216 if (M == -1)
2217 M = 0;
2218
2219 Imm <<= 2;
2220 Imm |= M & 0x3;
2221 }
2222
2223 MVT GRLenVT = Subtarget.getGRLenVT();
2224
2225 // Return vshuf4i.d
2226 if (VT == MVT::v2f64 || VT == MVT::v2i64)
2227 return DAG.getNode(LoongArchISD::VSHUF4I_D, DL, VT, V1, V2,
2228 DAG.getConstant(Imm, DL, GRLenVT));
2229
2230 return DAG.getNode(LoongArchISD::VSHUF4I, DL, VT, V1,
2231 DAG.getConstant(Imm, DL, GRLenVT));
2232}
2233
2234/// Lower VECTOR_SHUFFLE whose result is the reversed source vector.
2235///
2236/// It is possible to do optimization for VECTOR_SHUFFLE performing vector
2237/// reverse whose mask likes:
2238/// <7, 6, 5, 4, 3, 2, 1, 0>
2239///
2240/// When undef's appear in the mask they are treated as if they were whatever
2241/// value is necessary in order to fit the above forms.
2242static SDValue
2244 SDValue V1, SelectionDAG &DAG,
2246 // Only vectors with i8/i16 elements which cannot match other patterns
2247 // directly needs to do this.
2248 if (VT != MVT::v16i8 && VT != MVT::v8i16 && VT != MVT::v32i8 &&
2249 VT != MVT::v16i16)
2250 return SDValue();
2251
2252 if (!ShuffleVectorInst::isReverseMask(Mask, Mask.size()))
2253 return SDValue();
2254
2255 int WidenNumElts = VT.getVectorNumElements() / 4;
2256 SmallVector<int, 16> WidenMask(WidenNumElts, -1);
2257 for (int i = 0; i < WidenNumElts; ++i)
2258 WidenMask[i] = WidenNumElts - 1 - i;
2259
2260 MVT WidenVT = MVT::getVectorVT(
2261 VT.getVectorElementType() == MVT::i8 ? MVT::i32 : MVT::i64, WidenNumElts);
2262 SDValue NewV1 = DAG.getBitcast(WidenVT, V1);
2263 SDValue WidenRev = DAG.getVectorShuffle(WidenVT, DL, NewV1,
2264 DAG.getUNDEF(WidenVT), WidenMask);
2265
2266 return DAG.getNode(LoongArchISD::VSHUF4I, DL, VT,
2267 DAG.getBitcast(VT, WidenRev),
2268 DAG.getConstant(27, DL, Subtarget.getGRLenVT()));
2269}
2270
2271/// Lower VECTOR_SHUFFLE into VPACKEV (if possible).
2272///
2273/// VPACKEV interleaves the even elements from each vector.
2274///
2275/// It is possible to lower into VPACKEV when the mask consists of two of the
2276/// following forms interleaved:
2277/// <0, 2, 4, ...>
2278/// <n, n+2, n+4, ...>
2279/// where n is the number of elements in the vector.
2280/// For example:
2281/// <0, 0, 2, 2, 4, 4, ...>
2282/// <0, n, 2, n+2, 4, n+4, ...>
2283///
2284/// When undef's appear in the mask they are treated as if they were whatever
2285/// value is necessary in order to fit the above forms.
2287 MVT VT, SDValue V1, SDValue V2,
2288 SelectionDAG &DAG) {
2289
2290 const auto &Begin = Mask.begin();
2291 const auto &End = Mask.end();
2292 SDValue OriV1 = V1, OriV2 = V2;
2293
2294 if (fitsRegularPattern<int>(Begin, 2, End, 0, 2))
2295 V1 = OriV1;
2296 else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size(), 2))
2297 V1 = OriV2;
2298 else
2299 return SDValue();
2300
2301 if (fitsRegularPattern<int>(Begin + 1, 2, End, 0, 2))
2302 V2 = OriV1;
2303 else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size(), 2))
2304 V2 = OriV2;
2305 else
2306 return SDValue();
2307
2308 return DAG.getNode(LoongArchISD::VPACKEV, DL, VT, V2, V1);
2309}
2310
2311/// Lower VECTOR_SHUFFLE into VPACKOD (if possible).
2312///
2313/// VPACKOD interleaves the odd elements from each vector.
2314///
2315/// It is possible to lower into VPACKOD when the mask consists of two of the
2316/// following forms interleaved:
2317/// <1, 3, 5, ...>
2318/// <n+1, n+3, n+5, ...>
2319/// where n is the number of elements in the vector.
2320/// For example:
2321/// <1, 1, 3, 3, 5, 5, ...>
2322/// <1, n+1, 3, n+3, 5, n+5, ...>
2323///
2324/// When undef's appear in the mask they are treated as if they were whatever
2325/// value is necessary in order to fit the above forms.
2327 MVT VT, SDValue V1, SDValue V2,
2328 SelectionDAG &DAG) {
2329
2330 const auto &Begin = Mask.begin();
2331 const auto &End = Mask.end();
2332 SDValue OriV1 = V1, OriV2 = V2;
2333
2334 if (fitsRegularPattern<int>(Begin, 2, End, 1, 2))
2335 V1 = OriV1;
2336 else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size() + 1, 2))
2337 V1 = OriV2;
2338 else
2339 return SDValue();
2340
2341 if (fitsRegularPattern<int>(Begin + 1, 2, End, 1, 2))
2342 V2 = OriV1;
2343 else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size() + 1, 2))
2344 V2 = OriV2;
2345 else
2346 return SDValue();
2347
2348 return DAG.getNode(LoongArchISD::VPACKOD, DL, VT, V2, V1);
2349}
2350
2351/// Lower VECTOR_SHUFFLE into VILVH (if possible).
2352///
2353/// VILVH interleaves consecutive elements from the left (highest-indexed) half
2354/// of each vector.
2355///
2356/// It is possible to lower into VILVH when the mask consists of two of the
2357/// following forms interleaved:
2358/// <x, x+1, x+2, ...>
2359/// <n+x, n+x+1, n+x+2, ...>
2360/// where n is the number of elements in the vector and x is half n.
2361/// For example:
2362/// <x, x, x+1, x+1, x+2, x+2, ...>
2363/// <x, n+x, x+1, n+x+1, x+2, n+x+2, ...>
2364///
2365/// When undef's appear in the mask they are treated as if they were whatever
2366/// value is necessary in order to fit the above forms.
2368 MVT VT, SDValue V1, SDValue V2,
2369 SelectionDAG &DAG) {
2370
2371 const auto &Begin = Mask.begin();
2372 const auto &End = Mask.end();
2373 unsigned HalfSize = Mask.size() / 2;
2374 SDValue OriV1 = V1, OriV2 = V2;
2375
2376 if (fitsRegularPattern<int>(Begin, 2, End, HalfSize, 1))
2377 V1 = OriV1;
2378 else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size() + HalfSize, 1))
2379 V1 = OriV2;
2380 else
2381 return SDValue();
2382
2383 if (fitsRegularPattern<int>(Begin + 1, 2, End, HalfSize, 1))
2384 V2 = OriV1;
2385 else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size() + HalfSize,
2386 1))
2387 V2 = OriV2;
2388 else
2389 return SDValue();
2390
2391 return DAG.getNode(LoongArchISD::VILVH, DL, VT, V2, V1);
2392}
2393
2394/// Lower VECTOR_SHUFFLE into VILVL (if possible).
2395///
2396/// VILVL interleaves consecutive elements from the right (lowest-indexed) half
2397/// of each vector.
2398///
2399/// It is possible to lower into VILVL when the mask consists of two of the
2400/// following forms interleaved:
2401/// <0, 1, 2, ...>
2402/// <n, n+1, n+2, ...>
2403/// where n is the number of elements in the vector.
2404/// For example:
2405/// <0, 0, 1, 1, 2, 2, ...>
2406/// <0, n, 1, n+1, 2, n+2, ...>
2407///
2408/// When undef's appear in the mask they are treated as if they were whatever
2409/// value is necessary in order to fit the above forms.
2411 MVT VT, SDValue V1, SDValue V2,
2412 SelectionDAG &DAG) {
2413
2414 const auto &Begin = Mask.begin();
2415 const auto &End = Mask.end();
2416 SDValue OriV1 = V1, OriV2 = V2;
2417
2418 if (fitsRegularPattern<int>(Begin, 2, End, 0, 1))
2419 V1 = OriV1;
2420 else if (fitsRegularPattern<int>(Begin, 2, End, Mask.size(), 1))
2421 V1 = OriV2;
2422 else
2423 return SDValue();
2424
2425 if (fitsRegularPattern<int>(Begin + 1, 2, End, 0, 1))
2426 V2 = OriV1;
2427 else if (fitsRegularPattern<int>(Begin + 1, 2, End, Mask.size(), 1))
2428 V2 = OriV2;
2429 else
2430 return SDValue();
2431
2432 return DAG.getNode(LoongArchISD::VILVL, DL, VT, V2, V1);
2433}
2434
2435/// Lower VECTOR_SHUFFLE into VPICKEV (if possible).
2436///
2437/// VPICKEV copies the even elements of each vector into the result vector.
2438///
2439/// It is possible to lower into VPICKEV when the mask consists of two of the
2440/// following forms concatenated:
2441/// <0, 2, 4, ...>
2442/// <n, n+2, n+4, ...>
2443/// where n is the number of elements in the vector.
2444/// For example:
2445/// <0, 2, 4, ..., 0, 2, 4, ...>
2446/// <0, 2, 4, ..., n, n+2, n+4, ...>
2447///
2448/// When undef's appear in the mask they are treated as if they were whatever
2449/// value is necessary in order to fit the above forms.
2451 MVT VT, SDValue V1, SDValue V2,
2452 SelectionDAG &DAG) {
2453
2454 const auto &Begin = Mask.begin();
2455 const auto &Mid = Mask.begin() + Mask.size() / 2;
2456 const auto &End = Mask.end();
2457 SDValue OriV1 = V1, OriV2 = V2;
2458
2459 if (fitsRegularPattern<int>(Begin, 1, Mid, 0, 2))
2460 V1 = OriV1;
2461 else if (fitsRegularPattern<int>(Begin, 1, Mid, Mask.size(), 2))
2462 V1 = OriV2;
2463 else
2464 return SDValue();
2465
2466 if (fitsRegularPattern<int>(Mid, 1, End, 0, 2))
2467 V2 = OriV1;
2468 else if (fitsRegularPattern<int>(Mid, 1, End, Mask.size(), 2))
2469 V2 = OriV2;
2470
2471 else
2472 return SDValue();
2473
2474 return DAG.getNode(LoongArchISD::VPICKEV, DL, VT, V2, V1);
2475}
2476
2477/// Lower VECTOR_SHUFFLE into VPICKOD (if possible).
2478///
2479/// VPICKOD copies the odd elements of each vector into the result vector.
2480///
2481/// It is possible to lower into VPICKOD when the mask consists of two of the
2482/// following forms concatenated:
2483/// <1, 3, 5, ...>
2484/// <n+1, n+3, n+5, ...>
2485/// where n is the number of elements in the vector.
2486/// For example:
2487/// <1, 3, 5, ..., 1, 3, 5, ...>
2488/// <1, 3, 5, ..., n+1, n+3, n+5, ...>
2489///
2490/// When undef's appear in the mask they are treated as if they were whatever
2491/// value is necessary in order to fit the above forms.
2493 MVT VT, SDValue V1, SDValue V2,
2494 SelectionDAG &DAG) {
2495
2496 const auto &Begin = Mask.begin();
2497 const auto &Mid = Mask.begin() + Mask.size() / 2;
2498 const auto &End = Mask.end();
2499 SDValue OriV1 = V1, OriV2 = V2;
2500
2501 if (fitsRegularPattern<int>(Begin, 1, Mid, 1, 2))
2502 V1 = OriV1;
2503 else if (fitsRegularPattern<int>(Begin, 1, Mid, Mask.size() + 1, 2))
2504 V1 = OriV2;
2505 else
2506 return SDValue();
2507
2508 if (fitsRegularPattern<int>(Mid, 1, End, 1, 2))
2509 V2 = OriV1;
2510 else if (fitsRegularPattern<int>(Mid, 1, End, Mask.size() + 1, 2))
2511 V2 = OriV2;
2512 else
2513 return SDValue();
2514
2515 return DAG.getNode(LoongArchISD::VPICKOD, DL, VT, V2, V1);
2516}
2517
2518/// Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
2519///
2520/// VEXTRINS copies one element of a vector into any place of the result
2521/// vector and makes no change to the rest elements of the result vector.
2522///
2523/// It is possible to lower into VEXTRINS when the mask takes the form:
2524/// <0, 1, 2, ..., n+i, ..., n-1> or <n, n+1, n+2, ..., i, ..., 2n-1> or
2525/// <0, 1, 2, ..., i, ..., n-1> or <n, n+1, n+2, ..., n+i, ..., 2n-1>
2526/// where n is the number of elements in the vector and i is in [0, n).
2527/// For example:
2528/// <0, 1, 2, 3, 4, 5, 6, 8> , <2, 9, 10, 11, 12, 13, 14, 15> ,
2529/// <0, 1, 2, 6, 4, 5, 6, 7> , <8, 9, 10, 11, 12, 9, 14, 15>
2530///
2531/// When undef's appear in the mask they are treated as if they were whatever
2532/// value is necessary in order to fit the above forms.
2533static SDValue
2535 SDValue V1, SDValue V2, SelectionDAG &DAG,
2537 unsigned NumElts = VT.getVectorNumElements();
2538 MVT EltVT = VT.getVectorElementType();
2539 MVT GRLenVT = Subtarget.getGRLenVT();
2540
2541 if (Mask.size() != NumElts)
2542 return SDValue();
2543
2544 auto tryLowerToExtrAndIns = [&](unsigned Base) -> SDValue {
2545 int DiffCount = 0;
2546 int DiffPos = -1;
2547 for (unsigned i = 0; i < NumElts; ++i) {
2548 if (Mask[i] == -1)
2549 continue;
2550 if (Mask[i] != int(Base + i)) {
2551 ++DiffCount;
2552 DiffPos = int(i);
2553 if (DiffCount > 1)
2554 return SDValue();
2555 }
2556 }
2557
2558 // Need exactly one differing element to lower into VEXTRINS.
2559 if (DiffCount != 1)
2560 return SDValue();
2561
2562 // DiffMask must be in [0, 2N).
2563 int DiffMask = Mask[DiffPos];
2564 if (DiffMask < 0 || DiffMask >= int(2 * NumElts))
2565 return SDValue();
2566
2567 // Determine source vector and source index.
2568 SDValue SrcVec;
2569 unsigned SrcIdx;
2570 if (unsigned(DiffMask) < NumElts) {
2571 SrcVec = V1;
2572 SrcIdx = unsigned(DiffMask);
2573 } else {
2574 SrcVec = V2;
2575 SrcIdx = unsigned(DiffMask) - NumElts;
2576 }
2577
2578 // Replace with EXTRACT_VECTOR_ELT + INSERT_VECTOR_ELT, it will match the
2579 // patterns of VEXTRINS in tablegen.
2580 SDValue Extracted = DAG.getNode(
2581 ISD::EXTRACT_VECTOR_ELT, DL, EltVT.isFloatingPoint() ? EltVT : GRLenVT,
2582 SrcVec, DAG.getConstant(SrcIdx, DL, GRLenVT));
2583 SDValue Result =
2584 DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, (Base == 0) ? V1 : V2,
2585 Extracted, DAG.getConstant(DiffPos, DL, GRLenVT));
2586
2587 return Result;
2588 };
2589
2590 // Try [0, n-1) insertion then [n, 2n-1) insertion.
2591 if (SDValue Result = tryLowerToExtrAndIns(0))
2592 return Result;
2593 return tryLowerToExtrAndIns(NumElts);
2594}
2595
2596// Check the Mask and then build SrcVec and MaskImm infos which will
2597// be used to build LoongArchISD nodes for VPERMI_W or XVPERMI_W.
2598// On success, return true. Otherwise, return false.
2601 unsigned &MaskImm) {
2602 unsigned MaskSize = Mask.size();
2603
2604 auto isValid = [&](int M, int Off) {
2605 return (M == -1) || (M >= Off && M < Off + 4);
2606 };
2607
2608 auto buildImm = [&](int MLo, int MHi, unsigned Off, unsigned I) {
2609 auto immPart = [&](int M, unsigned Off) {
2610 return (M == -1 ? 0 : (M - Off)) & 0x3;
2611 };
2612 MaskImm |= immPart(MLo, Off) << (I * 2);
2613 MaskImm |= immPart(MHi, Off) << ((I + 1) * 2);
2614 };
2615
2616 for (unsigned i = 0; i < 4; i += 2) {
2617 int MLo = Mask[i];
2618 int MHi = Mask[i + 1];
2619
2620 if (MaskSize == 8) { // Only v8i32/v8f32 need this check.
2621 auto isValid2 = [&](int &M, int M2) {
2622 // If high half index is undef, it's always valid.
2623 if (M2 == -1)
2624 return true;
2625 if (M == -1) {
2626 // If low half index is undef, use index from high half,
2627 // remapped to low half.
2628 if ((M2 % MaskSize) < 4)
2629 return false;
2630 M = M2 - 4;
2631 return true;
2632 }
2633 // Index in low half must be same as index in high half.
2634 return M2 == M + 4;
2635 };
2636 if (!isValid2(MLo, Mask[i + 4]) || !isValid2(MHi, Mask[i + 5]))
2637 return false;
2638 }
2639
2640 if (isValid(MLo, 0) && isValid(MHi, 0)) {
2641 SrcVec.push_back(V1);
2642 buildImm(MLo, MHi, 0, i);
2643 } else if (isValid(MLo, MaskSize) && isValid(MHi, MaskSize)) {
2644 SrcVec.push_back(V2);
2645 buildImm(MLo, MHi, MaskSize, i);
2646 } else {
2647 return false;
2648 }
2649 }
2650
2651 return true;
2652}
2653
2654/// Lower VECTOR_SHUFFLE into VPERMI (if possible).
2655///
2656/// VPERMI selects two elements from each of the two vectors based on the
2657/// mask and places them in the corresponding positions of the result vector
2658/// in order. Only v4i32 and v4f32 types are allowed.
2659///
2660/// It is possible to lower into VPERMI when the mask consists of two of the
2661/// following forms concatenated:
2662/// <i, j, u, v>
2663/// <u, v, i, j>
2664/// where i,j are in [0,4) and u,v are in [4, 8).
2665/// For example:
2666/// <2, 3, 4, 5>
2667/// <5, 7, 0, 2>
2668///
2669/// When undef's appear in the mask they are treated as if they were whatever
2670/// value is necessary in order to fit the above forms.
2672 MVT VT, SDValue V1, SDValue V2,
2673 SelectionDAG &DAG,
2675 if ((VT != MVT::v4i32 && VT != MVT::v4f32) ||
2676 Mask.size() != VT.getVectorNumElements())
2677 return SDValue();
2678
2680 unsigned MaskImm = 0;
2681 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2682 return SDValue();
2683
2684 return DAG.getNode(LoongArchISD::VPERMI, DL, VT, SrcVec[1], SrcVec[0],
2685 DAG.getConstant(MaskImm, DL, Subtarget.getGRLenVT()));
2686}
2687
2688/// Lower VECTOR_SHUFFLE into VSHUF.
2689///
2690/// This mostly consists of converting the shuffle mask into a BUILD_VECTOR and
2691/// adding it as an operand to the resulting VSHUF.
2693 MVT VT, SDValue V1, SDValue V2,
2694 SelectionDAG &DAG,
2696
2698 for (auto M : Mask)
2699 Ops.push_back(DAG.getSignedConstant(M, DL, Subtarget.getGRLenVT()));
2700
2701 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
2702 SDValue MaskVec = DAG.getBuildVector(MaskVecTy, DL, Ops);
2703
2704 // VECTOR_SHUFFLE concatenates the vectors in an vectorwise fashion.
2705 // <0b00, 0b01> + <0b10, 0b11> -> <0b00, 0b01, 0b10, 0b11>
2706 // VSHF concatenates the vectors in a bitwise fashion:
2707 // <0b00, 0b01> + <0b10, 0b11> ->
2708 // 0b0100 + 0b1110 -> 0b01001110
2709 // <0b10, 0b11, 0b00, 0b01>
2710 // We must therefore swap the operands to get the correct result.
2711 return DAG.getNode(LoongArchISD::VSHUF, DL, VT, MaskVec, V2, V1);
2712}
2713
2714/// Dispatching routine to lower various 128-bit LoongArch vector shuffles.
2715///
2716/// This routine breaks down the specific type of 128-bit shuffle and
2717/// dispatches to the lowering routines accordingly.
2719 SDValue V1, SDValue V2, SelectionDAG &DAG,
2721 assert((VT.SimpleTy == MVT::v16i8 || VT.SimpleTy == MVT::v8i16 ||
2722 VT.SimpleTy == MVT::v4i32 || VT.SimpleTy == MVT::v2i64 ||
2723 VT.SimpleTy == MVT::v4f32 || VT.SimpleTy == MVT::v2f64) &&
2724 "Vector type is unsupported for lsx!");
2725 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
2726 "Two operands have different types!");
2727 assert(VT.getVectorNumElements() == Mask.size() &&
2728 "Unexpected mask size for shuffle!");
2729 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
2730
2731 APInt KnownUndef, KnownZero;
2732 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
2733 APInt Zeroable = KnownUndef | KnownZero;
2734
2735 SDValue Result;
2736 // TODO: Add more comparison patterns.
2737 if (V2.isUndef()) {
2738 if ((Result =
2739 lowerVECTOR_SHUFFLE_VREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
2740 return Result;
2741 if ((Result =
2742 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2743 return Result;
2744 if ((Result =
2745 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
2746 return Result;
2747
2748 // TODO: This comment may be enabled in the future to better match the
2749 // pattern for instruction selection.
2750 /* V2 = V1; */
2751 }
2752
2753 // It is recommended not to change the pattern comparison order for better
2754 // performance.
2755 if ((Result = lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG)))
2756 return Result;
2757 if ((Result = lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG)))
2758 return Result;
2759 if ((Result = lowerVECTOR_SHUFFLE_VILVH(DL, Mask, VT, V1, V2, DAG)))
2760 return Result;
2761 if ((Result = lowerVECTOR_SHUFFLE_VILVL(DL, Mask, VT, V1, V2, DAG)))
2762 return Result;
2763 if ((Result = lowerVECTOR_SHUFFLE_VPICKEV(DL, Mask, VT, V1, V2, DAG)))
2764 return Result;
2765 if ((Result = lowerVECTOR_SHUFFLE_VPICKOD(DL, Mask, VT, V1, V2, DAG)))
2766 return Result;
2767 if ((VT.SimpleTy == MVT::v2i64 || VT.SimpleTy == MVT::v2f64) &&
2768 (Result =
2769 lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2770 return Result;
2771 if ((Result =
2772 lowerVECTOR_SHUFFLE_VEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2773 return Result;
2774 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
2775 Zeroable)))
2776 return Result;
2777 if ((Result =
2778 lowerVECTOR_SHUFFLE_VPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2779 return Result;
2780 if ((Result = lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(DL, Mask, VT, V1, V2, DAG,
2781 Zeroable)))
2782 return Result;
2783 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
2784 Subtarget)))
2785 return Result;
2786 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
2787 return NewShuffle;
2788 if ((Result =
2789 lowerVECTOR_SHUFFLE_VSHUF(DL, Mask, VT, V1, V2, DAG, Subtarget)))
2790 return Result;
2791 return SDValue();
2792}
2793
2794/// Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
2795///
2796/// It is a XVREPLVEI when the mask is:
2797/// <x, x, x, ..., x+n, x+n, x+n, ...>
2798/// where the number of x is equal to n and n is half the length of vector.
2799///
2800/// When undef's appear in the mask they are treated as if they were whatever
2801/// value is necessary in order to fit the above form.
2802static SDValue
2804 SDValue V1, SelectionDAG &DAG,
2806 int SplatIndex = -1;
2807 for (const auto &M : Mask) {
2808 if (M != -1) {
2809 SplatIndex = M;
2810 break;
2811 }
2812 }
2813
2814 if (SplatIndex == -1)
2815 return DAG.getUNDEF(VT);
2816
2817 const auto &Begin = Mask.begin();
2818 const auto &End = Mask.end();
2819 int HalfSize = Mask.size() / 2;
2820
2821 if (SplatIndex >= HalfSize)
2822 return SDValue();
2823
2824 assert(SplatIndex < (int)Mask.size() && "Out of bounds mask index");
2825 if (fitsRegularPattern<int>(Begin, 1, End - HalfSize, SplatIndex, 0) &&
2826 fitsRegularPattern<int>(Begin + HalfSize, 1, End, SplatIndex + HalfSize,
2827 0)) {
2828 return DAG.getNode(LoongArchISD::VREPLVEI, DL, VT, V1,
2829 DAG.getConstant(SplatIndex, DL, Subtarget.getGRLenVT()));
2830 }
2831
2832 return SDValue();
2833}
2834
2835/// Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
2836static SDValue
2838 SDValue V1, SDValue V2, SelectionDAG &DAG,
2840 // XVSHUF4I_D must be handled separately because it is different from other
2841 // types of [X]VSHUF4I instructions.
2842 if (Mask.size() == 4) {
2843 unsigned MaskImm = 0;
2844 for (int i = 1; i >= 0; --i) {
2845 int MLo = Mask[i];
2846 int MHi = Mask[i + 2];
2847 if (!(MLo == -1 || (MLo >= 0 && MLo <= 1) || (MLo >= 4 && MLo <= 5)) ||
2848 !(MHi == -1 || (MHi >= 2 && MHi <= 3) || (MHi >= 6 && MHi <= 7)))
2849 return SDValue();
2850 if (MHi != -1 && MLo != -1 && MHi != MLo + 2)
2851 return SDValue();
2852
2853 MaskImm <<= 2;
2854 if (MLo != -1)
2855 MaskImm |= ((MLo <= 1) ? MLo : (MLo - 2)) & 0x3;
2856 else if (MHi != -1)
2857 MaskImm |= ((MHi <= 3) ? (MHi - 2) : (MHi - 4)) & 0x3;
2858 }
2859
2860 return DAG.getNode(LoongArchISD::VSHUF4I_D, DL, VT, V1, V2,
2861 DAG.getConstant(MaskImm, DL, Subtarget.getGRLenVT()));
2862 }
2863
2864 return lowerVECTOR_SHUFFLE_VSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget);
2865}
2866
2867/// Lower VECTOR_SHUFFLE into XVREPLVE0 (if possible).
2868///
2869/// It is a XVREPLVE0 when the VECTOR_SHUFFLE is a broadcast of element 0,
2870/// i.e. the mask is:
2871/// <0, 0, 0, ...>
2872///
2873/// When undef's appear in the mask they are treated as if they were whatever
2874/// value is necessary in order to fit the above form.
2876 ArrayRef<int> Mask, MVT VT,
2877 SDValue V1, SelectionDAG &DAG) {
2878 if (!ShuffleVectorInst::isZeroEltSplatMask(Mask, Mask.size()))
2879 return SDValue();
2880
2881 return DAG.getNode(LoongArchISD::XVREPLVE0, DL, VT, V1);
2882}
2883
2884/// Lower VECTOR_SHUFFLE into XVPERMI (if possible).
2885static SDValue
2887 SDValue V1, SDValue V2, SelectionDAG &DAG,
2889 MVT GRLenVT = Subtarget.getGRLenVT();
2890 unsigned MaskSize = Mask.size();
2891 if (MaskSize != VT.getVectorNumElements())
2892 return SDValue();
2893
2894 // Consider XVPERMI_W.
2895 if (VT == MVT::v8i32 || VT == MVT::v8f32) {
2897 unsigned MaskImm = 0;
2898 if (!buildVPERMIInfo(Mask, V1, V2, SrcVec, MaskImm))
2899 return SDValue();
2900
2901 return DAG.getNode(LoongArchISD::VPERMI, DL, VT, SrcVec[1], SrcVec[0],
2902 DAG.getConstant(MaskImm, DL, GRLenVT));
2903 }
2904
2905 // Consider XVPERMI_D.
2906 if (VT == MVT::v4i64 || VT == MVT::v4f64) {
2907 unsigned MaskImm = 0;
2908 for (unsigned i = 0; i < MaskSize; ++i) {
2909 if (Mask[i] == -1)
2910 continue;
2911 if (Mask[i] >= (int)MaskSize)
2912 return SDValue();
2913 MaskImm |= Mask[i] << (i * 2);
2914 }
2915
2916 return DAG.getNode(LoongArchISD::XVPERMI, DL, VT, V1,
2917 DAG.getConstant(MaskImm, DL, GRLenVT));
2918 }
2919
2920 return SDValue();
2921}
2922
2923/// Lower VECTOR_SHUFFLE into XVPERM (if possible).
2925 MVT VT, SDValue V1, SelectionDAG &DAG,
2927 // LoongArch LASX only have XVPERM_W.
2928 if (Mask.size() != 8 || (VT != MVT::v8i32 && VT != MVT::v8f32))
2929 return SDValue();
2930
2931 unsigned NumElts = VT.getVectorNumElements();
2932 unsigned HalfSize = NumElts / 2;
2933 bool FrontLo = true, FrontHi = true;
2934 bool BackLo = true, BackHi = true;
2935
2936 auto inRange = [](int val, int low, int high) {
2937 return (val == -1) || (val >= low && val < high);
2938 };
2939
2940 for (unsigned i = 0; i < HalfSize; ++i) {
2941 int Fronti = Mask[i];
2942 int Backi = Mask[i + HalfSize];
2943
2944 FrontLo &= inRange(Fronti, 0, HalfSize);
2945 FrontHi &= inRange(Fronti, HalfSize, NumElts);
2946 BackLo &= inRange(Backi, 0, HalfSize);
2947 BackHi &= inRange(Backi, HalfSize, NumElts);
2948 }
2949
2950 // If both the lower and upper 128-bit parts access only one half of the
2951 // vector (either lower or upper), avoid using xvperm.w. The latency of
2952 // xvperm.w(3) is higher than using xvshuf(1) and xvori(1).
2953 if ((FrontLo || FrontHi) && (BackLo || BackHi))
2954 return SDValue();
2955
2957 MVT GRLenVT = Subtarget.getGRLenVT();
2958 for (unsigned i = 0; i < NumElts; ++i)
2959 Masks.push_back(Mask[i] == -1 ? DAG.getUNDEF(GRLenVT)
2960 : DAG.getConstant(Mask[i], DL, GRLenVT));
2961 SDValue MaskVec = DAG.getBuildVector(MVT::v8i32, DL, Masks);
2962
2963 return DAG.getNode(LoongArchISD::XVPERM, DL, VT, V1, MaskVec);
2964}
2965
2966/// Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
2968 MVT VT, SDValue V1, SDValue V2,
2969 SelectionDAG &DAG) {
2970 return lowerVECTOR_SHUFFLE_VPACKEV(DL, Mask, VT, V1, V2, DAG);
2971}
2972
2973/// Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
2975 MVT VT, SDValue V1, SDValue V2,
2976 SelectionDAG &DAG) {
2977 return lowerVECTOR_SHUFFLE_VPACKOD(DL, Mask, VT, V1, V2, DAG);
2978}
2979
2980/// Lower VECTOR_SHUFFLE into XVILVH (if possible).
2982 MVT VT, SDValue V1, SDValue V2,
2983 SelectionDAG &DAG) {
2984
2985 const auto &Begin = Mask.begin();
2986 const auto &End = Mask.end();
2987 unsigned HalfSize = Mask.size() / 2;
2988 unsigned LeftSize = HalfSize / 2;
2989 SDValue OriV1 = V1, OriV2 = V2;
2990
2991 if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, HalfSize - LeftSize,
2992 1) &&
2993 fitsRegularPattern<int>(Begin + HalfSize, 2, End, HalfSize + LeftSize, 1))
2994 V1 = OriV1;
2995 else if (fitsRegularPattern<int>(Begin, 2, End - HalfSize,
2996 Mask.size() + HalfSize - LeftSize, 1) &&
2997 fitsRegularPattern<int>(Begin + HalfSize, 2, End,
2998 Mask.size() + HalfSize + LeftSize, 1))
2999 V1 = OriV2;
3000 else
3001 return SDValue();
3002
3003 if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, HalfSize - LeftSize,
3004 1) &&
3005 fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End, HalfSize + LeftSize,
3006 1))
3007 V2 = OriV1;
3008 else if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize,
3009 Mask.size() + HalfSize - LeftSize, 1) &&
3010 fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End,
3011 Mask.size() + HalfSize + LeftSize, 1))
3012 V2 = OriV2;
3013 else
3014 return SDValue();
3015
3016 return DAG.getNode(LoongArchISD::VILVH, DL, VT, V2, V1);
3017}
3018
3019/// Lower VECTOR_SHUFFLE into XVILVL (if possible).
3021 MVT VT, SDValue V1, SDValue V2,
3022 SelectionDAG &DAG) {
3023
3024 const auto &Begin = Mask.begin();
3025 const auto &End = Mask.end();
3026 unsigned HalfSize = Mask.size() / 2;
3027 SDValue OriV1 = V1, OriV2 = V2;
3028
3029 if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, 0, 1) &&
3030 fitsRegularPattern<int>(Begin + HalfSize, 2, End, HalfSize, 1))
3031 V1 = OriV1;
3032 else if (fitsRegularPattern<int>(Begin, 2, End - HalfSize, Mask.size(), 1) &&
3033 fitsRegularPattern<int>(Begin + HalfSize, 2, End,
3034 Mask.size() + HalfSize, 1))
3035 V1 = OriV2;
3036 else
3037 return SDValue();
3038
3039 if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, 0, 1) &&
3040 fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End, HalfSize, 1))
3041 V2 = OriV1;
3042 else if (fitsRegularPattern<int>(Begin + 1, 2, End - HalfSize, Mask.size(),
3043 1) &&
3044 fitsRegularPattern<int>(Begin + 1 + HalfSize, 2, End,
3045 Mask.size() + HalfSize, 1))
3046 V2 = OriV2;
3047 else
3048 return SDValue();
3049
3050 return DAG.getNode(LoongArchISD::VILVL, DL, VT, V2, V1);
3051}
3052
3053/// Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
3055 MVT VT, SDValue V1, SDValue V2,
3056 SelectionDAG &DAG) {
3057
3058 const auto &Begin = Mask.begin();
3059 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3060 const auto &Mid = Mask.begin() + Mask.size() / 2;
3061 const auto &RightMid = Mask.end() - Mask.size() / 4;
3062 const auto &End = Mask.end();
3063 unsigned HalfSize = Mask.size() / 2;
3064 SDValue OriV1 = V1, OriV2 = V2;
3065
3066 if (fitsRegularPattern<int>(Begin, 1, LeftMid, 0, 2) &&
3067 fitsRegularPattern<int>(Mid, 1, RightMid, HalfSize, 2))
3068 V1 = OriV1;
3069 else if (fitsRegularPattern<int>(Begin, 1, LeftMid, Mask.size(), 2) &&
3070 fitsRegularPattern<int>(Mid, 1, RightMid, Mask.size() + HalfSize, 2))
3071 V1 = OriV2;
3072 else
3073 return SDValue();
3074
3075 if (fitsRegularPattern<int>(LeftMid, 1, Mid, 0, 2) &&
3076 fitsRegularPattern<int>(RightMid, 1, End, HalfSize, 2))
3077 V2 = OriV1;
3078 else if (fitsRegularPattern<int>(LeftMid, 1, Mid, Mask.size(), 2) &&
3079 fitsRegularPattern<int>(RightMid, 1, End, Mask.size() + HalfSize, 2))
3080 V2 = OriV2;
3081
3082 else
3083 return SDValue();
3084
3085 return DAG.getNode(LoongArchISD::VPICKEV, DL, VT, V2, V1);
3086}
3087
3088/// Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
3090 MVT VT, SDValue V1, SDValue V2,
3091 SelectionDAG &DAG) {
3092
3093 const auto &Begin = Mask.begin();
3094 const auto &LeftMid = Mask.begin() + Mask.size() / 4;
3095 const auto &Mid = Mask.begin() + Mask.size() / 2;
3096 const auto &RightMid = Mask.end() - Mask.size() / 4;
3097 const auto &End = Mask.end();
3098 unsigned HalfSize = Mask.size() / 2;
3099 SDValue OriV1 = V1, OriV2 = V2;
3100
3101 if (fitsRegularPattern<int>(Begin, 1, LeftMid, 1, 2) &&
3102 fitsRegularPattern<int>(Mid, 1, RightMid, HalfSize + 1, 2))
3103 V1 = OriV1;
3104 else if (fitsRegularPattern<int>(Begin, 1, LeftMid, Mask.size() + 1, 2) &&
3105 fitsRegularPattern<int>(Mid, 1, RightMid, Mask.size() + HalfSize + 1,
3106 2))
3107 V1 = OriV2;
3108 else
3109 return SDValue();
3110
3111 if (fitsRegularPattern<int>(LeftMid, 1, Mid, 1, 2) &&
3112 fitsRegularPattern<int>(RightMid, 1, End, HalfSize + 1, 2))
3113 V2 = OriV1;
3114 else if (fitsRegularPattern<int>(LeftMid, 1, Mid, Mask.size() + 1, 2) &&
3115 fitsRegularPattern<int>(RightMid, 1, End, Mask.size() + HalfSize + 1,
3116 2))
3117 V2 = OriV2;
3118 else
3119 return SDValue();
3120
3121 return DAG.getNode(LoongArchISD::VPICKOD, DL, VT, V2, V1);
3122}
3123
3124/// Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
3125static SDValue
3127 SDValue V1, SDValue V2, SelectionDAG &DAG,
3129 int NumElts = VT.getVectorNumElements();
3130 int HalfSize = NumElts / 2;
3131 MVT EltVT = VT.getVectorElementType();
3132 MVT GRLenVT = Subtarget.getGRLenVT();
3133
3134 if ((int)Mask.size() != NumElts)
3135 return SDValue();
3136
3137 auto tryLowerToExtrAndIns = [&](int Base) -> SDValue {
3138 SmallVector<int> DiffPos;
3139 for (int i = 0; i < NumElts; ++i) {
3140 if (Mask[i] == -1)
3141 continue;
3142 if (Mask[i] != Base + i) {
3143 DiffPos.push_back(i);
3144 if (DiffPos.size() > 2)
3145 return SDValue();
3146 }
3147 }
3148
3149 // Need exactly two differing element to lower into XVEXTRINS.
3150 // If only one differing element, the element at a distance of
3151 // HalfSize from it must be undef.
3152 if (DiffPos.size() == 1) {
3153 if (DiffPos[0] < HalfSize && Mask[DiffPos[0] + HalfSize] == -1)
3154 DiffPos.push_back(DiffPos[0] + HalfSize);
3155 else if (DiffPos[0] >= HalfSize && Mask[DiffPos[0] - HalfSize] == -1)
3156 DiffPos.insert(DiffPos.begin(), DiffPos[0] - HalfSize);
3157 else
3158 return SDValue();
3159 }
3160 if (DiffPos.size() != 2 || DiffPos[1] != DiffPos[0] + HalfSize)
3161 return SDValue();
3162
3163 // DiffMask must be in its low or high part.
3164 int DiffMaskLo = Mask[DiffPos[0]];
3165 int DiffMaskHi = Mask[DiffPos[1]];
3166 DiffMaskLo = DiffMaskLo == -1 ? DiffMaskHi - HalfSize : DiffMaskLo;
3167 DiffMaskHi = DiffMaskHi == -1 ? DiffMaskLo + HalfSize : DiffMaskHi;
3168 if (!(DiffMaskLo >= 0 && DiffMaskLo < HalfSize) &&
3169 !(DiffMaskLo >= NumElts && DiffMaskLo < NumElts + HalfSize))
3170 return SDValue();
3171 if (!(DiffMaskHi >= HalfSize && DiffMaskHi < NumElts) &&
3172 !(DiffMaskHi >= NumElts + HalfSize && DiffMaskHi < 2 * NumElts))
3173 return SDValue();
3174 if (DiffMaskHi != DiffMaskLo + HalfSize)
3175 return SDValue();
3176
3177 // Determine source vector and source index.
3178 SDValue SrcVec = (DiffMaskLo < HalfSize) ? V1 : V2;
3179 int SrcIdxLo =
3180 (DiffMaskLo < HalfSize) ? DiffMaskLo : (DiffMaskLo - NumElts);
3181 bool IsEltFP = EltVT.isFloatingPoint();
3182
3183 // Replace with 2*EXTRACT_VECTOR_ELT + 2*INSERT_VECTOR_ELT, it will match
3184 // the patterns of XVEXTRINS in tablegen.
3185 SDValue BaseVec = (Base == 0) ? V1 : V2;
3186 SDValue EltLo =
3187 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IsEltFP ? EltVT : GRLenVT,
3188 SrcVec, DAG.getConstant(SrcIdxLo, DL, GRLenVT));
3189 SDValue InsLo = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, BaseVec, EltLo,
3190 DAG.getConstant(DiffPos[0], DL, GRLenVT));
3191 SDValue EltHi =
3192 DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, IsEltFP ? EltVT : GRLenVT,
3193 SrcVec, DAG.getConstant(SrcIdxLo + HalfSize, DL, GRLenVT));
3194 SDValue Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, VT, InsLo, EltHi,
3195 DAG.getConstant(DiffPos[1], DL, GRLenVT));
3196
3197 return Result;
3198 };
3199
3200 // Try [0, n-1) insertion then [n, 2n-1) insertion.
3201 if (SDValue Result = tryLowerToExtrAndIns(0))
3202 return Result;
3203 return tryLowerToExtrAndIns(NumElts);
3204}
3205
3206/// Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
3207static SDValue
3209 SDValue V1, SDValue V2, SelectionDAG &DAG,
3211 // LoongArch LASX only supports xvinsve0.{w/d}.
3212 if (VT != MVT::v8i32 && VT != MVT::v8f32 && VT != MVT::v4i64 &&
3213 VT != MVT::v4f64)
3214 return SDValue();
3215
3216 MVT GRLenVT = Subtarget.getGRLenVT();
3217 int MaskSize = Mask.size();
3218 assert(MaskSize == (int)VT.getVectorNumElements() && "Unexpected mask size");
3219
3220 // Check if exactly one element of the Mask is replaced by 'Replaced', while
3221 // all other elements are either 'Base + i' or undef (-1). On success, return
3222 // the index of the replaced element. Otherwise, just return -1.
3223 auto checkReplaceOne = [&](int Base, int Replaced) -> int {
3224 int Idx = -1;
3225 for (int i = 0; i < MaskSize; ++i) {
3226 if (Mask[i] == Base + i || Mask[i] == -1)
3227 continue;
3228 if (Mask[i] != Replaced)
3229 return -1;
3230 if (Idx == -1)
3231 Idx = i;
3232 else
3233 return -1;
3234 }
3235 return Idx;
3236 };
3237
3238 // Case 1: the lowest element of V2 replaces one element in V1.
3239 int Idx = checkReplaceOne(0, MaskSize);
3240 if (Idx != -1)
3241 return DAG.getNode(LoongArchISD::XVINSVE0, DL, VT, V1, V2,
3242 DAG.getConstant(Idx, DL, GRLenVT));
3243
3244 // Case 2: the lowest element of V1 replaces one element in V2.
3245 Idx = checkReplaceOne(MaskSize, 0);
3246 if (Idx != -1)
3247 return DAG.getNode(LoongArchISD::XVINSVE0, DL, VT, V2, V1,
3248 DAG.getConstant(Idx, DL, GRLenVT));
3249
3250 return SDValue();
3251}
3252
3253/// Lower VECTOR_SHUFFLE into XVSHUF (if possible).
3255 MVT VT, SDValue V1, SDValue V2,
3256 SelectionDAG &DAG) {
3257
3258 int MaskSize = Mask.size();
3259 int HalfSize = Mask.size() / 2;
3260 const auto &Begin = Mask.begin();
3261 const auto &Mid = Mask.begin() + HalfSize;
3262 const auto &End = Mask.end();
3263
3264 // VECTOR_SHUFFLE concatenates the vectors:
3265 // <0, 1, 2, 3, 4, 5, 6, 7> + <8, 9, 10, 11, 12, 13, 14, 15>
3266 // shuffling ->
3267 // <0, 1, 2, 3, 8, 9, 10, 11> <4, 5, 6, 7, 12, 13, 14, 15>
3268 //
3269 // XVSHUF concatenates the vectors:
3270 // <a0, a1, a2, a3, b0, b1, b2, b3> + <a4, a5, a6, a7, b4, b5, b6, b7>
3271 // shuffling ->
3272 // <a0, a1, a2, a3, a4, a5, a6, a7> + <b0, b1, b2, b3, b4, b5, b6, b7>
3273 SmallVector<SDValue, 8> MaskAlloc;
3274 for (auto it = Begin; it < Mid; it++) {
3275 if (*it < 0) // UNDEF
3276 MaskAlloc.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
3277 else if ((*it >= 0 && *it < HalfSize) ||
3278 (*it >= MaskSize && *it < MaskSize + HalfSize)) {
3279 int M = *it < HalfSize ? *it : *it - HalfSize;
3280 MaskAlloc.push_back(DAG.getTargetConstant(M, DL, MVT::i64));
3281 } else
3282 return SDValue();
3283 }
3284 assert((int)MaskAlloc.size() == HalfSize && "xvshuf convert failed!");
3285
3286 for (auto it = Mid; it < End; it++) {
3287 if (*it < 0) // UNDEF
3288 MaskAlloc.push_back(DAG.getTargetConstant(0, DL, MVT::i64));
3289 else if ((*it >= HalfSize && *it < MaskSize) ||
3290 (*it >= MaskSize + HalfSize && *it < MaskSize * 2)) {
3291 int M = *it < MaskSize ? *it - HalfSize : *it - MaskSize;
3292 MaskAlloc.push_back(DAG.getTargetConstant(M, DL, MVT::i64));
3293 } else
3294 return SDValue();
3295 }
3296 assert((int)MaskAlloc.size() == MaskSize && "xvshuf convert failed!");
3297
3298 EVT MaskVecTy = VT.changeVectorElementTypeToInteger();
3299 SDValue MaskVec = DAG.getBuildVector(MaskVecTy, DL, MaskAlloc);
3300 return DAG.getNode(LoongArchISD::VSHUF, DL, VT, MaskVec, V2, V1);
3301}
3302
3303/// Shuffle vectors by lane to generate more optimized instructions.
3304/// 256-bit shuffles are always considered as 2-lane 128-bit shuffles.
3305///
3306/// Therefore, except for the following four cases, other cases are regarded
3307/// as cross-lane shuffles, where optimization is relatively limited.
3308///
3309/// - Shuffle high, low lanes of two inputs vector
3310/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 3, 6>
3311/// - Shuffle low, high lanes of two inputs vector
3312/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 0, 5>
3313/// - Shuffle low, low lanes of two inputs vector
3314/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <3, 6, 3, 6>
3315/// - Shuffle high, high lanes of two inputs vector
3316/// <0, 1, 2, 3> + <4, 5, 6, 7> --- <0, 5, 0, 5>
3317///
3318/// The first case is the closest to LoongArch instructions and the other
3319/// cases need to be converted to it for processing.
3320///
3321/// This function will return true for the last three cases above and will
3322/// modify V1, V2 and Mask. Otherwise, return false for the first case and
3323/// cross-lane shuffle cases.
3325 const SDLoc &DL, MutableArrayRef<int> Mask, MVT VT, SDValue &V1,
3327
3328 enum HalfMaskType { HighLaneTy, LowLaneTy, None };
3329
3330 int MaskSize = Mask.size();
3331 int HalfSize = Mask.size() / 2;
3332 MVT GRLenVT = Subtarget.getGRLenVT();
3333
3334 HalfMaskType preMask = None, postMask = None;
3335
3336 if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](int M) {
3337 return M < 0 || (M >= 0 && M < HalfSize) ||
3338 (M >= MaskSize && M < MaskSize + HalfSize);
3339 }))
3340 preMask = HighLaneTy;
3341 else if (std::all_of(Mask.begin(), Mask.begin() + HalfSize, [&](int M) {
3342 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3343 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3344 }))
3345 preMask = LowLaneTy;
3346
3347 if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](int M) {
3348 return M < 0 || (M >= HalfSize && M < MaskSize) ||
3349 (M >= MaskSize + HalfSize && M < MaskSize * 2);
3350 }))
3351 postMask = LowLaneTy;
3352 else if (std::all_of(Mask.begin() + HalfSize, Mask.end(), [&](int M) {
3353 return M < 0 || (M >= 0 && M < HalfSize) ||
3354 (M >= MaskSize && M < MaskSize + HalfSize);
3355 }))
3356 postMask = HighLaneTy;
3357
3358 // The pre-half of mask is high lane type, and the post-half of mask
3359 // is low lane type, which is closest to the LoongArch instructions.
3360 //
3361 // Note: In the LoongArch architecture, the high lane of mask corresponds
3362 // to the lower 128-bit of vector register, and the low lane of mask
3363 // corresponds the higher 128-bit of vector register.
3364 if (preMask == HighLaneTy && postMask == LowLaneTy) {
3365 return false;
3366 }
3367 if (preMask == LowLaneTy && postMask == HighLaneTy) {
3368 V1 = DAG.getBitcast(MVT::v4i64, V1);
3369 V1 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V1,
3370 DAG.getConstant(0b01001110, DL, GRLenVT));
3371 V1 = DAG.getBitcast(VT, V1);
3372
3373 if (!V2.isUndef()) {
3374 V2 = DAG.getBitcast(MVT::v4i64, V2);
3375 V2 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V2,
3376 DAG.getConstant(0b01001110, DL, GRLenVT));
3377 V2 = DAG.getBitcast(VT, V2);
3378 }
3379
3380 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3381 *it = *it < 0 ? *it : *it - HalfSize;
3382 }
3383 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3384 *it = *it < 0 ? *it : *it + HalfSize;
3385 }
3386 } else if (preMask == LowLaneTy && postMask == LowLaneTy) {
3387 V1 = DAG.getBitcast(MVT::v4i64, V1);
3388 V1 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V1,
3389 DAG.getConstant(0b11101110, DL, GRLenVT));
3390 V1 = DAG.getBitcast(VT, V1);
3391
3392 if (!V2.isUndef()) {
3393 V2 = DAG.getBitcast(MVT::v4i64, V2);
3394 V2 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V2,
3395 DAG.getConstant(0b11101110, DL, GRLenVT));
3396 V2 = DAG.getBitcast(VT, V2);
3397 }
3398
3399 for (auto it = Mask.begin(); it < Mask.begin() + HalfSize; it++) {
3400 *it = *it < 0 ? *it : *it - HalfSize;
3401 }
3402 } else if (preMask == HighLaneTy && postMask == HighLaneTy) {
3403 V1 = DAG.getBitcast(MVT::v4i64, V1);
3404 V1 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V1,
3405 DAG.getConstant(0b01000100, DL, GRLenVT));
3406 V1 = DAG.getBitcast(VT, V1);
3407
3408 if (!V2.isUndef()) {
3409 V2 = DAG.getBitcast(MVT::v4i64, V2);
3410 V2 = DAG.getNode(LoongArchISD::XVPERMI, DL, MVT::v4i64, V2,
3411 DAG.getConstant(0b01000100, DL, GRLenVT));
3412 V2 = DAG.getBitcast(VT, V2);
3413 }
3414
3415 for (auto it = Mask.begin() + HalfSize; it < Mask.end(); it++) {
3416 *it = *it < 0 ? *it : *it + HalfSize;
3417 }
3418 } else { // cross-lane
3419 return false;
3420 }
3421
3422 return true;
3423}
3424
3425/// Lower VECTOR_SHUFFLE into XVPERMI_Q (if possible).
3426///
3427/// For example:
3428/// %2 = shufflevector <4 x i64> %0, <4 x i64> %1,
3429/// <4 x i32> <i32 2, i32 3, i32 4, i32 5>
3430/// is lowered to:
3431/// (XVPERMI_Q $xr1, $xr0, 33)
3432static SDValue
3434 SDValue V1, SDValue V2, SelectionDAG &DAG,
3436 // Each 128-bit lane of the result must be a whole lane of V1 or V2.
3437 SmallVector<int, 2> LaneMask;
3438 if (!widenShuffleMaskElts(Mask.size() / 2, Mask, LaneMask))
3439 return SDValue();
3440
3441 // Lanes 0-3 are V1.lo, V1.hi, V2.lo and V2.hi, the same as the XVPERMI_Q
3442 // selector with xj = V1 and xd = V2.
3443 int Lo = LaneMask[0] < 0 ? 0 : LaneMask[0];
3444 int Hi = LaneMask[1] < 0 ? 1 : LaneMask[1];
3445 return DAG.getNode(
3446 LoongArchISD::XVPERMIQ, DL, VT, V2, V1,
3447 DAG.getConstant(Lo | (Hi << 4), DL, Subtarget.getGRLenVT()));
3448}
3449
3450/// Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
3451/// Only for 256-bit vector.
3452///
3453/// Each 128-bit lane of the result may use at most two lanes of the inputs.
3454/// These lanes are first moved into place, then shuffled within the lane.
3455///
3456/// For example:
3457/// %2 = shufflevector <4 x i64> %0, <4 x i64> posion,
3458/// <4 x i64> <i32 0, i32 3, i32 2, i32 0>
3459/// is lowerded to:
3460/// (XVPERMI $xr2, $xr0, 78)
3461/// (XVSHUF $xr1, $xr2, $xr0)
3462/// (XVORI $xr0, $xr1, 0)
3463///
3464/// and the splice:
3465/// %2 = shufflevector <8 x i32> %0, <8 x i32> %1,
3466/// <8 x i32> <i32 7, i32 8, i32 9, i32 10,
3467/// i32 11, i32 12, i32 13, i32 14>
3468/// is lowered to:
3469/// (XVBSLL_V $xr2, $xr1, 4)
3470/// (XVPERMI_Q $xr1, $xr0, 33)
3471/// (XVBSRL_V $xr0, $xr1, 12)
3472/// (XVOR_V $xr0, $xr2, $xr0)
3474 ArrayRef<int> Mask,
3475 MVT VT, SDValue V1,
3476 SDValue V2,
3477 SelectionDAG &DAG) {
3478 assert(VT.is256BitVector() && "Only for 256-bit vector shuffles!");
3479 int Size = Mask.size();
3480 int LaneSize = Size / 2;
3481
3482 // For each result lane, the input lanes (0/1: V1, 2/3: V2) used by the two
3483 // operands of the in-lane shuffle.
3484 int LaneSrcs[2][2] = {{-1, -1}, {-1, -1}};
3485 SmallVector<int, 32> InLaneMask(Size, -1);
3486 bool LaneCrossing = false;
3487 for (int i = 0; i < Size; ++i) {
3488 int M = Mask[i];
3489 if (M < 0)
3490 continue;
3491 int Lane = i / LaneSize;
3492 int SrcLane = M / LaneSize;
3493
3494 // Put lanes that stay in place in the first operand and the others in the
3495 // second one, so a single input becomes V1 and V1 with lanes swapped.
3496 int Op = (SrcLane % 2) != Lane;
3497 LaneCrossing |= Op;
3498 if (LaneSrcs[Lane][Op] >= 0 && LaneSrcs[Lane][Op] != SrcLane)
3499 Op ^= 1;
3500 if (LaneSrcs[Lane][Op] >= 0 && LaneSrcs[Lane][Op] != SrcLane)
3501 return SDValue();
3502 LaneSrcs[Lane][Op] = SrcLane;
3503 InLaneMask[i] = Op * Size + Lane * LaneSize + M % LaneSize;
3504 }
3505
3506 if (!LaneCrossing)
3507 return SDValue();
3508
3509 // Swap the operands of the high lane if that makes both lanes use the same
3510 // in-lane mask, which more patterns (e.g. XVPICKEV) can match.
3511 SmallVector<int, 16> RepeatedMask;
3512 if (!isRepeatedShuffleMask(128, VT, InLaneMask, RepeatedMask)) {
3513 SmallVector<int, 32> SwappedMask(InLaneMask);
3514 for (int i = LaneSize; i < Size; ++i)
3515 if (SwappedMask[i] >= 0)
3516 SwappedMask[i] = (SwappedMask[i] + Size) % (2 * Size);
3517 if (isRepeatedShuffleMask(128, VT, SwappedMask, RepeatedMask)) {
3518 std::swap(LaneSrcs[1][0], LaneSrcs[1][1]);
3519 InLaneMask = SwappedMask;
3520 }
3521 }
3522
3523 // Move the input lanes of each operand into place.
3524 SDValue Ops[2];
3525 for (int Op = 0; Op < 2; ++Op) {
3526 SmallVector<int, 32> LaneMask(Size, -1);
3527 for (int Lane = 0; Lane < 2; ++Lane)
3528 if (LaneSrcs[Lane][Op] >= 0)
3529 for (int i = 0; i < LaneSize; ++i)
3530 LaneMask[Lane * LaneSize + i] = LaneSrcs[Lane][Op] * LaneSize + i;
3531 Ops[Op] = DAG.getVectorShuffle(VT, DL, V1, V2, LaneMask);
3532
3533 // Exit if we got back the shuffle we started with, to avoid infinite
3534 // recursion.
3535 if (auto *SVN = dyn_cast<ShuffleVectorSDNode>(Ops[Op]))
3536 if (SVN->getMask() == Mask)
3537 return SDValue();
3538 }
3539 return DAG.getVectorShuffle(VT, DL, Ops[0], Ops[1], InLaneMask);
3540}
3541
3542/// Dispatching routine to lower various 256-bit LoongArch vector shuffles.
3543///
3544/// This routine breaks down the specific type of 256-bit shuffle and
3545/// dispatches to the lowering routines accordingly.
3547 SDValue V1, SDValue V2, SelectionDAG &DAG,
3549 assert((VT.SimpleTy == MVT::v32i8 || VT.SimpleTy == MVT::v16i16 ||
3550 VT.SimpleTy == MVT::v8i32 || VT.SimpleTy == MVT::v4i64 ||
3551 VT.SimpleTy == MVT::v8f32 || VT.SimpleTy == MVT::v4f64) &&
3552 "Vector type is unsupported for lasx!");
3553 assert(V1.getSimpleValueType() == V2.getSimpleValueType() &&
3554 "Two operands have different types!");
3555 assert(VT.getVectorNumElements() == Mask.size() &&
3556 "Unexpected mask size for shuffle!");
3557 assert(Mask.size() % 2 == 0 && "Expected even mask size.");
3558 assert(Mask.size() >= 4 && "Mask size is less than 4.");
3559
3560 APInt KnownUndef, KnownZero;
3561 computeZeroableShuffleElements(Mask, V1, V2, KnownUndef, KnownZero);
3562 APInt Zeroable = KnownUndef | KnownZero;
3563
3564 SDValue Result;
3565 // TODO: Add more comparison patterns.
3566 if (V2.isUndef()) {
3567 if ((Result =
3568 lowerVECTOR_SHUFFLE_XVREPLVEI(DL, Mask, VT, V1, DAG, Subtarget)))
3569 return Result;
3570 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG,
3571 Subtarget)))
3572 return Result;
3573 if ((Result = lowerVECTOR_SHUFFLE_XVREPLVE0(DL, Mask, VT, V1, DAG)))
3574 return Result;
3575 // Try to widen vectors to gain more optimization opportunities.
3576 if (SDValue NewShuffle = widenShuffleMask(DL, Mask, VT, V1, V2, DAG))
3577 return NewShuffle;
3578 if ((Result =
3579 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3580 return Result;
3581 if ((Result = lowerVECTOR_SHUFFLE_XVPERM(DL, Mask, VT, V1, DAG, Subtarget)))
3582 return Result;
3583 if ((Result =
3584 lowerVECTOR_SHUFFLE_IsReverse(DL, Mask, VT, V1, DAG, Subtarget)))
3585 return Result;
3586
3587 // TODO: This comment may be enabled in the future to better match the
3588 // pattern for instruction selection.
3589 /* V2 = V1; */
3590 }
3591
3592 // It is recommended not to change the pattern comparison order for better
3593 // performance.
3594 if ((Result = lowerVECTOR_SHUFFLE_XVPACKEV(DL, Mask, VT, V1, V2, DAG)))
3595 return Result;
3596 if ((Result = lowerVECTOR_SHUFFLE_XVPACKOD(DL, Mask, VT, V1, V2, DAG)))
3597 return Result;
3598 if ((Result = lowerVECTOR_SHUFFLE_XVILVH(DL, Mask, VT, V1, V2, DAG)))
3599 return Result;
3600 if ((Result = lowerVECTOR_SHUFFLE_XVILVL(DL, Mask, VT, V1, V2, DAG)))
3601 return Result;
3602 if ((Result = lowerVECTOR_SHUFFLE_XVPICKEV(DL, Mask, VT, V1, V2, DAG)))
3603 return Result;
3604 if ((Result = lowerVECTOR_SHUFFLE_XVPICKOD(DL, Mask, VT, V1, V2, DAG)))
3605 return Result;
3606 if ((VT.SimpleTy == MVT::v4i64 || VT.SimpleTy == MVT::v4f64) &&
3607 (Result =
3608 lowerVECTOR_SHUFFLE_XVSHUF4I(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3609 return Result;
3610 if ((Result =
3611 lowerVECTOR_SHUFFLE_XVEXTRINS(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3612 return Result;
3613 if ((Result = lowerVECTOR_SHUFFLEAsShift(DL, Mask, VT, V1, V2, DAG, Subtarget,
3614 Zeroable)))
3615 return Result;
3616 if ((Result =
3617 lowerVECTOR_SHUFFLE_XVPERMI(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3618 return Result;
3619 if ((Result =
3620 lowerVECTOR_SHUFFLE_XVINSVE0(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3621 return Result;
3622 if ((Result = lowerVECTOR_SHUFFLEAsByteRotate(DL, Mask, VT, V1, V2, DAG,
3623 Subtarget)))
3624 return Result;
3625
3626 if ((Result =
3627 lowerVECTOR_SHUFFLE_XVPERMIQ(DL, Mask, VT, V1, V2, DAG, Subtarget)))
3628 return Result;
3629
3630 // canonicalize non cross-lane shuffle vector
3631 SmallVector<int> NewMask(Mask);
3632 if (canonicalizeShuffleVectorByLane(DL, NewMask, VT, V1, V2, DAG, Subtarget))
3633 return lower256BitShuffle(DL, NewMask, VT, V1, V2, DAG, Subtarget);
3634
3635 // FIXME: Handling the remaining cases earlier can degrade performance
3636 // in some situations. Further analysis is required to enable more
3637 // effective optimizations.
3638 if ((Result = lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(DL, NewMask, VT, V1,
3639 V2, DAG)))
3640 return Result;
3641
3642 if (SDValue NewShuffle = widenShuffleMask(DL, NewMask, VT, V1, V2, DAG))
3643 return NewShuffle;
3644 if ((Result = lowerVECTOR_SHUFFLE_XVSHUF(DL, NewMask, VT, V1, V2, DAG)))
3645 return Result;
3646
3647 return SDValue();
3648}
3649
3650SDValue LoongArchTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
3651 SelectionDAG &DAG) const {
3652 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
3653 ArrayRef<int> OrigMask = SVOp->getMask();
3654 SDValue V1 = Op.getOperand(0);
3655 SDValue V2 = Op.getOperand(1);
3656 MVT VT = Op.getSimpleValueType();
3657 int NumElements = VT.getVectorNumElements();
3658 SDLoc DL(Op);
3659
3660 bool V1IsUndef = V1.isUndef();
3661 bool V2IsUndef = V2.isUndef();
3662 if (V1IsUndef && V2IsUndef)
3663 return DAG.getUNDEF(VT);
3664
3665 // When we create a shuffle node we put the UNDEF node to second operand,
3666 // but in some cases the first operand may be transformed to UNDEF.
3667 // In this case we should just commute the node.
3668 if (V1IsUndef)
3669 return DAG.getCommutedVectorShuffle(*SVOp);
3670
3671 // Check for non-undef masks pointing at an undef vector and make the masks
3672 // undef as well. This makes it easier to match the shuffle based solely on
3673 // the mask.
3674 if (V2IsUndef &&
3675 any_of(OrigMask, [NumElements](int M) { return M >= NumElements; })) {
3676 SmallVector<int, 8> NewMask(OrigMask);
3677 for (int &M : NewMask)
3678 if (M >= NumElements)
3679 M = -1;
3680 return DAG.getVectorShuffle(VT, DL, V1, V2, NewMask);
3681 }
3682
3683 // Check for illegal shuffle mask element index values.
3684 int MaskUpperLimit = OrigMask.size() * (V2IsUndef ? 1 : 2);
3685 (void)MaskUpperLimit;
3686 assert(llvm::all_of(OrigMask,
3687 [&](int M) { return -1 <= M && M < MaskUpperLimit; }) &&
3688 "Out of bounds shuffle index");
3689
3690 // For each vector width, delegate to a specialized lowering routine.
3691 if (VT.is128BitVector())
3692 return lower128BitShuffle(DL, OrigMask, VT, V1, V2, DAG, Subtarget);
3693
3694 if (VT.is256BitVector())
3695 return lower256BitShuffle(DL, OrigMask, VT, V1, V2, DAG, Subtarget);
3696
3697 return SDValue();
3698}
3699
3700SDValue LoongArchTargetLowering::lowerFP_TO_FP16(SDValue Op,
3701 SelectionDAG &DAG) const {
3702 // Custom lower to ensure the libcall return is passed in an FPR on hard
3703 // float ABIs.
3704 SDLoc DL(Op);
3705 MakeLibCallOptions CallOptions;
3706 SDValue Op0 = Op.getOperand(0);
3707 SDValue Chain = SDValue();
3708 RTLIB::Libcall LC = RTLIB::getFPROUND(Op0.getValueType(), MVT::f16);
3709 SDValue Res;
3710 std::tie(Res, Chain) =
3711 makeLibCall(DAG, LC, MVT::f32, Op0, CallOptions, DL, Chain);
3712 if (Subtarget.is64Bit())
3713 return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Res);
3714 return DAG.getBitcast(MVT::i32, Res);
3715}
3716
3717SDValue LoongArchTargetLowering::lowerFP16_TO_FP(SDValue Op,
3718 SelectionDAG &DAG) const {
3719 // Custom lower to ensure the libcall argument is passed in an FPR on hard
3720 // float ABIs.
3721 SDLoc DL(Op);
3722 MakeLibCallOptions CallOptions;
3723 SDValue Op0 = Op.getOperand(0);
3724 SDValue Chain = SDValue();
3725 SDValue Arg = Subtarget.is64Bit() ? DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64,
3726 DL, MVT::f32, Op0)
3727 : DAG.getBitcast(MVT::f32, Op0);
3728 SDValue Res;
3729 std::tie(Res, Chain) = makeLibCall(DAG, RTLIB::FPEXT_F16_F32, MVT::f32, Arg,
3730 CallOptions, DL, Chain);
3731 return Res;
3732}
3733
3734SDValue LoongArchTargetLowering::lowerFP_TO_BF16(SDValue Op,
3735 SelectionDAG &DAG) const {
3736 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3737 SDLoc DL(Op);
3738 MakeLibCallOptions CallOptions;
3739 RTLIB::Libcall LC =
3740 RTLIB::getFPROUND(Op.getOperand(0).getValueType(), MVT::bf16);
3741 SDValue Res =
3742 makeLibCall(DAG, LC, MVT::f32, Op.getOperand(0), CallOptions, DL).first;
3743 if (Subtarget.is64Bit())
3744 return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Res);
3745 return DAG.getBitcast(MVT::i32, Res);
3746}
3747
3748SDValue LoongArchTargetLowering::lowerBF16_TO_FP(SDValue Op,
3749 SelectionDAG &DAG) const {
3750 assert(Subtarget.hasBasicF() && "Unexpected custom legalization");
3751 MVT VT = Op.getSimpleValueType();
3752 SDLoc DL(Op);
3753 Op = DAG.getNode(
3754 ISD::SHL, DL, Op.getOperand(0).getValueType(), Op.getOperand(0),
3755 DAG.getShiftAmountConstant(16, Op.getOperand(0).getValueType(), DL));
3756 SDValue Res = Subtarget.is64Bit() ? DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64,
3757 DL, MVT::f32, Op)
3758 : DAG.getBitcast(MVT::f32, Op);
3759 if (VT != MVT::f32)
3760 return DAG.getNode(ISD::FP_EXTEND, DL, VT, Res);
3761 return Res;
3762}
3763
3764// Lower BUILD_VECTOR as broadcast load (if possible).
3765// For example:
3766// %a = load i8, ptr %ptr
3767// %b = build_vector %a, %a, %a, %a
3768// is lowered to :
3769// (VLDREPL_B $a0, 0)
3771 const SDLoc &DL,
3772 SelectionDAG &DAG) {
3773 MVT VT = BVOp->getSimpleValueType(0);
3774 int NumOps = BVOp->getNumOperands();
3775
3776 assert((VT.is128BitVector() || VT.is256BitVector()) &&
3777 "Unsupported vector type for broadcast.");
3778
3779 SDValue IdentitySrc;
3780 bool IsIdeneity = true;
3781
3782 for (int i = 0; i != NumOps; i++) {
3783 SDValue Op = BVOp->getOperand(i);
3784 if (Op.getOpcode() != ISD::LOAD || (IdentitySrc && Op != IdentitySrc)) {
3785 IsIdeneity = false;
3786 break;
3787 }
3788 IdentitySrc = BVOp->getOperand(0);
3789 }
3790
3791 // make sure that this load is valid and only has one user.
3792 if (!IsIdeneity || !IdentitySrc || !BVOp->isOnlyUserOf(IdentitySrc.getNode()))
3793 return SDValue();
3794
3795 auto *LN = cast<LoadSDNode>(IdentitySrc);
3796 auto ExtType = LN->getExtensionType();
3797
3798 if ((ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) &&
3799 VT.getScalarSizeInBits() == LN->getMemoryVT().getScalarSizeInBits()) {
3800 // Indexed loads and stores are not supported on LoongArch.
3801 assert(LN->isUnindexed() && "Unexpected indexed load.");
3802
3803 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
3804 // The offset operand of unindexed load is always undefined, so there is
3805 // no need to pass it to VLDREPL.
3806 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
3807 SDValue BCast = DAG.getNode(LoongArchISD::VLDREPL, DL, Tys, Ops);
3808 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), BCast.getValue(1));
3809 return BCast;
3810 }
3811 return SDValue();
3812}
3813
3814// Sequentially insert elements from Ops into Vector, from low to high indices.
3815// Note: Ops can have fewer elements than Vector.
3818 EVT ResTy) {
3819 assert(Ops.size() <= ResTy.getVectorNumElements());
3820
3821 SDValue Op0 = Ops[0];
3822 if (!Op0.isUndef())
3823 Vector = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, ResTy, Op0);
3824 for (unsigned i = 1; i < Ops.size(); ++i) {
3825 SDValue Opi = Ops[i];
3826 if (Opi.isUndef())
3827 continue;
3828 Vector = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ResTy, Vector, Opi,
3829 DAG.getConstant(i, DL, Subtarget.getGRLenVT()));
3830 }
3831}
3832
3833// Build a ResTy subvector from Node, taking NumElts elements starting at index
3834// 'first'.
3836 SelectionDAG &DAG, SDLoc DL,
3838 EVT ResTy, unsigned first) {
3839 unsigned NumElts = ResTy.getVectorNumElements();
3840
3841 assert(first + NumElts <= Node->getSimpleValueType(0).getVectorNumElements());
3842
3843 SmallVector<SDValue, 16> Ops(Node->op_begin() + first,
3844 Node->op_begin() + first + NumElts);
3845 SDValue Vector = DAG.getUNDEF(ResTy);
3846 fillVector(Ops, DAG, DL, Subtarget, Vector, ResTy);
3847 return Vector;
3848}
3849
3850SDValue LoongArchTargetLowering::lowerBUILD_VECTOR(SDValue Op,
3851 SelectionDAG &DAG) const {
3852 BuildVectorSDNode *Node = cast<BuildVectorSDNode>(Op);
3853 MVT VT = Node->getSimpleValueType(0);
3854 EVT ResTy = Op->getValueType(0);
3855 unsigned NumElts = ResTy.getVectorNumElements();
3856 SDLoc DL(Op);
3857 APInt SplatValue, SplatUndef;
3858 unsigned SplatBitSize;
3859 bool HasAnyUndefs;
3860 bool IsConstant = false;
3861 bool UseSameConstant = true;
3862 SDValue ConstantValue;
3863 bool Is128Vec = ResTy.is128BitVector();
3864 bool Is256Vec = ResTy.is256BitVector();
3865
3866 if ((!Subtarget.hasExtLSX() || !Is128Vec) &&
3867 (!Subtarget.hasExtLASX() || !Is256Vec))
3868 return SDValue();
3869
3870 if (SDValue Result = lowerBUILD_VECTORAsBroadCastLoad(Node, DL, DAG))
3871 return Result;
3872
3873 if (Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize, HasAnyUndefs,
3874 /*MinSplatBits=*/8) &&
3875 SplatBitSize <= 64) {
3876 // We can only cope with 8, 16, 32, or 64-bit elements.
3877 if (SplatBitSize != 8 && SplatBitSize != 16 && SplatBitSize != 32 &&
3878 SplatBitSize != 64)
3879 return SDValue();
3880
3881 if (SplatBitSize == 64 && !Subtarget.is64Bit()) {
3882 // We can only handle 64-bit elements that are within
3883 // the signed 10-bit range or match vldi patterns on 32-bit targets.
3884 // See the BUILD_VECTOR case in LoongArchDAGToDAGISel::Select().
3885 if (!SplatValue.isSignedIntN(10) &&
3886 !isImmVLDILegalForMode1(SplatValue, SplatBitSize).first)
3887 return SDValue();
3888 if ((Is128Vec && ResTy == MVT::v4i32) ||
3889 (Is256Vec && ResTy == MVT::v8i32))
3890 return Op;
3891 }
3892
3893 EVT ViaVecTy;
3894
3895 switch (SplatBitSize) {
3896 default:
3897 return SDValue();
3898 case 8:
3899 ViaVecTy = Is128Vec ? MVT::v16i8 : MVT::v32i8;
3900 break;
3901 case 16:
3902 ViaVecTy = Is128Vec ? MVT::v8i16 : MVT::v16i16;
3903 break;
3904 case 32:
3905 ViaVecTy = Is128Vec ? MVT::v4i32 : MVT::v8i32;
3906 break;
3907 case 64:
3908 ViaVecTy = Is128Vec ? MVT::v2i64 : MVT::v4i64;
3909 break;
3910 }
3911
3912 // SelectionDAG::getConstant will promote SplatValue appropriately.
3913 SDValue Result = DAG.getConstant(SplatValue, DL, ViaVecTy);
3914
3915 // Bitcast to the type we originally wanted.
3916 if (ViaVecTy != ResTy)
3917 Result = DAG.getNode(ISD::BITCAST, SDLoc(Node), ResTy, Result);
3918
3919 return Result;
3920 }
3921
3922 if (DAG.isSplatValue(Op, /*AllowUndefs=*/false))
3923 return Op;
3924
3925 for (unsigned i = 0; i < NumElts; ++i) {
3926 SDValue Opi = Node->getOperand(i);
3927 if (isIntOrFPConstant(Opi)) {
3928 IsConstant = true;
3929 if (!ConstantValue.getNode())
3930 ConstantValue = Opi;
3931 else if (ConstantValue != Opi)
3932 UseSameConstant = false;
3933 }
3934 }
3935
3936 // If the type of BUILD_VECTOR is v2f64, custom legalizing it has no benefits.
3937 if (IsConstant && UseSameConstant && ResTy != MVT::v2f64) {
3938 SDValue Result = DAG.getSplatBuildVector(ResTy, DL, ConstantValue);
3939 for (unsigned i = 0; i < NumElts; ++i) {
3940 SDValue Opi = Node->getOperand(i);
3941 if (!isIntOrFPConstant(Opi))
3942 Result = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, ResTy, Result, Opi,
3943 DAG.getConstant(i, DL, Subtarget.getGRLenVT()));
3944 }
3945 return Result;
3946 }
3947
3948 if (!IsConstant) {
3950 BitVector UndefElements;
3951 bool IsRepeated = Node->getRepeatedSequence(Sequence, &UndefElements);
3952 unsigned SeqLen = Sequence.size();
3953 unsigned NumUndefElts = UndefElements.count();
3954
3955 // When a BUILD_VECTOR consists of the same element (ignoring undefs),
3956 // prefer emitting a broadcast instead of multiple insertions.
3957 if (IsRepeated && SeqLen == 1) {
3958 bool IsSplatBetter = true;
3959
3960 // Integer vectors benefit from splat except in the case where the only
3961 // non-undef element is at index 0 and is a load with a base+offset addr.
3962 if (ResTy.isInteger()) {
3963 SDValue Op0 = Node->getOperand(0);
3964 if (NumUndefElts == NumElts - 1 && Op0.getOpcode() == ISD::LOAD &&
3966 IsSplatBetter = false;
3967 }
3968
3969 // Several floating-point cases suffer performance regressions or have
3970 // no improvement when
3971 // only one element is non-undef,
3972 // one of the only two non-undefs at index 0,
3973 // the two non-undefs of v8f32 buildvector at index (1,2)/(1,3)/(2,3).
3974 if (ResTy.isFloatingPoint()) {
3975 if (NumUndefElts == NumElts - 1)
3976 IsSplatBetter = false;
3977 if (NumUndefElts == NumElts - 2 && !UndefElements[0])
3978 IsSplatBetter = false;
3979 if (ResTy == MVT::v8f32 && NumUndefElts == NumElts - 2 &&
3980 ((!UndefElements[1] && !UndefElements[2]) ||
3981 (!UndefElements[1] && !UndefElements[3]) ||
3982 (!UndefElements[2] && !UndefElements[3])))
3983 IsSplatBetter = false;
3984 }
3985
3986 if (IsSplatBetter)
3987 return DAG.getSplatBuildVector(ResTy, DL, Sequence[0]);
3988 }
3989
3990 // If the BUILD_VECTOR has a repeated pattern, use INSERT_VECTOR_ELT to fill
3991 // the sub-sequence of the vector and then broadcast the sub-sequence.
3992 //
3993 // TODO: If the BUILD_VECTOR contains undef elements, consider falling
3994 // back to use INSERT_VECTOR_ELT to materialize the vector, because it
3995 // generates worse code in some cases. This could be further optimized
3996 // with more consideration.
3997 if (IsRepeated && NumUndefElts == 0) {
3998 // Using LSX instructions to fill the sub-sequence of 256-bits vector,
3999 // because the high part can be simply treated as undef.
4000 SDValue Vector = DAG.getUNDEF(ResTy);
4001 EVT FillTy = Is256Vec
4003 : ResTy;
4004 SDValue FillVec =
4005 Is256Vec ? DAG.getExtractSubvector(DL, FillTy, Vector, 0) : Vector;
4006
4007 fillVector(Sequence, DAG, DL, Subtarget, FillVec, FillTy);
4008
4009 unsigned SplatLen = NumElts / SeqLen;
4010 MVT SplatEltTy = MVT::getIntegerVT(VT.getScalarSizeInBits() * SeqLen);
4011 MVT SplatTy = MVT::getVectorVT(SplatEltTy, SplatLen);
4012
4013 // If size of the sub-sequence is half of a 256-bits vector, bitcast the
4014 // vector to v4i64 type in order to match the pattern of XVREPLVE0Q.
4015 if (SplatEltTy == MVT::i128)
4016 SplatTy = MVT::v4i64;
4017
4018 SDValue SplatVec;
4019 SDValue SrcVec = DAG.getBitcast(
4020 SplatTy,
4021 Is256Vec ? DAG.getInsertSubvector(DL, Vector, FillVec, 0) : FillVec);
4022 if (Is256Vec) {
4023 SplatVec =
4024 DAG.getNode((SplatEltTy == MVT::i128) ? LoongArchISD::XVREPLVE0Q
4025 : LoongArchISD::XVREPLVE0,
4026 DL, SplatTy, SrcVec);
4027 } else {
4028 SplatVec = DAG.getNode(LoongArchISD::VREPLVEI, DL, SplatTy, SrcVec,
4029 DAG.getConstant(0, DL, Subtarget.getGRLenVT()));
4030 }
4031
4032 return DAG.getBitcast(ResTy, SplatVec);
4033 }
4034
4035 // Use INSERT_VECTOR_ELT operations rather than expand to stores, because
4036 // using memory operations is much lower.
4037 //
4038 // For 256-bit vectors, normally split into two halves and concatenate.
4039 // Special case: for v8i32/v8f32/v4i64/v4f64, if the upper half has only
4040 // one non-undef element, skip spliting to avoid a worse result.
4041 if (ResTy == MVT::v8i32 || ResTy == MVT::v8f32 || ResTy == MVT::v4i64 ||
4042 ResTy == MVT::v4f64) {
4043 unsigned NonUndefCount = 0;
4044 for (unsigned i = NumElts / 2; i < NumElts; ++i) {
4045 if (!Node->getOperand(i).isUndef()) {
4046 ++NonUndefCount;
4047 if (NonUndefCount > 1)
4048 break;
4049 }
4050 }
4051 if (NonUndefCount == 1)
4052 return fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, ResTy, 0);
4053 }
4054
4055 EVT VecTy =
4056 Is256Vec ? ResTy.getHalfNumVectorElementsVT(*DAG.getContext()) : ResTy;
4057 SDValue Vector =
4058 fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget, VecTy, 0);
4059
4060 if (Is128Vec)
4061 return Vector;
4062
4063 SDValue VectorHi = fillSubVectorFromBuildVector(Node, DAG, DL, Subtarget,
4064 VecTy, NumElts / 2);
4065
4066 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResTy, Vector, VectorHi);
4067 }
4068
4069 return SDValue();
4070}
4071
4072SDValue LoongArchTargetLowering::lowerCONCAT_VECTORS(SDValue Op,
4073 SelectionDAG &DAG) const {
4074 SDLoc DL(Op);
4075 MVT ResVT = Op.getSimpleValueType();
4076 assert(ResVT.is256BitVector() && Op.getNumOperands() == 2);
4077
4078 if (Op.getOperand(0).getOpcode() == ISD::TRUNCATE &&
4079 Op.getOperand(1).getOpcode() == ISD::TRUNCATE)
4080 return Op;
4081
4082 unsigned NumOperands = Op.getNumOperands();
4083 unsigned NumFreezeUndef = 0;
4084 unsigned NumZero = 0;
4085 unsigned NumNonZero = 0;
4086 unsigned NonZeros = 0;
4087 SmallSet<SDValue, 4> Undefs;
4088 for (unsigned i = 0; i != NumOperands; ++i) {
4089 SDValue SubVec = Op.getOperand(i);
4090 if (SubVec.isUndef())
4091 continue;
4092 if (ISD::isFreezeUndef(SubVec)) {
4093 // If the freeze(undef) has multiple uses then we must fold to zero.
4094 if (SubVec.hasOneUse()) {
4095 ++NumFreezeUndef;
4096 } else {
4097 ++NumZero;
4098 Undefs.insert(SubVec);
4099 }
4100 } else if (ISD::isBuildVectorAllZeros(SubVec))
4101 ++NumZero;
4102 else {
4103 assert(i < sizeof(NonZeros) * CHAR_BIT); // Ensure the shift is in range.
4104 NonZeros |= 1 << i;
4105 ++NumNonZero;
4106 }
4107 }
4108
4109 // If we have more than 2 non-zeros, build each half separately.
4110 if (NumNonZero > 2) {
4111 MVT HalfVT = ResVT.getHalfNumVectorElementsVT();
4112 ArrayRef<SDUse> Ops = Op->ops();
4113 SDValue Lo = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
4114 Ops.slice(0, NumOperands / 2));
4115 SDValue Hi = DAG.getNode(ISD::CONCAT_VECTORS, DL, HalfVT,
4116 Ops.slice(NumOperands / 2));
4117 return DAG.getNode(ISD::CONCAT_VECTORS, DL, ResVT, Lo, Hi);
4118 }
4119
4120 // Otherwise, build it up through insert_subvectors.
4121 SDValue Vec = NumZero ? DAG.getConstant(0, DL, ResVT)
4122 : (NumFreezeUndef ? DAG.getFreeze(DAG.getUNDEF(ResVT))
4123 : DAG.getUNDEF(ResVT));
4124
4125 // Replace Undef operands with ZeroVector.
4126 for (SDValue U : Undefs)
4127 DAG.ReplaceAllUsesWith(U, DAG.getConstant(0, DL, U.getSimpleValueType()));
4128
4129 MVT SubVT = Op.getOperand(0).getSimpleValueType();
4130 unsigned NumSubElems = SubVT.getVectorNumElements();
4131 for (unsigned i = 0; i != NumOperands; ++i) {
4132 if ((NonZeros & (1 << i)) == 0)
4133 continue;
4134
4135 Vec = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, ResVT, Vec, Op.getOperand(i),
4136 DAG.getVectorIdxConstant(i * NumSubElems, DL));
4137 }
4138
4139 return Vec;
4140}
4141
4142SDValue
4143LoongArchTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
4144 SelectionDAG &DAG) const {
4145 MVT EltVT = Op.getSimpleValueType();
4146 SDValue Vec = Op->getOperand(0);
4147 EVT VecTy = Vec->getValueType(0);
4148 SDValue Idx = Op->getOperand(1);
4149 SDLoc DL(Op);
4150 MVT GRLenVT = Subtarget.getGRLenVT();
4151
4152 assert(VecTy.is256BitVector() && "Unexpected EXTRACT_VECTOR_ELT vector type");
4153
4154 if (isa<ConstantSDNode>(Idx))
4155 return Op;
4156
4157 switch (VecTy.getSimpleVT().SimpleTy) {
4158 default:
4159 llvm_unreachable("Unexpected type");
4160 case MVT::v32i8:
4161 case MVT::v16i16:
4162 case MVT::v4i64:
4163 case MVT::v4f64: {
4164 // Extract the high half subvector and place it to the low half of a new
4165 // vector. It doesn't matter what the high half of the new vector is.
4166 EVT HalfTy = VecTy.getHalfNumVectorElementsVT(*DAG.getContext());
4167 SDValue VecHi =
4168 DAG.getExtractSubvector(DL, HalfTy, Vec, HalfTy.getVectorNumElements());
4169 SDValue TmpVec =
4170 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VecTy, DAG.getUNDEF(VecTy),
4171 VecHi, DAG.getConstant(0, DL, GRLenVT));
4172
4173 // Shuffle the origin Vec and the TmpVec using MaskVec, the lowest element
4174 // of MaskVec is Idx, the rest do not matter. ResVec[0] will hold the
4175 // desired element.
4176 SDValue IdxCp =
4177 Subtarget.is64Bit()
4178 ? DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, MVT::f32, Idx)
4179 : DAG.getBitcast(MVT::f32, Idx);
4180 SDValue IdxVec = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v8f32, IdxCp);
4181 SDValue MaskVec =
4182 DAG.getBitcast((VecTy == MVT::v4f64) ? MVT::v4i64 : VecTy, IdxVec);
4183 SDValue ResVec =
4184 DAG.getNode(LoongArchISD::VSHUF, DL, VecTy, MaskVec, TmpVec, Vec);
4185
4186 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, ResVec,
4187 DAG.getConstant(0, DL, GRLenVT));
4188 }
4189 case MVT::v8i32:
4190 case MVT::v8f32: {
4191 SDValue SplatIdx = DAG.getSplatBuildVector(MVT::v8i32, DL, Idx);
4192 SDValue SplatValue =
4193 DAG.getNode(LoongArchISD::XVPERM, DL, VecTy, Vec, SplatIdx);
4194
4195 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, SplatValue,
4196 DAG.getConstant(0, DL, GRLenVT));
4197 }
4198 }
4199}
4200
4201SDValue
4202LoongArchTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
4203 SelectionDAG &DAG) const {
4204 MVT VT = Op.getSimpleValueType();
4205 MVT EltVT = VT.getVectorElementType();
4206 unsigned NumElts = VT.getVectorNumElements();
4207 unsigned EltSizeInBits = EltVT.getScalarSizeInBits();
4208 SDLoc DL(Op);
4209 SDValue Op0 = Op.getOperand(0);
4210 SDValue Op1 = Op.getOperand(1);
4211 SDValue Op2 = Op.getOperand(2);
4212
4213 // Lower inserting a loaded integer into lane 0 of an undefined vector as a
4214 // vector load that writes the whole register: FLD.{S/D} for 32/64-bit
4215 // elements and [X]VLDREPL.{B/H} for 8/16-bit ones. Unlike a scalar load +
4216 // [X]VINSGR2VR, it has no false dependency on the previous value of the
4217 // destination and doesn't need to move the value from a GPR.
4218 // For example:
4219 // %a = load i64, ptr %ptr
4220 // %b = insert_vector_elt undef, %a, 0
4221 // is lowered to:
4222 // (FLD_D $a0, 0)
4223 if (Op0.isUndef() && isNullConstant(Op2) && EltVT.isInteger() &&
4224 Op1.getOpcode() == ISD::LOAD && Op1.hasOneUse()) {
4225 auto *LN = cast<LoadSDNode>(Op1);
4226 auto ExtType = LN->getExtensionType();
4227 if (LN->isSimple() &&
4228 (ExtType == ISD::EXTLOAD || ExtType == ISD::NON_EXTLOAD) &&
4229 LN->getMemoryVT().getSizeInBits() == EltSizeInBits) {
4230 // Indexed loads and stores are not supported on LoongArch.
4231 assert(LN->isUnindexed() && "Unexpected indexed load.");
4232 SDValue Res;
4233 if (EltSizeInBits >= 32) {
4234 MVT FltVT = MVT::getFloatingPointVT(EltSizeInBits);
4235 // Don't reuse the memory operand, its range metadata (if any) only
4236 // applies to integer loads.
4237 SDValue Ld = DAG.getLoad(
4238 FltVT, DL, LN->getChain(), LN->getBasePtr(), LN->getPointerInfo(),
4239 LN->getAlign(), LN->getMemOperand()->getFlags(), LN->getAAInfo());
4240 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL,
4241 MVT::getVectorVT(FltVT, NumElts), Ld);
4242 Res = DAG.getBitcast(VT, Res);
4243 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), Ld.getValue(1));
4244 } else {
4245 SDVTList Tys = DAG.getVTList(VT, MVT::Other);
4246 SDValue Ops[] = {LN->getChain(), LN->getBasePtr()};
4247 Res = DAG.getMemIntrinsicNode(LoongArchISD::VLDREPL, DL, Tys, Ops,
4248 LN->getMemoryVT(), LN->getMemOperand());
4249 DAG.ReplaceAllUsesOfValueWith(SDValue(LN, 1), Res.getValue(1));
4250 }
4251 return Res;
4252 }
4253 }
4254
4255 if (isa<ConstantSDNode>(Op2))
4256 return Op;
4257
4258 MVT IdxTy = MVT::getIntegerVT(EltSizeInBits);
4259 MVT IdxVTy = MVT::getVectorVT(IdxTy, NumElts);
4260
4261 if (!isTypeLegal(VT) || !isTypeLegal(IdxVTy))
4262 return SDValue();
4263
4264 SDValue SplatElt = DAG.getSplatBuildVector(VT, DL, Op1);
4265 SmallVector<SDValue, 32> RawIndices;
4266 SDValue SplatIdx;
4267 SDValue Indices;
4268
4269 if (!Subtarget.is64Bit() && IdxTy == MVT::i64) {
4270 MVT PairVTy = MVT::getVectorVT(MVT::i32, NumElts * 2);
4271 for (unsigned i = 0; i < NumElts; ++i) {
4272 RawIndices.push_back(Op2);
4273 RawIndices.push_back(DAG.getConstant(0, DL, MVT::i32));
4274 }
4275 SplatIdx = DAG.getBuildVector(PairVTy, DL, RawIndices);
4276 SplatIdx = DAG.getBitcast(IdxVTy, SplatIdx);
4277
4278 RawIndices.clear();
4279 for (unsigned i = 0; i < NumElts; ++i) {
4280 RawIndices.push_back(DAG.getConstant(i, DL, MVT::i32));
4281 RawIndices.push_back(DAG.getConstant(0, DL, MVT::i32));
4282 }
4283 Indices = DAG.getBuildVector(PairVTy, DL, RawIndices);
4284 Indices = DAG.getBitcast(IdxVTy, Indices);
4285 } else {
4286 SplatIdx = DAG.getSplatBuildVector(IdxVTy, DL, Op2);
4287
4288 for (unsigned i = 0; i < NumElts; ++i)
4289 RawIndices.push_back(DAG.getConstant(i, DL, Subtarget.getGRLenVT()));
4290 Indices = DAG.getBuildVector(IdxVTy, DL, RawIndices);
4291 }
4292
4293 // insert vec, elt, idx
4294 // =>
4295 // select (splatidx == {0,1,2...}) ? splatelt : vec
4296 SDValue SelectCC =
4297 DAG.getSetCC(DL, IdxVTy, SplatIdx, Indices, ISD::CondCode::SETEQ);
4298 return DAG.getNode(ISD::VSELECT, DL, VT, SelectCC, SplatElt, Op0);
4299}
4300
4301SDValue LoongArchTargetLowering::lowerATOMIC_FENCE(SDValue Op,
4302 SelectionDAG &DAG) const {
4303 SDLoc DL(Op);
4304 SyncScope::ID FenceSSID =
4305 static_cast<SyncScope::ID>(Op.getConstantOperandVal(2));
4306
4307 // singlethread fences only synchronize with signal handlers on the same
4308 // thread and thus only need to preserve instruction order, not actually
4309 // enforce memory ordering.
4310 if (FenceSSID == SyncScope::SingleThread)
4311 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
4312 return DAG.getNode(ISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
4313
4314 return Op;
4315}
4316
4318 MVT GRLenVT, SDValue RMValue) {
4319 // LLVM rounding mode encoding differs from LoongArch FCSR encoding:
4320 // LLVM: 0=RTZ, 1=RNE, 2=RUP, 3=RDN
4321 // FCSR: 0=RNE, 1=RZ, 2=RP, 3=RN
4322 //
4323 // The conversion swaps encodings 0 and 1 while preserving 2 and 3.
4324 // Since the transformation is self-inverse, it applies in both directions:
4325 // LLVM RM <-> LoongArch FCSR RM
4326 //
4327 // Transformation: RM ^ (~(RM >> 1) & 1)
4328 SDValue ShiftRight1 = DAG.getNode(ISD::SRL, DL, GRLenVT, RMValue,
4329 DAG.getConstant(1, DL, GRLenVT));
4330
4331 SDValue SwapMask = DAG.getNode(ISD::AND, DL, GRLenVT,
4332 DAG.getNode(ISD::XOR, DL, GRLenVT, ShiftRight1,
4333 DAG.getConstant(1, DL, GRLenVT)),
4334 DAG.getConstant(1, DL, GRLenVT));
4335
4336 return DAG.getNode(ISD::XOR, DL, GRLenVT, RMValue, SwapMask);
4337}
4338
4339SDValue LoongArchTargetLowering::lowerSET_ROUNDING(SDValue Op,
4340 SelectionDAG &DAG) const {
4341 MVT GRLenVT = Subtarget.getGRLenVT();
4342 SDLoc DL(Op);
4343 SDValue Chain = Op.getOperand(0);
4344 SDValue RMValue = Op.getOperand(1);
4345
4346 if (auto *CVal = dyn_cast<ConstantSDNode>(RMValue)) {
4347 uint64_t RM = CVal->getZExtValue();
4348 if (RM > 3) {
4350 LLVMContext &C = MF.getFunction().getContext();
4351 C.diagnose(DiagnosticInfoUnsupported(
4352 MF.getFunction(),
4353 "rounding mode is not supported by LoongArch hardware",
4354 DiagnosticLocation(DL.getDebugLoc()), DS_Error));
4355 return Chain;
4356 }
4357 }
4358
4359 RMValue = DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, RMValue);
4360 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4361
4362 // The RM field in FCSR is at bits [9:8]. Shift the rounding mode value
4363 // into position before writing via WRFCSR.
4364 RMValue = DAG.getNode(ISD::SHL, DL, GRLenVT, RMValue,
4365 DAG.getConstant(8, DL, GRLenVT));
4366
4367 // FCSR3 is an alias of the RM field; writing it avoids clobbering
4368 // unrelated fields in FCSR0.
4369 SDValue FCSRNo = DAG.getTargetConstant(3, DL, GRLenVT);
4370 MachineSDNode *RN = DAG.getMachineNode(LoongArch::WRFCSR, DL, MVT::Other,
4371 FCSRNo, RMValue, Chain);
4372 return SDValue(RN, 0);
4373}
4374
4375SDValue LoongArchTargetLowering::lowerGET_ROUNDING(SDValue Op,
4376 SelectionDAG &DAG) const {
4377 MVT GRLenVT = Subtarget.getGRLenVT();
4378 SDLoc DL(Op);
4379 SDValue Chain = Op->getOperand(0);
4380
4381 // FCSR3 is an alias of the RM field.
4382 SDValue FCSRNo = DAG.getTargetConstant(3, DL, GRLenVT);
4383 MachineSDNode *FCSR = DAG.getMachineNode(LoongArch::RDFCSR, DL, GRLenVT,
4384 MVT::Other, FCSRNo, Chain);
4385 SDValue RMValue = SDValue(FCSR, 0);
4386 Chain = SDValue(FCSR, 1);
4387
4388 // The RM field in FCSR is at bits [9:8].
4389 RMValue = DAG.getNode(ISD::SRL, DL, GRLenVT, RMValue,
4390 DAG.getConstant(8, DL, GRLenVT));
4391 RMValue = convertRMEncoding(DAG, DL, GRLenVT, RMValue);
4392
4393 SDValue RetVal = DAG.getZExtOrTrunc(RMValue, DL, Op.getValueType());
4394 return DAG.getMergeValues({RetVal, Chain}, DL);
4395}
4396
4397SDValue LoongArchTargetLowering::lowerWRITE_REGISTER(SDValue Op,
4398 SelectionDAG &DAG) const {
4399
4400 if (Subtarget.is64Bit() && Op.getOperand(2).getValueType() == MVT::i32) {
4401 DAG.getContext()->emitError(
4402 "On LA64, only 64-bit registers can be written.");
4403 return Op.getOperand(0);
4404 }
4405
4406 if (!Subtarget.is64Bit() && Op.getOperand(2).getValueType() == MVT::i64) {
4407 DAG.getContext()->emitError(
4408 "On LA32, only 32-bit registers can be written.");
4409 return Op.getOperand(0);
4410 }
4411
4412 return Op;
4413}
4414
4415SDValue LoongArchTargetLowering::lowerFRAMEADDR(SDValue Op,
4416 SelectionDAG &DAG) const {
4417 if (!isa<ConstantSDNode>(Op.getOperand(0))) {
4418 DAG.getContext()->emitError("argument to '__builtin_frame_address' must "
4419 "be a constant integer");
4420 return SDValue();
4421 }
4422
4425 Register FrameReg = Subtarget.getRegisterInfo()->getFrameRegister(MF);
4426 EVT VT = Op.getValueType();
4427 SDLoc DL(Op);
4428 SDValue FrameAddr = DAG.getCopyFromReg(DAG.getEntryNode(), DL, FrameReg, VT);
4429 unsigned Depth = Op.getConstantOperandVal(0);
4430 int GRLenInBytes = Subtarget.getGRLen() / 8;
4431
4432 while (Depth--) {
4433 int Offset = -(GRLenInBytes * 2);
4434 SDValue Ptr = DAG.getNode(ISD::ADD, DL, VT, FrameAddr,
4435 DAG.getSignedConstant(Offset, DL, VT));
4436 FrameAddr =
4437 DAG.getLoad(VT, DL, DAG.getEntryNode(), Ptr, MachinePointerInfo());
4438 }
4439 return FrameAddr;
4440}
4441
4442SDValue LoongArchTargetLowering::lowerRETURNADDR(SDValue Op,
4443 SelectionDAG &DAG) const {
4444 // Currently only support lowering return address for current frame.
4445 if (Op.getConstantOperandVal(0) != 0) {
4446 DAG.getContext()->emitError(
4447 "return address can only be determined for the current frame");
4448 return SDValue();
4449 }
4450
4453 MVT GRLenVT = Subtarget.getGRLenVT();
4454
4455 // Return the value of the return address register, marking it an implicit
4456 // live-in.
4457 Register Reg = MF.addLiveIn(Subtarget.getRegisterInfo()->getRARegister(),
4458 getRegClassFor(GRLenVT));
4459 return DAG.getCopyFromReg(DAG.getEntryNode(), SDLoc(Op), Reg, GRLenVT);
4460}
4461
4462SDValue LoongArchTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
4463 SelectionDAG &DAG) const {
4465 auto Size = Subtarget.getGRLen() / 8;
4466 auto FI = MF.getFrameInfo().CreateFixedObject(Size, 0, false);
4467 return DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
4468}
4469
4470SDValue LoongArchTargetLowering::lowerVASTART(SDValue Op,
4471 SelectionDAG &DAG) const {
4473 auto *FuncInfo = MF.getInfo<LoongArchMachineFunctionInfo>();
4474
4475 SDLoc DL(Op);
4476 SDValue FI = DAG.getFrameIndex(FuncInfo->getVarArgsFrameIndex(),
4478
4479 // vastart just stores the address of the VarArgsFrameIndex slot into the
4480 // memory location argument.
4481 const Value *SV = cast<SrcValueSDNode>(Op.getOperand(2))->getValue();
4482 return DAG.getStore(Op.getOperand(0), DL, FI, Op.getOperand(1),
4483 MachinePointerInfo(SV));
4484}
4485
4486// Lower an unsigned vXi64 -> vXf32 conversion to the same round-to-odd
4487// sequence that the generic legalizer uses for the scalar case:
4488//
4489// Odd = (x >> 1) | (x & 1)
4490// Doubled = (float)(int64_t)Odd + (float)(int64_t)Odd // exact, no rounding
4491// Signed = (float)(int64_t)x // valid for x < 2^63
4492// Result = x < 0 ? Doubled : Signed
4493//
4494// The generic half-word expansion is not usable here: it converts the high and
4495// low 32-bit halves separately and adds them, rounding three times, which is
4496// not enough to emulate the single rounding required by uitofp.
4497SDValue LoongArchTargetLowering::lowerUINT_TO_FPToF32(SDValue Op,
4498 SelectionDAG &DAG) const {
4499 SDLoc DL(Op);
4500 SDValue Src = Op.getOperand(0);
4501 EVT SrcVT = Src.getValueType();
4502 EVT DstVT = Op.getValueType();
4503
4504 SDValue Shift = DAG.getShiftAmountConstant(1, SrcVT, DL);
4505 SDValue Half = DAG.getNode(ISD::SRL, DL, SrcVT, Src, Shift);
4506 SDValue Sticky =
4507 DAG.getNode(ISD::AND, DL, SrcVT, Src, DAG.getConstant(1, DL, SrcVT));
4508 SDValue Odd = DAG.getNode(ISD::OR, DL, SrcVT, Half, Sticky);
4509
4510 // Doubling a value with at most 24 significant bits is exact.
4511 SDValue RoundToOdd = DAG.getNode(ISD::SINT_TO_FP, DL, DstVT, Odd);
4512 SDValue Doubled = DAG.getNode(ISD::FADD, DL, DstVT, RoundToOdd, RoundToOdd);
4513
4514 // Values below 2^63 convert directly; above that the signed conversion is
4515 // negative, so the round-to-odd path is used instead.
4516 SDValue Signed = DAG.getNode(ISD::SINT_TO_FP, DL, DstVT, Src);
4517
4518 // The signed conversion is negative exactly when the source did not fit, so
4519 // its sign bit is the mask: broadcasting it with an arithmetic shift avoids
4520 // a comparison followed by a truncate of the 64-bit result to the width of
4521 // the result vector.
4522 EVT MaskVT = DstVT.changeVectorElementTypeToInteger();
4523 SDValue Mask =
4524 DAG.getNode(ISD::SRA, DL, MaskVT, DAG.getBitcast(MaskVT, Signed),
4525 DAG.getShiftAmountConstant(31, MaskVT, DL));
4526 return DAG.getSelect(DL, DstVT, Mask, Doubled, Signed);
4527}
4528
4529SDValue LoongArchTargetLowering::lowerUINT_TO_FP(SDValue Op,
4530 SelectionDAG &DAG) const {
4531 SDLoc DL(Op);
4532 SDValue Op0 = Op.getOperand(0);
4533 EVT VT = Op.getValueType();
4534 EVT Op0VT = Op0.getValueType();
4535
4536 if (VT.isVector()) {
4537 if (VT.getScalarSizeInBits() == Op0VT.getScalarSizeInBits())
4538 return Op;
4539
4540 // There is no unsigned 64-bit to 32-bit conversion instruction, so
4541 // synthesize one with a single rounding. A value known to be
4542 // non-negative converts exactly as signed, and that form is native.
4543 if (Subtarget.hasExtLASX() && Op0VT.getScalarType() == MVT::i64 &&
4544 VT.getScalarType() == MVT::f32) {
4545 if (DAG.SignBitIsZero(Op0) || Op->getFlags().hasNonNeg())
4546 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, Op0);
4547 return lowerUINT_TO_FPToF32(Op, DAG);
4548 }
4549
4550 return SDValue();
4551 }
4552
4553 if ((DAG.SignBitIsZero(Op0) || Op->getFlags().hasNonNeg()) &&
4556 return DAG.getNode(ISD::SINT_TO_FP, DL, VT, Op0);
4557
4558 // We can't do uint64 -> double -> float because of double-rounding issue.
4559 if (Subtarget.hasExtLSX() && Op0VT == MVT::i64 && VT == MVT::f64) {
4560 Op0 = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2i64, Op0);
4561 SDValue Conv = DAG.getNode(ISD::UINT_TO_FP, DL, MVT::v2f64, Op0);
4562 Conv = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, MVT::f64, Conv,
4563 DAG.getIntPtrConstant(0, DL));
4564 return Conv;
4565 }
4566
4567 if (!Subtarget.is64Bit() || !Subtarget.hasBasicF() || Subtarget.hasBasicD())
4568 return SDValue();
4569
4570 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4571 !Subtarget.hasBasicD() && "unexpected target features");
4572
4573 if (Op0->getOpcode() == ISD::AND) {
4574 auto *C = dyn_cast<ConstantSDNode>(Op0.getOperand(1));
4575 if (C && C->getZExtValue() < UINT64_C(0xFFFFFFFF))
4576 return Op;
4577 }
4578
4579 if (Op0->getOpcode() == LoongArchISD::BSTRPICK &&
4580 Op0.getConstantOperandVal(1) < UINT64_C(0X1F) &&
4581 Op0.getConstantOperandVal(2) == UINT64_C(0))
4582 return Op;
4583
4584 if (Op0.getOpcode() == ISD::AssertZext &&
4585 dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLT(MVT::i32))
4586 return Op;
4587
4588 EVT OpVT = Op0.getValueType();
4589 EVT RetVT = Op.getValueType();
4590 RTLIB::Libcall LC = RTLIB::getUINTTOFP(OpVT, RetVT);
4591 MakeLibCallOptions CallOptions;
4592 CallOptions.setTypeListBeforeSoften(OpVT, RetVT);
4593 SDValue Chain = SDValue();
4595 std::tie(Result, Chain) =
4596 makeLibCall(DAG, LC, Op.getValueType(), Op0, CallOptions, DL, Chain);
4597 return Result;
4598}
4599
4600SDValue LoongArchTargetLowering::lowerSINT_TO_FP(SDValue Op,
4601 SelectionDAG &DAG) const {
4602 assert(Subtarget.is64Bit() && Subtarget.hasBasicF() &&
4603 !Subtarget.hasBasicD() && "unexpected target features");
4604
4605 SDLoc DL(Op);
4606 SDValue Op0 = Op.getOperand(0);
4607
4608 if ((Op0.getOpcode() == ISD::AssertSext ||
4610 dyn_cast<VTSDNode>(Op0.getOperand(1))->getVT().bitsLE(MVT::i32))
4611 return Op;
4612
4613 EVT OpVT = Op0.getValueType();
4614 EVT RetVT = Op.getValueType();
4615 RTLIB::Libcall LC = RTLIB::getSINTTOFP(OpVT, RetVT);
4616 MakeLibCallOptions CallOptions;
4617 CallOptions.setTypeListBeforeSoften(OpVT, RetVT);
4618 SDValue Chain = SDValue();
4620 std::tie(Result, Chain) =
4621 makeLibCall(DAG, LC, Op.getValueType(), Op0, CallOptions, DL, Chain);
4622 return Result;
4623}
4624
4625SDValue LoongArchTargetLowering::lowerBITCAST(SDValue Op,
4626 SelectionDAG &DAG) const {
4627
4628 SDLoc DL(Op);
4629 EVT VT = Op.getValueType();
4630 SDValue Op0 = Op.getOperand(0);
4631 EVT Op0VT = Op0.getValueType();
4632
4633 if (Op.getValueType() == MVT::f32 && Op0VT == MVT::i32 &&
4634 Subtarget.is64Bit() && Subtarget.hasBasicF()) {
4635 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op0);
4636 return DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, MVT::f32, NewOp0);
4637 }
4638 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit()) {
4639 SDValue Lo, Hi;
4640 std::tie(Lo, Hi) = DAG.SplitScalar(Op0, DL, MVT::i32, MVT::i32);
4641 return DAG.getNode(LoongArchISD::BUILD_PAIR_F64, DL, MVT::f64, Lo, Hi);
4642 }
4643 return Op;
4644}
4645
4646SDValue LoongArchTargetLowering::lowerFP_TO_SINT(SDValue Op,
4647 SelectionDAG &DAG) const {
4648
4649 SDLoc DL(Op);
4650 SDValue Op0 = Op.getOperand(0);
4651
4652 if (Op0.getValueType() == MVT::f16)
4653 Op0 = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Op0);
4654
4655 if (Op.getValueSizeInBits() > 32 && Subtarget.hasBasicF() &&
4656 !Subtarget.hasBasicD()) {
4657 SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, MVT::f32, Op0);
4658 return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Dst);
4659 }
4660
4661 EVT FPTy = EVT::getFloatingPointVT(Op.getValueSizeInBits());
4662 SDValue Trunc = DAG.getNode(LoongArchISD::FTINT, DL, FPTy, Op0);
4663 return DAG.getNode(ISD::BITCAST, DL, Op.getValueType(), Trunc);
4664}
4665
4666// [X]VFTINTRZ and FTINTRZ saturate out-of-range values and convert NaN to 0,
4667// which is exactly the semantics of FP_TO_SINT_SAT with a native saturation
4668// width.
4669SDValue LoongArchTargetLowering::lowerFP_TO_SINT_SAT(SDValue Op,
4670 SelectionDAG &DAG) const {
4671 SDLoc DL(Op);
4672 SDValue Src = Op.getOperand(0);
4673 EVT VT = Op.getValueType();
4674 EVT SrcVT = Src.getValueType();
4675 unsigned SatWidth = cast<VTSDNode>(Op.getOperand(1))->getVT().getSizeInBits();
4676
4677 // Vectors are selected by patterns.
4678 if (VT.isVector()) {
4679 if (SatWidth == VT.getScalarSizeInBits() &&
4680 SatWidth == SrcVT.getScalarSizeInBits())
4681 return Op;
4682 return SDValue();
4683 }
4684
4685 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4686 return SDValue();
4687
4688 if (SatWidth == 32) {
4689 SDValue Ftint = DAG.getNode(LoongArchISD::FTINT, DL, MVT::f32, Src);
4690 if (Subtarget.is64Bit())
4691 return DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, VT, Ftint);
4692 return DAG.getBitcast(VT, Ftint);
4693 }
4694
4695 if (SatWidth == 64 && Subtarget.is64Bit() && Subtarget.hasBasicD()) {
4696 SDValue Ftint = DAG.getNode(LoongArchISD::FTINT, DL, MVT::f64, Src);
4697 return DAG.getBitcast(VT, Ftint);
4698 }
4699
4700 return SDValue();
4701}
4702
4703SDValue LoongArchTargetLowering::lowerFP_TO_UINT(SDValue Op,
4704 SelectionDAG &DAG) const {
4705 if (!Subtarget.hasExtLSX())
4706 return SDValue();
4707
4708 SDLoc DL(Op);
4709 SDValue Src = Op.getOperand(0);
4710 EVT VT = Op.getValueType();
4711 EVT SrcVT = Src.getValueType();
4712
4713 if (VT != MVT::i64)
4714 return SDValue();
4715
4716 if (SrcVT != MVT::f32 && SrcVT != MVT::f64)
4717 return SDValue();
4718
4719 if (SrcVT == MVT::f32)
4720 Src = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f64, Src);
4721 Src = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, MVT::v2f64, Src);
4722 SDValue Conv = DAG.getNode(ISD::FP_TO_UINT, DL, MVT::v2i64, Src);
4723 return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Conv,
4724 DAG.getIntPtrConstant(0, DL));
4725}
4726
4728 SelectionDAG &DAG, unsigned Flags) {
4729 return DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, Flags);
4730}
4731
4733 SelectionDAG &DAG, unsigned Flags) {
4734 return DAG.getTargetBlockAddress(N->getBlockAddress(), Ty, N->getOffset(),
4735 Flags);
4736}
4737
4739 SelectionDAG &DAG, unsigned Flags) {
4740 return DAG.getTargetConstantPool(N->getConstVal(), Ty, N->getAlign(),
4741 N->getOffset(), Flags);
4742}
4743
4745 SelectionDAG &DAG, unsigned Flags) {
4746 return DAG.getTargetJumpTable(N->getIndex(), Ty, Flags);
4747}
4748
4749template <class NodeTy>
4750SDValue LoongArchTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
4752 bool IsLocal) const {
4753 SDLoc DL(N);
4754 EVT Ty = getPointerTy(DAG.getDataLayout());
4755 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
4756 SDValue Load;
4757
4758 switch (M) {
4759 default:
4760 report_fatal_error("Unsupported code model");
4761
4762 case CodeModel::Large: {
4763 assert(Subtarget.is64Bit() && "Large code model requires LA64");
4764
4765 // This is not actually used, but is necessary for successfully matching
4766 // the PseudoLA_*_LARGE nodes.
4767 SDValue Tmp = DAG.getConstant(0, DL, Ty);
4768 if (IsLocal) {
4769 // This generates the pattern (PseudoLA_PCREL_LARGE tmp sym), that
4770 // eventually becomes the desired 5-insn code sequence.
4771 Load = SDValue(DAG.getMachineNode(LoongArch::PseudoLA_PCREL_LARGE, DL, Ty,
4772 Tmp, Addr),
4773 0);
4774 } else {
4775 // This generates the pattern (PseudoLA_GOT_LARGE tmp sym), that
4776 // eventually becomes the desired 5-insn code sequence.
4777 Load = SDValue(
4778 DAG.getMachineNode(LoongArch::PseudoLA_GOT_LARGE, DL, Ty, Tmp, Addr),
4779 0);
4780 }
4781 break;
4782 }
4783
4784 case CodeModel::Small:
4785 case CodeModel::Medium:
4786 if (IsLocal) {
4787 // This generates the pattern (PseudoLA_PCREL sym), which
4788 //
4789 // for la32r expands to:
4790 // (addi.w (pcaddu12i %pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4791 //
4792 // for la32s and la64 expands to:
4793 // (addi.w/d (pcalau12i %pc_hi20(sym)) %pc_lo12(sym)).
4794 Load = SDValue(
4795 DAG.getMachineNode(LoongArch::PseudoLA_PCREL, DL, Ty, Addr), 0);
4796 } else {
4797 // This generates the pattern (PseudoLA_GOT sym), which
4798 //
4799 // for la32r expands to:
4800 // (ld.w (pcaddu12i %got_pcadd_hi20(sym)) %pcadd_lo12(.Lpcadd_hi)).
4801 //
4802 // for la32s and la64 expands to:
4803 // (ld.w/d (pcalau12i %got_pc_hi20(sym)) %got_pc_lo12(sym)).
4804 Load =
4805 SDValue(DAG.getMachineNode(LoongArch::PseudoLA_GOT, DL, Ty, Addr), 0);
4806 }
4807 }
4808
4809 if (!IsLocal) {
4810 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4812 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4816 LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
4817 DAG.setNodeMemRefs(cast<MachineSDNode>(Load.getNode()), {MemOp});
4818 }
4819
4820 return Load;
4821}
4822
4823SDValue LoongArchTargetLowering::lowerBlockAddress(SDValue Op,
4824 SelectionDAG &DAG) const {
4825 return getAddr(cast<BlockAddressSDNode>(Op), DAG,
4826 DAG.getTarget().getCodeModel());
4827}
4828
4829SDValue LoongArchTargetLowering::lowerJumpTable(SDValue Op,
4830 SelectionDAG &DAG) const {
4831 return getAddr(cast<JumpTableSDNode>(Op), DAG,
4832 DAG.getTarget().getCodeModel());
4833}
4834
4835SDValue LoongArchTargetLowering::lowerConstantPool(SDValue Op,
4836 SelectionDAG &DAG) const {
4837 return getAddr(cast<ConstantPoolSDNode>(Op), DAG,
4838 DAG.getTarget().getCodeModel());
4839}
4840
4841SDValue LoongArchTargetLowering::lowerGlobalAddress(SDValue Op,
4842 SelectionDAG &DAG) const {
4843 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
4844 assert(N->getOffset() == 0 && "unexpected offset in global node");
4845 auto CM = DAG.getTarget().getCodeModel();
4846 const GlobalValue *GV = N->getGlobal();
4847
4848 if (GV->isDSOLocal() && isa<GlobalVariable>(GV)) {
4849 if (auto GCM = dyn_cast<GlobalVariable>(GV)->getCodeModel())
4850 CM = *GCM;
4851 }
4852
4853 return getAddr(N, DAG, CM, GV->isDSOLocal());
4854}
4855
4856SDValue LoongArchTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
4857 SelectionDAG &DAG,
4858 unsigned Opc, bool UseGOT,
4859 bool Large) const {
4860 SDLoc DL(N);
4861 EVT Ty = getPointerTy(DAG.getDataLayout());
4862 MVT GRLenVT = Subtarget.getGRLenVT();
4863
4864 // This is not actually used, but is necessary for successfully matching the
4865 // PseudoLA_*_LARGE nodes.
4866 SDValue Tmp = DAG.getConstant(0, DL, Ty);
4867 SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0);
4868
4869 // Only IE needs an extra argument for large code model.
4870 SDValue Offset = Opc == LoongArch::PseudoLA_TLS_IE_LARGE
4871 ? SDValue(DAG.getMachineNode(Opc, DL, Ty, Tmp, Addr), 0)
4872 : SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0);
4873
4874 // If it is LE for normal/medium code model, the add tp operation will occur
4875 // during the pseudo-instruction expansion.
4876 if (Opc == LoongArch::PseudoLA_TLS_LE && !Large)
4877 return Offset;
4878
4879 if (UseGOT) {
4880 // Mark the load instruction as invariant to enable hoisting in MachineLICM.
4882 MachineMemOperand *MemOp = MF.getMachineMemOperand(
4886 LLT(Ty.getSimpleVT()), Align(Ty.getFixedSizeInBits() / 8));
4887 DAG.setNodeMemRefs(cast<MachineSDNode>(Offset.getNode()), {MemOp});
4888 }
4889
4890 // Add the thread pointer.
4891 return DAG.getNode(ISD::ADD, DL, Ty, Offset,
4892 DAG.getRegister(LoongArch::R2, GRLenVT));
4893}
4894
4895SDValue LoongArchTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
4896 SelectionDAG &DAG,
4897 unsigned Opc,
4898 bool Large) const {
4899 SDLoc DL(N);
4900 EVT Ty = getPointerTy(DAG.getDataLayout());
4901 IntegerType *CallTy = Type::getIntNTy(*DAG.getContext(), Ty.getSizeInBits());
4902
4903 // This is not actually used, but is necessary for successfully matching the
4904 // PseudoLA_*_LARGE nodes.
4905 SDValue Tmp = DAG.getConstant(0, DL, Ty);
4906
4907 // Use a PC-relative addressing mode to access the dynamic GOT address.
4908 SDValue Addr = DAG.getTargetGlobalAddress(N->getGlobal(), DL, Ty, 0, 0);
4909 SDValue Load = Large ? SDValue(DAG.getMachineNode(Opc, DL, Ty, Tmp, Addr), 0)
4910 : SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0);
4911
4912 // Prepare argument list to generate call.
4914 Args.emplace_back(Load, CallTy);
4915
4916 // Setup call to __tls_get_addr.
4917 TargetLowering::CallLoweringInfo CLI(DAG);
4918 CLI.setDebugLoc(DL)
4919 .setChain(DAG.getEntryNode())
4920 .setLibCallee(CallingConv::C, CallTy,
4921 DAG.getExternalSymbol("__tls_get_addr", Ty),
4922 std::move(Args));
4923
4924 return LowerCallTo(CLI).first;
4925}
4926
4927SDValue LoongArchTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
4928 SelectionDAG &DAG, unsigned Opc,
4929 bool Large) const {
4930 SDLoc DL(N);
4931 EVT Ty = getPointerTy(DAG.getDataLayout());
4932 const GlobalValue *GV = N->getGlobal();
4933
4934 // This is not actually used, but is necessary for successfully matching the
4935 // PseudoLA_*_LARGE nodes.
4936 SDValue Tmp = DAG.getConstant(0, DL, Ty);
4937
4938 // Use a PC-relative addressing mode to access the global dynamic GOT address.
4939 // This generates the pattern (PseudoLA_TLS_DESC_PC{,LARGE} sym).
4940 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, Ty, 0, 0);
4941 return Large ? SDValue(DAG.getMachineNode(Opc, DL, Ty, Tmp, Addr), 0)
4942 : SDValue(DAG.getMachineNode(Opc, DL, Ty, Addr), 0);
4943}
4944
4945SDValue
4946LoongArchTargetLowering::lowerGlobalTLSAddress(SDValue Op,
4947 SelectionDAG &DAG) const {
4950 report_fatal_error("In GHC calling convention TLS is not supported");
4951
4952 bool Large = DAG.getTarget().getCodeModel() == CodeModel::Large;
4953 assert((!Large || Subtarget.is64Bit()) && "Large code model requires LA64");
4954
4955 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Op);
4956 assert(N->getOffset() == 0 && "unexpected offset in global node");
4957
4958 if (DAG.getTarget().useEmulatedTLS())
4959 reportFatalUsageError("the emulated TLS is prohibited");
4960
4961 bool IsDesc = DAG.getTarget().useTLSDESC();
4962
4963 switch (getTargetMachine().getTLSModel(N->getGlobal())) {
4965 // In this model, application code calls the dynamic linker function
4966 // __tls_get_addr to locate TLS offsets into the dynamic thread vector at
4967 // runtime.
4968 if (!IsDesc)
4969 return getDynamicTLSAddr(N, DAG,
4970 Large ? LoongArch::PseudoLA_TLS_GD_LARGE
4971 : LoongArch::PseudoLA_TLS_GD,
4972 Large);
4973 break;
4975 // Same as GeneralDynamic, except for assembly modifiers and relocation
4976 // records.
4977 if (!IsDesc)
4978 return getDynamicTLSAddr(N, DAG,
4979 Large ? LoongArch::PseudoLA_TLS_LD_LARGE
4980 : LoongArch::PseudoLA_TLS_LD,
4981 Large);
4982 break;
4984 // This model uses the GOT to resolve TLS offsets.
4985 return getStaticTLSAddr(N, DAG,
4986 Large ? LoongArch::PseudoLA_TLS_IE_LARGE
4987 : LoongArch::PseudoLA_TLS_IE,
4988 /*UseGOT=*/true, Large);
4990 // This model is used when static linking as the TLS offsets are resolved
4991 // during program linking.
4992 //
4993 // This node doesn't need an extra argument for the large code model.
4994 return getStaticTLSAddr(N, DAG, LoongArch::PseudoLA_TLS_LE,
4995 /*UseGOT=*/false, Large);
4996 }
4997
4998 return getTLSDescAddr(N, DAG,
4999 Large ? LoongArch::PseudoLA_TLS_DESC_LARGE
5000 : LoongArch::PseudoLA_TLS_DESC,
5001 Large);
5002}
5003
5004template <unsigned N>
5006 SelectionDAG &DAG, bool IsSigned = false) {
5007 auto *CImm = cast<ConstantSDNode>(Op->getOperand(ImmOp));
5008 // Check the ImmArg.
5009 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
5010 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
5011 DAG.getContext()->emitError(Op->getOperationName(0) +
5012 ": argument out of range.");
5013 return DAG.getNode(ISD::UNDEF, SDLoc(Op), Op.getValueType());
5014 }
5015 return SDValue();
5016}
5017
5018SDValue
5019LoongArchTargetLowering::lowerINTRINSIC_WO_CHAIN(SDValue Op,
5020 SelectionDAG &DAG) const {
5021 switch (Op.getConstantOperandVal(0)) {
5022 default:
5023 return SDValue(); // Don't custom lower most intrinsics.
5024 case Intrinsic::thread_pointer: {
5025 EVT PtrVT = getPointerTy(DAG.getDataLayout());
5026 return DAG.getRegister(LoongArch::R2, PtrVT);
5027 }
5028 case Intrinsic::loongarch_lsx_vpickve2gr_d:
5029 case Intrinsic::loongarch_lsx_vpickve2gr_du:
5030 case Intrinsic::loongarch_lsx_vreplvei_d:
5031 case Intrinsic::loongarch_lasx_xvrepl128vei_d:
5032 return checkIntrinsicImmArg<1>(Op, 2, DAG);
5033 case Intrinsic::loongarch_lsx_vreplvei_w:
5034 case Intrinsic::loongarch_lasx_xvrepl128vei_w:
5035 case Intrinsic::loongarch_lasx_xvpickve2gr_d:
5036 case Intrinsic::loongarch_lasx_xvpickve2gr_du:
5037 case Intrinsic::loongarch_lasx_xvpickve_d:
5038 case Intrinsic::loongarch_lasx_xvpickve_d_f:
5039 return checkIntrinsicImmArg<2>(Op, 2, DAG);
5040 case Intrinsic::loongarch_lasx_xvinsve0_d:
5041 return checkIntrinsicImmArg<2>(Op, 3, DAG);
5042 case Intrinsic::loongarch_lsx_vsat_b:
5043 case Intrinsic::loongarch_lsx_vsat_bu:
5044 case Intrinsic::loongarch_lsx_vrotri_b:
5045 case Intrinsic::loongarch_lsx_vsllwil_h_b:
5046 case Intrinsic::loongarch_lsx_vsllwil_hu_bu:
5047 case Intrinsic::loongarch_lsx_vsrlri_b:
5048 case Intrinsic::loongarch_lsx_vsrari_b:
5049 case Intrinsic::loongarch_lsx_vreplvei_h:
5050 case Intrinsic::loongarch_lasx_xvsat_b:
5051 case Intrinsic::loongarch_lasx_xvsat_bu:
5052 case Intrinsic::loongarch_lasx_xvrotri_b:
5053 case Intrinsic::loongarch_lasx_xvsllwil_h_b:
5054 case Intrinsic::loongarch_lasx_xvsllwil_hu_bu:
5055 case Intrinsic::loongarch_lasx_xvsrlri_b:
5056 case Intrinsic::loongarch_lasx_xvsrari_b:
5057 case Intrinsic::loongarch_lasx_xvrepl128vei_h:
5058 case Intrinsic::loongarch_lasx_xvpickve_w:
5059 case Intrinsic::loongarch_lasx_xvpickve_w_f:
5060 return checkIntrinsicImmArg<3>(Op, 2, DAG);
5061 case Intrinsic::loongarch_lasx_xvinsve0_w:
5062 return checkIntrinsicImmArg<3>(Op, 3, DAG);
5063 case Intrinsic::loongarch_lsx_vsat_h:
5064 case Intrinsic::loongarch_lsx_vsat_hu:
5065 case Intrinsic::loongarch_lsx_vrotri_h:
5066 case Intrinsic::loongarch_lsx_vsllwil_w_h:
5067 case Intrinsic::loongarch_lsx_vsllwil_wu_hu:
5068 case Intrinsic::loongarch_lsx_vsrlri_h:
5069 case Intrinsic::loongarch_lsx_vsrari_h:
5070 case Intrinsic::loongarch_lsx_vreplvei_b:
5071 case Intrinsic::loongarch_lasx_xvsat_h:
5072 case Intrinsic::loongarch_lasx_xvsat_hu:
5073 case Intrinsic::loongarch_lasx_xvrotri_h:
5074 case Intrinsic::loongarch_lasx_xvsllwil_w_h:
5075 case Intrinsic::loongarch_lasx_xvsllwil_wu_hu:
5076 case Intrinsic::loongarch_lasx_xvsrlri_h:
5077 case Intrinsic::loongarch_lasx_xvsrari_h:
5078 case Intrinsic::loongarch_lasx_xvrepl128vei_b:
5079 return checkIntrinsicImmArg<4>(Op, 2, DAG);
5080 case Intrinsic::loongarch_lsx_vsrlni_b_h:
5081 case Intrinsic::loongarch_lsx_vsrani_b_h:
5082 case Intrinsic::loongarch_lsx_vsrlrni_b_h:
5083 case Intrinsic::loongarch_lsx_vsrarni_b_h:
5084 case Intrinsic::loongarch_lsx_vssrlni_b_h:
5085 case Intrinsic::loongarch_lsx_vssrani_b_h:
5086 case Intrinsic::loongarch_lsx_vssrlni_bu_h:
5087 case Intrinsic::loongarch_lsx_vssrani_bu_h:
5088 case Intrinsic::loongarch_lsx_vssrlrni_b_h:
5089 case Intrinsic::loongarch_lsx_vssrarni_b_h:
5090 case Intrinsic::loongarch_lsx_vssrlrni_bu_h:
5091 case Intrinsic::loongarch_lsx_vssrarni_bu_h:
5092 case Intrinsic::loongarch_lasx_xvsrlni_b_h:
5093 case Intrinsic::loongarch_lasx_xvsrani_b_h:
5094 case Intrinsic::loongarch_lasx_xvsrlrni_b_h:
5095 case Intrinsic::loongarch_lasx_xvsrarni_b_h:
5096 case Intrinsic::loongarch_lasx_xvssrlni_b_h:
5097 case Intrinsic::loongarch_lasx_xvssrani_b_h:
5098 case Intrinsic::loongarch_lasx_xvssrlni_bu_h:
5099 case Intrinsic::loongarch_lasx_xvssrani_bu_h:
5100 case Intrinsic::loongarch_lasx_xvssrlrni_b_h:
5101 case Intrinsic::loongarch_lasx_xvssrarni_b_h:
5102 case Intrinsic::loongarch_lasx_xvssrlrni_bu_h:
5103 case Intrinsic::loongarch_lasx_xvssrarni_bu_h:
5104 return checkIntrinsicImmArg<4>(Op, 3, DAG);
5105 case Intrinsic::loongarch_lsx_vsat_w:
5106 case Intrinsic::loongarch_lsx_vsat_wu:
5107 case Intrinsic::loongarch_lsx_vrotri_w:
5108 case Intrinsic::loongarch_lsx_vsllwil_d_w:
5109 case Intrinsic::loongarch_lsx_vsllwil_du_wu:
5110 case Intrinsic::loongarch_lsx_vsrlri_w:
5111 case Intrinsic::loongarch_lsx_vsrari_w:
5112 case Intrinsic::loongarch_lsx_vslei_bu:
5113 case Intrinsic::loongarch_lsx_vslei_hu:
5114 case Intrinsic::loongarch_lsx_vslei_wu:
5115 case Intrinsic::loongarch_lsx_vslei_du:
5116 case Intrinsic::loongarch_lsx_vslti_bu:
5117 case Intrinsic::loongarch_lsx_vslti_hu:
5118 case Intrinsic::loongarch_lsx_vslti_wu:
5119 case Intrinsic::loongarch_lsx_vslti_du:
5120 case Intrinsic::loongarch_lsx_vbsll_v:
5121 case Intrinsic::loongarch_lsx_vbsrl_v:
5122 case Intrinsic::loongarch_lasx_xvsat_w:
5123 case Intrinsic::loongarch_lasx_xvsat_wu:
5124 case Intrinsic::loongarch_lasx_xvrotri_w:
5125 case Intrinsic::loongarch_lasx_xvsllwil_d_w:
5126 case Intrinsic::loongarch_lasx_xvsllwil_du_wu:
5127 case Intrinsic::loongarch_lasx_xvsrlri_w:
5128 case Intrinsic::loongarch_lasx_xvsrari_w:
5129 case Intrinsic::loongarch_lasx_xvslei_bu:
5130 case Intrinsic::loongarch_lasx_xvslei_hu:
5131 case Intrinsic::loongarch_lasx_xvslei_wu:
5132 case Intrinsic::loongarch_lasx_xvslei_du:
5133 case Intrinsic::loongarch_lasx_xvslti_bu:
5134 case Intrinsic::loongarch_lasx_xvslti_hu:
5135 case Intrinsic::loongarch_lasx_xvslti_wu:
5136 case Intrinsic::loongarch_lasx_xvslti_du:
5137 case Intrinsic::loongarch_lasx_xvbsll_v:
5138 case Intrinsic::loongarch_lasx_xvbsrl_v:
5139 return checkIntrinsicImmArg<5>(Op, 2, DAG);
5140 case Intrinsic::loongarch_lsx_vseqi_b:
5141 case Intrinsic::loongarch_lsx_vseqi_h:
5142 case Intrinsic::loongarch_lsx_vseqi_w:
5143 case Intrinsic::loongarch_lsx_vseqi_d:
5144 case Intrinsic::loongarch_lsx_vslei_b:
5145 case Intrinsic::loongarch_lsx_vslei_h:
5146 case Intrinsic::loongarch_lsx_vslei_w:
5147 case Intrinsic::loongarch_lsx_vslei_d:
5148 case Intrinsic::loongarch_lsx_vslti_b:
5149 case Intrinsic::loongarch_lsx_vslti_h:
5150 case Intrinsic::loongarch_lsx_vslti_w:
5151 case Intrinsic::loongarch_lsx_vslti_d:
5152 case Intrinsic::loongarch_lasx_xvseqi_b:
5153 case Intrinsic::loongarch_lasx_xvseqi_h:
5154 case Intrinsic::loongarch_lasx_xvseqi_w:
5155 case Intrinsic::loongarch_lasx_xvseqi_d:
5156 case Intrinsic::loongarch_lasx_xvslei_b:
5157 case Intrinsic::loongarch_lasx_xvslei_h:
5158 case Intrinsic::loongarch_lasx_xvslei_w:
5159 case Intrinsic::loongarch_lasx_xvslei_d:
5160 case Intrinsic::loongarch_lasx_xvslti_b:
5161 case Intrinsic::loongarch_lasx_xvslti_h:
5162 case Intrinsic::loongarch_lasx_xvslti_w:
5163 case Intrinsic::loongarch_lasx_xvslti_d:
5164 return checkIntrinsicImmArg<5>(Op, 2, DAG, /*IsSigned=*/true);
5165 case Intrinsic::loongarch_lsx_vsrlni_h_w:
5166 case Intrinsic::loongarch_lsx_vsrani_h_w:
5167 case Intrinsic::loongarch_lsx_vsrlrni_h_w:
5168 case Intrinsic::loongarch_lsx_vsrarni_h_w:
5169 case Intrinsic::loongarch_lsx_vssrlni_h_w:
5170 case Intrinsic::loongarch_lsx_vssrani_h_w:
5171 case Intrinsic::loongarch_lsx_vssrlni_hu_w:
5172 case Intrinsic::loongarch_lsx_vssrani_hu_w:
5173 case Intrinsic::loongarch_lsx_vssrlrni_h_w:
5174 case Intrinsic::loongarch_lsx_vssrarni_h_w:
5175 case Intrinsic::loongarch_lsx_vssrlrni_hu_w:
5176 case Intrinsic::loongarch_lsx_vssrarni_hu_w:
5177 case Intrinsic::loongarch_lsx_vfrstpi_b:
5178 case Intrinsic::loongarch_lsx_vfrstpi_h:
5179 case Intrinsic::loongarch_lasx_xvsrlni_h_w:
5180 case Intrinsic::loongarch_lasx_xvsrani_h_w:
5181 case Intrinsic::loongarch_lasx_xvsrlrni_h_w:
5182 case Intrinsic::loongarch_lasx_xvsrarni_h_w:
5183 case Intrinsic::loongarch_lasx_xvssrlni_h_w:
5184 case Intrinsic::loongarch_lasx_xvssrani_h_w:
5185 case Intrinsic::loongarch_lasx_xvssrlni_hu_w:
5186 case Intrinsic::loongarch_lasx_xvssrani_hu_w:
5187 case Intrinsic::loongarch_lasx_xvssrlrni_h_w:
5188 case Intrinsic::loongarch_lasx_xvssrarni_h_w:
5189 case Intrinsic::loongarch_lasx_xvssrlrni_hu_w:
5190 case Intrinsic::loongarch_lasx_xvssrarni_hu_w:
5191 case Intrinsic::loongarch_lasx_xvfrstpi_b:
5192 case Intrinsic::loongarch_lasx_xvfrstpi_h:
5193 return checkIntrinsicImmArg<5>(Op, 3, DAG);
5194 case Intrinsic::loongarch_lsx_vsat_d:
5195 case Intrinsic::loongarch_lsx_vsat_du:
5196 case Intrinsic::loongarch_lsx_vrotri_d:
5197 case Intrinsic::loongarch_lsx_vsrlri_d:
5198 case Intrinsic::loongarch_lsx_vsrari_d:
5199 case Intrinsic::loongarch_lasx_xvsat_d:
5200 case Intrinsic::loongarch_lasx_xvsat_du:
5201 case Intrinsic::loongarch_lasx_xvrotri_d:
5202 case Intrinsic::loongarch_lasx_xvsrlri_d:
5203 case Intrinsic::loongarch_lasx_xvsrari_d:
5204 return checkIntrinsicImmArg<6>(Op, 2, DAG);
5205 case Intrinsic::loongarch_lsx_vsrlni_w_d:
5206 case Intrinsic::loongarch_lsx_vsrani_w_d:
5207 case Intrinsic::loongarch_lsx_vsrlrni_w_d:
5208 case Intrinsic::loongarch_lsx_vsrarni_w_d:
5209 case Intrinsic::loongarch_lsx_vssrlni_w_d:
5210 case Intrinsic::loongarch_lsx_vssrani_w_d:
5211 case Intrinsic::loongarch_lsx_vssrlni_wu_d:
5212 case Intrinsic::loongarch_lsx_vssrani_wu_d:
5213 case Intrinsic::loongarch_lsx_vssrlrni_w_d:
5214 case Intrinsic::loongarch_lsx_vssrarni_w_d:
5215 case Intrinsic::loongarch_lsx_vssrlrni_wu_d:
5216 case Intrinsic::loongarch_lsx_vssrarni_wu_d:
5217 case Intrinsic::loongarch_lasx_xvsrlni_w_d:
5218 case Intrinsic::loongarch_lasx_xvsrani_w_d:
5219 case Intrinsic::loongarch_lasx_xvsrlrni_w_d:
5220 case Intrinsic::loongarch_lasx_xvsrarni_w_d:
5221 case Intrinsic::loongarch_lasx_xvssrlni_w_d:
5222 case Intrinsic::loongarch_lasx_xvssrani_w_d:
5223 case Intrinsic::loongarch_lasx_xvssrlni_wu_d:
5224 case Intrinsic::loongarch_lasx_xvssrani_wu_d:
5225 case Intrinsic::loongarch_lasx_xvssrlrni_w_d:
5226 case Intrinsic::loongarch_lasx_xvssrarni_w_d:
5227 case Intrinsic::loongarch_lasx_xvssrlrni_wu_d:
5228 case Intrinsic::loongarch_lasx_xvssrarni_wu_d:
5229 return checkIntrinsicImmArg<6>(Op, 3, DAG);
5230 case Intrinsic::loongarch_lsx_vsrlni_d_q:
5231 case Intrinsic::loongarch_lsx_vsrani_d_q:
5232 case Intrinsic::loongarch_lsx_vsrlrni_d_q:
5233 case Intrinsic::loongarch_lsx_vsrarni_d_q:
5234 case Intrinsic::loongarch_lsx_vssrlni_d_q:
5235 case Intrinsic::loongarch_lsx_vssrani_d_q:
5236 case Intrinsic::loongarch_lsx_vssrlni_du_q:
5237 case Intrinsic::loongarch_lsx_vssrani_du_q:
5238 case Intrinsic::loongarch_lsx_vssrlrni_d_q:
5239 case Intrinsic::loongarch_lsx_vssrarni_d_q:
5240 case Intrinsic::loongarch_lsx_vssrlrni_du_q:
5241 case Intrinsic::loongarch_lsx_vssrarni_du_q:
5242 case Intrinsic::loongarch_lasx_xvsrlni_d_q:
5243 case Intrinsic::loongarch_lasx_xvsrani_d_q:
5244 case Intrinsic::loongarch_lasx_xvsrlrni_d_q:
5245 case Intrinsic::loongarch_lasx_xvsrarni_d_q:
5246 case Intrinsic::loongarch_lasx_xvssrlni_d_q:
5247 case Intrinsic::loongarch_lasx_xvssrani_d_q:
5248 case Intrinsic::loongarch_lasx_xvssrlni_du_q:
5249 case Intrinsic::loongarch_lasx_xvssrani_du_q:
5250 case Intrinsic::loongarch_lasx_xvssrlrni_d_q:
5251 case Intrinsic::loongarch_lasx_xvssrarni_d_q:
5252 case Intrinsic::loongarch_lasx_xvssrlrni_du_q:
5253 case Intrinsic::loongarch_lasx_xvssrarni_du_q:
5254 return checkIntrinsicImmArg<7>(Op, 3, DAG);
5255 case Intrinsic::loongarch_lsx_vnori_b:
5256 case Intrinsic::loongarch_lsx_vshuf4i_b:
5257 case Intrinsic::loongarch_lsx_vshuf4i_h:
5258 case Intrinsic::loongarch_lsx_vshuf4i_w:
5259 case Intrinsic::loongarch_lasx_xvnori_b:
5260 case Intrinsic::loongarch_lasx_xvshuf4i_b:
5261 case Intrinsic::loongarch_lasx_xvshuf4i_h:
5262 case Intrinsic::loongarch_lasx_xvshuf4i_w:
5263 case Intrinsic::loongarch_lasx_xvpermi_d:
5264 return checkIntrinsicImmArg<8>(Op, 2, DAG);
5265 case Intrinsic::loongarch_lsx_vshuf4i_d:
5266 case Intrinsic::loongarch_lsx_vpermi_w:
5267 case Intrinsic::loongarch_lsx_vbitseli_b:
5268 case Intrinsic::loongarch_lsx_vextrins_b:
5269 case Intrinsic::loongarch_lsx_vextrins_h:
5270 case Intrinsic::loongarch_lsx_vextrins_w:
5271 case Intrinsic::loongarch_lsx_vextrins_d:
5272 case Intrinsic::loongarch_lasx_xvshuf4i_d:
5273 case Intrinsic::loongarch_lasx_xvpermi_w:
5274 case Intrinsic::loongarch_lasx_xvpermi_q:
5275 case Intrinsic::loongarch_lasx_xvbitseli_b:
5276 case Intrinsic::loongarch_lasx_xvextrins_b:
5277 case Intrinsic::loongarch_lasx_xvextrins_h:
5278 case Intrinsic::loongarch_lasx_xvextrins_w:
5279 case Intrinsic::loongarch_lasx_xvextrins_d:
5280 return checkIntrinsicImmArg<8>(Op, 3, DAG);
5281 case Intrinsic::loongarch_lsx_vrepli_b:
5282 case Intrinsic::loongarch_lsx_vrepli_h:
5283 case Intrinsic::loongarch_lsx_vrepli_w:
5284 case Intrinsic::loongarch_lsx_vrepli_d:
5285 case Intrinsic::loongarch_lasx_xvrepli_b:
5286 case Intrinsic::loongarch_lasx_xvrepli_h:
5287 case Intrinsic::loongarch_lasx_xvrepli_w:
5288 case Intrinsic::loongarch_lasx_xvrepli_d:
5289 return checkIntrinsicImmArg<10>(Op, 1, DAG, /*IsSigned=*/true);
5290 case Intrinsic::loongarch_lsx_vldi:
5291 case Intrinsic::loongarch_lasx_xvldi:
5292 return checkIntrinsicImmArg<13>(Op, 1, DAG, /*IsSigned=*/true);
5293 }
5294}
5295
5296// Helper function that emits error message for intrinsics with chain and return
5297// merge values of a UNDEF and the chain.
5299 StringRef ErrorMsg,
5300 SelectionDAG &DAG) {
5301 DAG.getContext()->emitError(Op->getOperationName(0) + ": " + ErrorMsg + ".");
5302 return DAG.getMergeValues({DAG.getUNDEF(Op.getValueType()), Op.getOperand(0)},
5303 SDLoc(Op));
5304}
5305
5306SDValue
5307LoongArchTargetLowering::lowerINTRINSIC_W_CHAIN(SDValue Op,
5308 SelectionDAG &DAG) const {
5309 SDLoc DL(Op);
5310 MVT GRLenVT = Subtarget.getGRLenVT();
5311 EVT VT = Op.getValueType();
5312 SDValue Chain = Op.getOperand(0);
5313 const StringRef ErrorMsgOOR = "argument out of range";
5314 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5315 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5316
5317 switch (Op.getConstantOperandVal(1)) {
5318 default:
5319 return Op;
5320 case Intrinsic::loongarch_crc_w_b_w:
5321 case Intrinsic::loongarch_crc_w_h_w:
5322 case Intrinsic::loongarch_crc_w_w_w:
5323 case Intrinsic::loongarch_crc_w_d_w:
5324 case Intrinsic::loongarch_crcc_w_b_w:
5325 case Intrinsic::loongarch_crcc_w_h_w:
5326 case Intrinsic::loongarch_crcc_w_w_w:
5327 case Intrinsic::loongarch_crcc_w_d_w:
5328 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsgReqLA64, DAG);
5329 case Intrinsic::loongarch_csrrd_w:
5330 case Intrinsic::loongarch_csrrd_d: {
5331 unsigned Imm = Op.getConstantOperandVal(2);
5332 return !isUInt<14>(Imm)
5333 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
5334 : DAG.getNode(LoongArchISD::CSRRD, DL, {GRLenVT, MVT::Other},
5335 {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
5336 }
5337 case Intrinsic::loongarch_csrwr_w:
5338 case Intrinsic::loongarch_csrwr_d: {
5339 unsigned Imm = Op.getConstantOperandVal(3);
5340 return !isUInt<14>(Imm)
5341 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
5342 : DAG.getNode(LoongArchISD::CSRWR, DL, {GRLenVT, MVT::Other},
5343 {Chain, Op.getOperand(2),
5344 DAG.getConstant(Imm, DL, GRLenVT)});
5345 }
5346 case Intrinsic::loongarch_csrxchg_w:
5347 case Intrinsic::loongarch_csrxchg_d: {
5348 unsigned Imm = Op.getConstantOperandVal(4);
5349 return !isUInt<14>(Imm)
5350 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
5351 : DAG.getNode(LoongArchISD::CSRXCHG, DL, {GRLenVT, MVT::Other},
5352 {Chain, Op.getOperand(2), Op.getOperand(3),
5353 DAG.getConstant(Imm, DL, GRLenVT)});
5354 }
5355 case Intrinsic::loongarch_iocsrrd_d: {
5356 return DAG.getNode(
5357 LoongArchISD::IOCSRRD_D, DL, {GRLenVT, MVT::Other},
5358 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op.getOperand(2))});
5359 }
5360#define IOCSRRD_CASE(NAME, NODE) \
5361 case Intrinsic::loongarch_##NAME: { \
5362 return DAG.getNode(LoongArchISD::NODE, DL, {GRLenVT, MVT::Other}, \
5363 {Chain, Op.getOperand(2)}); \
5364 }
5365 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
5366 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
5367 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
5368#undef IOCSRRD_CASE
5369 case Intrinsic::loongarch_cpucfg: {
5370 return DAG.getNode(LoongArchISD::CPUCFG, DL, {GRLenVT, MVT::Other},
5371 {Chain, Op.getOperand(2)});
5372 }
5373 case Intrinsic::loongarch_lddir_d: {
5374 unsigned Imm = Op.getConstantOperandVal(3);
5375 return !isUInt<8>(Imm)
5376 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
5377 : Op;
5378 }
5379 case Intrinsic::loongarch_movfcsr2gr: {
5380 if (!Subtarget.hasBasicF())
5381 return emitIntrinsicWithChainErrorMessage(Op, ErrorMsgReqF, DAG);
5382 unsigned Imm = Op.getConstantOperandVal(2);
5383 return !isUInt<2>(Imm)
5384 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
5385 : DAG.getNode(LoongArchISD::MOVFCSR2GR, DL, {VT, MVT::Other},
5386 {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
5387 }
5388 case Intrinsic::loongarch_lsx_vld:
5389 case Intrinsic::loongarch_lsx_vldrepl_b:
5390 case Intrinsic::loongarch_lasx_xvld:
5391 case Intrinsic::loongarch_lasx_xvldrepl_b:
5392 return !isInt<12>(cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())
5393 ? emitIntrinsicWithChainErrorMessage(Op, ErrorMsgOOR, DAG)
5394 : SDValue();
5395 case Intrinsic::loongarch_lsx_vldrepl_h:
5396 case Intrinsic::loongarch_lasx_xvldrepl_h:
5397 return !isShiftedInt<11, 1>(
5398 cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())
5400 Op, "argument out of range or not a multiple of 2", DAG)
5401 : SDValue();
5402 case Intrinsic::loongarch_lsx_vldrepl_w:
5403 case Intrinsic::loongarch_lasx_xvldrepl_w:
5404 return !isShiftedInt<10, 2>(
5405 cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())
5407 Op, "argument out of range or not a multiple of 4", DAG)
5408 : SDValue();
5409 case Intrinsic::loongarch_lsx_vldrepl_d:
5410 case Intrinsic::loongarch_lasx_xvldrepl_d:
5411 return !isShiftedInt<9, 3>(
5412 cast<ConstantSDNode>(Op.getOperand(3))->getSExtValue())
5414 Op, "argument out of range or not a multiple of 8", DAG)
5415 : SDValue();
5416 }
5417}
5418
5419// Helper function that emits error message for intrinsics with void return
5420// value and return the chain.
5422 SelectionDAG &DAG) {
5423
5424 DAG.getContext()->emitError(Op->getOperationName(0) + ": " + ErrorMsg + ".");
5425 return Op.getOperand(0);
5426}
5427
5428SDValue LoongArchTargetLowering::lowerINTRINSIC_VOID(SDValue Op,
5429 SelectionDAG &DAG) const {
5430 SDLoc DL(Op);
5431 MVT GRLenVT = Subtarget.getGRLenVT();
5432 SDValue Chain = Op.getOperand(0);
5433 uint64_t IntrinsicEnum = Op.getConstantOperandVal(1);
5434 SDValue Op2 = Op.getOperand(2);
5435 const StringRef ErrorMsgOOR = "argument out of range";
5436 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
5437 const StringRef ErrorMsgReqLA32 = "requires loongarch32";
5438 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
5439
5440 switch (IntrinsicEnum) {
5441 default:
5442 // TODO: Add more Intrinsics.
5443 return SDValue();
5444 case Intrinsic::loongarch_cacop_d:
5445 case Intrinsic::loongarch_cacop_w: {
5446 if (IntrinsicEnum == Intrinsic::loongarch_cacop_d && !Subtarget.is64Bit())
5447 return emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG);
5448 if (IntrinsicEnum == Intrinsic::loongarch_cacop_w && Subtarget.is64Bit())
5449 return emitIntrinsicErrorMessage(Op, ErrorMsgReqLA32, DAG);
5450 // call void @llvm.loongarch.cacop.[d/w](uimm5, rj, simm12)
5451 unsigned Imm1 = Op2->getAsZExtVal();
5452 int Imm2 = cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue();
5453 if (!isUInt<5>(Imm1) || !isInt<12>(Imm2))
5454 return emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG);
5455 return Op;
5456 }
5457 case Intrinsic::loongarch_dbar: {
5458 unsigned Imm = Op2->getAsZExtVal();
5459 return !isUInt<15>(Imm)
5460 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5461 : DAG.getNode(LoongArchISD::DBAR, DL, MVT::Other, Chain,
5462 DAG.getConstant(Imm, DL, GRLenVT));
5463 }
5464 case Intrinsic::loongarch_ibar: {
5465 unsigned Imm = Op2->getAsZExtVal();
5466 return !isUInt<15>(Imm)
5467 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5468 : DAG.getNode(LoongArchISD::IBAR, DL, MVT::Other, Chain,
5469 DAG.getConstant(Imm, DL, GRLenVT));
5470 }
5471 case Intrinsic::loongarch_break: {
5472 unsigned Imm = Op2->getAsZExtVal();
5473 return !isUInt<15>(Imm)
5474 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5475 : DAG.getNode(LoongArchISD::BREAK, DL, MVT::Other, Chain,
5476 DAG.getConstant(Imm, DL, GRLenVT));
5477 }
5478 case Intrinsic::loongarch_movgr2fcsr: {
5479 if (!Subtarget.hasBasicF())
5480 return emitIntrinsicErrorMessage(Op, ErrorMsgReqF, DAG);
5481 unsigned Imm = Op2->getAsZExtVal();
5482 return !isUInt<2>(Imm)
5483 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5484 : DAG.getNode(LoongArchISD::MOVGR2FCSR, DL, MVT::Other, Chain,
5485 DAG.getConstant(Imm, DL, GRLenVT),
5486 DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT,
5487 Op.getOperand(3)));
5488 }
5489 case Intrinsic::loongarch_syscall: {
5490 unsigned Imm = Op2->getAsZExtVal();
5491 return !isUInt<15>(Imm)
5492 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5493 : DAG.getNode(LoongArchISD::SYSCALL, DL, MVT::Other, Chain,
5494 DAG.getConstant(Imm, DL, GRLenVT));
5495 }
5496#define IOCSRWR_CASE(NAME, NODE) \
5497 case Intrinsic::loongarch_##NAME: { \
5498 SDValue Op3 = Op.getOperand(3); \
5499 return Subtarget.is64Bit() \
5500 ? DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, \
5501 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
5502 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op3)) \
5503 : DAG.getNode(LoongArchISD::NODE, DL, MVT::Other, Chain, Op2, \
5504 Op3); \
5505 }
5506 IOCSRWR_CASE(iocsrwr_b, IOCSRWR_B);
5507 IOCSRWR_CASE(iocsrwr_h, IOCSRWR_H);
5508 IOCSRWR_CASE(iocsrwr_w, IOCSRWR_W);
5509#undef IOCSRWR_CASE
5510 case Intrinsic::loongarch_iocsrwr_d: {
5511 return !Subtarget.is64Bit()
5512 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG)
5513 : DAG.getNode(LoongArchISD::IOCSRWR_D, DL, MVT::Other, Chain,
5514 Op2,
5515 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64,
5516 Op.getOperand(3)));
5517 }
5518#define ASRT_LE_GT_CASE(NAME) \
5519 case Intrinsic::loongarch_##NAME: { \
5520 return !Subtarget.is64Bit() \
5521 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG) \
5522 : Op; \
5523 }
5524 ASRT_LE_GT_CASE(asrtle_d)
5525 ASRT_LE_GT_CASE(asrtgt_d)
5526#undef ASRT_LE_GT_CASE
5527 case Intrinsic::loongarch_ldpte_d: {
5528 unsigned Imm = Op.getConstantOperandVal(3);
5529 return !Subtarget.is64Bit()
5530 ? emitIntrinsicErrorMessage(Op, ErrorMsgReqLA64, DAG)
5531 : !isUInt<8>(Imm) ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5532 : Op;
5533 }
5534 case Intrinsic::loongarch_lsx_vst:
5535 case Intrinsic::loongarch_lasx_xvst:
5536 return !isInt<12>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue())
5537 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5538 : SDValue();
5539 case Intrinsic::loongarch_lasx_xvstelm_b:
5540 return (!isInt<8>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5541 !isUInt<5>(Op.getConstantOperandVal(5)))
5542 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5543 : SDValue();
5544 case Intrinsic::loongarch_lsx_vstelm_b:
5545 return (!isInt<8>(cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5546 !isUInt<4>(Op.getConstantOperandVal(5)))
5547 ? emitIntrinsicErrorMessage(Op, ErrorMsgOOR, DAG)
5548 : SDValue();
5549 case Intrinsic::loongarch_lasx_xvstelm_h:
5550 return (!isShiftedInt<8, 1>(
5551 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5552 !isUInt<4>(Op.getConstantOperandVal(5)))
5554 Op, "argument out of range or not a multiple of 2", DAG)
5555 : SDValue();
5556 case Intrinsic::loongarch_lsx_vstelm_h:
5557 return (!isShiftedInt<8, 1>(
5558 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5559 !isUInt<3>(Op.getConstantOperandVal(5)))
5561 Op, "argument out of range or not a multiple of 2", DAG)
5562 : SDValue();
5563 case Intrinsic::loongarch_lasx_xvstelm_w:
5564 return (!isShiftedInt<8, 2>(
5565 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5566 !isUInt<3>(Op.getConstantOperandVal(5)))
5568 Op, "argument out of range or not a multiple of 4", DAG)
5569 : SDValue();
5570 case Intrinsic::loongarch_lsx_vstelm_w:
5571 return (!isShiftedInt<8, 2>(
5572 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5573 !isUInt<2>(Op.getConstantOperandVal(5)))
5575 Op, "argument out of range or not a multiple of 4", DAG)
5576 : SDValue();
5577 case Intrinsic::loongarch_lasx_xvstelm_d:
5578 return (!isShiftedInt<8, 3>(
5579 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5580 !isUInt<2>(Op.getConstantOperandVal(5)))
5582 Op, "argument out of range or not a multiple of 8", DAG)
5583 : SDValue();
5584 case Intrinsic::loongarch_lsx_vstelm_d:
5585 return (!isShiftedInt<8, 3>(
5586 cast<ConstantSDNode>(Op.getOperand(4))->getSExtValue()) ||
5587 !isUInt<1>(Op.getConstantOperandVal(5)))
5589 Op, "argument out of range or not a multiple of 8", DAG)
5590 : SDValue();
5591 }
5592}
5593
5594SDValue LoongArchTargetLowering::lowerShiftLeftParts(SDValue Op,
5595 SelectionDAG &DAG) const {
5596 SDLoc DL(Op);
5597 SDValue Lo = Op.getOperand(0);
5598 SDValue Hi = Op.getOperand(1);
5599 SDValue Shamt = Op.getOperand(2);
5600 EVT VT = Lo.getValueType();
5601
5602 // if Shamt-GRLen < 0: // Shamt < GRLen
5603 // Lo = Lo << Shamt
5604 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (GRLen-1 ^ Shamt))
5605 // else:
5606 // Lo = 0
5607 // Hi = Lo << (Shamt-GRLen)
5608
5609 SDValue Zero = DAG.getConstant(0, DL, VT);
5610 SDValue One = DAG.getConstant(1, DL, VT);
5611 SDValue MinusGRLen =
5612 DAG.getSignedConstant(-(int)Subtarget.getGRLen(), DL, VT);
5613 SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT);
5614 SDValue ShamtMinusGRLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusGRLen);
5615 SDValue GRLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, GRLenMinus1);
5616
5617 SDValue LoTrue = DAG.getNode(ISD::SHL, DL, VT, Lo, Shamt);
5618 SDValue ShiftRight1Lo = DAG.getNode(ISD::SRL, DL, VT, Lo, One);
5619 SDValue ShiftRightLo =
5620 DAG.getNode(ISD::SRL, DL, VT, ShiftRight1Lo, GRLenMinus1Shamt);
5621 SDValue ShiftLeftHi = DAG.getNode(ISD::SHL, DL, VT, Hi, Shamt);
5622 SDValue HiTrue = DAG.getNode(ISD::OR, DL, VT, ShiftLeftHi, ShiftRightLo);
5623 SDValue HiFalse = DAG.getNode(ISD::SHL, DL, VT, Lo, ShamtMinusGRLen);
5624
5625 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusGRLen, Zero, ISD::SETLT);
5626
5627 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, Zero);
5628 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
5629
5630 SDValue Parts[2] = {Lo, Hi};
5631 return DAG.getMergeValues(Parts, DL);
5632}
5633
5634SDValue LoongArchTargetLowering::lowerShiftRightParts(SDValue Op,
5635 SelectionDAG &DAG,
5636 bool IsSRA) const {
5637 SDLoc DL(Op);
5638 SDValue Lo = Op.getOperand(0);
5639 SDValue Hi = Op.getOperand(1);
5640 SDValue Shamt = Op.getOperand(2);
5641 EVT VT = Lo.getValueType();
5642
5643 // SRA expansion:
5644 // if Shamt-GRLen < 0: // Shamt < GRLen
5645 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5646 // Hi = Hi >>s Shamt
5647 // else:
5648 // Lo = Hi >>s (Shamt-GRLen);
5649 // Hi = Hi >>s (GRLen-1)
5650 //
5651 // SRL expansion:
5652 // if Shamt-GRLen < 0: // Shamt < GRLen
5653 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (ShAmt ^ GRLen-1))
5654 // Hi = Hi >>u Shamt
5655 // else:
5656 // Lo = Hi >>u (Shamt-GRLen);
5657 // Hi = 0;
5658
5659 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
5660
5661 SDValue Zero = DAG.getConstant(0, DL, VT);
5662 SDValue One = DAG.getConstant(1, DL, VT);
5663 SDValue MinusGRLen =
5664 DAG.getSignedConstant(-(int)Subtarget.getGRLen(), DL, VT);
5665 SDValue GRLenMinus1 = DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT);
5666 SDValue ShamtMinusGRLen = DAG.getNode(ISD::ADD, DL, VT, Shamt, MinusGRLen);
5667 SDValue GRLenMinus1Shamt = DAG.getNode(ISD::XOR, DL, VT, Shamt, GRLenMinus1);
5668
5669 SDValue ShiftRightLo = DAG.getNode(ISD::SRL, DL, VT, Lo, Shamt);
5670 SDValue ShiftLeftHi1 = DAG.getNode(ISD::SHL, DL, VT, Hi, One);
5671 SDValue ShiftLeftHi =
5672 DAG.getNode(ISD::SHL, DL, VT, ShiftLeftHi1, GRLenMinus1Shamt);
5673 SDValue LoTrue = DAG.getNode(ISD::OR, DL, VT, ShiftRightLo, ShiftLeftHi);
5674 SDValue HiTrue = DAG.getNode(ShiftRightOp, DL, VT, Hi, Shamt);
5675 SDValue LoFalse = DAG.getNode(ShiftRightOp, DL, VT, Hi, ShamtMinusGRLen);
5676 SDValue HiFalse =
5677 IsSRA ? DAG.getNode(ISD::SRA, DL, VT, Hi, GRLenMinus1) : Zero;
5678
5679 SDValue CC = DAG.getSetCC(DL, VT, ShamtMinusGRLen, Zero, ISD::SETLT);
5680
5681 Lo = DAG.getNode(ISD::SELECT, DL, VT, CC, LoTrue, LoFalse);
5682 Hi = DAG.getNode(ISD::SELECT, DL, VT, CC, HiTrue, HiFalse);
5683
5684 SDValue Parts[2] = {Lo, Hi};
5685 return DAG.getMergeValues(Parts, DL);
5686}
5687
5688// Returns the opcode of the target-specific SDNode that implements the 32-bit
5689// form of the given Opcode.
5690static unsigned getLoongArchWOpcode(unsigned Opcode) {
5691 switch (Opcode) {
5692 default:
5693 llvm_unreachable("Unexpected opcode");
5694 case ISD::SDIV:
5695 return LoongArchISD::DIV_W;
5696 case ISD::UDIV:
5697 return LoongArchISD::DIV_WU;
5698 case ISD::SREM:
5699 return LoongArchISD::MOD_W;
5700 case ISD::UREM:
5701 return LoongArchISD::MOD_WU;
5702 case ISD::SHL:
5703 return LoongArchISD::SLL_W;
5704 case ISD::SRA:
5705 return LoongArchISD::SRA_W;
5706 case ISD::SRL:
5707 return LoongArchISD::SRL_W;
5708 case ISD::ROTL:
5709 case ISD::ROTR:
5710 return LoongArchISD::ROTR_W;
5711 case ISD::CTTZ:
5712 return LoongArchISD::CTZ_W;
5713 case ISD::CTLZ:
5714 return LoongArchISD::CLZ_W;
5715 }
5716}
5717
5718// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
5719// node. Because i8/i16/i32 isn't a legal type for LA64, these operations would
5720// otherwise be promoted to i64, making it difficult to select the
5721// SLL_W/.../*W later one because the fact the operation was originally of
5722// type i8/i16/i32 is lost.
5724 unsigned ExtOpc = ISD::ANY_EXTEND) {
5725 SDLoc DL(N);
5726 unsigned WOpcode = getLoongArchWOpcode(N->getOpcode());
5727 SDValue NewOp0, NewRes;
5728
5729 switch (NumOp) {
5730 default:
5731 llvm_unreachable("Unexpected NumOp");
5732 case 1: {
5733 NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
5734 NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0);
5735 break;
5736 }
5737 case 2: {
5738 NewOp0 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(0));
5739 SDValue NewOp1 = DAG.getNode(ExtOpc, DL, MVT::i64, N->getOperand(1));
5740 if (N->getOpcode() == ISD::ROTL) {
5741 SDValue TmpOp = DAG.getConstant(32, DL, MVT::i64);
5742 NewOp1 = DAG.getNode(ISD::SUB, DL, MVT::i64, TmpOp, NewOp1);
5743 }
5744 NewRes = DAG.getNode(WOpcode, DL, MVT::i64, NewOp0, NewOp1);
5745 break;
5746 }
5747 // TODO:Handle more NumOp.
5748 }
5749
5750 // ReplaceNodeResults requires we maintain the same type for the return
5751 // value.
5752 return DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), NewRes);
5753}
5754
5755// Converts the given 32-bit operation to a i64 operation with signed extension
5756// semantic to reduce the signed extension instructions.
5758 SDLoc DL(N);
5759 SDValue NewOp0 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(0));
5760 SDValue NewOp1 = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(1));
5761 SDValue NewWOp = DAG.getNode(N->getOpcode(), DL, MVT::i64, NewOp0, NewOp1);
5762 SDValue NewRes = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, MVT::i64, NewWOp,
5763 DAG.getValueType(MVT::i32));
5764 return DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, NewRes);
5765}
5766
5767// Helper function that emits error message for intrinsics with/without chain
5768// and return a UNDEF or and the chain as the results.
5771 StringRef ErrorMsg, bool WithChain = true) {
5772 DAG.getContext()->emitError(N->getOperationName(0) + ": " + ErrorMsg + ".");
5773 Results.push_back(DAG.getUNDEF(N->getValueType(0)));
5774 if (!WithChain)
5775 return;
5776 Results.push_back(N->getOperand(0));
5777}
5778
5779template <unsigned N>
5780static void
5783 unsigned ResOp) {
5784 const StringRef ErrorMsgOOR = "argument out of range";
5785 unsigned Imm = Node->getConstantOperandVal(2);
5786 if (!isUInt<N>(Imm)) {
5788 /*WithChain=*/false);
5789 return;
5790 }
5791 SDLoc DL(Node);
5792 SDValue Vec = Node->getOperand(1);
5793
5794 SDValue PickElt =
5795 DAG.getNode(ResOp, DL, Subtarget.getGRLenVT(), Vec,
5796 DAG.getConstant(Imm, DL, Subtarget.getGRLenVT()),
5798 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, Node->getValueType(0),
5799 PickElt.getValue(0)));
5800}
5801
5804 SelectionDAG &DAG,
5806 unsigned ResOp) {
5807 SDLoc DL(N);
5808 SDValue Vec = N->getOperand(1);
5809
5810 SDValue CB = DAG.getNode(ResOp, DL, Subtarget.getGRLenVT(), Vec);
5811 Results.push_back(
5812 DAG.getNode(ISD::TRUNCATE, DL, N->getValueType(0), CB.getValue(0)));
5813}
5814
5815static void
5817 SelectionDAG &DAG,
5819 switch (N->getConstantOperandVal(0)) {
5820 default:
5821 llvm_unreachable("Unexpected Intrinsic.");
5822 case Intrinsic::loongarch_lsx_vpickve2gr_b:
5824 LoongArchISD::VPICK_SEXT_ELT);
5825 break;
5826 case Intrinsic::loongarch_lsx_vpickve2gr_h:
5827 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
5829 LoongArchISD::VPICK_SEXT_ELT);
5830 break;
5831 case Intrinsic::loongarch_lsx_vpickve2gr_w:
5833 LoongArchISD::VPICK_SEXT_ELT);
5834 break;
5835 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
5837 LoongArchISD::VPICK_ZEXT_ELT);
5838 break;
5839 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
5840 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
5842 LoongArchISD::VPICK_ZEXT_ELT);
5843 break;
5844 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
5846 LoongArchISD::VPICK_ZEXT_ELT);
5847 break;
5848 case Intrinsic::loongarch_lsx_bz_b:
5849 case Intrinsic::loongarch_lsx_bz_h:
5850 case Intrinsic::loongarch_lsx_bz_w:
5851 case Intrinsic::loongarch_lsx_bz_d:
5852 case Intrinsic::loongarch_lasx_xbz_b:
5853 case Intrinsic::loongarch_lasx_xbz_h:
5854 case Intrinsic::loongarch_lasx_xbz_w:
5855 case Intrinsic::loongarch_lasx_xbz_d:
5857 LoongArchISD::VALL_ZERO);
5858 break;
5859 case Intrinsic::loongarch_lsx_bz_v:
5860 case Intrinsic::loongarch_lasx_xbz_v:
5862 LoongArchISD::VANY_ZERO);
5863 break;
5864 case Intrinsic::loongarch_lsx_bnz_b:
5865 case Intrinsic::loongarch_lsx_bnz_h:
5866 case Intrinsic::loongarch_lsx_bnz_w:
5867 case Intrinsic::loongarch_lsx_bnz_d:
5868 case Intrinsic::loongarch_lasx_xbnz_b:
5869 case Intrinsic::loongarch_lasx_xbnz_h:
5870 case Intrinsic::loongarch_lasx_xbnz_w:
5871 case Intrinsic::loongarch_lasx_xbnz_d:
5873 LoongArchISD::VALL_NONZERO);
5874 break;
5875 case Intrinsic::loongarch_lsx_bnz_v:
5876 case Intrinsic::loongarch_lasx_xbnz_v:
5878 LoongArchISD::VANY_NONZERO);
5879 break;
5880 }
5881}
5882
5885 SelectionDAG &DAG) {
5886 assert(N->getValueType(0) == MVT::i128 &&
5887 "AtomicCmpSwap on types less than 128 should be legal");
5888 MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand();
5889
5890 unsigned Opcode;
5891 switch (MemOp->getMergedOrdering()) {
5895 Opcode = LoongArch::PseudoCmpXchg128Acquire;
5896 break;
5899 Opcode = LoongArch::PseudoCmpXchg128;
5900 break;
5901 default:
5902 llvm_unreachable("Unexpected ordering!");
5903 }
5904
5905 SDLoc DL(N);
5906 auto CmpVal = DAG.SplitScalar(N->getOperand(2), DL, MVT::i64, MVT::i64);
5907 auto NewVal = DAG.SplitScalar(N->getOperand(3), DL, MVT::i64, MVT::i64);
5908 SDValue Ops[] = {N->getOperand(1), CmpVal.first, CmpVal.second,
5909 NewVal.first, NewVal.second, N->getOperand(0)};
5910
5911 SDNode *CmpSwap = DAG.getMachineNode(
5912 Opcode, SDLoc(N), DAG.getVTList(MVT::i64, MVT::i64, MVT::i64, MVT::Other),
5913 Ops);
5914 DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp});
5915 Results.push_back(DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i128,
5916 SDValue(CmpSwap, 0), SDValue(CmpSwap, 1)));
5917 Results.push_back(SDValue(CmpSwap, 3));
5918}
5919
5922 SDLoc DL(N);
5923 EVT VT = N->getValueType(0);
5924 switch (N->getOpcode()) {
5925 default:
5926 llvm_unreachable("Don't know how to legalize this operation");
5927 case ISD::ADD:
5928 case ISD::SUB:
5929 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
5930 "Unexpected custom legalisation");
5931 Results.push_back(customLegalizeToWOpWithSExt(N, DAG));
5932 break;
5933 case ISD::SDIV:
5934 case ISD::UDIV:
5935 case ISD::SREM:
5936 case ISD::UREM:
5937 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5938 "Unexpected custom legalisation");
5939 Results.push_back(customLegalizeToWOp(N, DAG, 2,
5940 Subtarget.hasDiv32() && VT == MVT::i32
5942 : ISD::SIGN_EXTEND));
5943 break;
5944 case ISD::SHL:
5945 case ISD::SRA:
5946 case ISD::SRL:
5947 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5948 "Unexpected custom legalisation");
5949 if (N->getOperand(1).getOpcode() != ISD::Constant) {
5950 Results.push_back(customLegalizeToWOp(N, DAG, 2));
5951 break;
5952 }
5953 break;
5954 case ISD::ROTL:
5955 case ISD::ROTR:
5956 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5957 "Unexpected custom legalisation");
5958 Results.push_back(customLegalizeToWOp(N, DAG, 2));
5959 break;
5960 case ISD::LOAD: {
5961 // Use an f64 load and a scalar_to_vector for v2f32 loads. This avoids
5962 // scalarizing in 32-bit mode. In 64-bit mode this avoids a int->fp
5963 // cast since type legalization will try to use an i64 load.
5964 MVT VT = N->getSimpleValueType(0);
5965 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
5966 "Unexpected custom legalisation");
5968 "Unexpected type action!");
5969 if (!ISD::isNON_EXTLoad(N))
5970 return;
5971 auto *Ld = cast<LoadSDNode>(N);
5972 SDValue Res = DAG.getLoad(MVT::f64, DL, Ld->getChain(), Ld->getBasePtr(),
5973 Ld->getPointerInfo(), Ld->getBaseAlign(),
5974 Ld->getMemOperand()->getFlags());
5975 SDValue Chain = Res.getValue(1);
5976 MVT VecVT = MVT::getVectorVT(MVT::f64, 2);
5977 Res = DAG.getNode(ISD::SCALAR_TO_VECTOR, DL, VecVT, Res);
5978 EVT WideVT = getTypeToTransformTo(*DAG.getContext(), VT);
5979 Res = DAG.getBitcast(WideVT, Res);
5980 Results.push_back(Res);
5981 Results.push_back(Chain);
5982 break;
5983 }
5984 case ISD::FP_TO_SINT: {
5985 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
5986 "Unexpected custom legalisation");
5987 SDValue Src = N->getOperand(0);
5988 EVT FVT = EVT::getFloatingPointVT(N->getValueSizeInBits(0));
5989 if (getTypeAction(*DAG.getContext(), Src.getValueType()) !=
5991 if (!isTypeLegal(Src.getValueType()))
5992 return;
5993 if (Src.getValueType() == MVT::f16)
5994 Src = DAG.getNode(ISD::FP_EXTEND, DL, MVT::f32, Src);
5995 SDValue Dst = DAG.getNode(LoongArchISD::FTINT, DL, FVT, Src);
5996 Results.push_back(DAG.getNode(ISD::BITCAST, DL, VT, Dst));
5997 return;
5998 }
5999 // If the FP type needs to be softened, emit a library call using the 'si'
6000 // version. If we left it to default legalization we'd end up with 'di'.
6001 RTLIB::Libcall LC;
6002 LC = RTLIB::getFPTOSINT(Src.getValueType(), VT);
6003 MakeLibCallOptions CallOptions;
6004 EVT OpVT = Src.getValueType();
6005 CallOptions.setTypeListBeforeSoften(OpVT, VT);
6006 SDValue Chain = SDValue();
6007 SDValue Result;
6008 std::tie(Result, Chain) =
6009 makeLibCall(DAG, LC, VT, Src, CallOptions, DL, Chain);
6010 Results.push_back(Result);
6011 break;
6012 }
6013 case ISD::BITCAST: {
6014 SDValue Src = N->getOperand(0);
6015 EVT SrcVT = Src.getValueType();
6016 if (VT == MVT::i32 && SrcVT == MVT::f32 && Subtarget.is64Bit() &&
6017 Subtarget.hasBasicF()) {
6018 SDValue Dst =
6019 DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Src);
6020 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Dst));
6021 } else if (VT == MVT::i64 && SrcVT == MVT::f64 && !Subtarget.is64Bit()) {
6022 SDValue NewReg = DAG.getNode(LoongArchISD::SPLIT_PAIR_F64, DL,
6023 DAG.getVTList(MVT::i32, MVT::i32), Src);
6024 SDValue RetReg = DAG.getNode(ISD::BUILD_PAIR, DL, MVT::i64,
6025 NewReg.getValue(0), NewReg.getValue(1));
6026 Results.push_back(RetReg);
6027 }
6028 break;
6029 }
6030 case ISD::FP_TO_UINT: {
6031 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
6032 "Unexpected custom legalisation");
6033 auto &TLI = DAG.getTargetLoweringInfo();
6034 SDValue Tmp1, Tmp2;
6035 TLI.expandFP_TO_UINT(N, Tmp1, Tmp2, DAG);
6036 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Tmp1));
6037 break;
6038 }
6039 case ISD::FP_ROUND: {
6040 assert(VT == MVT::v2f32 && Subtarget.hasExtLSX() &&
6041 "Unexpected custom legalisation");
6042 // On LSX platforms, rounding from v2f64 to v4f32 (after legalization from
6043 // v2f32) is scalarized. Add a customized v2f32 widening to convert it into
6044 // a target-specific LoongArchISD::VFCVT to optimize it.
6045 SDValue Op0 = N->getOperand(0);
6046 EVT OpVT = Op0.getValueType();
6047 if (OpVT == MVT::v2f64) {
6048 SDValue Undef = DAG.getUNDEF(OpVT);
6049 SDValue Dst =
6050 DAG.getNode(LoongArchISD::VFCVT, DL, MVT::v4f32, Undef, Op0);
6051 Results.push_back(Dst);
6052 }
6053 break;
6054 }
6055 case ISD::BSWAP: {
6056 SDValue Src = N->getOperand(0);
6057 assert((VT == MVT::i16 || VT == MVT::i32) &&
6058 "Unexpected custom legalization");
6059 MVT GRLenVT = Subtarget.getGRLenVT();
6060 SDValue NewSrc = DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, Src);
6061 SDValue Tmp;
6062 switch (VT.getSizeInBits()) {
6063 default:
6064 llvm_unreachable("Unexpected operand width");
6065 case 16:
6066 Tmp = DAG.getNode(LoongArchISD::REVB_2H, DL, GRLenVT, NewSrc);
6067 break;
6068 case 32:
6069 // Only LA64 will get to here due to the size mismatch between VT and
6070 // GRLenVT, LA32 lowering is directly defined in LoongArchInstrInfo.
6071 Tmp = DAG.getNode(LoongArchISD::REVB_2W, DL, GRLenVT, NewSrc);
6072 break;
6073 }
6074 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Tmp));
6075 break;
6076 }
6077 case ISD::BITREVERSE: {
6078 SDValue Src = N->getOperand(0);
6079 assert((VT == MVT::i8 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
6080 "Unexpected custom legalization");
6081 MVT GRLenVT = Subtarget.getGRLenVT();
6082 SDValue NewSrc = DAG.getNode(ISD::ANY_EXTEND, DL, GRLenVT, Src);
6083 SDValue Tmp;
6084 switch (VT.getSizeInBits()) {
6085 default:
6086 llvm_unreachable("Unexpected operand width");
6087 case 8:
6088 Tmp = DAG.getNode(LoongArchISD::BITREV_4B, DL, GRLenVT, NewSrc);
6089 break;
6090 case 32:
6091 Tmp = DAG.getNode(LoongArchISD::BITREV_W, DL, GRLenVT, NewSrc);
6092 break;
6093 }
6094 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, Tmp));
6095 break;
6096 }
6097 case ISD::CTLZ:
6098 case ISD::CTTZ: {
6099 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
6100 "Unexpected custom legalisation");
6101 Results.push_back(customLegalizeToWOp(N, DAG, 1));
6102 break;
6103 }
6105 SDValue Chain = N->getOperand(0);
6106 SDValue Op2 = N->getOperand(2);
6107 MVT GRLenVT = Subtarget.getGRLenVT();
6108 const StringRef ErrorMsgOOR = "argument out of range";
6109 const StringRef ErrorMsgReqLA64 = "requires loongarch64";
6110 const StringRef ErrorMsgReqF = "requires basic 'f' target feature";
6111
6112 switch (N->getConstantOperandVal(1)) {
6113 default:
6114 llvm_unreachable("Unexpected Intrinsic.");
6115 case Intrinsic::loongarch_movfcsr2gr: {
6116 if (!Subtarget.hasBasicF()) {
6117 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqF);
6118 return;
6119 }
6120 unsigned Imm = Op2->getAsZExtVal();
6121 if (!isUInt<2>(Imm)) {
6122 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);
6123 return;
6124 }
6125 SDValue MOVFCSR2GRResults = DAG.getNode(
6126 LoongArchISD::MOVFCSR2GR, SDLoc(N), {MVT::i64, MVT::Other},
6127 {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
6128 Results.push_back(
6129 DAG.getNode(ISD::TRUNCATE, DL, VT, MOVFCSR2GRResults.getValue(0)));
6130 Results.push_back(MOVFCSR2GRResults.getValue(1));
6131 break;
6132 }
6133#define CRC_CASE_EXT_BINARYOP(NAME, NODE) \
6134 case Intrinsic::loongarch_##NAME: { \
6135 SDValue NODE = DAG.getNode( \
6136 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6137 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2), \
6138 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
6139 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
6140 Results.push_back(NODE.getValue(1)); \
6141 break; \
6142 }
6143 CRC_CASE_EXT_BINARYOP(crc_w_b_w, CRC_W_B_W)
6144 CRC_CASE_EXT_BINARYOP(crc_w_h_w, CRC_W_H_W)
6145 CRC_CASE_EXT_BINARYOP(crc_w_w_w, CRC_W_W_W)
6146 CRC_CASE_EXT_BINARYOP(crcc_w_b_w, CRCC_W_B_W)
6147 CRC_CASE_EXT_BINARYOP(crcc_w_h_w, CRCC_W_H_W)
6148 CRC_CASE_EXT_BINARYOP(crcc_w_w_w, CRCC_W_W_W)
6149#undef CRC_CASE_EXT_BINARYOP
6150
6151#define CRC_CASE_EXT_UNARYOP(NAME, NODE) \
6152 case Intrinsic::loongarch_##NAME: { \
6153 SDValue NODE = DAG.getNode( \
6154 LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6155 {Chain, Op2, \
6156 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3))}); \
6157 Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, VT, NODE.getValue(0))); \
6158 Results.push_back(NODE.getValue(1)); \
6159 break; \
6160 }
6161 CRC_CASE_EXT_UNARYOP(crc_w_d_w, CRC_W_D_W)
6162 CRC_CASE_EXT_UNARYOP(crcc_w_d_w, CRCC_W_D_W)
6163#undef CRC_CASE_EXT_UNARYOP
6164#define CSR_CASE(ID) \
6165 case Intrinsic::loongarch_##ID: { \
6166 if (!Subtarget.is64Bit()) \
6167 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64); \
6168 break; \
6169 }
6170 CSR_CASE(csrrd_d);
6171 CSR_CASE(csrwr_d);
6172 CSR_CASE(csrxchg_d);
6173 CSR_CASE(iocsrrd_d);
6174#undef CSR_CASE
6175 case Intrinsic::loongarch_csrrd_w: {
6176 unsigned Imm = Op2->getAsZExtVal();
6177 if (!isUInt<14>(Imm)) {
6178 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);
6179 return;
6180 }
6181 SDValue CSRRDResults =
6182 DAG.getNode(LoongArchISD::CSRRD, DL, {GRLenVT, MVT::Other},
6183 {Chain, DAG.getConstant(Imm, DL, GRLenVT)});
6184 Results.push_back(
6185 DAG.getNode(ISD::TRUNCATE, DL, VT, CSRRDResults.getValue(0)));
6186 Results.push_back(CSRRDResults.getValue(1));
6187 break;
6188 }
6189 case Intrinsic::loongarch_csrwr_w: {
6190 unsigned Imm = N->getConstantOperandVal(3);
6191 if (!isUInt<14>(Imm)) {
6192 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);
6193 return;
6194 }
6195 SDValue CSRWRResults =
6196 DAG.getNode(LoongArchISD::CSRWR, DL, {GRLenVT, MVT::Other},
6197 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),
6198 DAG.getConstant(Imm, DL, GRLenVT)});
6199 Results.push_back(
6200 DAG.getNode(ISD::TRUNCATE, DL, VT, CSRWRResults.getValue(0)));
6201 Results.push_back(CSRWRResults.getValue(1));
6202 break;
6203 }
6204 case Intrinsic::loongarch_csrxchg_w: {
6205 unsigned Imm = N->getConstantOperandVal(4);
6206 if (!isUInt<14>(Imm)) {
6207 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgOOR);
6208 return;
6209 }
6210 SDValue CSRXCHGResults = DAG.getNode(
6211 LoongArchISD::CSRXCHG, DL, {GRLenVT, MVT::Other},
6212 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2),
6213 DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, N->getOperand(3)),
6214 DAG.getConstant(Imm, DL, GRLenVT)});
6215 Results.push_back(
6216 DAG.getNode(ISD::TRUNCATE, DL, VT, CSRXCHGResults.getValue(0)));
6217 Results.push_back(CSRXCHGResults.getValue(1));
6218 break;
6219 }
6220#define IOCSRRD_CASE(NAME, NODE) \
6221 case Intrinsic::loongarch_##NAME: { \
6222 SDValue IOCSRRDResults = \
6223 DAG.getNode(LoongArchISD::NODE, DL, {MVT::i64, MVT::Other}, \
6224 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)}); \
6225 Results.push_back( \
6226 DAG.getNode(ISD::TRUNCATE, DL, VT, IOCSRRDResults.getValue(0))); \
6227 Results.push_back(IOCSRRDResults.getValue(1)); \
6228 break; \
6229 }
6230 IOCSRRD_CASE(iocsrrd_b, IOCSRRD_B);
6231 IOCSRRD_CASE(iocsrrd_h, IOCSRRD_H);
6232 IOCSRRD_CASE(iocsrrd_w, IOCSRRD_W);
6233#undef IOCSRRD_CASE
6234 case Intrinsic::loongarch_cpucfg: {
6235 SDValue CPUCFGResults =
6236 DAG.getNode(LoongArchISD::CPUCFG, DL, {GRLenVT, MVT::Other},
6237 {Chain, DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i64, Op2)});
6238 Results.push_back(
6239 DAG.getNode(ISD::TRUNCATE, DL, VT, CPUCFGResults.getValue(0)));
6240 Results.push_back(CPUCFGResults.getValue(1));
6241 break;
6242 }
6243 case Intrinsic::loongarch_lddir_d: {
6244 if (!Subtarget.is64Bit()) {
6245 emitErrorAndReplaceIntrinsicResults(N, Results, DAG, ErrorMsgReqLA64);
6246 return;
6247 }
6248 break;
6249 }
6250 }
6251 break;
6252 }
6253 case ISD::READ_REGISTER: {
6254 if (Subtarget.is64Bit())
6255 DAG.getContext()->emitError(
6256 "On LA64, only 64-bit registers can be read.");
6257 else
6258 DAG.getContext()->emitError(
6259 "On LA32, only 32-bit registers can be read.");
6260 Results.push_back(DAG.getUNDEF(VT));
6261 Results.push_back(N->getOperand(0));
6262 break;
6263 }
6265 replaceINTRINSIC_WO_CHAINResults(N, Results, DAG, Subtarget);
6266 break;
6267 }
6268 case ISD::LROUND: {
6269 SDValue Op0 = N->getOperand(0);
6270 EVT OpVT = Op0.getValueType();
6271 RTLIB::Libcall LC =
6272 OpVT == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
6273 MakeLibCallOptions CallOptions;
6274 CallOptions.setTypeListBeforeSoften(OpVT, MVT::i64);
6275 SDValue Result = makeLibCall(DAG, LC, MVT::i64, Op0, CallOptions, DL).first;
6276 Result = DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Result);
6277 Results.push_back(Result);
6278 break;
6279 }
6280 case ISD::ATOMIC_CMP_SWAP: {
6282 break;
6283 }
6284 case ISD::TRUNCATE: {
6285 MVT VT = N->getSimpleValueType(0);
6286 if (getTypeAction(*DAG.getContext(), VT) != TypeWidenVector)
6287 return;
6288
6289 MVT WidenVT = getTypeToTransformTo(*DAG.getContext(), VT).getSimpleVT();
6290 SDValue In = N->getOperand(0);
6291 EVT InVT = In.getValueType();
6292 EVT InEltVT = InVT.getVectorElementType();
6293 EVT EltVT = VT.getVectorElementType();
6294 unsigned MinElts = VT.getVectorNumElements();
6295 unsigned WidenNumElts = WidenVT.getVectorNumElements();
6296 unsigned InBits = InVT.getSizeInBits();
6297
6298 // v8i64 -> (v8i32) -> v8i8
6299 if (InVT == MVT::v8i64 && WidenVT.is128BitVector()) {
6300 InVT = MVT::getVectorVT(MVT::getIntegerVT(256 / MinElts), MinElts);
6301 In = DAG.getNode(N->getOpcode(), DL, InVT, In);
6302 InBits = 256;
6303 }
6304
6305 // v8i32 -> v8i8 / v4i64 -> v4i16 / v4i64 -> v4i8
6306 if ((InVT == MVT::v8i32 || InVT == MVT::v4i64) &&
6307 WidenVT.is128BitVector()) {
6308 InVT = MVT::getVectorVT(MVT::getIntegerVT(128 / MinElts), MinElts);
6309 In = DAG.getNode(N->getOpcode(), DL, InVT, In);
6310 InBits = 128;
6311 InEltVT = InVT.getVectorElementType();
6312 }
6313
6314 if ((128 % InBits) == 0 && WidenVT.is128BitVector()) {
6315 if ((InEltVT.getSizeInBits() % EltVT.getSizeInBits()) == 0) {
6316 int Scale = InEltVT.getSizeInBits() / EltVT.getSizeInBits();
6317 SmallVector<int, 16> TruncMask(WidenNumElts, -1);
6318 for (unsigned I = 0; I < MinElts; ++I)
6319 TruncMask[I] = Scale * I;
6320
6321 unsigned WidenNumElts = 128 / In.getScalarValueSizeInBits();
6322 MVT SVT = In.getSimpleValueType().getScalarType();
6323 MVT VT = MVT::getVectorVT(SVT, WidenNumElts);
6324 SDValue WidenIn =
6325 DAG.getNode(ISD::INSERT_SUBVECTOR, DL, VT, DAG.getUNDEF(VT), In,
6326 DAG.getVectorIdxConstant(0, DL));
6327 assert(isTypeLegal(WidenVT) && isTypeLegal(WidenIn.getValueType()) &&
6328 "Illegal vector type in truncation");
6329 WidenIn = DAG.getBitcast(WidenVT, WidenIn);
6330 Results.push_back(
6331 DAG.getVectorShuffle(WidenVT, DL, WidenIn, WidenIn, TruncMask));
6332 return;
6333 }
6334 }
6335
6336 break;
6337 }
6338 case ISD::SIGN_EXTEND: {
6339 // LASX has native VEXT2XV_* for sign extension.
6340 if (!Subtarget.hasExtLSX() || Subtarget.hasExtLASX())
6341 return;
6342
6343 EVT DstVT = N->getValueType(0);
6344 SDValue Src = N->getOperand(0);
6345 MVT SrcVT = Src.getSimpleValueType();
6346
6347 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
6348 unsigned DstEltBits = DstVT.getScalarSizeInBits();
6349 unsigned NumElts = DstVT.getVectorNumElements();
6350
6351 if (SrcVT.getSizeInBits() > 128)
6352 return;
6353
6354 if (!DstVT.isVector() || DstVT.getSizeInBits() <= 128)
6355 return;
6356
6357 // If every source element is all-ones or all-zeros, so is every extended
6358 // element, and interleaving a block with itself extends its low half.
6359 bool IsSignSplat = DAG.ComputeNumSignBits(Src) == SrcEltBits;
6360
6361 // Legalize and extend the src to 128-bit first.
6362 if (SrcVT.getSizeInBits() < 128) {
6363 unsigned WidenSrcElts = 128 / SrcEltBits;
6364 MVT WidenSrcVT = MVT::getVectorVT(SrcVT.getScalarType(), WidenSrcElts);
6365 Src = DAG.getNode(ISD::INSERT_SUBVECTOR, DL, WidenSrcVT,
6366 DAG.getUNDEF(WidenSrcVT), Src,
6367 DAG.getVectorIdxConstant(0, DL));
6368 SrcVT = WidenSrcVT;
6369
6370 unsigned FirstStageEltBits = 128 / NumElts;
6371 MVT FirstStageEltVT = MVT::getIntegerVT(FirstStageEltBits);
6372 MVT FirstStageVT = MVT::getVectorVT(FirstStageEltVT, NumElts);
6373 Src = DAG.getNode(ISD::SIGN_EXTEND_VECTOR_INREG, DL, FirstStageVT, Src);
6374 SrcVT = FirstStageVT;
6375 SrcEltBits = FirstStageEltBits;
6376 }
6377
6379 Blocks.push_back(Src);
6380
6381 // Sign-extend the src by using VSLLWIL (or VILVL for a sign splat) + VEXTH
6382 // recursively.
6383 while (SrcEltBits < DstEltBits) {
6384 unsigned NextEltBits = SrcEltBits * 2;
6385 MVT NextEltVT = MVT::getIntegerVT(NextEltBits);
6386 unsigned CurEltsPerBlock = SrcVT.getVectorNumElements();
6387 unsigned NextEltsPerBlock = CurEltsPerBlock / 2;
6388 MVT NextBlockVT = MVT::getVectorVT(NextEltVT, NextEltsPerBlock);
6389
6390 SmallVector<SDValue, 8> NextBlocks;
6391 NextBlocks.reserve(Blocks.size() * 2);
6392 for (SDValue Block : Blocks) {
6393 SDValue Lo;
6394 if (IsSignSplat) {
6395 Lo = DAG.getNode(LoongArchISD::VILVL, DL, SrcVT, Block, Block);
6396 Lo = DAG.getBitcast(NextBlockVT, Lo);
6397 } else {
6398 Lo = DAG.getNode(LoongArchISD::VSLLWIL, DL, NextBlockVT, Block,
6399 DAG.getConstant(0, DL, Subtarget.getGRLenVT()));
6400 }
6401 NextBlocks.push_back(Lo);
6402 NextBlocks.push_back(
6403 DAG.getNode(LoongArchISD::VEXTH, DL, NextBlockVT, Block));
6404 }
6405
6406 Blocks = std::move(NextBlocks);
6407 SrcVT = NextBlockVT;
6408 SrcEltBits = NextEltBits;
6409 }
6410
6411 Results.push_back(DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Blocks));
6412 break;
6413 }
6414 case ISD::FP_EXTEND:
6415 // FP_EXTEND may reach here due to the Custom action for v2f32 results, but
6416 // no target-specific lowering is required. Leave it unchanged and rely on
6417 // the default type legalization.
6418 break;
6419 }
6420}
6421
6422/// Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
6424 SelectionDAG &DAG) {
6425 assert(N->getOpcode() == ISD::AND && "Unexpected opcode combine into ANDN");
6426
6427 MVT VT = N->getSimpleValueType(0);
6428 if (!VT.is128BitVector() && !VT.is256BitVector())
6429 return SDValue();
6430
6431 SDValue X, Y;
6432 SDValue N0 = N->getOperand(0);
6433 SDValue N1 = N->getOperand(1);
6434
6435 if (SDValue Not = isNOT(N0, DAG)) {
6436 X = Not;
6437 Y = N1;
6438 } else if (SDValue Not = isNOT(N1, DAG)) {
6439 X = Not;
6440 Y = N0;
6441 } else
6442 return SDValue();
6443
6444 X = DAG.getBitcast(VT, X);
6445 Y = DAG.getBitcast(VT, Y);
6446 return DAG.getNode(LoongArchISD::VANDN, DL, VT, X, Y);
6447}
6448
6449static bool isConstantSplatVector(SDValue N, APInt &SplatValue,
6450 unsigned MinSizeInBits) {
6453
6454 if (!Node)
6455 return false;
6456
6457 APInt SplatUndef;
6458 unsigned SplatBitSize;
6459 bool HasAnyUndefs;
6460
6461 return Node->isConstantSplat(SplatValue, SplatUndef, SplatBitSize,
6462 HasAnyUndefs, MinSizeInBits,
6463 /*IsBigEndian=*/false);
6464}
6465
6466static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex) {
6467 auto *BV = dyn_cast<BuildVectorSDNode>(N);
6468 if (!BV)
6469 return SDValue();
6470
6471 SDValue Src;
6472 int Start = -1;
6473
6474 for (unsigned i = 0, NumElts = BV->getNumOperands(); i < NumElts; ++i) {
6475 SDValue Op = BV->getOperand(i);
6476 if (Op.isUndef())
6477 continue;
6478 if (Op.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
6479 return SDValue();
6480
6481 auto *IdxC = dyn_cast<ConstantSDNode>(Op.getOperand(1));
6482 if (!IdxC)
6483 return SDValue();
6484
6485 unsigned EltIdx = IdxC->getZExtValue();
6486 if (Start < 0)
6487 Start = (int)EltIdx - (int)(i * 2);
6488 if (Start < 0 || Start > 1 || EltIdx != (unsigned)(Start + (int)(i * 2)))
6489 return SDValue();
6490
6491 SDValue CurSrc = Op.getOperand(0);
6492 if (!Src)
6493 Src = CurSrc;
6494 else if (Src != CurSrc)
6495 return SDValue();
6496 }
6497
6498 if (!Src || Start < 0)
6499 return SDValue();
6500
6501 StartIndex = (unsigned)Start;
6502 return Src;
6503}
6504
6505// Return true if values of type VT are kept in vector registers, i.e. VT is
6506// a vector type whose elements are legal in LSX vectors.
6507static bool isVectorRegisterVT(EVT VT, SelectionDAG &DAG) {
6508 if (!VT.isVector())
6509 return false;
6510 EVT EltVT = VT.getVectorElementType();
6511 unsigned EltBits = EltVT.getSizeInBits();
6512 if (EltBits < 8 || EltBits > 64)
6513 return false;
6514 EVT LSXVT = EVT::getVectorVT(*DAG.getContext(), EltVT, 128 / EltBits);
6515 return DAG.getTargetLoweringInfo().isTypeLegal(LSXVT);
6516}
6517
6518// Return true if all users of the value loaded by Ld can use it in vector
6519// registers.
6521 EVT VT = Ld->getValueType(0);
6522 for (SDUse &U : Ld->uses()) {
6523 if (U.getResNo() != 0)
6524 continue;
6525 SDNode *User = U.getUser();
6526 switch (User->getOpcode()) {
6527 case ISD::ADD:
6528 case ISD::SUB:
6529 if (User->getValueType(0) == VT)
6530 continue;
6531 break;
6532 case ISD::STORE:
6533 // The stored value, not the address.
6534 if (U.getOperandNo() == 1)
6535 continue;
6536 break;
6537 case ISD::BITCAST:
6538 if (isVectorRegisterVT(User->getValueType(0), DAG))
6539 continue;
6540 break;
6541 }
6542 return false;
6543 }
6544 return true;
6545}
6546
6547// Return true if V, a vNi128 value, is cheap to obtain in vector registers
6548// of type PartVT.
6550 SelectionDAG &DAG) {
6551 switch (V.getOpcode()) {
6552 case ISD::LOAD: {
6553 auto *Ld = cast<LoadSDNode>(V);
6554 unsigned Fast = 0;
6555 return ISD::isNormalLoad(Ld) && Ld->isSimple() &&
6557 *DAG.getContext(), DAG.getDataLayout(), PartVT,
6558 *Ld->getMemOperand(), &Fast) &&
6559 Fast && hasOnlyVectorUsers(Ld, DAG);
6560 }
6561 case ISD::BITCAST: {
6562 // Vectors built from GPRs are not in vector registers.
6563 SDValue Src = V.getOperand(0);
6564 switch (Src.getOpcode()) {
6565 case ISD::BUILD_VECTOR:
6569 return false;
6570 }
6571 return isVectorRegisterVT(Src.getValueType(), DAG);
6572 }
6573 default:
6575 }
6576}
6577
6578// Combine add/sub on vNi128 to [X]VADD.Q/[X]VSUB.Q:
6579//
6580// (add vNi128:a, vNi128:b)
6581// -> (bitcast (VADD_Q (bitcast a to vMi64), (bitcast b to vMi64)))
6582//
6583// vNi128 types are not legal and would otherwise be expanded to scalar
6584// add/sub with carry on GPRs, so this has to happen before type legalization.
6585// Vectors wider than a vector register are split into register-sized parts.
6586// Only do it when both operands are already in, or cheap to get into, vector
6587// registers, e.g. loaded from memory. Values passed in GPRs (such as function
6588// arguments) are better handled in GPRs.
6592 if (!Subtarget.hasExtLSX())
6593 return SDValue();
6594 if (!DCI.isBeforeLegalize())
6595 return SDValue();
6596
6597 EVT VT = N->getValueType(0);
6598 if (!VT.isFixedLengthVector() || VT.getVectorElementType() != MVT::i128)
6599 return SDValue();
6600
6601 // Use 256-bit parts with LASX and 128-bit parts for the rest.
6602 unsigned NumElts = VT.getVectorNumElements();
6603 bool UseLASX = Subtarget.hasExtLASX() && NumElts >= 2;
6604 MVT PartVT = UseLASX ? MVT::v4i64 : MVT::v2i64;
6605 EVT NewVT = EVT::getVectorVT(*DAG.getContext(), MVT::i64, NumElts * 2);
6606
6607 SDValue LHS = N->getOperand(0);
6608 SDValue RHS = N->getOperand(1);
6609 if (!isCheapVNi128VectorOperand(LHS, PartVT, DAG) ||
6610 !isCheapVNi128VectorOperand(RHS, PartVT, DAG))
6611 return SDValue();
6612
6613 // Replace loads with loads of the i64 vector type, so that the other users
6614 // of the loaded value (e.g. another add/sub) can use it directly as well.
6615 // The range metadata, if any, is dropped as it doesn't match the new type.
6616 // The old loads are only replaced after N, as replacing them may CSE N into
6617 // another node and delete it.
6619 auto ConvertOperand = [&](SDValue V) {
6620 auto *Ld = dyn_cast<LoadSDNode>(V);
6621 if (!Ld)
6622 return DAG.getBitcast(NewVT, V);
6623 SDValue NewLd =
6624 DAG.getLoad(NewVT, SDLoc(Ld), Ld->getChain(), Ld->getBasePtr(),
6625 Ld->getPointerInfo(), Ld->getBaseAlign(),
6626 Ld->getMemOperand()->getFlags(), Ld->getAAInfo());
6627 LoadsToReplace.push_back({Ld, NewLd});
6628 return NewLd;
6629 };
6630 SDValue NewLHS = ConvertOperand(LHS);
6631 SDValue NewRHS = LHS == RHS ? NewLHS : ConvertOperand(RHS);
6632
6633 SDLoc DL(N);
6634 unsigned Opc =
6635 N->getOpcode() == ISD::ADD ? LoongArchISD::VADD_Q : LoongArchISD::VSUB_Q;
6636 SDValue Res = DAG.getUNDEF(NewVT);
6637 for (unsigned Idx = 0, End = NumElts * 2; Idx != End;) {
6638 MVT CurVT = UseLASX && End - Idx >= 4 ? MVT::v4i64 : MVT::v2i64;
6639 SDValue PartLHS = DAG.getExtractSubvector(DL, CurVT, NewLHS, Idx);
6640 SDValue PartRHS = DAG.getExtractSubvector(DL, CurVT, NewRHS, Idx);
6641 SDValue Part = DAG.getNode(Opc, DL, CurVT, PartLHS, PartRHS);
6642 Res = DAG.getInsertSubvector(DL, Res, Part, Idx);
6643 Idx += CurVT.getVectorNumElements();
6644 }
6645 DCI.CombineTo(N, DAG.getBitcast(VT, Res));
6646 for (auto [Ld, NewLd] : LoadsToReplace)
6647 DCI.CombineTo(Ld, DAG.getBitcast(VT, NewLd), NewLd.getValue(1));
6648 return SDValue(N, 0);
6649}
6650
6651static SDValue
6654 if (!Subtarget.hasExtLSX())
6655 return SDValue();
6656
6657 unsigned Opc = N->getOpcode();
6658 assert((Opc == ISD::ADD || Opc == ISD::SUB) && "Unexpected opcode");
6659
6660 EVT VT = N->getValueType(0);
6661 SDLoc DL(N);
6662
6663 SDValue LHS = N->getOperand(0);
6664 SDValue RHS = N->getOperand(1);
6665
6666 bool isSigned;
6667 unsigned ExtOpc = LHS.getOpcode();
6668 if (ExtOpc == ISD::SIGN_EXTEND)
6669 isSigned = true;
6670 else if (ExtOpc == ISD::ZERO_EXTEND)
6671 isSigned = false;
6672 else
6673 return SDValue();
6674
6675 if (ExtOpc != RHS.getOpcode())
6676 return SDValue();
6677
6678 if (!LHS.hasOneUse() || !RHS.hasOneUse())
6679 return SDValue();
6680
6681 unsigned OddIdx, EvenIdx;
6682 SDValue LHSVec = matchDeinterleaveBuildVector(LHS.getOperand(0), OddIdx);
6683 SDValue RHSVec = matchDeinterleaveBuildVector(RHS.getOperand(0), EvenIdx);
6684
6685 if (!LHSVec || !RHSVec)
6686 return SDValue();
6687 if (OddIdx != 1 || EvenIdx != 0)
6688 return SDValue();
6689 if (LHSVec.getValueType() != RHSVec.getValueType())
6690 return SDValue();
6691
6692 EVT SrcVT = LHSVec.getValueType();
6693 EVT SrcEltVT = SrcVT.getVectorElementType();
6694 EVT DstEltVT = VT.getVectorElementType();
6695 auto &TLI = DAG.getTargetLoweringInfo();
6696
6697 if (!TLI.isTypeLegal(VT) || !TLI.isTypeLegal(SrcVT))
6698 return SDValue();
6699 if (!SrcVT.isVector() || !VT.isVector())
6700 return SDValue();
6701 if (SrcVT.getSizeInBits() != VT.getSizeInBits())
6702 return SDValue();
6703 if (DstEltVT.getSizeInBits() != SrcEltVT.getSizeInBits() * 2)
6704 return SDValue();
6705 if (!SrcEltVT.isInteger() || SrcEltVT.getSizeInBits() > 32)
6706 return SDValue();
6707
6708 unsigned TargetOpc;
6709 if (Opc == ISD::ADD)
6710 TargetOpc = isSigned ? LoongArchISD::VHADDW : LoongArchISD::VHADDW_U;
6711 else
6712 TargetOpc = isSigned ? LoongArchISD::VHSUBW : LoongArchISD::VHSUBW_U;
6713
6714 return DAG.getNode(TargetOpc, DL, VT, LHSVec, RHSVec);
6715}
6716
6721 return V;
6722
6724 return V;
6725
6726 if (DCI.isBeforeLegalizeOps())
6727 return SDValue();
6728
6729 EVT VT = N->getValueType(0);
6730 if (!VT.isVector())
6731 return SDValue();
6732
6733 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
6734 return SDValue();
6735
6736 EVT EltVT = VT.getVectorElementType();
6737 if (!EltVT.isInteger())
6738 return SDValue();
6739
6740 // match:
6741 //
6742 // add
6743 // (and
6744 // (srl X, shift-1) / X
6745 // 1)
6746 // (srl/sra X, shift)
6747
6748 SDValue Add0 = N->getOperand(0);
6749 SDValue Add1 = N->getOperand(1);
6750 SDValue And;
6751 SDValue Shr;
6752
6753 if (Add0.getOpcode() == ISD::AND) {
6754 And = Add0;
6755 Shr = Add1;
6756 } else if (Add1.getOpcode() == ISD::AND) {
6757 And = Add1;
6758 Shr = Add0;
6759 } else {
6760 return SDValue();
6761 }
6762
6763 // match:
6764 //
6765 // srl/sra X, shift
6766
6767 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
6768 return SDValue();
6769
6770 SDValue X = Shr.getOperand(0);
6771 SDValue Shift = Shr.getOperand(1);
6772 APInt ShiftVal;
6773
6774 if (!isConstantSplatVector(Shift, ShiftVal, EltVT.getSizeInBits()))
6775 return SDValue();
6776
6777 if (ShiftVal == 0)
6778 return SDValue();
6779
6780 // match:
6781 //
6782 // and
6783 // (srl X, shift-1) / X
6784 // 1
6785
6786 SDValue One = And.getOperand(1);
6787 APInt SplatVal;
6788
6789 if (!isConstantSplatVector(One, SplatVal, EltVT.getSizeInBits()))
6790 return SDValue();
6791
6792 if (SplatVal != 1)
6793 return SDValue();
6794
6795 if (And.getOperand(0) == X) {
6796 // match:
6797 //
6798 // shift == 1
6799
6800 if (ShiftVal != 1)
6801 return SDValue();
6802 } else {
6803 // match:
6804 //
6805 // srl X, shift-1
6806
6807 SDValue Srl = And.getOperand(0);
6808
6809 if (Srl.getOpcode() != ISD::SRL)
6810 return SDValue();
6811
6812 if (Srl.getOperand(0) != X)
6813 return SDValue();
6814
6815 // match:
6816 //
6817 // shift-1
6818
6819 SDValue ShiftMinus1 = Srl.getOperand(1);
6820
6821 if (!isConstantSplatVector(ShiftMinus1, SplatVal, EltVT.getSizeInBits()))
6822 return SDValue();
6823
6824 if (ShiftVal != (SplatVal + 1))
6825 return SDValue();
6826 }
6827
6828 // We matched a rounded right shift pattern and can lower it
6829 // to a single vector rounded shift instruction.
6830
6831 SDLoc DL(N);
6832 return DAG.getNode(Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
6833 : LoongArchISD::VSRAR,
6834 DL, VT, X, Shift);
6835}
6836
6840 if (DCI.isBeforeLegalizeOps())
6841 return SDValue();
6842
6843 SDValue FirstOperand = N->getOperand(0);
6844 SDValue SecondOperand = N->getOperand(1);
6845 unsigned FirstOperandOpc = FirstOperand.getOpcode();
6846 EVT ValTy = N->getValueType(0);
6847 SDLoc DL(N);
6848 uint64_t lsb, msb;
6849 unsigned SMIdx, SMLen;
6850 ConstantSDNode *CN;
6851 SDValue NewOperand;
6852 MVT GRLenVT = Subtarget.getGRLenVT();
6853
6854 if (SDValue R = combineAndNotIntoVANDN(N, DL, DAG))
6855 return R;
6856
6857 // BSTRPICK requires the 32S feature.
6858 if (!Subtarget.has32S())
6859 return SDValue();
6860
6861 // Op's second operand must be a shifted mask.
6862 if (!(CN = dyn_cast<ConstantSDNode>(SecondOperand)) ||
6863 !isShiftedMask_64(CN->getZExtValue(), SMIdx, SMLen))
6864 return SDValue();
6865
6866 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL) {
6867 // Pattern match BSTRPICK.
6868 // $dst = and ((sra or srl) $src , lsb), (2**len - 1)
6869 // => BSTRPICK $dst, $src, msb, lsb
6870 // where msb = lsb + len - 1
6871
6872 // The second operand of the shift must be an immediate.
6873 if (!(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))))
6874 return SDValue();
6875
6876 lsb = CN->getZExtValue();
6877
6878 // Return if the shifted mask does not start at bit 0 or the sum of its
6879 // length and lsb exceeds the word's size.
6880 if (SMIdx != 0 || lsb + SMLen > ValTy.getSizeInBits())
6881 return SDValue();
6882
6883 NewOperand = FirstOperand.getOperand(0);
6884 } else {
6885 // Pattern match BSTRPICK.
6886 // $dst = and $src, (2**len- 1) , if len > 12
6887 // => BSTRPICK $dst, $src, msb, lsb
6888 // where lsb = 0 and msb = len - 1
6889
6890 // If the mask is <= 0xfff, andi can be used instead.
6891 if (CN->getZExtValue() <= 0xfff)
6892 return SDValue();
6893
6894 // Return if the MSB exceeds.
6895 if (SMIdx + SMLen > ValTy.getSizeInBits())
6896 return SDValue();
6897
6898 if (SMIdx > 0) {
6899 // Omit if the constant has more than 2 uses. This a conservative
6900 // decision. Whether it is a win depends on the HW microarchitecture.
6901 // However it should always be better for 1 and 2 uses.
6902 if (CN->use_size() > 2)
6903 return SDValue();
6904 // Return if the constant can be composed by a single LU12I.W.
6905 if ((CN->getZExtValue() & 0xfff) == 0)
6906 return SDValue();
6907 // Return if the constand can be composed by a single ADDI with
6908 // the zero register.
6909 if (CN->getSExtValue() >= -2048 && CN->getSExtValue() < 0)
6910 return SDValue();
6911 }
6912
6913 lsb = SMIdx;
6914 NewOperand = FirstOperand;
6915 }
6916
6917 msb = lsb + SMLen - 1;
6918 SDValue NR0 = DAG.getNode(LoongArchISD::BSTRPICK, DL, ValTy, NewOperand,
6919 DAG.getConstant(msb, DL, GRLenVT),
6920 DAG.getConstant(lsb, DL, GRLenVT));
6921 if (FirstOperandOpc == ISD::SRA || FirstOperandOpc == ISD::SRL || lsb == 0)
6922 return NR0;
6923 // Try to optimize to
6924 // bstrpick $Rd, $Rs, msb, lsb
6925 // slli $Rd, $Rd, lsb
6926 return DAG.getNode(ISD::SHL, DL, ValTy, NR0,
6927 DAG.getConstant(lsb, DL, GRLenVT));
6928}
6929
6930// Return the original source vector if N consists of the half
6931// of each 128-bit lane.
6934
6935 EVT DstVT = N.getValueType();
6936 if (!DstVT.isVector())
6937 return SDValue();
6938
6939 unsigned NumElts = DstVT.getVectorNumElements();
6940
6941 // LSX canonical form:
6942 if (N.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
6943 SDValue Src = N.getOperand(0);
6944 EVT SrcVT = Src.getValueType();
6945
6946 if (!SrcVT.isVector() || !SrcVT.is128BitVector())
6947 return SDValue();
6948 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6949 return SDValue();
6950 if (SrcVT.getVectorNumElements() != NumElts * 2)
6951 return SDValue();
6952 if (N.getConstantOperandVal(1) != (isLow ? 0 : NumElts))
6953 return SDValue();
6954
6955 return Src;
6956 }
6957
6958 // LASX canonical form:
6959 auto *BV = dyn_cast<BuildVectorSDNode>(N);
6960 if (!BV)
6961 return SDValue();
6962
6963 if (NumElts % 2 != 0)
6964 return SDValue();
6965
6966 SDValue Src;
6967 EVT SrcVT;
6968
6969 for (unsigned I = 0; I != NumElts; ++I) {
6970 SDValue Elt = BV->getOperand(I);
6971 if (Elt.isUndef())
6972 continue;
6974 return SDValue();
6975
6976 SDValue ThisSrc = Elt.getOperand(0);
6977 SDValue Idx = Elt.getOperand(1);
6978 auto *CI = dyn_cast<ConstantSDNode>(Idx);
6979 if (!CI)
6980 return SDValue();
6981
6982 if (!Src) {
6983 Src = ThisSrc;
6984 SrcVT = Src.getValueType();
6985 if (!SrcVT.isVector())
6986 return SDValue();
6987
6988 if (!SrcVT.is256BitVector())
6989 return SDValue();
6990 if (SrcVT.getSizeInBits() != DstVT.getSizeInBits() * 2)
6991 return SDValue();
6992 if (SrcVT.getVectorNumElements() != NumElts * 2)
6993 return SDValue();
6994 } else if (ThisSrc != Src) {
6995 return SDValue();
6996 }
6997
6998 unsigned Half = NumElts / 2;
6999 unsigned ExpectedIdx = (I < Half) ? I : (I + Half);
7000 ExpectedIdx += isLow ? 0 : Half;
7001
7002 if (CI->getZExtValue() != ExpectedIdx)
7003 return SDValue();
7004 }
7005
7006 return Src;
7007}
7008
7012 assert(N->getOpcode() == ISD::SHL && "Unexpected opcode");
7013
7014 EVT VT = N->getValueType(0);
7015 SDLoc DL(N);
7016
7017 SDValue LHS = N->getOperand(0);
7018 SDValue RHS = N->getOperand(1);
7019
7020 bool isSigned;
7021 unsigned ExtOpc = LHS.getOpcode();
7022 if (ExtOpc == ISD::SIGN_EXTEND)
7023 isSigned = true;
7024 else if (ExtOpc == ISD::ZERO_EXTEND)
7025 isSigned = false;
7026 else
7027 return SDValue();
7028
7029 if (!LHS.hasOneUse())
7030 return SDValue();
7031
7032 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
7033 N->getValueSizeInBits(0) != LHS->getOperand(0).getValueSizeInBits() * 2)
7034 return SDValue();
7035
7036 SDValue Vec = matchHalfOf128BitLanes(LHS.getOperand(0), /*isLow=*/true);
7037 if (!Vec)
7038 return SDValue();
7039
7040 EVT SrcVT = Vec.getValueType();
7041 EVT SrcEltVT = SrcVT.getVectorElementType();
7042 EVT DstEltVT = VT.getVectorElementType();
7043 APInt Imm;
7044 if (!isConstantSplatVector(RHS, Imm, DstEltVT.getSizeInBits()))
7045 return SDValue();
7046 if (!Imm.ult(SrcEltVT.getSizeInBits()))
7047 return SDValue();
7048
7049 unsigned Opc = isSigned ? LoongArchISD::VSLLWIL : LoongArchISD::VSLLWIL_U;
7050 SDValue Sht = DAG.getConstant(Imm.getZExtValue(), DL, Subtarget.getGRLenVT());
7051 return DAG.getNode(Opc, DL, VT, Vec, Sht);
7052}
7053
7057 // BSTRPICK requires the 32S feature.
7058 if (!Subtarget.has32S())
7059 return SDValue();
7060
7061 if (DCI.isBeforeLegalizeOps())
7062 return SDValue();
7063
7064 // $dst = srl (and $src, Mask), Shamt
7065 // =>
7066 // BSTRPICK $dst, $src, MaskIdx+MaskLen-1, Shamt
7067 // when Mask is a shifted mask, and MaskIdx <= Shamt <= MaskIdx+MaskLen-1
7068 //
7069
7070 SDValue FirstOperand = N->getOperand(0);
7071 ConstantSDNode *CN;
7072 EVT ValTy = N->getValueType(0);
7073 SDLoc DL(N);
7074 MVT GRLenVT = Subtarget.getGRLenVT();
7075 unsigned MaskIdx, MaskLen;
7076 uint64_t Shamt;
7077
7078 // The first operand must be an AND and the second operand of the AND must be
7079 // a shifted mask.
7080 if (FirstOperand.getOpcode() != ISD::AND ||
7081 !(CN = dyn_cast<ConstantSDNode>(FirstOperand.getOperand(1))) ||
7082 !isShiftedMask_64(CN->getZExtValue(), MaskIdx, MaskLen))
7083 return SDValue();
7084
7085 // The second operand (shift amount) must be an immediate.
7086 if (!(CN = dyn_cast<ConstantSDNode>(N->getOperand(1))))
7087 return SDValue();
7088
7089 Shamt = CN->getZExtValue();
7090 if (MaskIdx <= Shamt && Shamt <= MaskIdx + MaskLen - 1)
7091 return DAG.getNode(LoongArchISD::BSTRPICK, DL, ValTy,
7092 FirstOperand->getOperand(0),
7093 DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT),
7094 DAG.getConstant(Shamt, DL, GRLenVT));
7095
7096 return SDValue();
7097}
7098
7099// Try to turn a sub boolean RHS and constant LHS into an addi. LoongArch's
7100// SUB has no immediate form, so (sub C, X) normally requires either
7101// materializing C or using sub with R0 plus an addi; rewriting it as
7102// (add X', C - 1) lets the immediate fold into a single ADDI.
7104 SDValue N0 = N->getOperand(0);
7105 SDValue N1 = N->getOperand(1);
7106 EVT VT = N->getValueType(0);
7107 SDLoc DL(N);
7108
7109 // Require a constant LHS.
7110 auto *N0C = dyn_cast<ConstantSDNode>(N0);
7111 if (!N0C)
7112 return SDValue();
7113
7114 // (sub 0, X) is a negation, not a constant materialization; rewriting it
7115 // as (add (setcc inv), -1) changes the condition code and can duplicate a
7116 // compare, without saving any instruction.
7117 if (N0C->isZero() && !N->hasOneUse())
7118 return SDValue();
7119
7120 // The result of this fold is only folded into an ADDI when it is used
7121 // directly. When it feeds an extend, the original (sub C, X) form usually
7122 // combines better with the surrounding zext/negate/xor.
7123 if (N->hasOneUse()) {
7124 unsigned UserOpc = N->use_begin()->getUser()->getOpcode();
7125 if (UserOpc == ISD::ZERO_EXTEND || UserOpc == ISD::SIGN_EXTEND ||
7126 UserOpc == ISD::ANY_EXTEND || UserOpc == ISD::SUB)
7127 return SDValue();
7128 }
7129
7130 // All our optimizations involve subtracting 1 from the immediate and
7131 // forming an ADDI. Make sure the new immediate is valid for an ADDI.
7132 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
7133 if (!ImmValMinus1.isSignedIntN(12))
7134 return SDValue();
7135
7136 SDValue NewLHS;
7137 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
7138 // (sub constant, (setcc x, y, eq/neq)) ->
7139 // (add (setcc x, y, neq/eq), constant - 1)
7140 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
7141 EVT SetCCOpVT = N1.getOperand(0).getValueType();
7142 if (!ISD::isIntEqualitySetCC(CCVal) || !SetCCOpVT.isInteger())
7143 return SDValue();
7144 CCVal = ISD::getSetCCInverse(CCVal, SetCCOpVT);
7145 NewLHS =
7146 DAG.getSetCC(SDLoc(N1), VT, N1.getOperand(0), N1.getOperand(1), CCVal);
7147 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(N1.getOperand(1)) &&
7148 N1.getOperand(0).getOpcode() == ISD::SETCC) {
7149 // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
7150 // Since setcc returns a bool the xor is equivalent to 1-setcc.
7151 NewLHS = N1.getOperand(0);
7152 } else
7153 return SDValue();
7154
7155 SDValue NewRHS = DAG.getConstant(ImmValMinus1, DL, VT);
7156 return DAG.getNode(ISD::ADD, DL, VT, NewLHS, NewRHS);
7157}
7158
7163 return V;
7164
7166 return V;
7167
7168 if (N->getValueType(0) != Subtarget.getGRLenVT())
7169 return SDValue();
7170
7171 SDValue N0 = N->getOperand(0);
7172 SDValue N1 = N->getOperand(1);
7173 EVT VT = N->getValueType(0);
7174
7175 if (SDValue V = combineSubOfBoolean(N, DAG))
7176 return V;
7177
7178 // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
7179 if (isNullConstant(N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
7180 isNullConstant(N1.getOperand(1)) &&
7181 N1.getValueType() == N1.getOperand(0).getValueType()) {
7182 ISD::CondCode CCVal = cast<CondCodeSDNode>(N1.getOperand(2))->get();
7183 if (CCVal == ISD::SETLT) {
7184 SDLoc DL(N);
7185 unsigned ShAmt = N0.getValueSizeInBits() - 1;
7186 return DAG.getNode(ISD::SRA, DL, VT, N1.getOperand(0),
7187 DAG.getConstant(ShAmt, DL, VT));
7188 }
7189 }
7190
7191 return SDValue();
7192}
7193
7194// Helper to peek through bitops/trunc/setcc to determine size of source vector.
7195// Allows BITCASTCombine to determine what size vector generated a <X x i1>.
7196static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size,
7197 unsigned Depth) {
7198 // Limit recursion.
7200 return false;
7201 switch (Src.getOpcode()) {
7202 case ISD::SETCC:
7203 case ISD::TRUNCATE:
7204 return Src.getOperand(0).getValueSizeInBits() == Size;
7205 case ISD::FREEZE:
7206 return checkBitcastSrcVectorSize(Src.getOperand(0), Size, Depth + 1);
7207 case ISD::AND:
7208 case ISD::XOR:
7209 case ISD::OR:
7210 return checkBitcastSrcVectorSize(Src.getOperand(0), Size, Depth + 1) &&
7211 checkBitcastSrcVectorSize(Src.getOperand(1), Size, Depth + 1);
7212 case ISD::SELECT:
7213 case ISD::VSELECT:
7214 return Src.getOperand(0).getScalarValueSizeInBits() == 1 &&
7215 checkBitcastSrcVectorSize(Src.getOperand(1), Size, Depth + 1) &&
7216 checkBitcastSrcVectorSize(Src.getOperand(2), Size, Depth + 1);
7217 case ISD::BUILD_VECTOR:
7219 }
7220 return false;
7221}
7222
7223// Helper to push sign extension of vXi1 SETCC result through bitops.
7225 SDValue Src, const SDLoc &DL) {
7226 switch (Src.getOpcode()) {
7227 case ISD::SETCC:
7228 case ISD::FREEZE:
7229 case ISD::TRUNCATE:
7230 case ISD::BUILD_VECTOR:
7231 return DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
7232 case ISD::AND:
7233 case ISD::XOR:
7234 case ISD::OR:
7235 return DAG.getNode(
7236 Src.getOpcode(), DL, SExtVT,
7237 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(0), DL),
7238 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL));
7239 case ISD::SELECT:
7240 case ISD::VSELECT:
7241 return DAG.getSelect(
7242 DL, SExtVT, Src.getOperand(0),
7243 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(1), DL),
7244 signExtendBitcastSrcVector(DAG, SExtVT, Src.getOperand(2), DL));
7245 }
7246 llvm_unreachable("Unexpected node type for vXi1 sign extension");
7247}
7248
7249static SDValue
7253 SDLoc DL(N);
7254 EVT VT = N->getValueType(0);
7255 SDValue Src = N->getOperand(0);
7256 EVT SrcVT = Src.getValueType();
7257
7258 if (Src.getOpcode() != ISD::SETCC || !Src.hasOneUse())
7259 return SDValue();
7260
7261 bool UseLASX;
7262 unsigned Opc = ISD::DELETED_NODE;
7263 EVT CmpVT = Src.getOperand(0).getValueType();
7264 EVT EltVT = CmpVT.getVectorElementType();
7265
7266 if (Subtarget.hasExtLSX() && CmpVT.getSizeInBits() == 128)
7267 UseLASX = false;
7268 else if (Subtarget.has32S() && Subtarget.hasExtLASX() &&
7269 CmpVT.getSizeInBits() == 256)
7270 UseLASX = true;
7271 else
7272 return SDValue();
7273
7274 SDValue SrcN1 = Src.getOperand(1);
7275 switch (Src.getOperandCondCode(2)) {
7276 default:
7277 break;
7278 case ISD::SETEQ:
7279 // x == 0 => not (vmsknez.b x)
7280 if (ISD::isBuildVectorAllZeros(SrcN1) && EltVT == MVT::i8)
7281 Opc = UseLASX ? LoongArchISD::XVMSKEQZ : LoongArchISD::VMSKEQZ;
7282 break;
7283 case ISD::SETGT:
7284 // x > -1 => vmskgez.b x
7285 if (ISD::isBuildVectorAllOnes(SrcN1) && EltVT == MVT::i8)
7286 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
7287 break;
7288 case ISD::SETGE:
7289 // x >= 0 => vmskgez.b x
7290 if (ISD::isBuildVectorAllZeros(SrcN1) && EltVT == MVT::i8)
7291 Opc = UseLASX ? LoongArchISD::XVMSKGEZ : LoongArchISD::VMSKGEZ;
7292 break;
7293 case ISD::SETLT:
7294 // x < 0 => vmskltz.{b,h,w,d} x
7295 if (ISD::isBuildVectorAllZeros(SrcN1) &&
7296 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
7297 EltVT == MVT::i64))
7298 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7299 break;
7300 case ISD::SETLE:
7301 // x <= -1 => vmskltz.{b,h,w,d} x
7302 if (ISD::isBuildVectorAllOnes(SrcN1) &&
7303 (EltVT == MVT::i8 || EltVT == MVT::i16 || EltVT == MVT::i32 ||
7304 EltVT == MVT::i64))
7305 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7306 break;
7307 case ISD::SETNE:
7308 // x != 0 => vmsknez.b x
7309 if (ISD::isBuildVectorAllZeros(SrcN1) && EltVT == MVT::i8)
7310 Opc = UseLASX ? LoongArchISD::XVMSKNEZ : LoongArchISD::VMSKNEZ;
7311 break;
7312 }
7313
7314 if (Opc == ISD::DELETED_NODE)
7315 return SDValue();
7316
7317 SDValue V = DAG.getNode(Opc, DL, Subtarget.getGRLenVT(), Src.getOperand(0));
7319 V = DAG.getZExtOrTrunc(V, DL, T);
7320 return DAG.getBitcast(VT, V);
7321}
7322
7326 SDLoc DL(N);
7327 EVT VT = N->getValueType(0);
7328 SDValue Src = N->getOperand(0);
7329 EVT SrcVT = Src.getValueType();
7330 MVT GRLenVT = Subtarget.getGRLenVT();
7331
7332 if (!DCI.isBeforeLegalizeOps())
7333 return SDValue();
7334
7335 if (!SrcVT.isSimple() || SrcVT.getScalarType() != MVT::i1)
7336 return SDValue();
7337
7338 // Combine SETCC and BITCAST into [X]VMSK{LT,GE,NE} when possible
7340 if (Res)
7341 return Res;
7342
7343 // Generate vXi1 using [X]VMSKLTZ
7344 MVT SExtVT;
7345 unsigned Opc;
7346 bool UseLASX = false;
7347 bool PropagateSExt = false;
7348
7349 if (Src.getOpcode() == ISD::SETCC && Src.hasOneUse()) {
7350 EVT CmpVT = Src.getOperand(0).getValueType();
7351 if (CmpVT.getSizeInBits() > 256)
7352 return SDValue();
7353 }
7354
7355 switch (SrcVT.getSimpleVT().SimpleTy) {
7356 default:
7357 return SDValue();
7358 case MVT::v2i1:
7359 SExtVT = MVT::v2i64;
7360 break;
7361 case MVT::v4i1:
7362 SExtVT = MVT::v4i32;
7363 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) {
7364 SExtVT = MVT::v4i64;
7365 UseLASX = true;
7366 PropagateSExt = true;
7367 }
7368 break;
7369 case MVT::v8i1:
7370 SExtVT = MVT::v8i16;
7371 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) {
7372 SExtVT = MVT::v8i32;
7373 UseLASX = true;
7374 PropagateSExt = true;
7375 }
7376 break;
7377 case MVT::v16i1:
7378 SExtVT = MVT::v16i8;
7379 if (Subtarget.hasExtLASX() && checkBitcastSrcVectorSize(Src, 256, 0)) {
7380 SExtVT = MVT::v16i16;
7381 UseLASX = true;
7382 PropagateSExt = true;
7383 }
7384 break;
7385 case MVT::v32i1:
7386 SExtVT = MVT::v32i8;
7387 UseLASX = true;
7388 break;
7389 };
7390 Src = PropagateSExt ? signExtendBitcastSrcVector(DAG, SExtVT, Src, DL)
7391 : DAG.getNode(ISD::SIGN_EXTEND, DL, SExtVT, Src);
7392
7393 SDValue V;
7394 if (!Subtarget.has32S() || !Subtarget.hasExtLASX()) {
7395 if (Src.getSimpleValueType() == MVT::v32i8) {
7396 SDValue Lo, Hi;
7397 std::tie(Lo, Hi) = DAG.SplitVector(Src, DL);
7398 Lo = DAG.getNode(LoongArchISD::VMSKLTZ, DL, GRLenVT, Lo);
7399 Hi = DAG.getNode(LoongArchISD::VMSKLTZ, DL, GRLenVT, Hi);
7400 Hi = DAG.getNode(ISD::SHL, DL, GRLenVT, Hi,
7401 DAG.getShiftAmountConstant(16, GRLenVT, DL));
7402 V = DAG.getNode(ISD::OR, DL, GRLenVT, Lo, Hi);
7403 } else if (UseLASX) {
7404 return SDValue();
7405 }
7406 }
7407
7408 if (!V) {
7409 Opc = UseLASX ? LoongArchISD::XVMSKLTZ : LoongArchISD::VMSKLTZ;
7410 V = DAG.getNode(Opc, DL, GRLenVT, Src);
7411 }
7412
7414 V = DAG.getZExtOrTrunc(V, DL, T);
7415 return DAG.getBitcast(VT, V);
7416}
7417
7421 MVT GRLenVT = Subtarget.getGRLenVT();
7422 EVT ValTy = N->getValueType(0);
7423 SDValue N0 = N->getOperand(0), N1 = N->getOperand(1);
7424 ConstantSDNode *CN0, *CN1;
7425 SDLoc DL(N);
7426 unsigned ValBits = ValTy.getSizeInBits();
7427 unsigned MaskIdx0, MaskLen0, MaskIdx1, MaskLen1;
7428 unsigned Shamt;
7429 bool SwapAndRetried = false;
7430
7431 // BSTRPICK requires the 32S feature.
7432 if (!Subtarget.has32S())
7433 return SDValue();
7434
7435 if (DCI.isBeforeLegalizeOps())
7436 return SDValue();
7437
7438 if (ValBits != 32 && ValBits != 64)
7439 return SDValue();
7440
7441Retry:
7442 // 1st pattern to match BSTRINS:
7443 // R = or (and X, mask0), (and (shl Y, lsb), mask1)
7444 // where mask1 = (2**size - 1) << lsb, mask0 = ~mask1
7445 // =>
7446 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
7447 if (N0.getOpcode() == ISD::AND &&
7448 (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&
7449 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&
7450 N1.getOpcode() == ISD::AND && N1.getOperand(0).getOpcode() == ISD::SHL &&
7451 (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&
7452 isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) &&
7453 MaskIdx0 == MaskIdx1 && MaskLen0 == MaskLen1 &&
7454 (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&
7455 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
7456 (MaskIdx0 + MaskLen0 <= ValBits)) {
7457 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 1\n");
7458 return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),
7459 N1.getOperand(0).getOperand(0),
7460 DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT),
7461 DAG.getConstant(MaskIdx0, DL, GRLenVT));
7462 }
7463
7464 // 2nd pattern to match BSTRINS:
7465 // R = or (and X, mask0), (shl (and Y, mask1), lsb)
7466 // where mask1 = (2**size - 1), mask0 = ~(mask1 << lsb)
7467 // =>
7468 // R = BSTRINS X, Y, msb, lsb (where msb = lsb + size - 1)
7469 if (N0.getOpcode() == ISD::AND &&
7470 (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&
7471 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&
7472 N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::AND &&
7473 (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&
7474 (Shamt = CN1->getZExtValue()) == MaskIdx0 &&
7475 (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&
7476 isShiftedMask_64(CN1->getZExtValue(), MaskIdx1, MaskLen1) &&
7477 MaskLen0 == MaskLen1 && MaskIdx1 == 0 &&
7478 (MaskIdx0 + MaskLen0 <= ValBits)) {
7479 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 2\n");
7480 return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),
7481 N1.getOperand(0).getOperand(0),
7482 DAG.getConstant((MaskIdx0 + MaskLen0 - 1), DL, GRLenVT),
7483 DAG.getConstant(MaskIdx0, DL, GRLenVT));
7484 }
7485
7486 // 3rd pattern to match BSTRINS:
7487 // R = or (and X, mask0), (and Y, mask1)
7488 // where ~mask0 = (2**size - 1) << lsb, mask0 & mask1 = 0
7489 // =>
7490 // R = BSTRINS X, (shr (and Y, mask1), lsb), msb, lsb
7491 // where msb = lsb + size - 1
7492 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::AND &&
7493 (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&
7494 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&
7495 (MaskIdx0 + MaskLen0 <= 64) &&
7496 (CN1 = dyn_cast<ConstantSDNode>(N1->getOperand(1))) &&
7497 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
7498 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 3\n");
7499 return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),
7500 DAG.getNode(ISD::SRL, DL, N1->getValueType(0), N1,
7501 DAG.getConstant(MaskIdx0, DL, GRLenVT)),
7502 DAG.getConstant(ValBits == 32
7503 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
7504 : (MaskIdx0 + MaskLen0 - 1),
7505 DL, GRLenVT),
7506 DAG.getConstant(MaskIdx0, DL, GRLenVT));
7507 }
7508
7509 // 4th pattern to match BSTRINS:
7510 // R = or (and X, mask), (shl Y, shamt)
7511 // where mask = (2**shamt - 1)
7512 // =>
7513 // R = BSTRINS X, Y, ValBits - 1, shamt
7514 // where ValBits = 32 or 64
7515 if (N0.getOpcode() == ISD::AND && N1.getOpcode() == ISD::SHL &&
7516 (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&
7517 isShiftedMask_64(CN0->getZExtValue(), MaskIdx0, MaskLen0) &&
7518 MaskIdx0 == 0 && (CN1 = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&
7519 (Shamt = CN1->getZExtValue()) == MaskLen0 &&
7520 (MaskIdx0 + MaskLen0 <= ValBits)) {
7521 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 4\n");
7522 return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),
7523 N1.getOperand(0),
7524 DAG.getConstant((ValBits - 1), DL, GRLenVT),
7525 DAG.getConstant(Shamt, DL, GRLenVT));
7526 }
7527
7528 // 5th pattern to match BSTRINS:
7529 // R = or (and X, mask), const
7530 // where ~mask = (2**size - 1) << lsb, mask & const = 0
7531 // =>
7532 // R = BSTRINS X, (const >> lsb), msb, lsb
7533 // where msb = lsb + size - 1
7534 if (N0.getOpcode() == ISD::AND &&
7535 (CN0 = dyn_cast<ConstantSDNode>(N0.getOperand(1))) &&
7536 isShiftedMask_64(~CN0->getSExtValue(), MaskIdx0, MaskLen0) &&
7537 (CN1 = dyn_cast<ConstantSDNode>(N1)) &&
7538 (CN1->getSExtValue() & CN0->getSExtValue()) == 0) {
7539 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 5\n");
7540 return DAG.getNode(
7541 LoongArchISD::BSTRINS, DL, ValTy, N0.getOperand(0),
7542 DAG.getSignedConstant(CN1->getSExtValue() >> MaskIdx0, DL, ValTy),
7543 DAG.getConstant(ValBits == 32 ? (MaskIdx0 + (MaskLen0 & 31) - 1)
7544 : (MaskIdx0 + MaskLen0 - 1),
7545 DL, GRLenVT),
7546 DAG.getConstant(MaskIdx0, DL, GRLenVT));
7547 }
7548
7549 // 6th pattern.
7550 // a = b | ((c & mask) << shamt), where all positions in b to be overwritten
7551 // by the incoming bits are known to be zero.
7552 // =>
7553 // a = BSTRINS b, c, shamt + MaskLen - 1, shamt
7554 //
7555 // Note that the 1st pattern is a special situation of the 6th, i.e. the 6th
7556 // pattern is more common than the 1st. So we put the 1st before the 6th in
7557 // order to match as many nodes as possible.
7558 ConstantSDNode *CNMask, *CNShamt;
7559 unsigned MaskIdx, MaskLen;
7560 if (N1.getOpcode() == ISD::SHL && N1.getOperand(0).getOpcode() == ISD::AND &&
7561 (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&
7562 isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) &&
7563 MaskIdx == 0 && (CNShamt = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&
7564 CNShamt->getZExtValue() + MaskLen <= ValBits) {
7565 Shamt = CNShamt->getZExtValue();
7566 APInt ShMask(ValBits, CNMask->getZExtValue() << Shamt);
7567 if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) {
7568 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 6\n");
7569 return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0,
7570 N1.getOperand(0).getOperand(0),
7571 DAG.getConstant(Shamt + MaskLen - 1, DL, GRLenVT),
7572 DAG.getConstant(Shamt, DL, GRLenVT));
7573 }
7574 }
7575
7576 // 7th pattern.
7577 // a = b | ((c << shamt) & shifted_mask), where all positions in b to be
7578 // overwritten by the incoming bits are known to be zero.
7579 // =>
7580 // a = BSTRINS b, c, MaskIdx + MaskLen - 1, MaskIdx
7581 //
7582 // Similarly, the 7th pattern is more common than the 2nd. So we put the 2nd
7583 // before the 7th in order to match as many nodes as possible.
7584 if (N1.getOpcode() == ISD::AND &&
7585 (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&
7586 isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen) &&
7587 N1.getOperand(0).getOpcode() == ISD::SHL &&
7588 (CNShamt = dyn_cast<ConstantSDNode>(N1.getOperand(0).getOperand(1))) &&
7589 CNShamt->getZExtValue() == MaskIdx) {
7590 APInt ShMask(ValBits, CNMask->getZExtValue());
7591 if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) {
7592 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 7\n");
7593 return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0,
7594 N1.getOperand(0).getOperand(0),
7595 DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT),
7596 DAG.getConstant(MaskIdx, DL, GRLenVT));
7597 }
7598 }
7599
7600 // (or a, b) and (or b, a) are equivalent, so swap the operands and retry.
7601 if (!SwapAndRetried) {
7602 std::swap(N0, N1);
7603 SwapAndRetried = true;
7604 goto Retry;
7605 }
7606
7607 SwapAndRetried = false;
7608Retry2:
7609 // 8th pattern.
7610 // a = b | (c & shifted_mask), where all positions in b to be overwritten by
7611 // the incoming bits are known to be zero.
7612 // =>
7613 // a = BSTRINS b, c >> MaskIdx, MaskIdx + MaskLen - 1, MaskIdx
7614 //
7615 // Similarly, the 8th pattern is more common than the 4th and 5th patterns. So
7616 // we put it here in order to match as many nodes as possible or generate less
7617 // instructions.
7618 if (N1.getOpcode() == ISD::AND &&
7619 (CNMask = dyn_cast<ConstantSDNode>(N1.getOperand(1))) &&
7620 isShiftedMask_64(CNMask->getZExtValue(), MaskIdx, MaskLen)) {
7621 APInt ShMask(ValBits, CNMask->getZExtValue());
7622 if (ShMask.isSubsetOf(DAG.computeKnownBits(N0).Zero)) {
7623 LLVM_DEBUG(dbgs() << "Perform OR combine: match pattern 8\n");
7624 return DAG.getNode(LoongArchISD::BSTRINS, DL, ValTy, N0,
7625 DAG.getNode(ISD::SRL, DL, N1->getValueType(0),
7626 N1->getOperand(0),
7627 DAG.getConstant(MaskIdx, DL, GRLenVT)),
7628 DAG.getConstant(MaskIdx + MaskLen - 1, DL, GRLenVT),
7629 DAG.getConstant(MaskIdx, DL, GRLenVT));
7630 }
7631 }
7632 // Swap N0/N1 and retry.
7633 if (!SwapAndRetried) {
7634 std::swap(N0, N1);
7635 SwapAndRetried = true;
7636 goto Retry2;
7637 }
7638
7639 return SDValue();
7640}
7641
7644 EVT VT = N->getValueType(0);
7645 // LA64 promotes i32 integer operations to GPR-width operations during
7646 // legalization. Match the idiom before that promotion as well as after
7647 // it, otherwise common unsigned 32-bit bit tests miss this combine.
7648 if (VT != Subtarget.getGRLenVT() && !(Subtarget.is64Bit() && VT == MVT::i32))
7649 return SDValue();
7650
7651 SDValue XorOps[] = {N->getOperand(0), N->getOperand(1)};
7652 for (unsigned AndIdx = 0; AndIdx != 2; ++AndIdx) {
7653 SDValue And = XorOps[AndIdx];
7654 SDValue Mask = XorOps[1 - AndIdx];
7655 if (And.getOpcode() != ISD::AND || !And.hasOneUse())
7656 continue;
7657
7658 for (unsigned MaskIdx = 0; MaskIdx != 2; ++MaskIdx) {
7659 // Match by SDValue identity so that (X & M1) ^ M2 is never folded
7660 // unless M1 and M2 are the same value. Canonicalize before
7661 // instruction selection, where the AND operand may already have been
7662 // selected to a target machine node.
7663 if (And.getOperand(MaskIdx) != Mask)
7664 continue;
7665 SDLoc DL(N);
7666 SDValue X = And.getOperand(1 - MaskIdx);
7667 return DAG.getNode(ISD::AND, DL, VT, Mask, DAG.getNOT(DL, X, VT));
7668 }
7669 }
7670 return SDValue();
7671}
7672
7674 ExtType = ISD::NON_EXTLOAD;
7675
7676 switch (V.getNode()->getOpcode()) {
7677 case ISD::LOAD: {
7678 LoadSDNode *LoadNode = cast<LoadSDNode>(V.getNode());
7679 if ((LoadNode->getMemoryVT() == MVT::i8) ||
7680 (LoadNode->getMemoryVT() == MVT::i16)) {
7681 ExtType = LoadNode->getExtensionType();
7682 return true;
7683 }
7684 return false;
7685 }
7686 case ISD::AssertSext: {
7687 VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
7688 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7689 ExtType = ISD::SEXTLOAD;
7690 return true;
7691 }
7692 return false;
7693 }
7694 case ISD::AssertZext: {
7695 VTSDNode *TypeNode = cast<VTSDNode>(V.getNode()->getOperand(1));
7696 if ((TypeNode->getVT() == MVT::i8) || (TypeNode->getVT() == MVT::i16)) {
7697 ExtType = ISD::ZEXTLOAD;
7698 return true;
7699 }
7700 return false;
7701 }
7702 default:
7703 return false;
7704 }
7705
7706 return false;
7707}
7708
7709// Fold a lane-mask extraction that is only compared against zero into one of
7710// all-lane check instructions, which write the results into fcc register, so
7711// BCEQZ could use it directly.
7712//
7713// (VMSKLTZ X) != 0 -> VSETNEZ.V X
7714// (VMSKLTZ X) == 0 -> VSETEQZ.V X
7715//
7716// This is valid when X is the result of a vector compare instruction,
7717// so each lane of X has to be all-ones or all-zeros.
7719 const SDLoc &DL, SelectionDAG &DAG,
7721 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7722 return SDValue();
7723 if (!isNullConstant(RHS))
7724 return SDValue();
7725
7726 unsigned MskOpc = LHS.getOpcode();
7727 if (MskOpc != LoongArchISD::VMSKLTZ && MskOpc != LoongArchISD::XVMSKLTZ)
7728 return SDValue();
7729 // Keeping the mask alive for another user would defeat the purpose.
7730 if (!LHS.hasOneUse())
7731 return SDValue();
7732
7733 SDValue Src = LHS.getOperand(0);
7734 EVT SrcVT = Src.getValueType();
7735 // Make sure every lane is all-ones or all-zeros.
7736 if (!SrcVT.isVector() ||
7737 DAG.ComputeNumSignBits(Src) != SrcVT.getScalarSizeInBits())
7738 return SDValue();
7739
7740 return DAG.getNode(CC == ISD::SETNE ? LoongArchISD::VANYNONZERO
7741 : LoongArchISD::VALLZERO,
7742 DL, Subtarget.getGRLenVT(), Src);
7743}
7744
7745// Eliminate redundant truncation and zero-extension nodes.
7746// * Case 1:
7747// +------------+ +------------+ +------------+
7748// | Input1 | | Input2 | | CC |
7749// +------------+ +------------+ +------------+
7750// | | |
7751// V V +----+
7752// +------------+ +------------+ |
7753// | TRUNCATE | | TRUNCATE | |
7754// +------------+ +------------+ |
7755// | | |
7756// V V |
7757// +------------+ +------------+ |
7758// | ZERO_EXT | | ZERO_EXT | |
7759// +------------+ +------------+ |
7760// | | |
7761// | +-------------+ |
7762// V V | |
7763// +----------------+ | |
7764// | AND | | |
7765// +----------------+ | |
7766// | | |
7767// +---------------+ | |
7768// | | |
7769// V V V
7770// +-------------+
7771// | CMP |
7772// +-------------+
7773// * Case 2:
7774// +------------+ +------------+ +-------------+ +------------+ +------------+
7775// | Input1 | | Input2 | | Constant -1 | | Constant 0 | | CC |
7776// +------------+ +------------+ +-------------+ +------------+ +------------+
7777// | | | | |
7778// V | | | |
7779// +------------+ | | | |
7780// | XOR |<---------------------+ | |
7781// +------------+ | | |
7782// | | | |
7783// V V +---------------+ |
7784// +------------+ +------------+ | |
7785// | TRUNCATE | | TRUNCATE | | +-------------------------+
7786// +------------+ +------------+ | |
7787// | | | |
7788// V V | |
7789// +------------+ +------------+ | |
7790// | ZERO_EXT | | ZERO_EXT | | |
7791// +------------+ +------------+ | |
7792// | | | |
7793// V V | |
7794// +----------------+ | |
7795// | AND | | |
7796// +----------------+ | |
7797// | | |
7798// +---------------+ | |
7799// | | |
7800// V V V
7801// +-------------+
7802// | CMP |
7803// +-------------+
7807 ISD::CondCode CC = N->getOperandCondCode(2);
7808
7809 if (N->getValueType(0) == Subtarget.getGRLenVT())
7810 if (SDValue V = foldVMskZeroTest(N->getOperand(0), N->getOperand(1), CC,
7811 SDLoc(N), DAG, Subtarget))
7812 return V;
7813
7814 SDNode *AndNode = N->getOperand(0).getNode();
7815 if (AndNode->getOpcode() != ISD::AND)
7816 return SDValue();
7817
7818 SDValue AndInputValue2 = AndNode->getOperand(1);
7819 if (AndInputValue2.getOpcode() != ISD::ZERO_EXTEND)
7820 return SDValue();
7821
7822 SDValue CmpInputValue = N->getOperand(1);
7823 SDValue AndInputValue1 = AndNode->getOperand(0);
7824 if (AndInputValue1.getOpcode() == ISD::XOR) {
7825 if (CC != ISD::SETEQ && CC != ISD::SETNE)
7826 return SDValue();
7827 ConstantSDNode *CN = dyn_cast<ConstantSDNode>(AndInputValue1.getOperand(1));
7828 if (!CN || !CN->isAllOnes())
7829 return SDValue();
7830 CN = dyn_cast<ConstantSDNode>(CmpInputValue);
7831 if (!CN || !CN->isZero())
7832 return SDValue();
7833 AndInputValue1 = AndInputValue1.getOperand(0);
7834 if (AndInputValue1.getOpcode() != ISD::ZERO_EXTEND)
7835 return SDValue();
7836 } else if (AndInputValue1.getOpcode() == ISD::ZERO_EXTEND) {
7837 if (AndInputValue2 != CmpInputValue)
7838 return SDValue();
7839 } else {
7840 return SDValue();
7841 }
7842
7843 SDValue TruncValue1 = AndInputValue1.getNode()->getOperand(0);
7844 if (TruncValue1.getOpcode() != ISD::TRUNCATE)
7845 return SDValue();
7846
7847 SDValue TruncValue2 = AndInputValue2.getNode()->getOperand(0);
7848 if (TruncValue2.getOpcode() != ISD::TRUNCATE)
7849 return SDValue();
7850
7851 SDValue TruncInputValue1 = TruncValue1.getNode()->getOperand(0);
7852 SDValue TruncInputValue2 = TruncValue2.getNode()->getOperand(0);
7853 ISD::LoadExtType ExtType1;
7854 ISD::LoadExtType ExtType2;
7855
7856 if (!checkValueWidth(TruncInputValue1, ExtType1) ||
7857 !checkValueWidth(TruncInputValue2, ExtType2))
7858 return SDValue();
7859
7860 if (TruncInputValue1->getValueType(0) != TruncInputValue2->getValueType(0) ||
7861 AndNode->getValueType(0) != TruncInputValue1->getValueType(0))
7862 return SDValue();
7863
7864 if ((ExtType2 != ISD::ZEXTLOAD) &&
7865 ((ExtType2 != ISD::SEXTLOAD) && (ExtType1 != ISD::SEXTLOAD)))
7866 return SDValue();
7867
7868 // These truncation and zero-extension nodes are not necessary, remove them.
7869 SDValue NewAnd = DAG.getNode(ISD::AND, SDLoc(N), AndNode->getValueType(0),
7870 TruncInputValue1, TruncInputValue2);
7871 SDValue NewSetCC =
7872 DAG.getSetCC(SDLoc(N), N->getValueType(0), NewAnd, TruncInputValue2, CC);
7873 DAG.ReplaceAllUsesWith(N, NewSetCC.getNode());
7874 return SDValue(N, 0);
7875}
7876
7877// Strip a single outer ISD::SIGN_EXTEND_INREG from \p V, if present, and
7878// return the inner value together with the narrow VT it was extending
7879// from. If no such node is present, returns \p V unchanged and an invalid
7880// EVT.
7881//
7882// i32 (and other sub-GRLen) arithmetic is legalized to operate on the full
7883// GRLen-width register, with a `sign_extend_inreg` re-normalizing the
7884// result back into the narrow type's range afterwards (see e.g. the
7885// `add i32` -> `add` + `sign_extend_inreg ..., i32` legalization). Any
7886// combine that reassociates such a binop must track and reapply this
7887// extension, otherwise the transformed code can produce a value whose
7888// high bits no longer match the narrow-type semantics.
7889static std::pair<SDValue, EVT> stripSignExtendInReg(SDValue V) {
7890 if (V.getOpcode() == ISD::SIGN_EXTEND_INREG)
7891 return {V.getOperand(0), cast<VTSDNode>(V.getOperand(1))->getVT()};
7892 return {V, EVT()};
7893}
7894
7895// Try to match \p BinV (after optionally stripping an outer
7896// sign_extend_inreg) as a supported binary operation that has \p X as one
7897// of its operands, returning the matched opcode, the other operand (the
7898// "delta"), and the narrow VT of the sign_extend_inreg that was stripped
7899// (invalid EVT if none was present).
7900//
7901// For commutative ops (add/or/xor), \p X may be either operand, since
7902// `binop(X, Y) == binop(Y, X)` and the identity element (0) works on
7903// either side.
7904//
7905// For `sub`, the operation is NOT commutative: `sub(X, Y) != sub(Y, X)`.
7906// Only `sub(X, Y)` (i.e. \p X is the *minuend*, the first operand) can be
7907// rewritten using the identity `X - 0 == X`. If \p X were the *subtrahend*
7908// (second operand, i.e. the pattern is actually `sub(Y, X)`), there is no
7909// way to express `cond ? (Y - X) : X` (or the symmetric case) as
7910// `X op (select ...)` without introducing an extra negation, so that case
7911// must be rejected instead of "optimized" into worse code.
7912static std::tuple<unsigned, SDValue, EVT>
7914 auto [Inner, ExtVT] = stripSignExtendInReg(BinV);
7915
7916 unsigned Opc = Inner.getOpcode();
7917 switch (Opc) {
7918 case ISD::ADD:
7919 case ISD::OR:
7920 case ISD::XOR:
7921 if (Inner.getOperand(0) == X)
7922 return {Opc, Inner.getOperand(1), ExtVT};
7923 if (Inner.getOperand(1) == X)
7924 return {Opc, Inner.getOperand(0), ExtVT};
7925 return {0, SDValue(), EVT()};
7926 case ISD::SUB:
7927 // Only accept X as the minuend (first operand); see comment above.
7928 if (Inner.getOperand(0) == X)
7929 return {Opc, Inner.getOperand(1), ExtVT};
7930 return {0, SDValue(), EVT()};
7931 default:
7932 return {0, SDValue(), EVT()};
7933 }
7934}
7935
7936// Try to combine:
7937// select cond, binop(X, Y), X -> binop X, (select cond, Y, 0)
7938// select cond, X, binop(X, Y) -> binop X, (select cond, 0, Y)
7939// for binop in {add, or, xor, sub}, where 0 is the identity element of the
7940// respective operation, additionally handling the common legalized form
7941// where the binop result is wrapped in a `sign_extend_inreg` (as happens
7942// for sub-GRLen types such as i32 on a 64-bit GRLen target). See
7943// matchBinOpWithSharedOperand() for the restrictions applied to
7944// non-commutative operations (currently only `sub`).
7948 if (DCI.isBeforeLegalizeOps())
7949 return SDValue();
7950
7951 EVT VT = N->getValueType(0);
7952 // Restrict to the scalar GRLen integer type that maskeqz/masknez operate
7953 // on; this also naturally excludes float and vector selects.
7954 if (VT != Subtarget.getGRLenVT())
7955 return SDValue();
7956
7957 SDValue Cond = N->getOperand(0);
7958 SDValue TrueV = N->getOperand(1);
7959 SDValue FalseV = N->getOperand(2);
7960 SDLoc DL(N);
7961
7962 auto TryFold = [&](SDValue BinV, SDValue SharedV,
7963 bool BinIsTrueArm) -> SDValue {
7964 auto [Opc, Delta, ExtVT] = matchBinOpWithSharedOperand(BinV, SharedV);
7965 if (!Opc)
7966 return SDValue();
7967
7968 // Avoid infinite combine loops: bail out if Delta is trivially the
7969 // same node we would otherwise be selecting on (shouldn't normally
7970 // happen, but guards against degenerate/self-referential IR).
7971 if (Delta.getNode() == N)
7972 return SDValue();
7973
7974 SDValue Zero = DAG.getConstant(0, DL, VT);
7975 SDValue NewSel = BinIsTrueArm ? DAG.getSelect(DL, VT, Cond, Delta, Zero)
7976 : DAG.getSelect(DL, VT, Cond, Zero, Delta);
7977 SDValue NewBin = DAG.getNode(Opc, DL, VT, SharedV, NewSel);
7978
7979 // If the original binop result was normalized back into a narrower
7980 // type via sign_extend_inreg (e.g. i32 arithmetic on a 64-bit GRLen
7981 // target), the new binop must be re-normalized the same way: SharedV
7982 // is already known-sign-extended for that narrow type, but NewSel
7983 // (Delta or 0, selected) combined with SharedV via Opc can still
7984 // produce a 64-bit result whose high bits don't match the narrow
7985 // type's sign-extended representation.
7986 if (ExtVT != EVT())
7987 NewBin = DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, NewBin,
7988 DAG.getValueType(ExtVT));
7989
7990 return NewBin;
7991 };
7992
7993 if (SDValue R = TryFold(TrueV, FalseV, /*BinIsTrueArm=*/true))
7994 return R;
7995 if (SDValue R = TryFold(FalseV, TrueV, /*BinIsTrueArm=*/false))
7996 return R;
7997
7998 return SDValue();
7999}
8000
8001// Combine (loongarch_bitrev_w (loongarch_revb_2w X)) to loongarch_bitrev_4b.
8005 if (DCI.isBeforeLegalizeOps())
8006 return SDValue();
8007
8008 SDValue Src = N->getOperand(0);
8009 if (Src.getOpcode() != LoongArchISD::REVB_2W)
8010 return SDValue();
8011
8012 return DAG.getNode(LoongArchISD::BITREV_4B, SDLoc(N), N->getValueType(0),
8013 Src.getOperand(0));
8014}
8015
8016// Perform common combines for BR_CC and SELECT_CC conditions.
8017static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
8019 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
8020
8021 // As far as arithmetic right shift always saves the sign,
8022 // shift can be omitted.
8023 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
8024 // setge (sra X, N), 0 -> setge X, 0
8025 if (isNullConstant(RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
8026 LHS.getOpcode() == ISD::SRA) {
8027 LHS = LHS.getOperand(0);
8028 return true;
8029 }
8030
8031 if (!ISD::isIntEqualitySetCC(CCVal))
8032 return false;
8033
8034 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
8035 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
8036 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(RHS) &&
8037 LHS.getOperand(0).getValueType() == Subtarget.getGRLenVT()) {
8038 // If we're looking for eq 0 instead of ne 0, we need to invert the
8039 // condition.
8040 bool Invert = CCVal == ISD::SETEQ;
8041 CCVal = LHS.getOperandCondCode(2);
8042 if (Invert)
8043 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
8044
8045 RHS = LHS.getOperand(1);
8046 LHS = LHS.getOperand(0);
8047 translateSetCCForBranch(DL, LHS, RHS, CCVal, DAG);
8048
8049 CC = DAG.getCondCode(CCVal);
8050 return true;
8051 }
8052
8053 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, GRLen-1-C), 0, ge/lt)
8054 if (isNullConstant(RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
8055 LHS.getOperand(1).getOpcode() == ISD::Constant) {
8056 SDValue LHS0 = LHS.getOperand(0);
8057 if (LHS0.getOpcode() == ISD::AND &&
8058 LHS0.getOperand(1).getOpcode() == ISD::Constant) {
8059 uint64_t Mask = LHS0.getConstantOperandVal(1);
8060 uint64_t ShAmt = LHS.getConstantOperandVal(1);
8061 if (isPowerOf2_64(Mask) && Log2_64(Mask) == ShAmt) {
8062 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
8063 CC = DAG.getCondCode(CCVal);
8064
8065 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
8066 LHS = LHS0.getOperand(0);
8067 if (ShAmt != 0)
8068 LHS =
8069 DAG.getNode(ISD::SHL, DL, LHS.getValueType(), LHS0.getOperand(0),
8070 DAG.getConstant(ShAmt, DL, LHS.getValueType()));
8071 return true;
8072 }
8073 }
8074 }
8075
8076 // (X, 1, setne) -> (X, 0, seteq) if we can prove X is 0/1.
8077 // This can occur when legalizing some floating point comparisons.
8078 APInt Mask = APInt::getBitsSetFrom(LHS.getValueSizeInBits(), 1);
8079 if (isOneConstant(RHS) && DAG.MaskedValueIsZero(LHS, Mask)) {
8080 CCVal = ISD::getSetCCInverse(CCVal, LHS.getValueType());
8081 CC = DAG.getCondCode(CCVal);
8082 RHS = DAG.getConstant(0, DL, LHS.getValueType());
8083 return true;
8084 }
8085
8086 // Fold ((shl (extract_vector_elt X, I), GRLen - EleBits)), 0, eq/ne) ->
8087 // ((extract_vector_elt X, I), 0, eq/ne)
8088 if (isNullConstant(RHS) && (CCVal == ISD::SETEQ || CCVal == ISD::SETNE) &&
8089 LHS.getOpcode() == ISD::SHL && LHS.hasOneUse() &&
8090 isa<ConstantSDNode>(LHS.getOperand(1))) {
8091 SDValue Ext = LHS.getOperand(0);
8092 unsigned Sht = LHS.getConstantOperandVal(1);
8093 if (Ext.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
8094 SDValue Vec = Ext.getOperand(0);
8095 unsigned EleBits = Vec.getScalarValueSizeInBits();
8096 if ((EleBits + Sht) == Subtarget.getGRLen()) {
8097 LHS = Ext;
8098 return true;
8099 }
8100 }
8101 }
8102
8103 // Fold (C1, C2, cond) -> (0, 0, seteq/setne)
8105 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
8106 EVT VT = LHS.getValueType();
8107 EVT SetCCResVT =
8108 TLI->getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), VT);
8109 if (SDValue Folded = DAG.FoldSetCC(SetCCResVT, LHS, RHS, CCVal, DL)) {
8110 LHS = DAG.getConstant(0, DL, VT);
8111 RHS = DAG.getConstant(0, DL, VT);
8113 return true;
8114 }
8115 }
8116
8117 return false;
8118}
8119
8123 SDValue LHS = N->getOperand(1);
8124 SDValue RHS = N->getOperand(2);
8125 SDValue CC = N->getOperand(3);
8126 SDLoc DL(N);
8127
8128 // CC was folded into V, so always return ISD::SETNE is fine.
8130 DL, DAG, Subtarget))
8131 return DAG.getNode(LoongArchISD::BR_CC, DL, N->getValueType(0),
8132 N->getOperand(0), V,
8133 DAG.getConstant(0, DL, Subtarget.getGRLenVT()),
8134 DAG.getCondCode(ISD::SETNE), N->getOperand(4));
8135
8136 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
8137 return DAG.getNode(LoongArchISD::BR_CC, DL, N->getValueType(0),
8138 N->getOperand(0), LHS, RHS, CC, N->getOperand(4));
8139
8140 return SDValue();
8141}
8142
8146 // Transform
8147 SDValue LHS = N->getOperand(0);
8148 SDValue RHS = N->getOperand(1);
8149 SDValue CC = N->getOperand(2);
8150 ISD::CondCode CCVal = cast<CondCodeSDNode>(CC)->get();
8151 SDValue TrueV = N->getOperand(3);
8152 SDValue FalseV = N->getOperand(4);
8153 SDLoc DL(N);
8154 EVT VT = N->getValueType(0);
8155
8156 // If the True and False values are the same, we don't need a select_cc.
8157 if (TrueV == FalseV)
8158 return TrueV;
8159
8160 // (select (x < 0), y, z) -> x >> (GRLEN - 1) & (y - z) + z
8161 // (select (x >= 0), y, z) -> x >> (GRLEN - 1) & (z - y) + y
8162 if (isa<ConstantSDNode>(TrueV) && isa<ConstantSDNode>(FalseV) &&
8164 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
8165 if (CCVal == ISD::CondCode::SETGE)
8166 std::swap(TrueV, FalseV);
8167
8168 int64_t TrueSImm = cast<ConstantSDNode>(TrueV)->getSExtValue();
8169 int64_t FalseSImm = cast<ConstantSDNode>(FalseV)->getSExtValue();
8170 // Only handle simm12, if it is not in this range, it can be considered as
8171 // register.
8172 if (isInt<12>(TrueSImm) && isInt<12>(FalseSImm) &&
8173 isInt<12>(TrueSImm - FalseSImm)) {
8174 SDValue SRA =
8175 DAG.getNode(ISD::SRA, DL, VT, LHS,
8176 DAG.getConstant(Subtarget.getGRLen() - 1, DL, VT));
8177 SDValue AND =
8178 DAG.getNode(ISD::AND, DL, VT, SRA,
8179 DAG.getSignedConstant(TrueSImm - FalseSImm, DL, VT));
8180 return DAG.getNode(ISD::ADD, DL, VT, AND, FalseV);
8181 }
8182
8183 if (CCVal == ISD::CondCode::SETGE)
8184 std::swap(TrueV, FalseV);
8185 }
8186
8187 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
8188 return DAG.getNode(LoongArchISD::SELECT_CC, DL, N->getValueType(0),
8189 {LHS, RHS, CC, TrueV, FalseV});
8190
8191 return SDValue();
8192}
8193
8194template <unsigned N>
8196 SelectionDAG &DAG,
8198 bool IsSigned = false) {
8199 SDLoc DL(Node);
8200 auto *CImm = cast<ConstantSDNode>(Node->getOperand(ImmOp));
8201 // Check the ImmArg.
8202 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
8203 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
8204 DAG.getContext()->emitError(Node->getOperationName(0) +
8205 ": argument out of range.");
8206 return DAG.getNode(ISD::UNDEF, DL, Subtarget.getGRLenVT());
8207 }
8208 return DAG.getConstant(CImm->getZExtValue(), DL, Subtarget.getGRLenVT());
8209}
8210
8211template <unsigned N>
8212static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp,
8213 SelectionDAG &DAG, bool IsSigned = false) {
8214 SDLoc DL(Node);
8215 EVT ResTy = Node->getValueType(0);
8216 auto *CImm = cast<ConstantSDNode>(Node->getOperand(ImmOp));
8217
8218 // Check the ImmArg.
8219 if ((IsSigned && !isInt<N>(CImm->getSExtValue())) ||
8220 (!IsSigned && !isUInt<N>(CImm->getZExtValue()))) {
8221 DAG.getContext()->emitError(Node->getOperationName(0) +
8222 ": argument out of range.");
8223 return DAG.getNode(ISD::UNDEF, DL, ResTy);
8224 }
8225 return DAG.getConstant(
8227 IsSigned ? CImm->getSExtValue() : CImm->getZExtValue(), IsSigned),
8228 DL, ResTy);
8229}
8230
8232 SDLoc DL(Node);
8233 EVT ResTy = Node->getValueType(0);
8234 SDValue Vec = Node->getOperand(2);
8235 SDValue Mask = DAG.getConstant(Vec.getScalarValueSizeInBits() - 1, DL, ResTy);
8236 return DAG.getNode(ISD::AND, DL, ResTy, Vec, Mask);
8237}
8238
8240 SDLoc DL(Node);
8241 EVT ResTy = Node->getValueType(0);
8242 SDValue One = DAG.getConstant(1, DL, ResTy);
8243 SDValue Bit =
8244 DAG.getNode(ISD::SHL, DL, ResTy, One, truncateVecElts(Node, DAG));
8245
8246 return DAG.getNode(ISD::AND, DL, ResTy, Node->getOperand(1),
8247 DAG.getNOT(DL, Bit, ResTy));
8248}
8249
8250template <unsigned N>
8252 SDLoc DL(Node);
8253 EVT ResTy = Node->getValueType(0);
8254 auto *CImm = cast<ConstantSDNode>(Node->getOperand(2));
8255 // Check the unsigned ImmArg.
8256 if (!isUInt<N>(CImm->getZExtValue())) {
8257 DAG.getContext()->emitError(Node->getOperationName(0) +
8258 ": argument out of range.");
8259 return DAG.getNode(ISD::UNDEF, DL, ResTy);
8260 }
8261
8262 APInt BitImm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
8263 SDValue Mask = DAG.getConstant(~BitImm, DL, ResTy);
8264
8265 return DAG.getNode(ISD::AND, DL, ResTy, Node->getOperand(1), Mask);
8266}
8267
8268template <unsigned N>
8270 SDLoc DL(Node);
8271 EVT ResTy = Node->getValueType(0);
8272 auto *CImm = cast<ConstantSDNode>(Node->getOperand(2));
8273 // Check the unsigned ImmArg.
8274 if (!isUInt<N>(CImm->getZExtValue())) {
8275 DAG.getContext()->emitError(Node->getOperationName(0) +
8276 ": argument out of range.");
8277 return DAG.getNode(ISD::UNDEF, DL, ResTy);
8278 }
8279
8280 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
8281 SDValue BitImm = DAG.getConstant(Imm, DL, ResTy);
8282 return DAG.getNode(ISD::OR, DL, ResTy, Node->getOperand(1), BitImm);
8283}
8284
8285template <unsigned N>
8287 SDLoc DL(Node);
8288 EVT ResTy = Node->getValueType(0);
8289 auto *CImm = cast<ConstantSDNode>(Node->getOperand(2));
8290 // Check the unsigned ImmArg.
8291 if (!isUInt<N>(CImm->getZExtValue())) {
8292 DAG.getContext()->emitError(Node->getOperationName(0) +
8293 ": argument out of range.");
8294 return DAG.getNode(ISD::UNDEF, DL, ResTy);
8295 }
8296
8297 APInt Imm = APInt(ResTy.getScalarSizeInBits(), 1) << CImm->getAPIntValue();
8298 SDValue BitImm = DAG.getConstant(Imm, DL, ResTy);
8299 return DAG.getNode(ISD::XOR, DL, ResTy, Node->getOperand(1), BitImm);
8300}
8301
8302template <unsigned W>
8304 unsigned ResOp) {
8305 unsigned Imm = N->getConstantOperandVal(2);
8306 if (!isUInt<W>(Imm)) {
8307 const StringRef ErrorMsg = "argument out of range";
8308 DAG.getContext()->emitError(N->getOperationName(0) + ": " + ErrorMsg + ".");
8309 return DAG.getUNDEF(N->getValueType(0));
8310 }
8311 SDLoc DL(N);
8312 SDValue Vec = N->getOperand(1);
8313 SDValue Idx = DAG.getConstant(Imm, DL, MVT::i32);
8315 return DAG.getNode(ResOp, DL, N->getValueType(0), Vec, Idx, EltVT);
8316}
8317
8318static SDValue
8322 SDLoc DL(N);
8323 switch (N->getConstantOperandVal(0)) {
8324 default:
8325 break;
8326 case Intrinsic::loongarch_lsx_vadd_b:
8327 case Intrinsic::loongarch_lsx_vadd_h:
8328 case Intrinsic::loongarch_lsx_vadd_w:
8329 case Intrinsic::loongarch_lsx_vadd_d:
8330 case Intrinsic::loongarch_lasx_xvadd_b:
8331 case Intrinsic::loongarch_lasx_xvadd_h:
8332 case Intrinsic::loongarch_lasx_xvadd_w:
8333 case Intrinsic::loongarch_lasx_xvadd_d:
8334 return DAG.getNode(ISD::ADD, DL, N->getValueType(0), N->getOperand(1),
8335 N->getOperand(2));
8336 case Intrinsic::loongarch_lsx_vaddi_bu:
8337 case Intrinsic::loongarch_lsx_vaddi_hu:
8338 case Intrinsic::loongarch_lsx_vaddi_wu:
8339 case Intrinsic::loongarch_lsx_vaddi_du:
8340 case Intrinsic::loongarch_lasx_xvaddi_bu:
8341 case Intrinsic::loongarch_lasx_xvaddi_hu:
8342 case Intrinsic::loongarch_lasx_xvaddi_wu:
8343 case Intrinsic::loongarch_lasx_xvaddi_du:
8344 return DAG.getNode(ISD::ADD, DL, N->getValueType(0), N->getOperand(1),
8345 lowerVectorSplatImm<5>(N, 2, DAG));
8346 case Intrinsic::loongarch_lsx_vsub_b:
8347 case Intrinsic::loongarch_lsx_vsub_h:
8348 case Intrinsic::loongarch_lsx_vsub_w:
8349 case Intrinsic::loongarch_lsx_vsub_d:
8350 case Intrinsic::loongarch_lasx_xvsub_b:
8351 case Intrinsic::loongarch_lasx_xvsub_h:
8352 case Intrinsic::loongarch_lasx_xvsub_w:
8353 case Intrinsic::loongarch_lasx_xvsub_d:
8354 return DAG.getNode(ISD::SUB, DL, N->getValueType(0), N->getOperand(1),
8355 N->getOperand(2));
8356 case Intrinsic::loongarch_lsx_vsubi_bu:
8357 case Intrinsic::loongarch_lsx_vsubi_hu:
8358 case Intrinsic::loongarch_lsx_vsubi_wu:
8359 case Intrinsic::loongarch_lsx_vsubi_du:
8360 case Intrinsic::loongarch_lasx_xvsubi_bu:
8361 case Intrinsic::loongarch_lasx_xvsubi_hu:
8362 case Intrinsic::loongarch_lasx_xvsubi_wu:
8363 case Intrinsic::loongarch_lasx_xvsubi_du:
8364 return DAG.getNode(ISD::SUB, DL, N->getValueType(0), N->getOperand(1),
8365 lowerVectorSplatImm<5>(N, 2, DAG));
8366 case Intrinsic::loongarch_lsx_vneg_b:
8367 case Intrinsic::loongarch_lsx_vneg_h:
8368 case Intrinsic::loongarch_lsx_vneg_w:
8369 case Intrinsic::loongarch_lsx_vneg_d:
8370 case Intrinsic::loongarch_lasx_xvneg_b:
8371 case Intrinsic::loongarch_lasx_xvneg_h:
8372 case Intrinsic::loongarch_lasx_xvneg_w:
8373 case Intrinsic::loongarch_lasx_xvneg_d:
8374 return DAG.getNode(
8375 ISD::SUB, DL, N->getValueType(0),
8376 DAG.getConstant(
8377 APInt(N->getValueType(0).getScalarType().getSizeInBits(), 0,
8378 /*isSigned=*/true),
8379 SDLoc(N), N->getValueType(0)),
8380 N->getOperand(1));
8381 case Intrinsic::loongarch_lsx_vmax_b:
8382 case Intrinsic::loongarch_lsx_vmax_h:
8383 case Intrinsic::loongarch_lsx_vmax_w:
8384 case Intrinsic::loongarch_lsx_vmax_d:
8385 case Intrinsic::loongarch_lasx_xvmax_b:
8386 case Intrinsic::loongarch_lasx_xvmax_h:
8387 case Intrinsic::loongarch_lasx_xvmax_w:
8388 case Intrinsic::loongarch_lasx_xvmax_d:
8389 return DAG.getNode(ISD::SMAX, DL, N->getValueType(0), N->getOperand(1),
8390 N->getOperand(2));
8391 case Intrinsic::loongarch_lsx_vmax_bu:
8392 case Intrinsic::loongarch_lsx_vmax_hu:
8393 case Intrinsic::loongarch_lsx_vmax_wu:
8394 case Intrinsic::loongarch_lsx_vmax_du:
8395 case Intrinsic::loongarch_lasx_xvmax_bu:
8396 case Intrinsic::loongarch_lasx_xvmax_hu:
8397 case Intrinsic::loongarch_lasx_xvmax_wu:
8398 case Intrinsic::loongarch_lasx_xvmax_du:
8399 return DAG.getNode(ISD::UMAX, DL, N->getValueType(0), N->getOperand(1),
8400 N->getOperand(2));
8401 case Intrinsic::loongarch_lsx_vmaxi_b:
8402 case Intrinsic::loongarch_lsx_vmaxi_h:
8403 case Intrinsic::loongarch_lsx_vmaxi_w:
8404 case Intrinsic::loongarch_lsx_vmaxi_d:
8405 case Intrinsic::loongarch_lasx_xvmaxi_b:
8406 case Intrinsic::loongarch_lasx_xvmaxi_h:
8407 case Intrinsic::loongarch_lasx_xvmaxi_w:
8408 case Intrinsic::loongarch_lasx_xvmaxi_d:
8409 return DAG.getNode(ISD::SMAX, DL, N->getValueType(0), N->getOperand(1),
8410 lowerVectorSplatImm<5>(N, 2, DAG, /*IsSigned=*/true));
8411 case Intrinsic::loongarch_lsx_vmaxi_bu:
8412 case Intrinsic::loongarch_lsx_vmaxi_hu:
8413 case Intrinsic::loongarch_lsx_vmaxi_wu:
8414 case Intrinsic::loongarch_lsx_vmaxi_du:
8415 case Intrinsic::loongarch_lasx_xvmaxi_bu:
8416 case Intrinsic::loongarch_lasx_xvmaxi_hu:
8417 case Intrinsic::loongarch_lasx_xvmaxi_wu:
8418 case Intrinsic::loongarch_lasx_xvmaxi_du:
8419 return DAG.getNode(ISD::UMAX, DL, N->getValueType(0), N->getOperand(1),
8420 lowerVectorSplatImm<5>(N, 2, DAG));
8421 case Intrinsic::loongarch_lsx_vmin_b:
8422 case Intrinsic::loongarch_lsx_vmin_h:
8423 case Intrinsic::loongarch_lsx_vmin_w:
8424 case Intrinsic::loongarch_lsx_vmin_d:
8425 case Intrinsic::loongarch_lasx_xvmin_b:
8426 case Intrinsic::loongarch_lasx_xvmin_h:
8427 case Intrinsic::loongarch_lasx_xvmin_w:
8428 case Intrinsic::loongarch_lasx_xvmin_d:
8429 return DAG.getNode(ISD::SMIN, DL, N->getValueType(0), N->getOperand(1),
8430 N->getOperand(2));
8431 case Intrinsic::loongarch_lsx_vmin_bu:
8432 case Intrinsic::loongarch_lsx_vmin_hu:
8433 case Intrinsic::loongarch_lsx_vmin_wu:
8434 case Intrinsic::loongarch_lsx_vmin_du:
8435 case Intrinsic::loongarch_lasx_xvmin_bu:
8436 case Intrinsic::loongarch_lasx_xvmin_hu:
8437 case Intrinsic::loongarch_lasx_xvmin_wu:
8438 case Intrinsic::loongarch_lasx_xvmin_du:
8439 return DAG.getNode(ISD::UMIN, DL, N->getValueType(0), N->getOperand(1),
8440 N->getOperand(2));
8441 case Intrinsic::loongarch_lsx_vmini_b:
8442 case Intrinsic::loongarch_lsx_vmini_h:
8443 case Intrinsic::loongarch_lsx_vmini_w:
8444 case Intrinsic::loongarch_lsx_vmini_d:
8445 case Intrinsic::loongarch_lasx_xvmini_b:
8446 case Intrinsic::loongarch_lasx_xvmini_h:
8447 case Intrinsic::loongarch_lasx_xvmini_w:
8448 case Intrinsic::loongarch_lasx_xvmini_d:
8449 return DAG.getNode(ISD::SMIN, DL, N->getValueType(0), N->getOperand(1),
8450 lowerVectorSplatImm<5>(N, 2, DAG, /*IsSigned=*/true));
8451 case Intrinsic::loongarch_lsx_vmini_bu:
8452 case Intrinsic::loongarch_lsx_vmini_hu:
8453 case Intrinsic::loongarch_lsx_vmini_wu:
8454 case Intrinsic::loongarch_lsx_vmini_du:
8455 case Intrinsic::loongarch_lasx_xvmini_bu:
8456 case Intrinsic::loongarch_lasx_xvmini_hu:
8457 case Intrinsic::loongarch_lasx_xvmini_wu:
8458 case Intrinsic::loongarch_lasx_xvmini_du:
8459 return DAG.getNode(ISD::UMIN, DL, N->getValueType(0), N->getOperand(1),
8460 lowerVectorSplatImm<5>(N, 2, DAG));
8461 case Intrinsic::loongarch_lsx_vmul_b:
8462 case Intrinsic::loongarch_lsx_vmul_h:
8463 case Intrinsic::loongarch_lsx_vmul_w:
8464 case Intrinsic::loongarch_lsx_vmul_d:
8465 case Intrinsic::loongarch_lasx_xvmul_b:
8466 case Intrinsic::loongarch_lasx_xvmul_h:
8467 case Intrinsic::loongarch_lasx_xvmul_w:
8468 case Intrinsic::loongarch_lasx_xvmul_d:
8469 return DAG.getNode(ISD::MUL, DL, N->getValueType(0), N->getOperand(1),
8470 N->getOperand(2));
8471 case Intrinsic::loongarch_lsx_vmadd_b:
8472 case Intrinsic::loongarch_lsx_vmadd_h:
8473 case Intrinsic::loongarch_lsx_vmadd_w:
8474 case Intrinsic::loongarch_lsx_vmadd_d:
8475 case Intrinsic::loongarch_lasx_xvmadd_b:
8476 case Intrinsic::loongarch_lasx_xvmadd_h:
8477 case Intrinsic::loongarch_lasx_xvmadd_w:
8478 case Intrinsic::loongarch_lasx_xvmadd_d: {
8479 EVT ResTy = N->getValueType(0);
8480 return DAG.getNode(ISD::ADD, SDLoc(N), ResTy, N->getOperand(1),
8481 DAG.getNode(ISD::MUL, SDLoc(N), ResTy, N->getOperand(2),
8482 N->getOperand(3)));
8483 }
8484 case Intrinsic::loongarch_lsx_vmsub_b:
8485 case Intrinsic::loongarch_lsx_vmsub_h:
8486 case Intrinsic::loongarch_lsx_vmsub_w:
8487 case Intrinsic::loongarch_lsx_vmsub_d:
8488 case Intrinsic::loongarch_lasx_xvmsub_b:
8489 case Intrinsic::loongarch_lasx_xvmsub_h:
8490 case Intrinsic::loongarch_lasx_xvmsub_w:
8491 case Intrinsic::loongarch_lasx_xvmsub_d: {
8492 EVT ResTy = N->getValueType(0);
8493 return DAG.getNode(ISD::SUB, SDLoc(N), ResTy, N->getOperand(1),
8494 DAG.getNode(ISD::MUL, SDLoc(N), ResTy, N->getOperand(2),
8495 N->getOperand(3)));
8496 }
8497 case Intrinsic::loongarch_lsx_vdiv_b:
8498 case Intrinsic::loongarch_lsx_vdiv_h:
8499 case Intrinsic::loongarch_lsx_vdiv_w:
8500 case Intrinsic::loongarch_lsx_vdiv_d:
8501 case Intrinsic::loongarch_lasx_xvdiv_b:
8502 case Intrinsic::loongarch_lasx_xvdiv_h:
8503 case Intrinsic::loongarch_lasx_xvdiv_w:
8504 case Intrinsic::loongarch_lasx_xvdiv_d:
8505 return DAG.getNode(ISD::SDIV, DL, N->getValueType(0), N->getOperand(1),
8506 N->getOperand(2));
8507 case Intrinsic::loongarch_lsx_vdiv_bu:
8508 case Intrinsic::loongarch_lsx_vdiv_hu:
8509 case Intrinsic::loongarch_lsx_vdiv_wu:
8510 case Intrinsic::loongarch_lsx_vdiv_du:
8511 case Intrinsic::loongarch_lasx_xvdiv_bu:
8512 case Intrinsic::loongarch_lasx_xvdiv_hu:
8513 case Intrinsic::loongarch_lasx_xvdiv_wu:
8514 case Intrinsic::loongarch_lasx_xvdiv_du:
8515 return DAG.getNode(ISD::UDIV, DL, N->getValueType(0), N->getOperand(1),
8516 N->getOperand(2));
8517 case Intrinsic::loongarch_lsx_vmod_b:
8518 case Intrinsic::loongarch_lsx_vmod_h:
8519 case Intrinsic::loongarch_lsx_vmod_w:
8520 case Intrinsic::loongarch_lsx_vmod_d:
8521 case Intrinsic::loongarch_lasx_xvmod_b:
8522 case Intrinsic::loongarch_lasx_xvmod_h:
8523 case Intrinsic::loongarch_lasx_xvmod_w:
8524 case Intrinsic::loongarch_lasx_xvmod_d:
8525 return DAG.getNode(ISD::SREM, DL, N->getValueType(0), N->getOperand(1),
8526 N->getOperand(2));
8527 case Intrinsic::loongarch_lsx_vmod_bu:
8528 case Intrinsic::loongarch_lsx_vmod_hu:
8529 case Intrinsic::loongarch_lsx_vmod_wu:
8530 case Intrinsic::loongarch_lsx_vmod_du:
8531 case Intrinsic::loongarch_lasx_xvmod_bu:
8532 case Intrinsic::loongarch_lasx_xvmod_hu:
8533 case Intrinsic::loongarch_lasx_xvmod_wu:
8534 case Intrinsic::loongarch_lasx_xvmod_du:
8535 return DAG.getNode(ISD::UREM, DL, N->getValueType(0), N->getOperand(1),
8536 N->getOperand(2));
8537 case Intrinsic::loongarch_lsx_vand_v:
8538 case Intrinsic::loongarch_lasx_xvand_v:
8539 return DAG.getNode(ISD::AND, DL, N->getValueType(0), N->getOperand(1),
8540 N->getOperand(2));
8541 case Intrinsic::loongarch_lsx_vor_v:
8542 case Intrinsic::loongarch_lasx_xvor_v:
8543 return DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1),
8544 N->getOperand(2));
8545 case Intrinsic::loongarch_lsx_vxor_v:
8546 case Intrinsic::loongarch_lasx_xvxor_v:
8547 return DAG.getNode(ISD::XOR, DL, N->getValueType(0), N->getOperand(1),
8548 N->getOperand(2));
8549 case Intrinsic::loongarch_lsx_vnor_v:
8550 case Intrinsic::loongarch_lasx_xvnor_v: {
8551 SDValue Res = DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1),
8552 N->getOperand(2));
8553 return DAG.getNOT(DL, Res, Res->getValueType(0));
8554 }
8555 case Intrinsic::loongarch_lsx_vandi_b:
8556 case Intrinsic::loongarch_lasx_xvandi_b:
8557 return DAG.getNode(ISD::AND, DL, N->getValueType(0), N->getOperand(1),
8558 lowerVectorSplatImm<8>(N, 2, DAG));
8559 case Intrinsic::loongarch_lsx_vori_b:
8560 case Intrinsic::loongarch_lasx_xvori_b:
8561 return DAG.getNode(ISD::OR, DL, N->getValueType(0), N->getOperand(1),
8562 lowerVectorSplatImm<8>(N, 2, DAG));
8563 case Intrinsic::loongarch_lsx_vxori_b:
8564 case Intrinsic::loongarch_lasx_xvxori_b:
8565 return DAG.getNode(ISD::XOR, DL, N->getValueType(0), N->getOperand(1),
8566 lowerVectorSplatImm<8>(N, 2, DAG));
8567 case Intrinsic::loongarch_lsx_vsll_b:
8568 case Intrinsic::loongarch_lsx_vsll_h:
8569 case Intrinsic::loongarch_lsx_vsll_w:
8570 case Intrinsic::loongarch_lsx_vsll_d:
8571 case Intrinsic::loongarch_lasx_xvsll_b:
8572 case Intrinsic::loongarch_lasx_xvsll_h:
8573 case Intrinsic::loongarch_lasx_xvsll_w:
8574 case Intrinsic::loongarch_lasx_xvsll_d:
8575 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),
8576 truncateVecElts(N, DAG));
8577 case Intrinsic::loongarch_lsx_vslli_b:
8578 case Intrinsic::loongarch_lasx_xvslli_b:
8579 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),
8580 lowerVectorSplatImm<3>(N, 2, DAG));
8581 case Intrinsic::loongarch_lsx_vslli_h:
8582 case Intrinsic::loongarch_lasx_xvslli_h:
8583 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),
8584 lowerVectorSplatImm<4>(N, 2, DAG));
8585 case Intrinsic::loongarch_lsx_vslli_w:
8586 case Intrinsic::loongarch_lasx_xvslli_w:
8587 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),
8588 lowerVectorSplatImm<5>(N, 2, DAG));
8589 case Intrinsic::loongarch_lsx_vslli_d:
8590 case Intrinsic::loongarch_lasx_xvslli_d:
8591 return DAG.getNode(ISD::SHL, DL, N->getValueType(0), N->getOperand(1),
8592 lowerVectorSplatImm<6>(N, 2, DAG));
8593 case Intrinsic::loongarch_lsx_vsrl_b:
8594 case Intrinsic::loongarch_lsx_vsrl_h:
8595 case Intrinsic::loongarch_lsx_vsrl_w:
8596 case Intrinsic::loongarch_lsx_vsrl_d:
8597 case Intrinsic::loongarch_lasx_xvsrl_b:
8598 case Intrinsic::loongarch_lasx_xvsrl_h:
8599 case Intrinsic::loongarch_lasx_xvsrl_w:
8600 case Intrinsic::loongarch_lasx_xvsrl_d:
8601 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),
8602 truncateVecElts(N, DAG));
8603 case Intrinsic::loongarch_lsx_vsrli_b:
8604 case Intrinsic::loongarch_lasx_xvsrli_b:
8605 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),
8606 lowerVectorSplatImm<3>(N, 2, DAG));
8607 case Intrinsic::loongarch_lsx_vsrli_h:
8608 case Intrinsic::loongarch_lasx_xvsrli_h:
8609 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),
8610 lowerVectorSplatImm<4>(N, 2, DAG));
8611 case Intrinsic::loongarch_lsx_vsrli_w:
8612 case Intrinsic::loongarch_lasx_xvsrli_w:
8613 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),
8614 lowerVectorSplatImm<5>(N, 2, DAG));
8615 case Intrinsic::loongarch_lsx_vsrli_d:
8616 case Intrinsic::loongarch_lasx_xvsrli_d:
8617 return DAG.getNode(ISD::SRL, DL, N->getValueType(0), N->getOperand(1),
8618 lowerVectorSplatImm<6>(N, 2, DAG));
8619 case Intrinsic::loongarch_lsx_vsra_b:
8620 case Intrinsic::loongarch_lsx_vsra_h:
8621 case Intrinsic::loongarch_lsx_vsra_w:
8622 case Intrinsic::loongarch_lsx_vsra_d:
8623 case Intrinsic::loongarch_lasx_xvsra_b:
8624 case Intrinsic::loongarch_lasx_xvsra_h:
8625 case Intrinsic::loongarch_lasx_xvsra_w:
8626 case Intrinsic::loongarch_lasx_xvsra_d:
8627 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),
8628 truncateVecElts(N, DAG));
8629 case Intrinsic::loongarch_lsx_vsrai_b:
8630 case Intrinsic::loongarch_lasx_xvsrai_b:
8631 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),
8632 lowerVectorSplatImm<3>(N, 2, DAG));
8633 case Intrinsic::loongarch_lsx_vsrai_h:
8634 case Intrinsic::loongarch_lasx_xvsrai_h:
8635 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),
8636 lowerVectorSplatImm<4>(N, 2, DAG));
8637 case Intrinsic::loongarch_lsx_vsrai_w:
8638 case Intrinsic::loongarch_lasx_xvsrai_w:
8639 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),
8640 lowerVectorSplatImm<5>(N, 2, DAG));
8641 case Intrinsic::loongarch_lsx_vsrai_d:
8642 case Intrinsic::loongarch_lasx_xvsrai_d:
8643 return DAG.getNode(ISD::SRA, DL, N->getValueType(0), N->getOperand(1),
8644 lowerVectorSplatImm<6>(N, 2, DAG));
8645 case Intrinsic::loongarch_lsx_vclz_b:
8646 case Intrinsic::loongarch_lsx_vclz_h:
8647 case Intrinsic::loongarch_lsx_vclz_w:
8648 case Intrinsic::loongarch_lsx_vclz_d:
8649 case Intrinsic::loongarch_lasx_xvclz_b:
8650 case Intrinsic::loongarch_lasx_xvclz_h:
8651 case Intrinsic::loongarch_lasx_xvclz_w:
8652 case Intrinsic::loongarch_lasx_xvclz_d:
8653 return DAG.getNode(ISD::CTLZ, DL, N->getValueType(0), N->getOperand(1));
8654 case Intrinsic::loongarch_lsx_vpcnt_b:
8655 case Intrinsic::loongarch_lsx_vpcnt_h:
8656 case Intrinsic::loongarch_lsx_vpcnt_w:
8657 case Intrinsic::loongarch_lsx_vpcnt_d:
8658 case Intrinsic::loongarch_lasx_xvpcnt_b:
8659 case Intrinsic::loongarch_lasx_xvpcnt_h:
8660 case Intrinsic::loongarch_lasx_xvpcnt_w:
8661 case Intrinsic::loongarch_lasx_xvpcnt_d:
8662 return DAG.getNode(ISD::CTPOP, DL, N->getValueType(0), N->getOperand(1));
8663 case Intrinsic::loongarch_lsx_vbitclr_b:
8664 case Intrinsic::loongarch_lsx_vbitclr_h:
8665 case Intrinsic::loongarch_lsx_vbitclr_w:
8666 case Intrinsic::loongarch_lsx_vbitclr_d:
8667 case Intrinsic::loongarch_lasx_xvbitclr_b:
8668 case Intrinsic::loongarch_lasx_xvbitclr_h:
8669 case Intrinsic::loongarch_lasx_xvbitclr_w:
8670 case Intrinsic::loongarch_lasx_xvbitclr_d:
8671 return lowerVectorBitClear(N, DAG);
8672 case Intrinsic::loongarch_lsx_vbitclri_b:
8673 case Intrinsic::loongarch_lasx_xvbitclri_b:
8674 return lowerVectorBitClearImm<3>(N, DAG);
8675 case Intrinsic::loongarch_lsx_vbitclri_h:
8676 case Intrinsic::loongarch_lasx_xvbitclri_h:
8677 return lowerVectorBitClearImm<4>(N, DAG);
8678 case Intrinsic::loongarch_lsx_vbitclri_w:
8679 case Intrinsic::loongarch_lasx_xvbitclri_w:
8680 return lowerVectorBitClearImm<5>(N, DAG);
8681 case Intrinsic::loongarch_lsx_vbitclri_d:
8682 case Intrinsic::loongarch_lasx_xvbitclri_d:
8683 return lowerVectorBitClearImm<6>(N, DAG);
8684 case Intrinsic::loongarch_lsx_vbitset_b:
8685 case Intrinsic::loongarch_lsx_vbitset_h:
8686 case Intrinsic::loongarch_lsx_vbitset_w:
8687 case Intrinsic::loongarch_lsx_vbitset_d:
8688 case Intrinsic::loongarch_lasx_xvbitset_b:
8689 case Intrinsic::loongarch_lasx_xvbitset_h:
8690 case Intrinsic::loongarch_lasx_xvbitset_w:
8691 case Intrinsic::loongarch_lasx_xvbitset_d: {
8692 EVT VecTy = N->getValueType(0);
8693 SDValue One = DAG.getConstant(1, DL, VecTy);
8694 return DAG.getNode(
8695 ISD::OR, DL, VecTy, N->getOperand(1),
8696 DAG.getNode(ISD::SHL, DL, VecTy, One, truncateVecElts(N, DAG)));
8697 }
8698 case Intrinsic::loongarch_lsx_vbitseti_b:
8699 case Intrinsic::loongarch_lasx_xvbitseti_b:
8700 return lowerVectorBitSetImm<3>(N, DAG);
8701 case Intrinsic::loongarch_lsx_vbitseti_h:
8702 case Intrinsic::loongarch_lasx_xvbitseti_h:
8703 return lowerVectorBitSetImm<4>(N, DAG);
8704 case Intrinsic::loongarch_lsx_vbitseti_w:
8705 case Intrinsic::loongarch_lasx_xvbitseti_w:
8706 return lowerVectorBitSetImm<5>(N, DAG);
8707 case Intrinsic::loongarch_lsx_vbitseti_d:
8708 case Intrinsic::loongarch_lasx_xvbitseti_d:
8709 return lowerVectorBitSetImm<6>(N, DAG);
8710 case Intrinsic::loongarch_lsx_vbitrev_b:
8711 case Intrinsic::loongarch_lsx_vbitrev_h:
8712 case Intrinsic::loongarch_lsx_vbitrev_w:
8713 case Intrinsic::loongarch_lsx_vbitrev_d:
8714 case Intrinsic::loongarch_lasx_xvbitrev_b:
8715 case Intrinsic::loongarch_lasx_xvbitrev_h:
8716 case Intrinsic::loongarch_lasx_xvbitrev_w:
8717 case Intrinsic::loongarch_lasx_xvbitrev_d: {
8718 EVT VecTy = N->getValueType(0);
8719 SDValue One = DAG.getConstant(1, DL, VecTy);
8720 return DAG.getNode(
8721 ISD::XOR, DL, VecTy, N->getOperand(1),
8722 DAG.getNode(ISD::SHL, DL, VecTy, One, truncateVecElts(N, DAG)));
8723 }
8724 case Intrinsic::loongarch_lsx_vbitrevi_b:
8725 case Intrinsic::loongarch_lasx_xvbitrevi_b:
8726 return lowerVectorBitRevImm<3>(N, DAG);
8727 case Intrinsic::loongarch_lsx_vbitrevi_h:
8728 case Intrinsic::loongarch_lasx_xvbitrevi_h:
8729 return lowerVectorBitRevImm<4>(N, DAG);
8730 case Intrinsic::loongarch_lsx_vbitrevi_w:
8731 case Intrinsic::loongarch_lasx_xvbitrevi_w:
8732 return lowerVectorBitRevImm<5>(N, DAG);
8733 case Intrinsic::loongarch_lsx_vbitrevi_d:
8734 case Intrinsic::loongarch_lasx_xvbitrevi_d:
8735 return lowerVectorBitRevImm<6>(N, DAG);
8736 case Intrinsic::loongarch_lsx_vfadd_s:
8737 case Intrinsic::loongarch_lsx_vfadd_d:
8738 case Intrinsic::loongarch_lasx_xvfadd_s:
8739 case Intrinsic::loongarch_lasx_xvfadd_d:
8740 return DAG.getNode(ISD::FADD, DL, N->getValueType(0), N->getOperand(1),
8741 N->getOperand(2));
8742 case Intrinsic::loongarch_lsx_vfsub_s:
8743 case Intrinsic::loongarch_lsx_vfsub_d:
8744 case Intrinsic::loongarch_lasx_xvfsub_s:
8745 case Intrinsic::loongarch_lasx_xvfsub_d:
8746 return DAG.getNode(ISD::FSUB, DL, N->getValueType(0), N->getOperand(1),
8747 N->getOperand(2));
8748 case Intrinsic::loongarch_lsx_vfmul_s:
8749 case Intrinsic::loongarch_lsx_vfmul_d:
8750 case Intrinsic::loongarch_lasx_xvfmul_s:
8751 case Intrinsic::loongarch_lasx_xvfmul_d:
8752 return DAG.getNode(ISD::FMUL, DL, N->getValueType(0), N->getOperand(1),
8753 N->getOperand(2));
8754 case Intrinsic::loongarch_lsx_vfdiv_s:
8755 case Intrinsic::loongarch_lsx_vfdiv_d:
8756 case Intrinsic::loongarch_lasx_xvfdiv_s:
8757 case Intrinsic::loongarch_lasx_xvfdiv_d:
8758 return DAG.getNode(ISD::FDIV, DL, N->getValueType(0), N->getOperand(1),
8759 N->getOperand(2));
8760 case Intrinsic::loongarch_lsx_vfmadd_s:
8761 case Intrinsic::loongarch_lsx_vfmadd_d:
8762 case Intrinsic::loongarch_lasx_xvfmadd_s:
8763 case Intrinsic::loongarch_lasx_xvfmadd_d:
8764 return DAG.getNode(ISD::FMA, DL, N->getValueType(0), N->getOperand(1),
8765 N->getOperand(2), N->getOperand(3));
8766 case Intrinsic::loongarch_lsx_vinsgr2vr_b:
8767 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
8768 N->getOperand(1), N->getOperand(2),
8770 case Intrinsic::loongarch_lsx_vinsgr2vr_h:
8771 case Intrinsic::loongarch_lasx_xvinsgr2vr_w:
8772 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
8773 N->getOperand(1), N->getOperand(2),
8775 case Intrinsic::loongarch_lsx_vinsgr2vr_w:
8776 case Intrinsic::loongarch_lasx_xvinsgr2vr_d:
8777 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
8778 N->getOperand(1), N->getOperand(2),
8780 case Intrinsic::loongarch_lsx_vinsgr2vr_d:
8781 return DAG.getNode(ISD::INSERT_VECTOR_ELT, SDLoc(N), N->getValueType(0),
8782 N->getOperand(1), N->getOperand(2),
8784 case Intrinsic::loongarch_lsx_vreplgr2vr_b:
8785 case Intrinsic::loongarch_lsx_vreplgr2vr_h:
8786 case Intrinsic::loongarch_lsx_vreplgr2vr_w:
8787 case Intrinsic::loongarch_lsx_vreplgr2vr_d:
8788 case Intrinsic::loongarch_lasx_xvreplgr2vr_b:
8789 case Intrinsic::loongarch_lasx_xvreplgr2vr_h:
8790 case Intrinsic::loongarch_lasx_xvreplgr2vr_w:
8791 case Intrinsic::loongarch_lasx_xvreplgr2vr_d:
8792 return DAG.getNode(LoongArchISD::VREPLGR2VR, DL, N->getValueType(0),
8793 DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getGRLenVT(),
8794 N->getOperand(1)));
8795 case Intrinsic::loongarch_lsx_vreplve_b:
8796 case Intrinsic::loongarch_lsx_vreplve_h:
8797 case Intrinsic::loongarch_lsx_vreplve_w:
8798 case Intrinsic::loongarch_lsx_vreplve_d:
8799 case Intrinsic::loongarch_lasx_xvreplve_b:
8800 case Intrinsic::loongarch_lasx_xvreplve_h:
8801 case Intrinsic::loongarch_lasx_xvreplve_w:
8802 case Intrinsic::loongarch_lasx_xvreplve_d:
8803 return DAG.getNode(LoongArchISD::VREPLVE, DL, N->getValueType(0),
8804 N->getOperand(1),
8805 DAG.getNode(ISD::ANY_EXTEND, DL, Subtarget.getGRLenVT(),
8806 N->getOperand(2)));
8807 case Intrinsic::loongarch_lsx_vpickve2gr_b:
8808 if (!Subtarget.is64Bit())
8809 return lowerVectorPickVE2GR<4>(N, DAG, LoongArchISD::VPICK_SEXT_ELT);
8810 break;
8811 case Intrinsic::loongarch_lsx_vpickve2gr_h:
8812 case Intrinsic::loongarch_lasx_xvpickve2gr_w:
8813 if (!Subtarget.is64Bit())
8814 return lowerVectorPickVE2GR<3>(N, DAG, LoongArchISD::VPICK_SEXT_ELT);
8815 break;
8816 case Intrinsic::loongarch_lsx_vpickve2gr_w:
8817 if (!Subtarget.is64Bit())
8818 return lowerVectorPickVE2GR<2>(N, DAG, LoongArchISD::VPICK_SEXT_ELT);
8819 break;
8820 case Intrinsic::loongarch_lsx_vpickve2gr_bu:
8821 if (!Subtarget.is64Bit())
8822 return lowerVectorPickVE2GR<4>(N, DAG, LoongArchISD::VPICK_ZEXT_ELT);
8823 break;
8824 case Intrinsic::loongarch_lsx_vpickve2gr_hu:
8825 case Intrinsic::loongarch_lasx_xvpickve2gr_wu:
8826 if (!Subtarget.is64Bit())
8827 return lowerVectorPickVE2GR<3>(N, DAG, LoongArchISD::VPICK_ZEXT_ELT);
8828 break;
8829 case Intrinsic::loongarch_lsx_vpickve2gr_wu:
8830 if (!Subtarget.is64Bit())
8831 return lowerVectorPickVE2GR<2>(N, DAG, LoongArchISD::VPICK_ZEXT_ELT);
8832 break;
8833 case Intrinsic::loongarch_lsx_bz_b:
8834 case Intrinsic::loongarch_lsx_bz_h:
8835 case Intrinsic::loongarch_lsx_bz_w:
8836 case Intrinsic::loongarch_lsx_bz_d:
8837 case Intrinsic::loongarch_lasx_xbz_b:
8838 case Intrinsic::loongarch_lasx_xbz_h:
8839 case Intrinsic::loongarch_lasx_xbz_w:
8840 case Intrinsic::loongarch_lasx_xbz_d:
8841 if (!Subtarget.is64Bit())
8842 return DAG.getNode(LoongArchISD::VALL_ZERO, DL, N->getValueType(0),
8843 N->getOperand(1));
8844 break;
8845 case Intrinsic::loongarch_lsx_bz_v:
8846 case Intrinsic::loongarch_lasx_xbz_v:
8847 if (!Subtarget.is64Bit())
8848 return DAG.getNode(LoongArchISD::VANY_ZERO, DL, N->getValueType(0),
8849 N->getOperand(1));
8850 break;
8851 case Intrinsic::loongarch_lsx_bnz_b:
8852 case Intrinsic::loongarch_lsx_bnz_h:
8853 case Intrinsic::loongarch_lsx_bnz_w:
8854 case Intrinsic::loongarch_lsx_bnz_d:
8855 case Intrinsic::loongarch_lasx_xbnz_b:
8856 case Intrinsic::loongarch_lasx_xbnz_h:
8857 case Intrinsic::loongarch_lasx_xbnz_w:
8858 case Intrinsic::loongarch_lasx_xbnz_d:
8859 if (!Subtarget.is64Bit())
8860 return DAG.getNode(LoongArchISD::VALL_NONZERO, DL, N->getValueType(0),
8861 N->getOperand(1));
8862 break;
8863 case Intrinsic::loongarch_lsx_bnz_v:
8864 case Intrinsic::loongarch_lasx_xbnz_v:
8865 if (!Subtarget.is64Bit())
8866 return DAG.getNode(LoongArchISD::VANY_NONZERO, DL, N->getValueType(0),
8867 N->getOperand(1));
8868 break;
8869 case Intrinsic::loongarch_lasx_concat_128_s:
8870 case Intrinsic::loongarch_lasx_concat_128_d:
8871 case Intrinsic::loongarch_lasx_concat_128:
8872 return DAG.getNode(ISD::CONCAT_VECTORS, DL, N->getValueType(0),
8873 N->getOperand(1), N->getOperand(2));
8874 }
8875 return SDValue();
8876}
8877
8881 // If the input to MOVGR2FR_W_LA64 is just MOVFR2GR_S_LA64 the the
8882 // conversion is unnecessary and can be replaced with the
8883 // MOVFR2GR_S_LA64 operand.
8884 SDValue Op0 = N->getOperand(0);
8885 if (Op0.getOpcode() == LoongArchISD::MOVFR2GR_S_LA64)
8886 return Op0.getOperand(0);
8887 return SDValue();
8888}
8889
8893 // If the input to MOVFR2GR_S_LA64 is just MOVGR2FR_W_LA64 then the
8894 // conversion is unnecessary and can be replaced with the MOVGR2FR_W_LA64
8895 // operand.
8896 SDValue Op0 = N->getOperand(0);
8897 if (Op0->getOpcode() == LoongArchISD::MOVGR2FR_W_LA64) {
8898 assert(Op0.getOperand(0).getValueType() == N->getSimpleValueType(0) &&
8899 "Unexpected value type!");
8900 return Op0.getOperand(0);
8901 }
8902 return SDValue();
8903}
8904
8905static SDValue
8908 MVT VT = N->getSimpleValueType(0);
8909 unsigned NumBits = VT.getScalarSizeInBits();
8910
8911 // Simplify the inputs.
8913 APInt DemandedMask(APInt::getAllOnes(NumBits));
8914 if (TLI.SimplifyDemandedBits(SDValue(N, 0), DemandedMask, DCI))
8915 return SDValue(N, 0);
8916
8917 return SDValue();
8918}
8919
8920static SDValue
8924 SDValue Op0 = N->getOperand(0);
8925 SDLoc DL(N);
8926
8927 // If the input to SplitPairF64 is just BuildPairF64 then the operation is
8928 // redundant. Instead, use BuildPairF64's operands directly.
8929 if (Op0->getOpcode() == LoongArchISD::BUILD_PAIR_F64)
8930 return DCI.CombineTo(N, Op0.getOperand(0), Op0.getOperand(1));
8931
8932 if (Op0->isUndef()) {
8933 SDValue Lo = DAG.getUNDEF(MVT::i32);
8934 SDValue Hi = DAG.getUNDEF(MVT::i32);
8935 return DCI.CombineTo(N, Lo, Hi);
8936 }
8937
8938 // It's cheaper to materialise two 32-bit integers than to load a double
8939 // from the constant pool and transfer it to integer registers through the
8940 // stack.
8942 APInt V = C->getValueAPF().bitcastToAPInt();
8943 SDValue Lo = DAG.getConstant(V.trunc(32), DL, MVT::i32);
8944 SDValue Hi = DAG.getConstant(V.lshr(32).trunc(32), DL, MVT::i32);
8945 return DCI.CombineTo(N, Lo, Hi);
8946 }
8947
8948 return SDValue();
8949}
8950
8951/// Do target-specific dag combines on LoongArchISD::VANDN nodes.
8955 SDValue N0 = N->getOperand(0);
8956 SDValue N1 = N->getOperand(1);
8957 MVT VT = N->getSimpleValueType(0);
8958 SDLoc DL(N);
8959
8960 // VANDN(undef, x) -> 0
8961 // VANDN(x, undef) -> 0
8962 if (N0.isUndef() || N1.isUndef())
8963 return DAG.getConstant(0, DL, VT);
8964
8965 // VANDN(0, x) -> x
8967 return N1;
8968
8969 // VANDN(x, 0) -> 0
8971 return DAG.getConstant(0, DL, VT);
8972
8973 // VANDN(x, -1) -> NOT(x) -> XOR(x, -1)
8975 return DAG.getNOT(DL, N0, VT);
8976
8977 // Turn VANDN back to AND if input is inverted.
8978 if (SDValue Not = isNOT(N0, DAG))
8979 return DAG.getNode(ISD::AND, DL, VT, DAG.getBitcast(VT, Not), N1);
8980
8981 // VANDN(bitcast(shl(splat(1), y)), x)
8982 // -> bitcast(VANDN(shl(splat(1), y), bitcast(x)))
8983 // Perform the VANDN in the element type of the shift so that it can be
8984 // selected as [x]vbitclr.{b/h/w/d}.
8985 if (N0.getOpcode() == ISD::BITCAST && N0.hasOneUse()) {
8986 SDValue Shl = N0.getOperand(0);
8987 EVT ShlVT = Shl.getValueType();
8988 APInt SplatVal;
8989 if (Shl.getOpcode() == ISD::SHL && ShlVT.isVector() && ShlVT.isInteger() &&
8990 ShlVT != VT && ShlVT.getSizeInBits() == VT.getSizeInBits() &&
8991 DAG.getTargetLoweringInfo().isTypeLegal(ShlVT) &&
8992 ISD::isConstantSplatVector(Shl.getOperand(0), SplatVal) &&
8993 SplatVal.isOne()) {
8994 SDValue V = DAG.getNode(LoongArchISD::VANDN, DL, ShlVT, Shl,
8995 DAG.getBitcast(ShlVT, N1));
8996 return DAG.getBitcast(VT, V);
8997 }
8998 }
8999
9000 // Folds for better commutativity:
9001 if (N1->hasOneUse()) {
9002 // VANDN(x,NOT(y)) -> AND(NOT(x),NOT(y)) -> NOT(OR(X,Y)).
9003 if (SDValue Not = isNOT(N1, DAG))
9004 return DAG.getNOT(
9005 DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)), VT);
9006
9007 // VANDN(x, SplatVector(Imm)) -> AND(NOT(x), NOT(SplatVector(~Imm)))
9008 // -> NOT(OR(x, SplatVector(-Imm))
9009 // Combination is performed only when VT is v16i8/v32i8, using `vnori.b` to
9010 // gain benefits.
9011 if (!DCI.isBeforeLegalizeOps() && (VT == MVT::v16i8 || VT == MVT::v32i8) &&
9012 N1.getOpcode() == ISD::BUILD_VECTOR) {
9013 if (SDValue SplatValue =
9014 cast<BuildVectorSDNode>(N1.getNode())->getSplatValue()) {
9015 if (!N1->isOnlyUserOf(SplatValue.getNode()))
9016 return SDValue();
9017
9018 if (auto *C = dyn_cast<ConstantSDNode>(SplatValue)) {
9019 uint8_t NCVal = static_cast<uint8_t>(~(C->getSExtValue()));
9020 SDValue Not =
9021 DAG.getSplat(VT, DL, DAG.getTargetConstant(NCVal, DL, MVT::i8));
9022 return DAG.getNOT(
9023 DL, DAG.getNode(ISD::OR, DL, VT, N0, DAG.getBitcast(VT, Not)),
9024 VT);
9025 }
9026 }
9027 }
9028 }
9029
9030 return SDValue();
9031}
9032
9033static SDValue ExtendSrcToDst(SDNode *N, SelectionDAG &DAG, unsigned ExtendOp) {
9034 SDLoc DL(N);
9035 EVT VT = N->getValueType(0);
9036 SDValue Src = N->getOperand(0);
9037 EVT SrcVT = Src.getValueType();
9038
9039 unsigned DstElts = VT.getVectorNumElements();
9040 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9041 unsigned DstEltBits = VT.getScalarSizeInBits();
9042
9043 if (SrcEltBits >= DstEltBits)
9044 return SDValue();
9045
9046 MVT WidenEltVT = MVT::getIntegerVT(DstEltBits);
9047 EVT WidenSrcVT = EVT::getVectorVT(*DAG.getContext(), WidenEltVT, DstElts);
9048
9049 SDValue Extend = DAG.getNode(ExtendOp, DL, WidenSrcVT, Src);
9050 return DAG.getNode(N->getOpcode(), DL, VT, Extend);
9051}
9052
9053// Merge two 64 to 32 convert instructions into one,
9054// e.g.
9055// vffint.s.l $vr0, $vr1, $vr2
9056// will convert 4 si64 into 4 float at once.
9057// or
9058// vftintrz.w.d $vr0, $vr1, $vr2
9059// which will convert 4 double into 4 si32 at once.
9060// also deal with their 256-bits LASX version.
9061static SDValue MergeBlocksConvert(SDNode *N, SelectionDAG &DAG, unsigned Opcode,
9062 unsigned BlockBits) {
9063 SDLoc DL(N);
9064 MVT DstVT = N->getSimpleValueType(0);
9065 SDValue Src = N->getOperand(0);
9066 MVT SrcVT = Src.getSimpleValueType();
9067 unsigned SrcBits = SrcVT.getSizeInBits();
9068
9070 unsigned BlockNumElts = BlockBits / SrcVT.getScalarSizeInBits();
9071 MVT BlockVT = MVT::getVectorVT(SrcVT.getScalarType(), BlockNumElts);
9072 if (Src.getOpcode() == ISD::CONCAT_VECTORS &&
9073 Src.getOperand(0).getValueType() == BlockVT) {
9074 for (unsigned i = 0; i < Src.getNumOperands(); ++i)
9075 Blocks.push_back(Src.getOperand(i));
9076 } else if (SrcBits > BlockBits) {
9077 // Wider than one register: extract each BlockBits-wide sub-vector.
9078 for (unsigned i = 0; i < SrcBits / BlockBits; ++i)
9079 Blocks.push_back(
9080 DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, BlockVT, Src,
9081 DAG.getVectorIdxConstant(i * BlockNumElts, DL)));
9082 } else {
9083 BlockBits = SrcBits;
9084 Blocks.push_back(Src);
9085 }
9086
9087 MVT NativeVecVT = MVT::getVectorVT(DstVT.getScalarType(),
9088 BlockBits / DstVT.getScalarSizeInBits());
9090 for (unsigned i = 0; i < Blocks.size(); i += 2) {
9091 SDValue Lo = Blocks[i];
9092 SDValue Hi = Blocks.size() > 1 ? Blocks[i + 1] : Lo;
9093 SDValue Res = DAG.getNode(Opcode, DL, NativeVecVT, Hi, Lo);
9094
9095 if (BlockBits == 256) {
9096 SDValue Undef = DAG.getUNDEF(NativeVecVT);
9097 SmallVector<int, 8> Mask = {0, 1, 4, 5, 2, 3, 6, 7};
9098 Res = DAG.getVectorShuffle(NativeVecVT, DL, Res, Undef, Mask);
9099 Res = DAG.getBitcast(NativeVecVT, Res);
9100 }
9101
9102 Parts.push_back(Res);
9103 }
9104
9105 if (Blocks.size() == 1)
9106 return DAG.getNode(ISD::EXTRACT_SUBVECTOR, DL, DstVT, Parts[0],
9107 DAG.getVectorIdxConstant(0, DL));
9108 return DAG.getNode(ISD::CONCAT_VECTORS, DL, DstVT, Parts);
9109}
9110
9114 SDLoc DL(N);
9115 EVT VT = N->getValueType(0);
9116 SDValue Src = N->getOperand(0);
9117 EVT SrcVT = Src.getValueType();
9118
9119 if (VT.isVector()) {
9120 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9121 unsigned DstEltBits = VT.getScalarSizeInBits();
9122 unsigned NumElts = VT.getVectorNumElements();
9123 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
9124
9125 // Sign-extend src to avoid scalarization.
9126 if (SrcEltBits <= DstEltBits)
9127 return ExtendSrcToDst(N, DAG, ISD::SIGN_EXTEND);
9128
9129 if (SrcEltBits != 64 || DstEltBits != 32 || !isPowerOf2_32(NumElts))
9130 return SDValue();
9131
9132 if (!SrcVT.isSimple() || !VT.isSimple())
9133 return SDValue();
9134
9135 // [x]vffint.s.l needs LSX and at least one full 128-bit source register.
9136 if (!Subtarget.hasExtLSX() || SrcVT.getSizeInBits() < 128)
9137 return SDValue();
9138
9139 // Combine [x]vffint.s.l for vector si64 to float conversion.
9140 return MergeBlocksConvert(N, DAG, LoongArchISD::VFFINT, BlockBits);
9141 }
9142
9143 if (VT != MVT::f32 && VT != MVT::f64)
9144 return SDValue();
9145 if (VT == MVT::f32 && !Subtarget.hasBasicF())
9146 return SDValue();
9147 if (VT == MVT::f64 && !Subtarget.hasBasicD())
9148 return SDValue();
9149
9150 // Only optimize when the source and destination types have the same width.
9151 if (VT.getSizeInBits() != N->getOperand(0).getValueSizeInBits())
9152 return SDValue();
9153
9154 // If the result of an integer load is only used by an integer-to-float
9155 // conversion, use a fp load instead. This eliminates an integer-to-float-move
9156 // (movgr2fr) instruction.
9157 if (ISD::isNormalLoad(Src.getNode()) && Src.hasOneUse() &&
9158 // Do not change the width of a volatile load. This condition check is
9159 // inspired by AArch64.
9160 !cast<LoadSDNode>(Src)->isVolatile()) {
9161 LoadSDNode *LN0 = cast<LoadSDNode>(Src);
9162 SDValue Load = DAG.getLoad(VT, DL, LN0->getChain(), LN0->getBasePtr(),
9163 LN0->getPointerInfo(), LN0->getAlign(),
9164 LN0->getMemOperand()->getFlags());
9165
9166 // Make sure successors of the original load stay after it by updating them
9167 // to use the new Chain.
9168 DAG.ReplaceAllUsesOfValueWith(SDValue(LN0, 1), Load.getValue(1));
9169 return DAG.getNode(LoongArchISD::SITOF, SDLoc(N), VT, Load);
9170 }
9171
9172 return SDValue();
9173}
9174
9178 SDLoc DL(N);
9179 EVT VT = N->getValueType(0);
9180
9181 // Zero-extend src to avoid scalarization.
9182 if (VT.isVector())
9183 return ExtendSrcToDst(N, DAG, ISD::ZERO_EXTEND);
9184
9185 return SDValue();
9186}
9187
9188// Using [X]VFTINTRZ_W_D for double to signed 32-bit integer conversion.
9189// For example:
9190// v4i32 = fp_to_sint (concat_vectors v2f64, v2f64)
9191// Can be combined into:
9192// v4i32 = VFTINTRZ_W_D v2f64. v2f64
9196 if (!Subtarget.hasExtLSX())
9197 return SDValue();
9198
9199 SDLoc DL(N);
9200 EVT DstVT = N->getValueType(0);
9201 SDValue Src = N->getOperand(0);
9202 EVT SrcVT = Src.getValueType();
9203 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
9204
9205 if (!DstVT.isVector() || !DstVT.isSimple() || !SrcVT.isSimple())
9206 return SDValue();
9207
9208 unsigned SrcEltBits = SrcVT.getScalarSizeInBits();
9209 unsigned SrcBits = SrcVT.getSizeInBits();
9210 unsigned DstEltBits = DstVT.getScalarSizeInBits();
9211 unsigned NumElts = DstVT.getVectorNumElements();
9212 unsigned BlockBits = Subtarget.hasExtLASX() ? 256 : 128;
9213
9214 if (!isPowerOf2_32(NumElts) || !isPowerOf2_32(DstEltBits))
9215 return SDValue();
9216
9217 if (SrcBits % BlockBits != 0 && SrcBits != 128)
9218 return SDValue();
9219
9220 if (DstEltBits < 32) {
9221 MVT PromoteVT = MVT::getVectorVT(MVT::getIntegerVT(32), NumElts);
9222 SDValue Conv = DAG.getNode(N->getOpcode(), DL, PromoteVT, Src);
9223 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Conv);
9224 }
9225
9226 if (SrcEltBits != 64 || DstEltBits != 32)
9227 return SDValue();
9228
9229 if (!IsSigned) {
9230 // LASX already has pattern for double convert to uint32.
9231 if (Subtarget.hasExtLASX())
9232 return SDValue();
9233 MVT TmpVT = MVT::getVectorVT(MVT::i64, NumElts);
9234 SDValue Tmp = DAG.getNode(ISD::FP_TO_SINT, DL, TmpVT, Src);
9235 return DAG.getNode(ISD::TRUNCATE, DL, DstVT, Tmp);
9236 }
9237
9238 return MergeBlocksConvert(N, DAG, LoongArchISD::VFTINTRZ, BlockBits);
9239}
9240
9241// Try to widen AND, OR and XOR nodes to VT in order to remove casts around
9242// logical operations, like in the example below.
9243// or (and (truncate x, truncate y)),
9244// (xor (truncate z, build_vector (constants)))
9245// Given a target type \p VT, we generate
9246// or (and x, y), (xor z, zext(build_vector (constants)))
9247// given x, y and z are of type \p VT. We can do so, if operands are either
9248// truncates from VT types, the second operand is a vector of constants, can
9249// be recursively promoted or is an existing extension we can extend further.
9251 SelectionDAG &DAG,
9253 unsigned Depth) {
9254 // Limit recursion to avoid excessive compile times.
9256 return SDValue();
9257
9258 if (!ISD::isBitwiseLogicOp(N.getOpcode()))
9259 return SDValue();
9260
9261 SDValue N0 = N.getOperand(0);
9262 SDValue N1 = N.getOperand(1);
9263
9265 if (!TLI.isOperationLegalOrPromote(N.getOpcode(), VT))
9266 return SDValue();
9267
9268 if (SDValue NN0 =
9269 PromoteMaskArithmetic(N0, DL, VT, DAG, Subtarget, Depth + 1))
9270 N0 = NN0;
9271 else {
9272 // The left side has to be a 'trunc'.
9273 bool LHSTrunc = N0.getOpcode() == ISD::TRUNCATE &&
9274 N0.getOperand(0).getValueType() == VT;
9275 if (LHSTrunc)
9276 N0 = N0.getOperand(0);
9277 else
9278 return SDValue();
9279 }
9280
9281 if (SDValue NN1 =
9282 PromoteMaskArithmetic(N1, DL, VT, DAG, Subtarget, Depth + 1))
9283 N1 = NN1;
9284 else {
9285 // The right side has to be a 'trunc', a (foldable) constant or an
9286 // existing extension we can extend further.
9287 bool RHSTrunc = N1.getOpcode() == ISD::TRUNCATE &&
9288 N1.getOperand(0).getValueType() == VT;
9289 if (RHSTrunc)
9290 N1 = N1.getOperand(0);
9291 else if (ISD::isExtVecInRegOpcode(N1.getOpcode()) && VT.is256BitVector() &&
9292 Subtarget.hasExtLASX() && N1.hasOneUse())
9293 N1 = DAG.getNode(N1.getOpcode(), DL, VT, N1.getOperand(0));
9294 // On 32-bit platform, i64 is an illegal integer scalar type, and
9295 // FoldConstantArithmetic will fail for v4i64. This may be optimized in the
9296 // future.
9297 else if (SDValue Cst =
9299 N1 = Cst;
9300 else
9301 return SDValue();
9302 }
9303
9304 return DAG.getNode(N.getOpcode(), DL, VT, N0, N1);
9305}
9306
9307// On LASX the type v4i1/v8i1/v16i1 may be legalized to v4i32/v8i16/v16i8, which
9308// is LSX-sized register. In most cases we actually compare or select LASX-sized
9309// registers and mixing the two types creates horrible code. This method
9310// optimizes some of the transition sequences.
9312 SelectionDAG &DAG,
9314 EVT VT = N.getValueType();
9315 assert(VT.isVector() && "Expected vector type");
9316 assert((N.getOpcode() == ISD::ANY_EXTEND ||
9317 N.getOpcode() == ISD::ZERO_EXTEND ||
9318 N.getOpcode() == ISD::SIGN_EXTEND) &&
9319 "Invalid Node");
9320
9321 if (!Subtarget.hasExtLASX() || !VT.is256BitVector())
9322 return SDValue();
9323
9324 SDValue Narrow = N.getOperand(0);
9325 EVT NarrowVT = Narrow.getValueType();
9326
9327 // Generate the wide operation.
9328 SDValue Op = PromoteMaskArithmetic(Narrow, DL, VT, DAG, Subtarget, 0);
9329 if (!Op)
9330 return SDValue();
9331 switch (N.getOpcode()) {
9332 default:
9333 llvm_unreachable("Unexpected opcode");
9334 case ISD::ANY_EXTEND:
9335 return Op;
9336 case ISD::ZERO_EXTEND:
9337 return DAG.getZeroExtendInReg(Op, DL, NarrowVT);
9338 case ISD::SIGN_EXTEND:
9339 return DAG.getNode(ISD::SIGN_EXTEND_INREG, DL, VT, Op,
9340 DAG.getValueType(NarrowVT));
9341 }
9342}
9343
9347 EVT VT = N->getValueType(0);
9348 SDLoc DL(N);
9349
9350 if (VT.isVector()) {
9351 if (SDValue R = PromoteMaskArithmetic(SDValue(N, 0), DL, DAG, Subtarget))
9352 return R;
9353
9354 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT) ||
9355 N->getValueSizeInBits(0) != N->getOperand(0).getValueSizeInBits() * 2)
9356 return SDValue();
9357
9358 if (SDValue R = matchHalfOf128BitLanes(N->getOperand(0), /*isLow=*/false)) {
9359 if (N->getOpcode() == ISD::SIGN_EXTEND)
9360 return DAG.getNode(LoongArchISD::VEXTH, DL, VT, R);
9361 if (N->getOpcode() == ISD::ZERO_EXTEND)
9362 return DAG.getNode(LoongArchISD::VEXTH_U, DL, VT, R);
9363 }
9364 }
9365
9366 return SDValue();
9367}
9368
9369static SDValue
9373 SDLoc DL(N);
9374 EVT VT = N->getValueType(0);
9375
9376 if (VT.isVector() && N->getNumOperands() == 2)
9377 if (SDValue R = combineFP_ROUND(SDValue(N, 0), DL, DAG, Subtarget))
9378 return R;
9379
9380 return SDValue();
9381}
9382
9386 if (DCI.isBeforeLegalizeOps())
9387 return SDValue();
9388
9389 EVT VT = N->getValueType(0);
9390 if (!VT.isVector())
9391 return SDValue();
9392
9393 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT))
9394 return SDValue();
9395
9396 EVT EltVT = VT.getVectorElementType();
9397 if (!EltVT.isInteger())
9398 return SDValue();
9399
9400 SDValue Cond = N->getOperand(0);
9401 SDValue TrueVal = N->getOperand(1);
9402 SDValue FalseVal = N->getOperand(2);
9403
9404 // match:
9405 //
9406 // vselect (setcc shift, 0, seteq),
9407 // x,
9408 // rounded_shift
9409
9410 if (Cond.getOpcode() != ISD::SETCC)
9411 return SDValue();
9412
9413 if (!ISD::isConstantSplatVectorAllZeros(Cond.getOperand(1)))
9414 return SDValue();
9415
9416 if (Cond.getOperandCondCode(2) != ISD::SETEQ)
9417 return SDValue();
9418
9419 SDValue Shift = Cond.getOperand(0);
9420
9421 // True branch must be original value:
9422 //
9423 // vselect cond, x, ...
9424
9425 SDValue X = TrueVal;
9426
9427 // Now match rounded shift pattern:
9428 //
9429 // add
9430 // (and
9431 // (srl X, shift-1)
9432 // 1)
9433 // (srl/sra X, shift)
9434
9435 if (FalseVal.getOpcode() != ISD::ADD)
9436 return SDValue();
9437
9438 SDValue Add0 = FalseVal.getOperand(0);
9439 SDValue Add1 = FalseVal.getOperand(1);
9440 SDValue And;
9441 SDValue Shr;
9442
9443 if (Add0.getOpcode() == ISD::AND) {
9444 And = Add0;
9445 Shr = Add1;
9446 } else if (Add1.getOpcode() == ISD::AND) {
9447 And = Add1;
9448 Shr = Add0;
9449 } else {
9450 return SDValue();
9451 }
9452
9453 // match:
9454 //
9455 // srl/sra X, shift
9456
9457 if (Shr.getOpcode() != ISD::SRL && Shr.getOpcode() != ISD::SRA)
9458 return SDValue();
9459
9460 if (Shr.getOperand(0) != X)
9461 return SDValue();
9462
9463 if (Shr.getOperand(1) != Shift)
9464 return SDValue();
9465
9466 // match:
9467 //
9468 // and
9469 // (srl X, shift-1)
9470 // 1
9471
9472 SDValue Srl = And.getOperand(0);
9473 SDValue One = And.getOperand(1);
9474 APInt SplatVal;
9475
9476 if (Srl.getOpcode() != ISD::SRL)
9477 return SDValue();
9478
9479 One = peekThroughBitcasts(One);
9480 if (!isConstantSplatVector(One, SplatVal, EltVT.getSizeInBits()))
9481 return SDValue();
9482
9483 if (SplatVal != 1)
9484 return SDValue();
9485
9486 if (Srl.getOperand(0) != X)
9487 return SDValue();
9488
9489 // match:
9490 //
9491 // shift-1
9492
9493 SDValue ShiftMinus1 = Srl.getOperand(1);
9494
9495 if (ShiftMinus1.getOpcode() != ISD::ADD)
9496 return SDValue();
9497
9498 if (ShiftMinus1.getOperand(0) != Shift)
9499 return SDValue();
9500
9501 if (!ISD::isConstantSplatVectorAllOnes(ShiftMinus1.getOperand(1)))
9502 return SDValue();
9503
9504 // We matched a rounded right shift pattern and can lower it
9505 // to a single vector rounded shift instruction.
9506
9507 SDLoc DL(N);
9508 return DAG.getNode(Shr.getOpcode() == ISD::SRL ? LoongArchISD::VSRLR
9509 : LoongArchISD::VSRAR,
9510 DL, VT, X, Shift);
9511}
9512
9516 if (!DCI.isBeforeLegalize())
9517 return SDValue();
9518
9519 SDLoc DL(N);
9520 EVT ResTy = N->getValueType(0);
9521 SDValue N0 = N->getOperand(0);
9522 SDValue N1 = N->getOperand(1);
9523
9524 bool HasLSXOnly = Subtarget.hasExtLSX() && !Subtarget.hasExtLASX();
9525 if (ResTy != MVT::v8i16 && ResTy != MVT::v4i32 && ResTy != MVT::v2i64 &&
9526 ResTy != MVT::i128 && ResTy != MVT::v16i16 && ResTy != MVT::v8i32 &&
9527 ResTy != MVT::v4i64 && ResTy != MVT::v2i128)
9528 return SDValue();
9529 if (HasLSXOnly && (ResTy.is256BitVector() || ResTy == MVT::v2i128))
9530 return SDValue();
9531
9532 // Combine:
9533 // ti,tii,...,tx = extract_vector_elt t0, {0,2,4,.../1,3,5,...}
9534 // tj,tjj,...,ty = extract_vector_elt t1, {0,2,4,.../1,3,5,...}
9535 // tm = BUILD_VECTOR ti,tii,...,tx (Only when ResTy != MVT::i128)
9536 // tn = BUILD_VECTOR tj,tjj,...,ty (Only when ResTy != MVT::i128)
9537 // ta = {sign/zero}_extend tm
9538 // tb = {sign/zero}_extend tn
9539 // tr = mul ta, tb
9540 // to:
9541 // tr = VMULW{EV/OD}[U/US] t0, t1
9542 auto getExtType = [](unsigned Op0, unsigned Op1) -> int {
9543 if (Op0 == ISD::SIGN_EXTEND && Op1 == ISD::SIGN_EXTEND)
9544 return 0;
9545 if (Op0 == ISD::ZERO_EXTEND && Op1 == ISD::ZERO_EXTEND)
9546 return 1;
9547 if (Op0 == ISD::ZERO_EXTEND && Op1 == ISD::SIGN_EXTEND)
9548 return 2;
9549 if (Op0 == ISD::SIGN_EXTEND && Op1 == ISD::ZERO_EXTEND)
9550 return 3;
9551 return -1;
9552 };
9553
9554 int ExtType = getExtType(N0.getOpcode(), N1.getOpcode());
9555 if (ExtType < 0)
9556 return SDValue();
9557
9558 // Avoid the regressions on LA64 introduced by combining
9559 // i128-type multiplications.
9560 bool IsScalar = (ResTy == MVT::i128);
9561 if (IsScalar && ExtType < 2 && Subtarget.is64Bit())
9562 return SDValue();
9563
9564 SDValue Src0 = N0.getOperand(0);
9565 SDValue Src1 = N1.getOperand(0);
9566 if (IsScalar && (Src0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9567 Src1.getOpcode() != ISD::EXTRACT_VECTOR_ELT))
9568 return SDValue();
9569 if (!IsScalar && (Src0.getOpcode() != ISD::BUILD_VECTOR ||
9570 Src1.getOpcode() != ISD::BUILD_VECTOR))
9571 return SDValue();
9572
9573 unsigned ResBits = ResTy.getScalarSizeInBits();
9574 unsigned Src0Bits = Src0.getValueType().getScalarSizeInBits();
9575 unsigned Src1Bits = Src1.getValueType().getScalarSizeInBits();
9576 if (Src0Bits != Src1Bits || ResBits != Src0Bits * 2)
9577 return SDValue();
9578
9579 // Collect all EXTRACT_VECTOR_ELT.
9581 if (IsScalar) {
9582 Elems.emplace_back(Src0, Src1);
9583 } else {
9584 for (unsigned i = 0; i < Src0.getNumOperands(); ++i)
9585 Elems.emplace_back(Src0.getOperand(i), Src1.getOperand(i));
9586 }
9587
9588 unsigned Index;
9589 SDValue OrigN0, OrigN1;
9590 bool First = true;
9591 for (auto &[Op0, Op1] : Elems) {
9592 // Each element must be EXTRACT_VECTOR_ELT.
9593 if (Op0.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
9595 return SDValue();
9596
9597 // Check each EXTRACT_VECTOR_ELT's source vector and index.
9598 if (Op0.getOperand(1) != Op1.getOperand(1))
9599 return SDValue();
9600
9601 auto *IdxC = dyn_cast<ConstantSDNode>(Op0.getOperand(1));
9602 if (!IdxC)
9603 return SDValue();
9604
9605 unsigned CurIdx = IdxC->getZExtValue();
9606 if (First) {
9607 if (CurIdx != 0 && CurIdx != 1)
9608 return SDValue();
9609 OrigN0 = Op0.getOperand(0);
9610 OrigN1 = Op1.getOperand(0);
9611 First = false;
9612 } else {
9613 if (CurIdx != Index + 2 || Op0.getOperand(0) != OrigN0 ||
9614 Op1.getOperand(0) != OrigN1)
9615 return SDValue();
9616 }
9617 Index = CurIdx;
9618 }
9619
9620 if (OrigN0.getValueType() != OrigN1.getValueType())
9621 return SDValue();
9622 if (OrigN0.getValueType().getVectorNumElements() !=
9623 (IsScalar ? 1 : ResTy.getVectorNumElements()) * 2)
9624 return SDValue();
9625
9626 SDValue Result;
9627 EVT OrigTy = OrigN0.getValueType();
9628 bool IsEven = (Index % 2 == 0);
9629
9630 static const unsigned OpcTable[3][2] = {
9631 {LoongArchISD::VMULWOD, LoongArchISD::VMULWEV},
9632 {LoongArchISD::VMULWODU, LoongArchISD::VMULWEVU},
9633 {LoongArchISD::VMULWODUS, LoongArchISD::VMULWEVUS}};
9634
9635 if (ExtType == 3)
9636 Result = DAG.getNode(OpcTable[2][IsEven], DL, OrigTy, OrigN1, OrigN0);
9637 else
9638 Result = DAG.getNode(OpcTable[ExtType][IsEven], DL, OrigTy, OrigN0, OrigN1);
9639
9640 return DAG.getBitcast(ResTy, Result);
9641}
9642
9643static SDValue
9646 SelectionDAG &DAG = DCI.DAG;
9647 SDValue Src = N->getOperand(0);
9648 EVT VT = N->getValueType(0);
9649 EVT SrcVT = cast<VTSDNode>(N->getOperand(1))->getVT();
9650 unsigned Opc = Src.getOpcode();
9651 SDLoc DL(N);
9652
9653 // Leave the sign extension in place when it feeds an xor with another
9654 // sign-extended value; the generic xor-of-sext_inregs combine can then
9655 // fold both extensions into a single one.
9656 if (N->hasOneUse()) {
9657 SDNode *User = N->use_begin()->getUser();
9658 if (User->getOpcode() == ISD::XOR) {
9659 SDValue Other = User->getOperand(0).getNode() == N ? User->getOperand(1)
9660 : User->getOperand(0);
9661 if (Other.getOpcode() == ISD::SIGN_EXTEND_INREG)
9662 return SDValue();
9663 }
9664 }
9665
9666 // Do not hide a sign-extended boolean from a compare against zero: the
9667 // generic combiner folds (setcc (sext_inreg X, i1), 0, setlt) back to X.
9668 for (SDNode *User : N->users()) {
9669 if (User->getOpcode() != ISD::SETCC)
9670 continue;
9671 if ((User->getOperand(0).getNode() == N &&
9673 (User->getOperand(1).getNode() == N &&
9675 return SDValue();
9676 }
9677
9678 // Fold (sext_inreg (shl X, Y), i32) -> (sll.w X, Y) iff Y u< 32.
9679 if (DCI.isAfterLegalizeDAG() && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
9680 VT == MVT::i64 && Opc == ISD::SHL &&
9681 !isa<ConstantSDNode>(Src.getOperand(1)) &&
9682 DAG.computeKnownBits(Src.getOperand(1)).countMaxActiveBits() <= 5) {
9683 return DAG.getNode(LoongArchISD::SLL_W, DL, VT, Src.getOperand(0),
9684 Src.getOperand(1));
9685 }
9686
9687 // The SETCC/XOR folds replace the sign extension itself. When the result
9688 // has several users they do not save an instruction and can duplicate the
9689 // compare; leave it to the generic combiner.
9690 if (!N->hasOneUse())
9691 return SDValue();
9692
9693 // Similarly, a sign-extended boolean that feeds a sign-bit extraction is
9694 // usually part of a compare that the generic combiner can fold away.
9695 for (SDNode *User : N->users()) {
9696 if ((User->getOpcode() == ISD::SRL || User->getOpcode() == ISD::SRA) &&
9698 return SDValue();
9699 }
9700
9701 // Fold (sext_inreg (setcc X, Y, CC), i1) -> (sub 0, (setcc X, Y, CC)) if CC
9702 // can be selected with a single instruction, or
9703 // (add (setcc X, Y, inv(CC)), -1) if the inverted CC is cheaper. LoongArch
9704 // selects setge/setle/setuge/setule as slt + xori, so inverting such an
9705 // integer compare avoids the extra xor. Inverting replaces the original
9706 // setcc, so only do it when it has a single use; otherwise (e.g. an FP
9707 // compare shared with another user) this would introduce a second compare.
9708 if (Opc == ISD::SETCC && SrcVT == MVT::i1) {
9709 ISD::CondCode CC = cast<CondCodeSDNode>(Src.getOperand(2))->get();
9710 if (Src.hasOneUse() && Src.getOperand(0).getValueType().isInteger()) {
9711 switch (CC) {
9712 case ISD::SETGE:
9713 case ISD::SETLE:
9714 case ISD::SETUGE:
9715 case ISD::SETULE:
9716 // (sub 0, X >= Y) == (add (X < Y), -1), etc.
9717 return DAG.getNode(
9718 ISD::ADD, DL, VT,
9719 DAG.getSetCC(
9720 SDLoc(N), Src.getValueType(), Src.getOperand(0),
9721 Src.getOperand(1),
9722 ISD::getSetCCInverse(CC, Src.getOperand(0).getValueType())),
9723 DAG.getAllOnesConstant(DL, VT));
9724 default:
9725 break;
9726 }
9727 }
9728 return DAG.getNegative(Src, DL, VT);
9729 }
9730
9731 // Fold (sext_inreg (xor (setcc), C), i1) -> (add (setcc), -1) where C is
9732 // either 1 or -1. Since setcc only produces 0 or 1, the xor only flips the
9733 // low bit, so the sign extension of that bit is -(1 - setcc) == setcc - 1.
9734 // The setcc may be wrapped in a truncate/any_extend when the boolean is
9735 // extended to VT (e.g. by a zext, which knows the upper bits are zero).
9736 if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
9737 (isOneConstant(Src.getOperand(1)) ||
9738 isAllOnesConstant(Src.getOperand(1)))) {
9739 SDValue XorOp = Src.getOperand(0);
9740 if (XorOp.getOpcode() == ISD::TRUNCATE ||
9741 XorOp.getOpcode() == ISD::ANY_EXTEND)
9742 XorOp = XorOp.getOperand(0);
9743 if (XorOp.getOpcode() != ISD::SETCC)
9744 return SDValue();
9745 // The setcc may still have an i1 result type, in which case rebuild it
9746 // with VT so that both ADD operands have the same type. A setcc produces
9747 // a canonical 0/1 value, so this is exact while the any_extend leaves the
9748 // upper bits undefined.
9750 EVT OpVT = XorOp.getOperand(0).getValueType();
9751
9752 // Inverting an FP compare is free, so this fold does not save anything
9753 // and can duplicate a shared fcmp. Keep the original form.
9754 if (OpVT.isFloatingPoint())
9755 return SDValue();
9756
9757 // Reusing the original compare is only possible when the setcc is
9758 // otherwise dead. Inverting a shared compare duplicates it (e.g. an FP
9759 // compare used by a zext as well).
9760 if (!XorOp.hasOneUse())
9761 return SDValue();
9762
9763 // A constant compare with a greater-than style condition code cannot use
9764 // the slti/sltui immediate patterns; leave it to the generic combiner,
9765 // which canonicalizes it to setlt/setult first.
9766 if (isa<ConstantSDNode>(XorOp.getOperand(1)) && CC != ISD::SETLT &&
9767 CC != ISD::SETULT)
9768 return SDValue();
9769
9770 // The generic (sub X, (0 - Y)) -> (add X, Y) fold is better than
9771 // rewriting the sign-extended boolean here.
9772 if (N->hasOneUse() && N->use_begin()->getUser()->getOpcode() == ISD::SUB)
9773 return SDValue();
9774
9775 // For integer compares, sext(!setcc CC) == -(setcc inv(CC)). Only keep
9776 // CC (as add -1) when CC itself is the single-instruction form; otherwise
9777 // negate the inverted setcc, which also CSEs with an existing one.
9778 if (OpVT.isInteger() && CC != ISD::SETLT && CC != ISD::SETGT &&
9779 CC != ISD::SETULT && CC != ISD::SETUGT)
9780 return DAG.getNegative(DAG.getSetCC(DL, VT, XorOp.getOperand(0),
9781 XorOp.getOperand(1),
9782 ISD::getSetCCInverse(CC, OpVT)),
9783 DL, VT);
9784 if (XorOp.getValueType() != VT)
9785 XorOp =
9786 DAG.getSetCC(DL, VT, XorOp.getOperand(0), XorOp.getOperand(1), CC);
9787 return DAG.getNode(ISD::ADD, DL, VT, XorOp, DAG.getAllOnesConstant(DL, VT));
9788 }
9789
9790 return SDValue();
9791}
9792
9794 DAGCombinerInfo &DCI) const {
9795 SelectionDAG &DAG = DCI.DAG;
9796 switch (N->getOpcode()) {
9797 default:
9798 break;
9799 case ISD::ADD:
9800 return performADDCombine(N, DAG, DCI, Subtarget);
9801 case ISD::AND:
9802 return performANDCombine(N, DAG, DCI, Subtarget);
9803 case ISD::OR:
9804 return performORCombine(N, DAG, DCI, Subtarget);
9805 case ISD::XOR:
9806 return performXORCombine(N, DAG, Subtarget);
9807 case ISD::SETCC:
9808 return performSETCCCombine(N, DAG, DCI, Subtarget);
9809 case ISD::SELECT:
9810 return performSELECTCombine(N, DAG, DCI, Subtarget);
9811 case ISD::SHL:
9812 return performSHLCombine(N, DAG, DCI, Subtarget);
9813 case ISD::SRL:
9814 return performSRLCombine(N, DAG, DCI, Subtarget);
9815 case ISD::SUB:
9816 return performSUBCombine(N, DAG, DCI, Subtarget);
9817 case ISD::BITCAST:
9818 return performBITCASTCombine(N, DAG, DCI, Subtarget);
9819 case ISD::ANY_EXTEND:
9820 case ISD::ZERO_EXTEND:
9821 case ISD::SIGN_EXTEND:
9822 return performEXTENDCombine(N, DAG, DCI, Subtarget);
9824 return performSIGN_EXTEND_INREGCombine(N, DCI, Subtarget);
9825 case ISD::SINT_TO_FP:
9826 return performSINT_TO_FPCombine(N, DAG, DCI, Subtarget);
9827 case ISD::UINT_TO_FP:
9828 return performUINT_TO_FPCombine(N, DAG, DCI, Subtarget);
9829 case ISD::FP_TO_SINT:
9830 case ISD::FP_TO_UINT:
9831 return performFP_TO_INTCombine(N, DAG, DCI, Subtarget);
9832 case LoongArchISD::BITREV_W:
9833 return performBITREV_WCombine(N, DAG, DCI, Subtarget);
9834 case LoongArchISD::BR_CC:
9835 return performBR_CCCombine(N, DAG, DCI, Subtarget);
9836 case LoongArchISD::SELECT_CC:
9837 return performSELECT_CCCombine(N, DAG, DCI, Subtarget);
9839 return performINTRINSIC_WO_CHAINCombine(N, DAG, DCI, Subtarget);
9840 case LoongArchISD::MOVGR2FR_W_LA64:
9841 return performMOVGR2FR_WCombine(N, DAG, DCI, Subtarget);
9842 case LoongArchISD::MOVFR2GR_S_LA64:
9843 return performMOVFR2GR_SCombine(N, DAG, DCI, Subtarget);
9844 case LoongArchISD::CRC_W_B_W:
9845 case LoongArchISD::CRC_W_H_W:
9846 case LoongArchISD::CRCC_W_B_W:
9847 case LoongArchISD::CRCC_W_H_W:
9848 case LoongArchISD::VMSKLTZ:
9849 case LoongArchISD::XVMSKLTZ:
9850 return performDemandedBitsCombine(N, DAG, DCI);
9851 case LoongArchISD::SPLIT_PAIR_F64:
9852 return performSPLIT_PAIR_F64Combine(N, DAG, DCI, Subtarget);
9853 case LoongArchISD::VANDN:
9854 return performVANDNCombine(N, DAG, DCI, Subtarget);
9856 return performCONCAT_VECTORSCombine(N, DAG, DCI, Subtarget);
9857 case ISD::VSELECT:
9858 return performVSELECTCombine(N, DAG, DCI, Subtarget);
9859 case LoongArchISD::VPACKEV:
9860 case LoongArchISD::VPERMI:
9861 if (SDValue Result =
9862 combineFP_ROUND(SDValue(N, 0), SDLoc(N), DAG, Subtarget))
9863 return Result;
9864 break;
9865 case ISD::MUL:
9866 return performMULCombine(N, DAG, DCI, Subtarget);
9867 }
9868 return SDValue();
9869}
9870
9873 if (!ZeroDivCheck)
9874 return MBB;
9875
9876 // Build instructions:
9877 // MBB:
9878 // div(or mod) $dst, $dividend, $divisor
9879 // bne $divisor, $zero, SinkMBB
9880 // BreakMBB:
9881 // break 7 // BRK_DIVZERO
9882 // SinkMBB:
9883 // fallthrough
9884 const BasicBlock *LLVM_BB = MBB->getBasicBlock();
9885 MachineFunction::iterator It = ++MBB->getIterator();
9886 MachineFunction *MF = MBB->getParent();
9887 auto BreakMBB = MF->CreateMachineBasicBlock(LLVM_BB);
9888 auto SinkMBB = MF->CreateMachineBasicBlock(LLVM_BB);
9889 MF->insert(It, BreakMBB);
9890 MF->insert(It, SinkMBB);
9891
9892 // Transfer the remainder of MBB and its successor edges to SinkMBB.
9893 SinkMBB->splice(SinkMBB->end(), MBB, std::next(MI.getIterator()), MBB->end());
9894 SinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
9895
9896 const TargetInstrInfo &TII = *MF->getSubtarget().getInstrInfo();
9897 DebugLoc DL = MI.getDebugLoc();
9898 MachineOperand &Divisor = MI.getOperand(2);
9899 Register DivisorReg = Divisor.getReg();
9900
9901 // MBB:
9902 BuildMI(MBB, DL, TII.get(LoongArch::BNE))
9903 .addReg(DivisorReg, getKillRegState(Divisor.isKill()))
9904 .addReg(LoongArch::R0)
9905 .addMBB(SinkMBB);
9906 MBB->addSuccessor(BreakMBB);
9907 MBB->addSuccessor(SinkMBB);
9908
9909 // BreakMBB:
9910 // See linux header file arch/loongarch/include/uapi/asm/break.h for the
9911 // definition of BRK_DIVZERO.
9912 BuildMI(BreakMBB, DL, TII.get(LoongArch::BREAK)).addImm(7 /*BRK_DIVZERO*/);
9913 BreakMBB->addSuccessor(SinkMBB);
9914
9915 // Clear Divisor's kill flag.
9916 Divisor.setIsKill(false);
9917
9918 return SinkMBB;
9919}
9920
9921static MachineBasicBlock *
9924 unsigned InsOp;
9925 unsigned BroadcastOp;
9926 unsigned HalfSize;
9927 switch (MI.getOpcode()) {
9928 default:
9929 llvm_unreachable("Unexpected opcode");
9930 case LoongArch::PseudoXVINSGR2VR_B:
9931 HalfSize = 16;
9932 BroadcastOp = LoongArch::XVREPLGR2VR_B;
9933 InsOp = LoongArch::XVEXTRINS_B;
9934 break;
9935 case LoongArch::PseudoXVINSGR2VR_H:
9936 HalfSize = 8;
9937 BroadcastOp = LoongArch::XVREPLGR2VR_H;
9938 InsOp = LoongArch::XVEXTRINS_H;
9939 break;
9940 }
9941 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9942 const TargetRegisterClass *RC = &LoongArch::LASX256RegClass;
9943 const TargetRegisterClass *SubRC = &LoongArch::LSX128RegClass;
9944 DebugLoc DL = MI.getDebugLoc();
9946 // XDst = vector_insert XSrc, Elt, Idx
9947 Register XDst = MI.getOperand(0).getReg();
9948 Register XSrc = MI.getOperand(1).getReg();
9949 Register Elt = MI.getOperand(2).getReg();
9950 unsigned Idx = MI.getOperand(3).getImm();
9951
9952 if (XSrc.isVirtual() && MRI.getVRegDef(XSrc)->isImplicitDef() &&
9953 Idx < HalfSize) {
9954 Register ScratchSubReg1 = MRI.createVirtualRegister(SubRC);
9955 Register ScratchSubReg2 = MRI.createVirtualRegister(SubRC);
9956
9957 BuildMI(*BB, MI, DL, TII->get(LoongArch::COPY), ScratchSubReg1)
9958 .addReg(XSrc, {}, LoongArch::sub_128);
9959 BuildMI(*BB, MI, DL,
9960 TII->get(HalfSize == 8 ? LoongArch::VINSGR2VR_H
9961 : LoongArch::VINSGR2VR_B),
9962 ScratchSubReg2)
9963 .addReg(ScratchSubReg1)
9964 .addReg(Elt)
9965 .addImm(Idx);
9966
9967 BuildMI(*BB, MI, DL, TII->get(LoongArch::SUBREG_TO_REG), XDst)
9968 .addReg(ScratchSubReg2)
9969 .addImm(LoongArch::sub_128);
9970 } else {
9971 Register ScratchReg1 = MRI.createVirtualRegister(RC);
9972 Register ScratchReg2 = MRI.createVirtualRegister(RC);
9973
9974 BuildMI(*BB, MI, DL, TII->get(BroadcastOp), ScratchReg1).addReg(Elt);
9975
9976 BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPERMI_Q), ScratchReg2)
9977 .addReg(ScratchReg1)
9978 .addReg(XSrc)
9979 .addImm(Idx >= HalfSize ? 48 : 18);
9980
9981 BuildMI(*BB, MI, DL, TII->get(InsOp), XDst)
9982 .addReg(XSrc)
9983 .addReg(ScratchReg2)
9984 .addImm((Idx >= HalfSize ? Idx - HalfSize : Idx) * 17);
9985 }
9986
9987 MI.eraseFromParent();
9988 return BB;
9989}
9990
9994 assert(Subtarget.hasExtLSX());
9995 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
9996 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
9997 DebugLoc DL = MI.getDebugLoc();
9999 Register Dst = MI.getOperand(0).getReg();
10000 Register Src = MI.getOperand(1).getReg();
10001
10002 unsigned BroadcastOp, CTOp, PickOp;
10003 switch (MI.getOpcode()) {
10004 default:
10005 llvm_unreachable("Unexpected opcode");
10006 case LoongArch::PseudoCTPOP_B:
10007 BroadcastOp = LoongArch::VREPLGR2VR_B;
10008 CTOp = LoongArch::VPCNT_B;
10009 PickOp = LoongArch::VPICKVE2GR_B;
10010 break;
10011 case LoongArch::PseudoCTPOP_H:
10012 case LoongArch::PseudoCTPOP_H_LA32:
10013 BroadcastOp = LoongArch::VREPLGR2VR_H;
10014 CTOp = LoongArch::VPCNT_H;
10015 PickOp = LoongArch::VPICKVE2GR_H;
10016 break;
10017 case LoongArch::PseudoCTPOP_W:
10018 case LoongArch::PseudoCTPOP_W_LA32:
10019 BroadcastOp = LoongArch::VREPLGR2VR_W;
10020 CTOp = LoongArch::VPCNT_W;
10021 PickOp = LoongArch::VPICKVE2GR_W;
10022 break;
10023 case LoongArch::PseudoCTPOP_D:
10024 BroadcastOp = LoongArch::VREPLGR2VR_D;
10025 CTOp = LoongArch::VPCNT_D;
10026 PickOp = LoongArch::VPICKVE2GR_D;
10027 break;
10028 }
10029
10030 Register ScratchReg1 = MRI.createVirtualRegister(RC);
10031 Register ScratchReg2 = MRI.createVirtualRegister(RC);
10032 BuildMI(*BB, MI, DL, TII->get(BroadcastOp), ScratchReg1).addReg(Src);
10033 BuildMI(*BB, MI, DL, TII->get(CTOp), ScratchReg2).addReg(ScratchReg1);
10034 BuildMI(*BB, MI, DL, TII->get(PickOp), Dst).addReg(ScratchReg2).addImm(0);
10035
10036 MI.eraseFromParent();
10037 return BB;
10038}
10039
10040static MachineBasicBlock *
10043 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
10044 const TargetRegisterClass *RC = &LoongArch::LSX128RegClass;
10045 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
10047 Register Dst = MI.getOperand(0).getReg();
10048 Register Src = MI.getOperand(1).getReg();
10049 DebugLoc DL = MI.getDebugLoc();
10050 unsigned EleBits = 8;
10051 unsigned NotOpc = 0;
10052 unsigned MskOpc;
10053
10054 switch (MI.getOpcode()) {
10055 default:
10056 llvm_unreachable("Unexpected opcode");
10057 case LoongArch::PseudoVMSKLTZ_B:
10058 MskOpc = LoongArch::VMSKLTZ_B;
10059 break;
10060 case LoongArch::PseudoVMSKLTZ_H:
10061 MskOpc = LoongArch::VMSKLTZ_H;
10062 EleBits = 16;
10063 break;
10064 case LoongArch::PseudoVMSKLTZ_W:
10065 MskOpc = LoongArch::VMSKLTZ_W;
10066 EleBits = 32;
10067 break;
10068 case LoongArch::PseudoVMSKLTZ_D:
10069 MskOpc = LoongArch::VMSKLTZ_D;
10070 EleBits = 64;
10071 break;
10072 case LoongArch::PseudoVMSKGEZ_B:
10073 MskOpc = LoongArch::VMSKGEZ_B;
10074 break;
10075 case LoongArch::PseudoVMSKEQZ_B:
10076 MskOpc = LoongArch::VMSKNZ_B;
10077 NotOpc = LoongArch::VNOR_V;
10078 break;
10079 case LoongArch::PseudoVMSKNEZ_B:
10080 MskOpc = LoongArch::VMSKNZ_B;
10081 break;
10082 case LoongArch::PseudoXVMSKLTZ_B:
10083 MskOpc = LoongArch::XVMSKLTZ_B;
10084 RC = &LoongArch::LASX256RegClass;
10085 break;
10086 case LoongArch::PseudoXVMSKLTZ_H:
10087 MskOpc = LoongArch::XVMSKLTZ_H;
10088 RC = &LoongArch::LASX256RegClass;
10089 EleBits = 16;
10090 break;
10091 case LoongArch::PseudoXVMSKLTZ_W:
10092 MskOpc = LoongArch::XVMSKLTZ_W;
10093 RC = &LoongArch::LASX256RegClass;
10094 EleBits = 32;
10095 break;
10096 case LoongArch::PseudoXVMSKLTZ_D:
10097 MskOpc = LoongArch::XVMSKLTZ_D;
10098 RC = &LoongArch::LASX256RegClass;
10099 EleBits = 64;
10100 break;
10101 case LoongArch::PseudoXVMSKGEZ_B:
10102 MskOpc = LoongArch::XVMSKGEZ_B;
10103 RC = &LoongArch::LASX256RegClass;
10104 break;
10105 case LoongArch::PseudoXVMSKEQZ_B:
10106 MskOpc = LoongArch::XVMSKNZ_B;
10107 NotOpc = LoongArch::XVNOR_V;
10108 RC = &LoongArch::LASX256RegClass;
10109 break;
10110 case LoongArch::PseudoXVMSKNEZ_B:
10111 MskOpc = LoongArch::XVMSKNZ_B;
10112 RC = &LoongArch::LASX256RegClass;
10113 break;
10114 }
10115
10116 Register Msk = MRI.createVirtualRegister(RC);
10117 if (NotOpc) {
10118 Register Tmp = MRI.createVirtualRegister(RC);
10119 BuildMI(*BB, MI, DL, TII->get(MskOpc), Tmp).addReg(Src);
10120 BuildMI(*BB, MI, DL, TII->get(NotOpc), Msk).addReg(Tmp).addReg(Tmp);
10121 } else {
10122 BuildMI(*BB, MI, DL, TII->get(MskOpc), Msk).addReg(Src);
10123 }
10124
10125 if (TRI->getRegSizeInBits(*RC) > 128) {
10126 Register Lo = MRI.createVirtualRegister(&LoongArch::GPRRegClass);
10127 Register Hi = MRI.createVirtualRegister(&LoongArch::GPRRegClass);
10128 BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPICKVE2GR_WU), Lo)
10129 .addReg(Msk)
10130 .addImm(0);
10131 BuildMI(*BB, MI, DL, TII->get(LoongArch::XVPICKVE2GR_WU), Hi)
10132 .addReg(Msk)
10133 .addImm(4);
10134 BuildMI(*BB, MI, DL,
10135 TII->get(Subtarget.is64Bit() ? LoongArch::BSTRINS_D
10136 : LoongArch::BSTRINS_W),
10137 Dst)
10138 .addReg(Lo)
10139 .addReg(Hi)
10140 .addImm(256 / EleBits - 1)
10141 .addImm(128 / EleBits);
10142 } else {
10143 BuildMI(*BB, MI, DL, TII->get(LoongArch::VPICKVE2GR_HU), Dst)
10144 .addReg(Msk)
10145 .addImm(0);
10146 }
10147
10148 MI.eraseFromParent();
10149 return BB;
10150}
10151
10152static MachineBasicBlock *
10155 assert(MI.getOpcode() == LoongArch::SplitPairF64Pseudo &&
10156 "Unexpected instruction");
10157
10158 MachineFunction &MF = *BB->getParent();
10159 DebugLoc DL = MI.getDebugLoc();
10161 Register LoReg = MI.getOperand(0).getReg();
10162 Register HiReg = MI.getOperand(1).getReg();
10163 Register SrcReg = MI.getOperand(2).getReg();
10164
10165 BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVFR2GR_S_64), LoReg).addReg(SrcReg);
10166 BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVFRH2GR_S), HiReg)
10167 .addReg(SrcReg, getKillRegState(MI.getOperand(2).isKill()));
10168 MI.eraseFromParent(); // The pseudo instruction is gone now.
10169 return BB;
10170}
10171
10172static MachineBasicBlock *
10175 assert(MI.getOpcode() == LoongArch::BuildPairF64Pseudo &&
10176 "Unexpected instruction");
10177
10178 MachineFunction &MF = *BB->getParent();
10179 DebugLoc DL = MI.getDebugLoc();
10182 Register TmpReg = MRI.createVirtualRegister(&LoongArch::FPR64RegClass);
10183 Register DstReg = MI.getOperand(0).getReg();
10184 Register LoReg = MI.getOperand(1).getReg();
10185 Register HiReg = MI.getOperand(2).getReg();
10186
10187 BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVGR2FR_W_64), TmpReg)
10188 .addReg(LoReg, getKillRegState(MI.getOperand(1).isKill()));
10189 BuildMI(*BB, MI, DL, TII.get(LoongArch::MOVGR2FRH_W), DstReg)
10190 .addReg(TmpReg)
10191 .addReg(HiReg, getKillRegState(MI.getOperand(2).isKill()));
10192 MI.eraseFromParent(); // The pseudo instruction is gone now.
10193 return BB;
10194}
10195
10197 switch (MI.getOpcode()) {
10198 default:
10199 return false;
10200 case LoongArch::Select_GPR_Using_CC_GPR:
10201 return true;
10202 }
10203}
10204
10205static MachineBasicBlock *
10208 // To "insert" Select_* instructions, we actually have to insert the triangle
10209 // control-flow pattern. The incoming instructions know the destination vreg
10210 // to set, the condition code register to branch on, the true/false values to
10211 // select between, and the condcode to use to select the appropriate branch.
10212 //
10213 // We produce the following control flow:
10214 // HeadMBB
10215 // | \
10216 // | IfFalseMBB
10217 // | /
10218 // TailMBB
10219 //
10220 // When we find a sequence of selects we attempt to optimize their emission
10221 // by sharing the control flow. Currently we only handle cases where we have
10222 // multiple selects with the exact same condition (same LHS, RHS and CC).
10223 // The selects may be interleaved with other instructions if the other
10224 // instructions meet some requirements we deem safe:
10225 // - They are not pseudo instructions.
10226 // - They are debug instructions. Otherwise,
10227 // - They do not have side-effects, do not access memory and their inputs do
10228 // not depend on the results of the select pseudo-instructions.
10229 // The TrueV/FalseV operands of the selects cannot depend on the result of
10230 // previous selects in the sequence.
10231 // These conditions could be further relaxed. See the X86 target for a
10232 // related approach and more information.
10233
10234 Register LHS = MI.getOperand(1).getReg();
10235 Register RHS;
10236 if (MI.getOperand(2).isReg())
10237 RHS = MI.getOperand(2).getReg();
10238 auto CC = static_cast<unsigned>(MI.getOperand(3).getImm());
10239
10240 SmallVector<MachineInstr *, 4> SelectDebugValues;
10241 SmallSet<Register, 4> SelectDests;
10242 SelectDests.insert(MI.getOperand(0).getReg());
10243
10244 MachineInstr *LastSelectPseudo = &MI;
10245 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
10246 SequenceMBBI != E; ++SequenceMBBI) {
10247 if (SequenceMBBI->isDebugInstr())
10248 continue;
10249 if (isSelectPseudo(*SequenceMBBI)) {
10250 if (SequenceMBBI->getOperand(1).getReg() != LHS ||
10251 !SequenceMBBI->getOperand(2).isReg() ||
10252 SequenceMBBI->getOperand(2).getReg() != RHS ||
10253 SequenceMBBI->getOperand(3).getImm() != CC ||
10254 SelectDests.count(SequenceMBBI->getOperand(4).getReg()) ||
10255 SelectDests.count(SequenceMBBI->getOperand(5).getReg()))
10256 break;
10257 LastSelectPseudo = &*SequenceMBBI;
10258 SequenceMBBI->collectDebugValues(SelectDebugValues);
10259 SelectDests.insert(SequenceMBBI->getOperand(0).getReg());
10260 continue;
10261 }
10262 if (SequenceMBBI->hasUnmodeledSideEffects() ||
10263 SequenceMBBI->mayLoadOrStore() ||
10264 SequenceMBBI->usesCustomInsertionHook())
10265 break;
10266 if (llvm::any_of(SequenceMBBI->operands(), [&](MachineOperand &MO) {
10267 return MO.isReg() && MO.isUse() && SelectDests.count(MO.getReg());
10268 }))
10269 break;
10270 }
10271
10272 const LoongArchInstrInfo &TII = *Subtarget.getInstrInfo();
10273 const BasicBlock *LLVM_BB = BB->getBasicBlock();
10274 DebugLoc DL = MI.getDebugLoc();
10276
10277 MachineBasicBlock *HeadMBB = BB;
10278 MachineFunction *F = BB->getParent();
10279 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(LLVM_BB);
10280 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(LLVM_BB);
10281
10282 F->insert(I, IfFalseMBB);
10283 F->insert(I, TailMBB);
10284
10285 // Set the call frame size on entry to the new basic blocks.
10286 unsigned CallFrameSize = TII.getCallFrameSizeAt(*LastSelectPseudo);
10287 IfFalseMBB->setCallFrameSize(CallFrameSize);
10288 TailMBB->setCallFrameSize(CallFrameSize);
10289
10290 // Transfer debug instructions associated with the selects to TailMBB.
10291 for (MachineInstr *DebugInstr : SelectDebugValues) {
10292 TailMBB->push_back(DebugInstr->removeFromParent());
10293 }
10294
10295 // Move all instructions after the sequence to TailMBB.
10296 TailMBB->splice(TailMBB->end(), HeadMBB,
10297 std::next(LastSelectPseudo->getIterator()), HeadMBB->end());
10298 // Update machine-CFG edges by transferring all successors of the current
10299 // block to the new block which will contain the Phi nodes for the selects.
10300 TailMBB->transferSuccessorsAndUpdatePHIs(HeadMBB);
10301 // Set the successors for HeadMBB.
10302 HeadMBB->addSuccessor(IfFalseMBB);
10303 HeadMBB->addSuccessor(TailMBB);
10304
10305 // Insert appropriate branch.
10306 if (MI.getOperand(2).isImm())
10307 BuildMI(HeadMBB, DL, TII.get(CC))
10308 .addReg(LHS)
10309 .addImm(MI.getOperand(2).getImm())
10310 .addMBB(TailMBB);
10311 else
10312 BuildMI(HeadMBB, DL, TII.get(CC)).addReg(LHS).addReg(RHS).addMBB(TailMBB);
10313
10314 // IfFalseMBB just falls through to TailMBB.
10315 IfFalseMBB->addSuccessor(TailMBB);
10316
10317 // Create PHIs for all of the select pseudo-instructions.
10318 auto SelectMBBI = MI.getIterator();
10319 auto SelectEnd = std::next(LastSelectPseudo->getIterator());
10320 auto InsertionPoint = TailMBB->begin();
10321 while (SelectMBBI != SelectEnd) {
10322 auto Next = std::next(SelectMBBI);
10323 if (isSelectPseudo(*SelectMBBI)) {
10324 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
10325 BuildMI(*TailMBB, InsertionPoint, SelectMBBI->getDebugLoc(),
10326 TII.get(LoongArch::PHI), SelectMBBI->getOperand(0).getReg())
10327 .addReg(SelectMBBI->getOperand(4).getReg())
10328 .addMBB(HeadMBB)
10329 .addReg(SelectMBBI->getOperand(5).getReg())
10330 .addMBB(IfFalseMBB);
10331 SelectMBBI->eraseFromParent();
10332 }
10333 SelectMBBI = Next;
10334 }
10335
10336 F->getProperties().resetNoPHIs();
10337 return TailMBB;
10338}
10339
10340MachineBasicBlock *LoongArchTargetLowering::EmitInstrWithCustomInserter(
10341 MachineInstr &MI, MachineBasicBlock *BB) const {
10342 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
10343 DebugLoc DL = MI.getDebugLoc();
10344
10345 switch (MI.getOpcode()) {
10346 default:
10347 llvm_unreachable("Unexpected instr type to insert");
10348 case LoongArch::DIV_W:
10349 case LoongArch::DIV_WU:
10350 case LoongArch::MOD_W:
10351 case LoongArch::MOD_WU:
10352 case LoongArch::DIV_D:
10353 case LoongArch::DIV_DU:
10354 case LoongArch::MOD_D:
10355 case LoongArch::MOD_DU:
10356 return insertDivByZeroTrap(MI, BB);
10357 break;
10358 case LoongArch::WRFCSR: {
10359 BuildMI(*BB, MI, DL, TII->get(LoongArch::MOVGR2FCSR),
10360 LoongArch::FCSR0 + MI.getOperand(0).getImm())
10361 .addReg(MI.getOperand(1).getReg());
10362 MI.eraseFromParent();
10363 return BB;
10364 }
10365 case LoongArch::RDFCSR: {
10366 MachineInstr *ReadFCSR =
10367 BuildMI(*BB, MI, DL, TII->get(LoongArch::MOVFCSR2GR),
10368 MI.getOperand(0).getReg())
10369 .addReg(LoongArch::FCSR0 + MI.getOperand(1).getImm());
10370 ReadFCSR->getOperand(1).setIsUndef();
10371 MI.eraseFromParent();
10372 return BB;
10373 }
10374 case LoongArch::Select_GPR_Using_CC_GPR:
10375 return emitSelectPseudo(MI, BB, Subtarget);
10376 case LoongArch::BuildPairF64Pseudo:
10377 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
10378 case LoongArch::SplitPairF64Pseudo:
10379 return emitSplitPairF64Pseudo(MI, BB, Subtarget);
10380 case LoongArch::PseudoXVINSGR2VR_B:
10381 case LoongArch::PseudoXVINSGR2VR_H:
10382 return emitPseudoXVINSGR2VR(MI, BB, Subtarget);
10383 case LoongArch::PseudoCTPOP_B:
10384 case LoongArch::PseudoCTPOP_H:
10385 case LoongArch::PseudoCTPOP_W:
10386 case LoongArch::PseudoCTPOP_D:
10387 case LoongArch::PseudoCTPOP_H_LA32:
10388 case LoongArch::PseudoCTPOP_W_LA32:
10389 return emitPseudoCTPOP(MI, BB, Subtarget);
10390 case LoongArch::PseudoVMSKLTZ_B:
10391 case LoongArch::PseudoVMSKLTZ_H:
10392 case LoongArch::PseudoVMSKLTZ_W:
10393 case LoongArch::PseudoVMSKLTZ_D:
10394 case LoongArch::PseudoVMSKGEZ_B:
10395 case LoongArch::PseudoVMSKEQZ_B:
10396 case LoongArch::PseudoVMSKNEZ_B:
10397 case LoongArch::PseudoXVMSKLTZ_B:
10398 case LoongArch::PseudoXVMSKLTZ_H:
10399 case LoongArch::PseudoXVMSKLTZ_W:
10400 case LoongArch::PseudoXVMSKLTZ_D:
10401 case LoongArch::PseudoXVMSKGEZ_B:
10402 case LoongArch::PseudoXVMSKEQZ_B:
10403 case LoongArch::PseudoXVMSKNEZ_B:
10404 return emitPseudoVMSKCOND(MI, BB, Subtarget);
10405 case TargetOpcode::STATEPOINT:
10406 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
10407 // while bl call instruction (where statepoint will be lowered at the
10408 // end) has implicit def. This def is early-clobber as it will be set at
10409 // the moment of the call and earlier than any use is read.
10410 // Add this implicit dead def here as a workaround.
10411 MI.addOperand(*MI.getMF(),
10413 LoongArch::R1, /*isDef*/ true,
10414 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
10415 /*isUndef*/ false, /*isEarlyClobber*/ true));
10416 if (!Subtarget.is64Bit())
10417 report_fatal_error("STATEPOINT is only supported on 64-bit targets");
10418 return emitPatchPoint(MI, BB);
10419 case LoongArch::PROBED_STACKALLOC_DYN:
10420 return emitDynamicProbedAlloc(MI, BB);
10421 }
10422}
10423
10425 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
10426 unsigned *Fast) const {
10427 if (!Subtarget.hasUAL())
10428 return false;
10429
10430 // TODO: set reasonable speed number.
10431 if (Fast)
10432 *Fast = 1;
10433 return true;
10434}
10435
10436//===----------------------------------------------------------------------===//
10437// Calling Convention Implementation
10438//===----------------------------------------------------------------------===//
10439
10440// Eight general-purpose registers a0-a7 used for passing integer arguments,
10441// with a0-a1 reused to return values. Generally, the GPRs are used to pass
10442// fixed-point arguments, and floating-point arguments when no FPR is available
10443// or with soft float ABI.
10444const MCPhysReg ArgGPRs[] = {LoongArch::R4, LoongArch::R5, LoongArch::R6,
10445 LoongArch::R7, LoongArch::R8, LoongArch::R9,
10446 LoongArch::R10, LoongArch::R11};
10447
10448// PreserveNone calling convention:
10449// Arguments may be passed in any general-purpose registers except:
10450// - R1 : return address register
10451// - R22 : frame pointer
10452// - R31 : base pointer
10453//
10454// All general-purpose registers are treated as caller-saved,
10455// except R1 (RA) and R22 (FP).
10456//
10457// Non-volatile registers are allocated first so that a function
10458// can call normal functions without having to spill and reload
10459// argument registers.
10461 LoongArch::R23, LoongArch::R24, LoongArch::R25, LoongArch::R26,
10462 LoongArch::R27, LoongArch::R28, LoongArch::R29, LoongArch::R30,
10463 LoongArch::R4, LoongArch::R5, LoongArch::R6, LoongArch::R7,
10464 LoongArch::R8, LoongArch::R9, LoongArch::R10, LoongArch::R11,
10465 LoongArch::R12, LoongArch::R13, LoongArch::R14, LoongArch::R15,
10466 LoongArch::R16, LoongArch::R17, LoongArch::R18, LoongArch::R19,
10467 LoongArch::R20};
10468
10469// Eight floating-point registers fa0-fa7 used for passing floating-point
10470// arguments, and fa0-fa1 are also used to return values.
10471const MCPhysReg ArgFPR32s[] = {LoongArch::F0, LoongArch::F1, LoongArch::F2,
10472 LoongArch::F3, LoongArch::F4, LoongArch::F5,
10473 LoongArch::F6, LoongArch::F7};
10474// FPR32 and FPR64 alias each other.
10476 LoongArch::F0_64, LoongArch::F1_64, LoongArch::F2_64, LoongArch::F3_64,
10477 LoongArch::F4_64, LoongArch::F5_64, LoongArch::F6_64, LoongArch::F7_64};
10478
10479const MCPhysReg ArgVRs[] = {LoongArch::VR0, LoongArch::VR1, LoongArch::VR2,
10480 LoongArch::VR3, LoongArch::VR4, LoongArch::VR5,
10481 LoongArch::VR6, LoongArch::VR7};
10482
10483const MCPhysReg ArgXRs[] = {LoongArch::XR0, LoongArch::XR1, LoongArch::XR2,
10484 LoongArch::XR3, LoongArch::XR4, LoongArch::XR5,
10485 LoongArch::XR6, LoongArch::XR7};
10486
10488 switch (State.getCallingConv()) {
10490 if (!State.isVarArg())
10491 return State.AllocateReg(PreserveNoneArgGPRs);
10492 [[fallthrough]];
10493 default:
10494 return State.AllocateReg(ArgGPRs);
10495 }
10496}
10497
10498// Pass a 2*GRLen argument that has been split into two GRLen values through
10499// registers or the stack as necessary.
10500static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State,
10501 CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1,
10502 unsigned ValNo2, MVT ValVT2, MVT LocVT2,
10503 ISD::ArgFlagsTy ArgFlags2) {
10504 unsigned GRLenInBytes = GRLen / 8;
10505 if (Register Reg = allocateArgGPR(State)) {
10506 // At least one half can be passed via register.
10507 State.addLoc(CCValAssign::getReg(VA1.getValNo(), VA1.getValVT(), Reg,
10508 VA1.getLocVT(), CCValAssign::Full));
10509 } else {
10510 // Both halves must be passed on the stack, with proper alignment.
10511 Align StackAlign =
10512 std::max(Align(GRLenInBytes), ArgFlags1.getNonZeroOrigAlign());
10513 State.addLoc(
10515 State.AllocateStack(GRLenInBytes, StackAlign),
10516 VA1.getLocVT(), CCValAssign::Full));
10517 State.addLoc(CCValAssign::getMem(
10518 ValNo2, ValVT2, State.AllocateStack(GRLenInBytes, Align(GRLenInBytes)),
10519 LocVT2, CCValAssign::Full));
10520 return false;
10521 }
10522 if (Register Reg = allocateArgGPR(State)) {
10523 // The second half can also be passed via register.
10524 State.addLoc(
10525 CCValAssign::getReg(ValNo2, ValVT2, Reg, LocVT2, CCValAssign::Full));
10526 } else {
10527 // The second half is passed via the stack, without additional alignment.
10528 State.addLoc(CCValAssign::getMem(
10529 ValNo2, ValVT2, State.AllocateStack(GRLenInBytes, Align(GRLenInBytes)),
10530 LocVT2, CCValAssign::Full));
10531 }
10532 return false;
10533}
10534
10535// Implements the LoongArch calling convention. Returns true upon failure.
10537 unsigned ValNo, MVT ValVT,
10538 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
10539 CCState &State, bool IsRet, Type *OrigTy) {
10540 unsigned GRLen = DL.getLargestLegalIntTypeSizeInBits();
10541 assert((GRLen == 32 || GRLen == 64) && "Unspport GRLen");
10542 MVT GRLenVT = GRLen == 32 ? MVT::i32 : MVT::i64;
10543 MVT LocVT = ValVT;
10544
10545 // Any return value split into more than two values can't be returned
10546 // directly.
10547 if (IsRet && ValNo > 1)
10548 return true;
10549
10550 // If passing a variadic argument, or if no FPR is available.
10551 bool UseGPRForFloat = true;
10552
10553 switch (ABI) {
10554 default:
10555 llvm_unreachable("Unexpected ABI");
10556 break;
10561 UseGPRForFloat = ArgFlags.isVarArg();
10562 break;
10565 break;
10566 }
10567
10568 // If this is a variadic argument, the LoongArch calling convention requires
10569 // that it is assigned an 'even' or 'aligned' register if it has (2*GRLen)/8
10570 // byte alignment. An aligned register should be used regardless of whether
10571 // the original argument was split during legalisation or not. The argument
10572 // will not be passed by registers if the original type is larger than
10573 // 2*GRLen, so the register alignment rule does not apply.
10574 unsigned TwoGRLenInBytes = (2 * GRLen) / 8;
10575 if (ArgFlags.isVarArg() &&
10576 ArgFlags.getNonZeroOrigAlign() == TwoGRLenInBytes &&
10577 DL.getTypeAllocSize(OrigTy) == TwoGRLenInBytes) {
10578 unsigned RegIdx = State.getFirstUnallocated(ArgGPRs);
10579 // Skip 'odd' register if necessary.
10580 if (RegIdx != std::size(ArgGPRs) && RegIdx % 2 == 1)
10581 State.AllocateReg(ArgGPRs);
10582 }
10583
10584 SmallVectorImpl<CCValAssign> &PendingLocs = State.getPendingLocs();
10585 SmallVectorImpl<ISD::ArgFlagsTy> &PendingArgFlags =
10586 State.getPendingArgFlags();
10587
10588 assert(PendingLocs.size() == PendingArgFlags.size() &&
10589 "PendingLocs and PendingArgFlags out of sync");
10590
10591 // FPR32 and FPR64 alias each other.
10592 if (State.getFirstUnallocated(ArgFPR32s) == std::size(ArgFPR32s))
10593 UseGPRForFloat = true;
10594
10595 if (UseGPRForFloat && ValVT == MVT::f32) {
10596 LocVT = GRLenVT;
10597 LocInfo = CCValAssign::BCvt;
10598 } else if (UseGPRForFloat && GRLen == 64 && ValVT == MVT::f64) {
10599 LocVT = MVT::i64;
10600 LocInfo = CCValAssign::BCvt;
10601 } else if (UseGPRForFloat && GRLen == 32 && ValVT == MVT::f64) {
10602 // Handle passing f64 on LA32D with a soft float ABI or when floating point
10603 // registers are exhausted.
10604 assert(PendingLocs.empty() && "Can't lower f64 if it is split");
10605 // Depending on available argument GPRS, f64 may be passed in a pair of
10606 // GPRs, split between a GPR and the stack, or passed completely on the
10607 // stack. LowerCall/LowerFormalArguments/LowerReturn must recognise these
10608 // cases.
10609 MCRegister Reg = allocateArgGPR(State);
10610 if (!Reg) {
10611 int64_t StackOffset = State.AllocateStack(8, Align(8));
10612 State.addLoc(
10613 CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
10614 return false;
10615 }
10616 LocVT = MVT::i32;
10617 State.addLoc(CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT, LocInfo));
10618 MCRegister HiReg = allocateArgGPR(State);
10619 if (HiReg) {
10620 State.addLoc(
10621 CCValAssign::getCustomReg(ValNo, ValVT, HiReg, LocVT, LocInfo));
10622 } else {
10623 int64_t StackOffset = State.AllocateStack(4, Align(4));
10624 State.addLoc(
10625 CCValAssign::getCustomMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
10626 }
10627 return false;
10628 }
10629
10630 // Split arguments might be passed indirectly, so keep track of the pending
10631 // values.
10632 if (ValVT.isScalarInteger() && (ArgFlags.isSplit() || !PendingLocs.empty())) {
10633 LocVT = GRLenVT;
10634 LocInfo = CCValAssign::Indirect;
10635 PendingLocs.push_back(
10636 CCValAssign::getPending(ValNo, ValVT, LocVT, LocInfo));
10637 PendingArgFlags.push_back(ArgFlags);
10638 if (!ArgFlags.isSplitEnd()) {
10639 return false;
10640 }
10641 }
10642
10643 // If the split argument only had two elements, it should be passed directly
10644 // in registers or on the stack.
10645 if (ValVT.isScalarInteger() && ArgFlags.isSplitEnd() &&
10646 PendingLocs.size() <= 2) {
10647 assert(PendingLocs.size() == 2 && "Unexpected PendingLocs.size()");
10648 // Apply the normal calling convention rules to the first half of the
10649 // split argument.
10650 CCValAssign VA = PendingLocs[0];
10651 ISD::ArgFlagsTy AF = PendingArgFlags[0];
10652 PendingLocs.clear();
10653 PendingArgFlags.clear();
10654 return CC_LoongArchAssign2GRLen(GRLen, State, VA, AF, ValNo, ValVT, LocVT,
10655 ArgFlags);
10656 }
10657
10658 // Allocate to a register if possible, or else a stack slot.
10659 Register Reg;
10660 unsigned StoreSizeBytes = GRLen / 8;
10661 Align StackAlign = Align(GRLen / 8);
10662
10663 if (ValVT == MVT::f32 && !UseGPRForFloat) {
10664 Reg = State.AllocateReg(ArgFPR32s);
10665 } else if (ValVT == MVT::f64 && !UseGPRForFloat) {
10666 Reg = State.AllocateReg(ArgFPR64s);
10667 } else if (ValVT.is128BitVector()) {
10668 Reg = State.AllocateReg(ArgVRs);
10669 UseGPRForFloat = false;
10670 StoreSizeBytes = 16;
10671 StackAlign = Align(16);
10672 } else if (ValVT.is256BitVector()) {
10673 Reg = State.AllocateReg(ArgXRs);
10674 UseGPRForFloat = false;
10675 StoreSizeBytes = 32;
10676 StackAlign = Align(32);
10677 } else {
10678 Reg = allocateArgGPR(State);
10679 }
10680
10681 unsigned StackOffset =
10682 Reg ? 0 : State.AllocateStack(StoreSizeBytes, StackAlign);
10683
10684 // If we reach this point and PendingLocs is non-empty, we must be at the
10685 // end of a split argument that must be passed indirectly.
10686 if (!PendingLocs.empty()) {
10687 assert(ArgFlags.isSplitEnd() && "Expected ArgFlags.isSplitEnd()");
10688 assert(PendingLocs.size() > 2 && "Unexpected PendingLocs.size()");
10689 for (auto &It : PendingLocs) {
10690 if (Reg)
10691 It.convertToReg(Reg);
10692 else
10693 It.convertToMem(StackOffset);
10694 State.addLoc(It);
10695 }
10696 PendingLocs.clear();
10697 PendingArgFlags.clear();
10698 return false;
10699 }
10700 assert((!UseGPRForFloat || LocVT == GRLenVT) &&
10701 "Expected an GRLenVT at this stage");
10702
10703 if (Reg) {
10704 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
10705 return false;
10706 }
10707
10708 // When a floating-point value is passed on the stack, no bit-cast is needed.
10709 if (ValVT.isFloatingPoint()) {
10710 LocVT = ValVT;
10711 LocInfo = CCValAssign::Full;
10712 }
10713
10714 State.addLoc(CCValAssign::getMem(ValNo, ValVT, StackOffset, LocVT, LocInfo));
10715 return false;
10716}
10717
10718void LoongArchTargetLowering::analyzeInputArgs(
10719 MachineFunction &MF, CCState &CCInfo,
10720 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
10721 LoongArchCCAssignFn Fn) const {
10722 FunctionType *FType = MF.getFunction().getFunctionType();
10723 for (unsigned i = 0, e = Ins.size(); i != e; ++i) {
10724 MVT ArgVT = Ins[i].VT;
10725 Type *ArgTy = nullptr;
10726 if (IsRet)
10727 ArgTy = FType->getReturnType();
10728 else if (Ins[i].isOrigArg())
10729 ArgTy = FType->getParamType(Ins[i].getOrigArgIndex());
10731 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
10732 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Ins[i].Flags,
10733 CCInfo, IsRet, ArgTy)) {
10734 LLVM_DEBUG(dbgs() << "InputArg #" << i << " has unhandled type " << ArgVT
10735 << '\n');
10736 llvm_unreachable("");
10737 }
10738 }
10739}
10740
10741void LoongArchTargetLowering::analyzeOutputArgs(
10742 MachineFunction &MF, CCState &CCInfo,
10743 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
10744 CallLoweringInfo *CLI, LoongArchCCAssignFn Fn) const {
10745 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
10746 MVT ArgVT = Outs[i].VT;
10747 Type *OrigTy = CLI ? CLI->getArgs()[Outs[i].OrigArgIndex].Ty : nullptr;
10749 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
10750 if (Fn(MF.getDataLayout(), ABI, i, ArgVT, CCValAssign::Full, Outs[i].Flags,
10751 CCInfo, IsRet, OrigTy)) {
10752 LLVM_DEBUG(dbgs() << "OutputArg #" << i << " has unhandled type " << ArgVT
10753 << "\n");
10754 llvm_unreachable("");
10755 }
10756 }
10757}
10758
10759// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
10760// values.
10762 const CCValAssign &VA, const SDLoc &DL) {
10763 switch (VA.getLocInfo()) {
10764 default:
10765 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10766 case CCValAssign::Full:
10768 break;
10769 case CCValAssign::BCvt:
10770 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
10771 Val = DAG.getNode(LoongArchISD::MOVGR2FR_W_LA64, DL, MVT::f32, Val);
10772 else
10773 Val = DAG.getNode(ISD::BITCAST, DL, VA.getValVT(), Val);
10774 break;
10775 }
10776 return Val;
10777}
10778
10780 const CCValAssign &VA, const SDLoc &DL,
10781 const ISD::InputArg &In,
10785 EVT LocVT = VA.getLocVT();
10786 SDValue Val;
10787 const TargetRegisterClass *RC = TLI.getRegClassFor(LocVT.getSimpleVT());
10788 Register VReg = RegInfo.createVirtualRegister(RC);
10789 RegInfo.addLiveIn(VA.getLocReg(), VReg);
10790 Val = DAG.getCopyFromReg(Chain, DL, VReg, LocVT);
10791
10792 // If input is sign extended from 32 bits, note it for the OptW pass.
10793 if (In.isOrigArg()) {
10794 Argument *OrigArg = MF.getFunction().getArg(In.getOrigArgIndex());
10795 if (OrigArg->getType()->isIntegerTy()) {
10796 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
10797 // An input zero extended from i31 can also be considered sign extended.
10798 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
10799 (BitWidth < 32 && In.Flags.isZExt())) {
10802 LAFI->addSExt32Register(VReg);
10803 }
10804 }
10805 }
10806
10807 return convertLocVTToValVT(DAG, Val, VA, DL);
10808}
10809
10810// The caller is responsible for loading the full value if the argument is
10811// passed with CCValAssign::Indirect.
10813 const CCValAssign &VA, const SDLoc &DL) {
10815 MachineFrameInfo &MFI = MF.getFrameInfo();
10816 EVT ValVT = VA.getValVT();
10817 int FI = MFI.CreateFixedObject(ValVT.getStoreSize(), VA.getLocMemOffset(),
10818 /*IsImmutable=*/true);
10819 SDValue FIN = DAG.getFrameIndex(
10821
10822 ISD::LoadExtType ExtType;
10823 switch (VA.getLocInfo()) {
10824 default:
10825 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10826 case CCValAssign::Full:
10828 case CCValAssign::BCvt:
10829 ExtType = ISD::NON_EXTLOAD;
10830 break;
10831 }
10832 return DAG.getExtLoad(
10833 ExtType, DL, VA.getLocVT(), Chain, FIN,
10835}
10836
10838 const CCValAssign &VA,
10839 const CCValAssign &HiVA,
10840 const SDLoc &DL) {
10841 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
10842 "Unexpected VA");
10844 MachineFrameInfo &MFI = MF.getFrameInfo();
10846
10847 assert(VA.isRegLoc() && "Expected register VA assignment");
10848
10849 Register LoVReg = RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
10850 RegInfo.addLiveIn(VA.getLocReg(), LoVReg);
10851 SDValue Lo = DAG.getCopyFromReg(Chain, DL, LoVReg, MVT::i32);
10852 SDValue Hi;
10853 if (HiVA.isMemLoc()) {
10854 // Second half of f64 is passed on the stack.
10855 int FI = MFI.CreateFixedObject(4, HiVA.getLocMemOffset(),
10856 /*IsImmutable=*/true);
10857 SDValue FIN = DAG.getFrameIndex(FI, MVT::i32);
10858 Hi = DAG.getLoad(MVT::i32, DL, Chain, FIN,
10860 } else {
10861 // Second half of f64 is passed in another GPR.
10862 Register HiVReg = RegInfo.createVirtualRegister(&LoongArch::GPRRegClass);
10863 RegInfo.addLiveIn(HiVA.getLocReg(), HiVReg);
10864 Hi = DAG.getCopyFromReg(Chain, DL, HiVReg, MVT::i32);
10865 }
10866 return DAG.getNode(LoongArchISD::BUILD_PAIR_F64, DL, MVT::f64, Lo, Hi);
10867}
10868
10870 const CCValAssign &VA, const SDLoc &DL) {
10871 EVT LocVT = VA.getLocVT();
10872
10873 switch (VA.getLocInfo()) {
10874 default:
10875 llvm_unreachable("Unexpected CCValAssign::LocInfo");
10876 case CCValAssign::Full:
10877 break;
10878 case CCValAssign::BCvt:
10879 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
10880 Val = DAG.getNode(LoongArchISD::MOVFR2GR_S_LA64, DL, MVT::i64, Val);
10881 else
10882 Val = DAG.getNode(ISD::BITCAST, DL, LocVT, Val);
10883 break;
10884 }
10885 return Val;
10886}
10887
10888static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT,
10889 CCValAssign::LocInfo LocInfo,
10890 ISD::ArgFlagsTy ArgFlags, Type *OrigTy,
10891 CCState &State) {
10892 if (LocVT == MVT::i32 || LocVT == MVT::i64) {
10893 // Pass in STG registers: Base, Sp, Hp, R1, R2, R3, R4, R5, SpLim
10894 // s0 s1 s2 s3 s4 s5 s6 s7 s8
10895 static const MCPhysReg GPRList[] = {
10896 LoongArch::R23, LoongArch::R24, LoongArch::R25,
10897 LoongArch::R26, LoongArch::R27, LoongArch::R28,
10898 LoongArch::R29, LoongArch::R30, LoongArch::R31};
10899 if (MCRegister Reg = State.AllocateReg(GPRList)) {
10900 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
10901 return false;
10902 }
10903 }
10904
10905 if (LocVT == MVT::f32) {
10906 // Pass in STG registers: F1, F2, F3, F4
10907 // fs0,fs1,fs2,fs3
10908 static const MCPhysReg FPR32List[] = {LoongArch::F24, LoongArch::F25,
10909 LoongArch::F26, LoongArch::F27};
10910 if (MCRegister Reg = State.AllocateReg(FPR32List)) {
10911 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
10912 return false;
10913 }
10914 }
10915
10916 if (LocVT == MVT::f64) {
10917 // Pass in STG registers: D1, D2, D3, D4
10918 // fs4,fs5,fs6,fs7
10919 static const MCPhysReg FPR64List[] = {LoongArch::F28_64, LoongArch::F29_64,
10920 LoongArch::F30_64, LoongArch::F31_64};
10921 if (MCRegister Reg = State.AllocateReg(FPR64List)) {
10922 State.addLoc(CCValAssign::getReg(ValNo, ValVT, Reg, LocVT, LocInfo));
10923 return false;
10924 }
10925 }
10926
10927 report_fatal_error("No registers left in GHC calling convention");
10928 return true;
10929}
10930
10931// Transform physical registers into virtual registers.
10933 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
10934 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
10935 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
10936
10938
10939 switch (CallConv) {
10940 default:
10941 llvm_unreachable("Unsupported calling convention");
10942 case CallingConv::C:
10943 case CallingConv::Fast:
10946 break;
10947 case CallingConv::GHC:
10948 if (!MF.getSubtarget().hasFeature(LoongArch::FeatureBasicF) ||
10949 !MF.getSubtarget().hasFeature(LoongArch::FeatureBasicD))
10951 "GHC calling convention requires the F and D extensions");
10952 }
10953
10954 const Function &Func = MF.getFunction();
10955 EVT PtrVT = getPointerTy(DAG.getDataLayout());
10956 MVT GRLenVT = Subtarget.getGRLenVT();
10957 unsigned GRLenInBytes = Subtarget.getGRLen() / 8;
10958
10959 // Check if this function has any musttail calls. If so, incoming indirect
10960 // arg pointers must be saved in virtual registers so they survive across
10961 // basic blocks (the SelectionDAG is cleared between BBs). Only do this
10962 // when needed to avoid adding register pressure to non-musttail functions.
10963 bool HasMusttail = llvm::any_of(Func, [](const BasicBlock &BB) {
10964 return llvm::any_of(BB, [](const Instruction &I) {
10965 if (const auto *CI = dyn_cast<CallInst>(&I))
10966 return CI->isMustTailCall();
10967 return false;
10968 });
10969 });
10970 // Used with varargs to acumulate store chains.
10971 std::vector<SDValue> OutChains;
10972
10973 // Assign locations to all of the incoming arguments.
10975 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
10976
10977 if (CallConv == CallingConv::GHC)
10979 else
10980 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false, CC_LoongArch);
10981
10982 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
10983 CCValAssign &VA = ArgLocs[i];
10984 SDValue ArgValue;
10985 // Passing f64 on LA32D with a soft float ABI must be handled as a special
10986 // case.
10987 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
10988 assert(VA.needsCustom());
10989 ArgValue = unpackF64OnLA32DSoftABI(DAG, Chain, VA, ArgLocs[++i], DL);
10990 } else if (VA.isRegLoc())
10991 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, Ins[InsIdx], *this);
10992 else
10993 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL);
10994 if (VA.getLocInfo() == CCValAssign::Indirect) {
10995 // If the original argument was split and passed by reference, we need to
10996 // load all parts of it here (using the same address).
10997 InVals.push_back(DAG.getLoad(VA.getValVT(), DL, Chain, ArgValue,
10999 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
11000 if (HasMusttail) {
11003 Register VReg =
11004 MF.getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass);
11005 Chain = DAG.getCopyToReg(Chain, DL, VReg, ArgValue);
11006 LAFI->setIncomingIndirectArg(ArgIndex, VReg);
11007 }
11008 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
11009 assert(ArgPartOffset == 0);
11010 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
11011 CCValAssign &PartVA = ArgLocs[i + 1];
11012 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
11013 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
11014 SDValue Address = DAG.getNode(ISD::ADD, DL, PtrVT, ArgValue, Offset);
11015 InVals.push_back(DAG.getLoad(PartVA.getValVT(), DL, Chain, Address,
11017 ++i;
11018 ++InsIdx;
11019 }
11020 continue;
11021 }
11022 InVals.push_back(ArgValue);
11023 }
11024
11025 if (IsVarArg) {
11027 unsigned Idx = CCInfo.getFirstUnallocated(ArgRegs);
11028 const TargetRegisterClass *RC = &LoongArch::GPRRegClass;
11029 MachineFrameInfo &MFI = MF.getFrameInfo();
11030 MachineRegisterInfo &RegInfo = MF.getRegInfo();
11031 auto *LoongArchFI = MF.getInfo<LoongArchMachineFunctionInfo>();
11032
11033 // Offset of the first variable argument from stack pointer, and size of
11034 // the vararg save area. For now, the varargs save area is either zero or
11035 // large enough to hold a0-a7.
11036 int VaArgOffset, VarArgsSaveSize;
11037
11038 // If all registers are allocated, then all varargs must be passed on the
11039 // stack and we don't need to save any argregs.
11040 if (ArgRegs.size() == Idx) {
11041 VaArgOffset = CCInfo.getStackSize();
11042 VarArgsSaveSize = 0;
11043 } else {
11044 VarArgsSaveSize = GRLenInBytes * (ArgRegs.size() - Idx);
11045 VaArgOffset = -VarArgsSaveSize;
11046 }
11047
11048 // Record the frame index of the first variable argument
11049 // which is a value necessary to VASTART.
11050 int FI = MFI.CreateFixedObject(GRLenInBytes, VaArgOffset, true);
11051 LoongArchFI->setVarArgsFrameIndex(FI);
11052
11053 // If saving an odd number of registers then create an extra stack slot to
11054 // ensure that the frame pointer is 2*GRLen-aligned, which in turn ensures
11055 // offsets to even-numbered registered remain 2*GRLen-aligned.
11056 if (Idx % 2) {
11057 MFI.CreateFixedObject(GRLenInBytes, VaArgOffset - (int)GRLenInBytes,
11058 true);
11059 VarArgsSaveSize += GRLenInBytes;
11060 }
11061
11062 // Copy the integer registers that may have been used for passing varargs
11063 // to the vararg save area.
11064 for (unsigned I = Idx; I < ArgRegs.size();
11065 ++I, VaArgOffset += GRLenInBytes) {
11066 const Register Reg = RegInfo.createVirtualRegister(RC);
11067 RegInfo.addLiveIn(ArgRegs[I], Reg);
11068 SDValue ArgValue = DAG.getCopyFromReg(Chain, DL, Reg, GRLenVT);
11069 FI = MFI.CreateFixedObject(GRLenInBytes, VaArgOffset, true);
11070 SDValue PtrOff = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
11071 SDValue Store = DAG.getStore(Chain, DL, ArgValue, PtrOff,
11073 cast<StoreSDNode>(Store.getNode())
11074 ->getMemOperand()
11075 ->setValue((Value *)nullptr);
11076 OutChains.push_back(Store);
11077 }
11078 LoongArchFI->setVarArgsSaveSize(VarArgsSaveSize);
11079 }
11080
11081 // All stores are grouped in one node to allow the matching between
11082 // the size of Ins and InVals. This only happens for vararg functions.
11083 if (!OutChains.empty()) {
11084 OutChains.push_back(Chain);
11085 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, OutChains);
11086 }
11087
11088 return Chain;
11089}
11090
11092 return CI->isTailCall();
11093}
11094
11095// Check if the return value is used as only a return value, as otherwise
11096// we can't perform a tail-call.
11098 SDValue &Chain) const {
11099 if (N->getNumValues() != 1)
11100 return false;
11101 if (!N->hasNUsesOfValue(1, 0))
11102 return false;
11103
11104 SDNode *Copy = *N->user_begin();
11105 if (Copy->getOpcode() != ISD::CopyToReg)
11106 return false;
11107
11108 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
11109 // isn't safe to perform a tail call.
11110 if (Copy->getGluedNode())
11111 return false;
11112
11113 // The copy must be used by a LoongArchISD::RET, and nothing else.
11114 bool HasRet = false;
11115 for (SDNode *Node : Copy->users()) {
11116 if (Node->getOpcode() != LoongArchISD::RET)
11117 return false;
11118 HasRet = true;
11119 }
11120
11121 if (!HasRet)
11122 return false;
11123
11124 Chain = Copy->getOperand(0);
11125 return true;
11126}
11127
11128// Check whether the call is eligible for tail call optimization.
11129bool LoongArchTargetLowering::isEligibleForTailCallOptimization(
11130 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
11131 const SmallVectorImpl<CCValAssign> &ArgLocs) const {
11132
11133 auto CalleeCC = CLI.CallConv;
11134 auto &Outs = CLI.Outs;
11135 auto &Caller = MF.getFunction();
11136 auto CallerCC = Caller.getCallingConv();
11137
11138 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
11139
11140 // Byval parameters hand the function a pointer directly into the stack area
11141 // we want to reuse during a tail call. Working around this *is* possible
11142 // but less efficient and uglier in LowerCall. For musttail, there is no
11143 // workaround today: a byval arg requires a local copy that becomes invalid
11144 // after the tail call deallocates the caller's frame, so rejecting here
11145 // (and triggering reportFatalInternalError in LowerCall) is safer than
11146 // miscompiling.
11147 for (auto &Arg : Outs)
11148 if (Arg.Flags.isByVal())
11149 return false;
11150
11151 // musttail bypasses the remaining checks: the checks either reject cases
11152 // we handle specially (indirect args are forwarded via incoming pointers,
11153 // stack-passed args reuse the matching incoming layout, sret is forwarded
11154 // like any other pointer arg) or are optimizations not applicable to
11155 // mandatory tail calls.
11156 if (IsMustTail)
11157 return true;
11158
11159 // Do not tail call opt if the stack is used to pass parameters.
11160 if (CCInfo.getStackSize() != 0)
11161 return false;
11162
11163 // Do not tail call opt if any parameters need to be passed indirectly.
11164 for (auto &VA : ArgLocs)
11165 if (VA.getLocInfo() == CCValAssign::Indirect)
11166 return false;
11167
11168 // Do not tail call opt if either caller or callee uses struct return
11169 // semantics.
11170 auto IsCallerStructRet = Caller.hasStructRetAttr();
11171 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
11172 if (IsCallerStructRet || IsCalleeStructRet)
11173 return false;
11174
11175 // The callee has to preserve all registers the caller needs to preserve.
11176 const LoongArchRegisterInfo *TRI = Subtarget.getRegisterInfo();
11177 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
11178 if (CalleeCC != CallerCC) {
11179 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
11180 if (!TRI->regmaskSubsetEqual(CallerPreserved, CalleePreserved))
11181 return false;
11182 }
11183 return true;
11184}
11185
11187 return DAG.getDataLayout().getPrefTypeAlign(
11188 VT.getTypeForEVT(*DAG.getContext()));
11189}
11190
11191// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
11192// and output parameter nodes.
11193SDValue
11195 SmallVectorImpl<SDValue> &InVals) const {
11196 SelectionDAG &DAG = CLI.DAG;
11197 SDLoc &DL = CLI.DL;
11199 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
11201 SDValue Chain = CLI.Chain;
11202 SDValue Callee = CLI.Callee;
11203 CallingConv::ID CallConv = CLI.CallConv;
11204 bool IsVarArg = CLI.IsVarArg;
11205 EVT PtrVT = getPointerTy(DAG.getDataLayout());
11206 MVT GRLenVT = Subtarget.getGRLenVT();
11207 bool &IsTailCall = CLI.IsTailCall;
11208
11210
11211 // Analyze the operands of the call, assigning locations to each operand.
11213 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
11214
11215 if (CallConv == CallingConv::GHC)
11216 ArgCCInfo.AnalyzeCallOperands(Outs, CC_LoongArch_GHC);
11217 else
11218 analyzeOutputArgs(MF, ArgCCInfo, Outs, /*IsRet=*/false, &CLI, CC_LoongArch);
11219
11220 // Check if it's really possible to do a tail call.
11221 if (IsTailCall)
11222 IsTailCall = isEligibleForTailCallOptimization(ArgCCInfo, CLI, MF, ArgLocs);
11223
11224 if (IsTailCall)
11225 ++NumTailCalls;
11226 else if (CLI.CB && CLI.CB->isMustTailCall())
11227 report_fatal_error("failed to perform tail call elimination on a call "
11228 "site marked musttail");
11229
11230 // Get a count of how many bytes are to be pushed on the stack.
11231 unsigned NumBytes = ArgCCInfo.getStackSize();
11232
11233 // Create local copies for byval args.
11234 SmallVector<SDValue> ByValArgs;
11235 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
11236 ISD::ArgFlagsTy Flags = Outs[i].Flags;
11237 if (!Flags.isByVal())
11238 continue;
11239
11240 SDValue Arg = OutVals[i];
11241 unsigned Size = Flags.getByValSize();
11242 Align Alignment = Flags.getNonZeroByValAlign();
11243
11244 int FI =
11245 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/false);
11246 SDValue FIPtr = DAG.getFrameIndex(FI, getPointerTy(DAG.getDataLayout()));
11247 SDValue SizeNode = DAG.getConstant(Size, DL, GRLenVT);
11248
11249 Chain = DAG.getMemcpy(Chain, DL, FIPtr, Arg, SizeNode, Alignment, Alignment,
11250 /*IsVolatile=*/false,
11251 /*AlwaysInline=*/false, /*CI=*/nullptr, std::nullopt,
11253 ByValArgs.push_back(FIPtr);
11254 }
11255
11256 if (!IsTailCall)
11257 Chain = DAG.getCALLSEQ_START(Chain, NumBytes, 0, CLI.DL);
11258
11259 // Copy argument values to their designated locations.
11261 SmallVector<SDValue> MemOpChains;
11262 SDValue StackPtr;
11263 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
11264 ++i, ++OutIdx) {
11265 CCValAssign &VA = ArgLocs[i];
11266 SDValue ArgValue = OutVals[OutIdx];
11267 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
11268
11269 // Handle passing f64 on LA32D with a soft float ABI as a special case.
11270 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
11271 assert(VA.isRegLoc() && "Expected register VA assignment");
11272 assert(VA.needsCustom());
11273 SDValue SplitF64 =
11274 DAG.getNode(LoongArchISD::SPLIT_PAIR_F64, DL,
11275 DAG.getVTList(MVT::i32, MVT::i32), ArgValue);
11276 SDValue Lo = SplitF64.getValue(0);
11277 SDValue Hi = SplitF64.getValue(1);
11278
11279 Register RegLo = VA.getLocReg();
11280 RegsToPass.push_back(std::make_pair(RegLo, Lo));
11281
11282 // Get the CCValAssign for the Hi part.
11283 CCValAssign &HiVA = ArgLocs[++i];
11284
11285 if (HiVA.isMemLoc()) {
11286 // Second half of f64 is passed on the stack.
11287 if (!StackPtr.getNode())
11288 StackPtr = DAG.getCopyFromReg(Chain, DL, LoongArch::R3, PtrVT);
11290 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
11291 DAG.getIntPtrConstant(HiVA.getLocMemOffset(), DL));
11292 // Emit the store.
11293 MemOpChains.push_back(DAG.getStore(
11294 Chain, DL, Hi, Address,
11296 } else {
11297 // Second half of f64 is passed in another GPR.
11298 Register RegHigh = HiVA.getLocReg();
11299 RegsToPass.push_back(std::make_pair(RegHigh, Hi));
11300 }
11301 continue;
11302 }
11303
11304 // Promote the value if needed.
11305 // For now, only handle fully promoted and indirect arguments.
11306 if (VA.getLocInfo() == CCValAssign::Indirect) {
11307 // For musttail calls, reuse incoming indirect pointers instead of
11308 // creating new stack temporaries. The incoming pointers point to the
11309 // caller's caller's frame, which remains valid after a tail call.
11310 if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
11313 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
11314
11315 // Resolve which formal parameter is being passed at this call
11316 // position.
11317 //
11318 // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered),
11319 // but Outs[].OrigArgIndex is an index into a filtered arg list
11320 // (empty types removed, via CallLoweringInfo in the target-
11321 // independent layer). IncomingIndirectArgs is keyed by the
11322 // caller's unfiltered Argument::getArgNo(), so we have to walk
11323 // the caller's formals (same filter) to translate the index.
11324 // This target-independent asymmetry should be normalized so
11325 // backends do not need to re-derive the mapping.
11326 //
11327 // Steps:
11328 // 1. Find the call operand at filtered position CallArgIdx.
11329 // 2. If it is an Argument, use getArgNo() directly (same filter
11330 // for caller formals and call operands).
11331 // 3. Otherwise (computed value), walk the caller's formals and
11332 // skip empty types to map the filtered index to getArgNo().
11333 const Argument *FormalArg = nullptr;
11334 unsigned FilteredIdx = 0;
11335 for (const auto &CallArg : CLI.CB->args()) {
11336 if (CallArg->getType()->isEmptyTy())
11337 continue;
11338 if (FilteredIdx == CallArgIdx) {
11339 FormalArg = dyn_cast<Argument>(CallArg);
11340 break;
11341 }
11342 ++FilteredIdx;
11343 }
11344
11345 // For forwarded args, getArgNo() gives the unfiltered index directly.
11346 // For computed args, walk the caller's formals to resolve it.
11347 unsigned FormalArgIdx = CallArgIdx;
11348 if (FormalArg) {
11349 FormalArgIdx = FormalArg->getArgNo();
11350 } else {
11351 FilteredIdx = 0;
11352 for (const auto &Arg : MF.getFunction().args()) {
11353 if (Arg.getType()->isEmptyTy())
11354 continue;
11355 if (FilteredIdx == CallArgIdx) {
11356 FormalArgIdx = Arg.getArgNo();
11357 break;
11358 }
11359 ++FilteredIdx;
11360 }
11361 }
11362
11363 Register VReg = LAFI->getIncomingIndirectArg(FormalArgIdx);
11364 SDValue CopyOp = DAG.getCopyFromReg(Chain, DL, VReg, PtrVT);
11365 // Thread the CopyFromReg output chain through MemOpChains so the
11366 // TokenFactor below sequences the copy with any stores we emit
11367 // for this argument.
11368 MemOpChains.push_back(CopyOp.getValue(1));
11369 SDValue IncomingPtr = CopyOp;
11370
11371 if (!FormalArg) {
11372 // Computed value: store into the incoming indirect pointer for the
11373 // same-position formal parameter (musttail guarantees matching
11374 // prototypes, so types match). The pointer survives the tail call
11375 // since it points to the caller's caller's frame.
11376 //
11377 // The data-flow edge through IncomingPtr already prevents the
11378 // store from being scheduled before the CopyFromReg. Threading
11379 // CopyOp.getValue(1) (the copy's output chain) into the store
11380 // makes that ordering explicit on the chain edge as well, which
11381 // is the convention for memory ops chaining off their producers.
11382 MemOpChains.push_back(
11383 DAG.getStore(CopyOp.getValue(1), DL, ArgValue, IncomingPtr,
11385 // Store any split parts at their respective offsets.
11386 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
11387 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
11388 SDValue PartValue = OutVals[OutIdx + 1];
11389 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
11390 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
11391 SDValue Addr =
11392 DAG.getNode(ISD::ADD, DL, PtrVT, IncomingPtr, Offset);
11393 MemOpChains.push_back(
11394 DAG.getStore(CopyOp.getValue(1), DL, PartValue, Addr,
11396 ++i;
11397 ++OutIdx;
11398 }
11399 }
11400 ArgValue = IncomingPtr;
11401
11402 // Skip any remaining split parts (for forwarded args, they are
11403 // covered by the forwarded pointer).
11404 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
11405 ++i;
11406 ++OutIdx;
11407 }
11408 } else {
11409 // Store the argument in a stack slot and pass its address.
11410 Align StackAlign =
11411 std::max(getPrefTypeAlign(Outs[OutIdx].ArgVT, DAG),
11412 getPrefTypeAlign(ArgValue.getValueType(), DAG));
11413 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
11414 // If the original argument was split and passed by reference, we need
11415 // to store the required parts of it here (and pass just one address).
11416 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
11417 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
11418 assert(ArgPartOffset == 0);
11419 // Calculate the total size to store. We don't have access to what we're
11420 // actually storing other than performing the loop and collecting the
11421 // info.
11423 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
11424 SDValue PartValue = OutVals[OutIdx + 1];
11425 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
11426 SDValue Offset = DAG.getIntPtrConstant(PartOffset, DL);
11427 EVT PartVT = PartValue.getValueType();
11428 StoredSize += PartVT.getStoreSize();
11429 StackAlign = std::max(StackAlign, getPrefTypeAlign(PartVT, DAG));
11430 Parts.push_back(std::make_pair(PartValue, Offset));
11431 ++i;
11432 ++OutIdx;
11433 }
11434 SDValue SpillSlot = DAG.CreateStackTemporary(StoredSize, StackAlign);
11435 int FI = cast<FrameIndexSDNode>(SpillSlot)->getIndex();
11436 MemOpChains.push_back(
11437 DAG.getStore(Chain, DL, ArgValue, SpillSlot,
11439 for (const auto &Part : Parts) {
11440 SDValue PartValue = Part.first;
11441 SDValue PartOffset = Part.second;
11443 DAG.getNode(ISD::ADD, DL, PtrVT, SpillSlot, PartOffset);
11444 MemOpChains.push_back(
11445 DAG.getStore(Chain, DL, PartValue, Address,
11447 }
11448 ArgValue = SpillSlot;
11449 }
11450 } else {
11451 ArgValue = convertValVTToLocVT(DAG, ArgValue, VA, DL);
11452 }
11453
11454 // Use local copy if it is a byval arg.
11455 if (Flags.isByVal())
11456 ArgValue = ByValArgs[j++];
11457
11458 if (VA.isRegLoc()) {
11459 // Queue up the argument copies and emit them at the end.
11460 RegsToPass.push_back(std::make_pair(VA.getLocReg(), ArgValue));
11461 } else {
11462 assert(VA.isMemLoc() && "Argument not register or memory");
11463 assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) &&
11464 "Tail call not allowed if stack is used for passing parameters");
11465
11466 // Work out the address of the stack slot.
11467 if (!StackPtr.getNode())
11468 StackPtr = DAG.getCopyFromReg(Chain, DL, LoongArch::R3, PtrVT);
11470 DAG.getNode(ISD::ADD, DL, PtrVT, StackPtr,
11472
11473 // Emit the store.
11474 MemOpChains.push_back(
11475 DAG.getStore(Chain, DL, ArgValue, Address, MachinePointerInfo()));
11476 }
11477 }
11478
11479 // Join the stores, which are independent of one another.
11480 if (!MemOpChains.empty())
11481 Chain = DAG.getNode(ISD::TokenFactor, DL, MVT::Other, MemOpChains);
11482
11483 SDValue Glue;
11484
11485 // Build a sequence of copy-to-reg nodes, chained and glued together.
11486 for (auto &Reg : RegsToPass) {
11487 Chain = DAG.getCopyToReg(Chain, DL, Reg.first, Reg.second, Glue);
11488 Glue = Chain.getValue(1);
11489 }
11490
11491 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
11492 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
11493 // split it and then direct call can be matched by PseudoCALL_SMALL.
11495 const GlobalValue *GV = S->getGlobal();
11496 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(GV)
11499 Callee = DAG.getTargetGlobalAddress(S->getGlobal(), DL, PtrVT, 0, OpFlags);
11500 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Callee)) {
11501 unsigned OpFlags = getTargetMachine().shouldAssumeDSOLocal(nullptr)
11504 Callee = DAG.getTargetExternalSymbol(S->getSymbol(), PtrVT, OpFlags);
11505 }
11506
11507 // The first call operand is the chain and the second is the target address.
11509 Ops.push_back(Chain);
11510 Ops.push_back(Callee);
11511
11512 // Add argument registers to the end of the list so that they are
11513 // known live into the call.
11514 for (auto &Reg : RegsToPass)
11515 Ops.push_back(DAG.getRegister(Reg.first, Reg.second.getValueType()));
11516
11517 if (!IsTailCall) {
11518 // Add a register mask operand representing the call-preserved registers.
11519 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
11520 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
11521 assert(Mask && "Missing call preserved mask for calling convention");
11522 Ops.push_back(DAG.getRegisterMask(Mask));
11523 }
11524
11525 // Glue the call to the argument copies, if any.
11526 if (Glue.getNode())
11527 Ops.push_back(Glue);
11528
11529 // Emit the call.
11530 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);
11531 unsigned Op;
11532 switch (DAG.getTarget().getCodeModel()) {
11533 default:
11534 report_fatal_error("Unsupported code model");
11535 case CodeModel::Small:
11536 Op = IsTailCall ? LoongArchISD::TAIL : LoongArchISD::CALL;
11537 break;
11538 case CodeModel::Medium:
11539 Op = IsTailCall ? LoongArchISD::TAIL_MEDIUM : LoongArchISD::CALL_MEDIUM;
11540 break;
11541 case CodeModel::Large:
11542 assert(Subtarget.is64Bit() && "Large code model requires LA64");
11543 Op = IsTailCall ? LoongArchISD::TAIL_LARGE : LoongArchISD::CALL_LARGE;
11544 break;
11545 }
11546
11547 if (IsTailCall) {
11549 SDValue Ret = DAG.getNode(Op, DL, NodeTys, Ops);
11550 DAG.addNoMergeSiteInfo(Ret.getNode(), CLI.NoMerge);
11551 return Ret;
11552 }
11553
11554 Chain = DAG.getNode(Op, DL, NodeTys, Ops);
11555 DAG.addNoMergeSiteInfo(Chain.getNode(), CLI.NoMerge);
11556 Glue = Chain.getValue(1);
11557
11558 // Mark the end of the call, which is glued to the call itself.
11559 Chain = DAG.getCALLSEQ_END(Chain, NumBytes, 0, Glue, DL);
11560 Glue = Chain.getValue(1);
11561
11562 // Assign locations to each value returned by this call.
11564 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
11565 analyzeInputArgs(MF, RetCCInfo, Ins, /*IsRet=*/true, CC_LoongArch);
11566
11567 // Copy all of the result registers out of their specified physreg.
11568 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
11569 auto &VA = RVLocs[i];
11570 // Copy the value out.
11571 SDValue RetValue =
11572 DAG.getCopyFromReg(Chain, DL, VA.getLocReg(), VA.getLocVT(), Glue);
11573 // Glue the RetValue to the end of the call sequence.
11574 Chain = RetValue.getValue(1);
11575 Glue = RetValue.getValue(2);
11576
11577 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
11578 assert(VA.needsCustom());
11579 SDValue RetValue2 = DAG.getCopyFromReg(Chain, DL, RVLocs[++i].getLocReg(),
11580 MVT::i32, Glue);
11581 Chain = RetValue2.getValue(1);
11582 Glue = RetValue2.getValue(2);
11583 RetValue = DAG.getNode(LoongArchISD::BUILD_PAIR_F64, DL, MVT::f64,
11584 RetValue, RetValue2);
11585 } else
11586 RetValue = convertLocVTToValVT(DAG, RetValue, VA, DL);
11587
11588 InVals.push_back(RetValue);
11589 }
11590
11591 return Chain;
11592}
11593
11595 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
11596 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
11597 const Type *RetTy) const {
11599 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
11600
11601 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
11602 LoongArchABI::ABI ABI =
11603 MF.getSubtarget<LoongArchSubtarget>().getTargetABI();
11604 if (CC_LoongArch(MF.getDataLayout(), ABI, i, Outs[i].VT, CCValAssign::Full,
11605 Outs[i].Flags, CCInfo, /*IsRet=*/true, nullptr))
11606 return false;
11607 }
11608 return true;
11609}
11610
11612 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
11614 const SmallVectorImpl<SDValue> &OutVals, const SDLoc &DL,
11615 SelectionDAG &DAG) const {
11616 // Stores the assignment of the return value to a location.
11618
11619 // Info about the registers and stack slot.
11620 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
11621 *DAG.getContext());
11622
11623 analyzeOutputArgs(DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
11624 nullptr, CC_LoongArch);
11625 if (CallConv == CallingConv::GHC && !RVLocs.empty())
11626 report_fatal_error("GHC functions return void only");
11627 SDValue Glue;
11628 SmallVector<SDValue, 4> RetOps(1, Chain);
11629
11630 // Copy the result values into the output registers.
11631 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
11632 SDValue Val = OutVals[OutIdx];
11633 CCValAssign &VA = RVLocs[i];
11634 assert(VA.isRegLoc() && "Can only return in registers!");
11635
11636 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
11637 // Handle returning f64 on LA32D with a soft float ABI.
11638 assert(VA.isRegLoc() && "Expected return via registers");
11639 assert(VA.needsCustom());
11640 SDValue SplitF64 = DAG.getNode(LoongArchISD::SPLIT_PAIR_F64, DL,
11641 DAG.getVTList(MVT::i32, MVT::i32), Val);
11642 SDValue Lo = SplitF64.getValue(0);
11643 SDValue Hi = SplitF64.getValue(1);
11644 Register RegLo = VA.getLocReg();
11645 Register RegHi = RVLocs[++i].getLocReg();
11646
11647 Chain = DAG.getCopyToReg(Chain, DL, RegLo, Lo, Glue);
11648 Glue = Chain.getValue(1);
11649 RetOps.push_back(DAG.getRegister(RegLo, MVT::i32));
11650 Chain = DAG.getCopyToReg(Chain, DL, RegHi, Hi, Glue);
11651 Glue = Chain.getValue(1);
11652 RetOps.push_back(DAG.getRegister(RegHi, MVT::i32));
11653 } else {
11654 // Handle a 'normal' return.
11655 Val = convertValVTToLocVT(DAG, Val, VA, DL);
11656 Chain = DAG.getCopyToReg(Chain, DL, VA.getLocReg(), Val, Glue);
11657
11658 // Guarantee that all emitted copies are stuck together.
11659 Glue = Chain.getValue(1);
11660 RetOps.push_back(DAG.getRegister(VA.getLocReg(), VA.getLocVT()));
11661 }
11662 }
11663
11664 RetOps[0] = Chain; // Update chain.
11665
11666 // Add the glue node if we have it.
11667 if (Glue.getNode())
11668 RetOps.push_back(Glue);
11669
11670 return DAG.getNode(LoongArchISD::RET, DL, MVT::Other, RetOps);
11671}
11672
11673// Check if a constant splat can be generated using [x]vldi, where imm[12] == 1.
11674// Note: The following prefixes are excluded:
11675// imm[11:8] == 4'b0000, 4'b0100, 4'b1000
11676// as they can be represented using [x]vrepli.[whb]
11678 const APInt &SplatValue, const unsigned SplatBitSize) const {
11679 uint64_t RequiredImm = 0;
11680 uint64_t V = SplatValue.getZExtValue();
11681 if (SplatBitSize == 16 && !(V & 0x00FF)) {
11682 // 4'b0101
11683 RequiredImm = (0b10101 << 8) | (V >> 8);
11684 return {true, RequiredImm};
11685 } else if (SplatBitSize == 32) {
11686 // 4'b0001
11687 if (!(V & 0xFFFF00FF)) {
11688 RequiredImm = (0b10001 << 8) | (V >> 8);
11689 return {true, RequiredImm};
11690 }
11691 // 4'b0010
11692 if (!(V & 0xFF00FFFF)) {
11693 RequiredImm = (0b10010 << 8) | (V >> 16);
11694 return {true, RequiredImm};
11695 }
11696 // 4'b0011
11697 if (!(V & 0x00FFFFFF)) {
11698 RequiredImm = (0b10011 << 8) | (V >> 24);
11699 return {true, RequiredImm};
11700 }
11701 // 4'b0110
11702 if ((V & 0xFFFF00FF) == 0xFF) {
11703 RequiredImm = (0b10110 << 8) | (V >> 8);
11704 return {true, RequiredImm};
11705 }
11706 // 4'b0111
11707 if ((V & 0xFF00FFFF) == 0xFFFF) {
11708 RequiredImm = (0b10111 << 8) | (V >> 16);
11709 return {true, RequiredImm};
11710 }
11711 // 4'b1010
11712 if ((V & 0x7E07FFFF) == 0x3E000000 || (V & 0x7E07FFFF) == 0x40000000) {
11713 RequiredImm =
11714 (0b11010 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
11715 return {true, RequiredImm};
11716 }
11717 } else if (SplatBitSize == 64) {
11718 // 4'b1011
11719 if ((V & 0xFFFFFFFF7E07FFFFULL) == 0x3E000000ULL ||
11720 (V & 0xFFFFFFFF7E07FFFFULL) == 0x40000000ULL) {
11721 RequiredImm =
11722 (0b11011 << 8) | (((V >> 24) & 0xC0) ^ 0x40) | ((V >> 19) & 0x3F);
11723 return {true, RequiredImm};
11724 }
11725 // 4'b1100
11726 if ((V & 0x7FC0FFFFFFFFFFFFULL) == 0x4000000000000000ULL ||
11727 (V & 0x7FC0FFFFFFFFFFFFULL) == 0x3FC0000000000000ULL) {
11728 RequiredImm =
11729 (0b11100 << 8) | (((V >> 56) & 0xC0) ^ 0x40) | ((V >> 48) & 0x3F);
11730 return {true, RequiredImm};
11731 }
11732 // 4'b1001
11733 auto sameBitsPreByte = [](uint64_t x) -> std::pair<bool, uint8_t> {
11734 uint8_t res = 0;
11735 for (int i = 0; i < 8; ++i) {
11736 uint8_t byte = x & 0xFF;
11737 if (byte == 0 || byte == 0xFF)
11738 res |= ((byte & 1) << i);
11739 else
11740 return {false, 0};
11741 x >>= 8;
11742 }
11743 return {true, res};
11744 };
11745 auto [IsSame, Suffix] = sameBitsPreByte(V);
11746 if (IsSame) {
11747 RequiredImm = (0b11001 << 8) | Suffix;
11748 return {true, RequiredImm};
11749 }
11750 }
11751 return {false, RequiredImm};
11752}
11753
11755 EVT VT) const {
11756 if (!Subtarget.hasExtLSX())
11757 return false;
11758
11759 if (VT == MVT::f32) {
11760 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7e07ffff;
11761 return (masked == 0x3e000000 || masked == 0x40000000);
11762 }
11763
11764 if (VT == MVT::f64) {
11765 uint64_t masked = Imm.bitcastToAPInt().getZExtValue() & 0x7fc0ffffffffffff;
11766 return (masked == 0x3fc0000000000000 || masked == 0x4000000000000000);
11767 }
11768
11769 return false;
11770}
11771
11772// Only form FP_TO_SINT_SAT from clamp/NaN-guard idioms when it will be lowered
11773// to a single [X]VFTINTRZ/FTINTRZ, otherwise the generic expansion is worse
11774// than the original sequence.
11775bool LoongArchTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
11776 EVT VT) const {
11777 if (!isOperationLegalOrCustom(Op, VT))
11778 return false;
11779 if (VT.isVector())
11780 return FPVT.getScalarSizeInBits() == VT.getScalarSizeInBits();
11781 if (FPVT == MVT::f32)
11782 return VT.getSizeInBits() == 32 || Subtarget.hasBasicD();
11783 return FPVT == MVT::f64 && Subtarget.hasBasicD();
11784}
11785
11787LoongArchTargetLowering::getFPImmAction(const APFloat &Imm, EVT VT,
11788 bool ForCodeSize) const {
11789 // TODO: Maybe need more checks here after vector extension is supported.
11790 if (VT == MVT::f32 && !Subtarget.hasBasicF())
11792 if (VT == MVT::f64 && !Subtarget.hasBasicD())
11794 if (Imm.isZero() || isFPImmVLDILegal(Imm, VT))
11795 return FPImmAction::Legal;
11796
11797 // Immediates the size of a GPR are built from their bits. Others are built in
11798 // lowerConstantFP.
11799 if (VT.getSizeInBits() == Subtarget.getGRLen() && isFPImmCheapInGPR(Imm, VT))
11800 return FPImmAction::Bitcast;
11802}
11803
11805 EVT VT) const {
11806 // Count the instructions to generate the bits and move them to a float
11807 // register.
11808 auto Seq =
11809 LoongArchMatInt::generateInstSeq(Imm.bitcastToAPInt().getSExtValue());
11810 int InsNum = Seq.size() + ((VT == MVT::f64 && !Subtarget.is64Bit()) ? 2 : 1);
11811 return InsNum <= MaterializeFPImmInsNum || Imm.isOne();
11812}
11813
11815 return true;
11816}
11817
11819 return true;
11820}
11821
11822bool LoongArchTargetLowering::shouldInsertFencesForAtomic(
11823 const Instruction *I) const {
11824 if (!Subtarget.is64Bit())
11825 return isa<LoadInst>(I) || isa<StoreInst>(I);
11826
11827 if (isa<LoadInst>(I))
11828 return true;
11829
11830 // On LA64, atomic store operations with IntegerBitWidth of 32 and 64 do not
11831 // require fences beacuse we can use amswap_db.[w/d].
11832 Type *Ty = I->getOperand(0)->getType();
11833 if (isa<StoreInst>(I) && Ty->isIntegerTy()) {
11834 unsigned Size = Ty->getIntegerBitWidth();
11835 return (Size == 8 || Size == 16);
11836 }
11837
11838 return false;
11839}
11840
11842 LLVMContext &Context,
11843 EVT VT) const {
11844 if (!VT.isVector())
11845 return getPointerTy(DL);
11847}
11848
11850 unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const {
11851 // Do not merge to float value size (128 or 256 bits) if no implicit
11852 // float attribute is set.
11853 bool NoFloat = MF.getFunction().hasFnAttribute(Attribute::NoImplicitFloat);
11854 unsigned MaxIntSize = Subtarget.is64Bit() ? 64 : 32;
11855 if (NoFloat)
11856 return MemVT.getSizeInBits() <= MaxIntSize;
11857
11858 // Make sure we don't merge greater than our maximum supported vector width.
11859 if (Subtarget.hasExtLASX())
11860 MaxIntSize = 256;
11861 else if (Subtarget.hasExtLSX())
11862 MaxIntSize = 128;
11863
11864 return MemVT.getSizeInBits() <= MaxIntSize;
11865}
11866
11868 EVT VT = Y.getValueType();
11869
11870 if (VT.isVector())
11871 return Subtarget.hasExtLSX() && VT.isInteger();
11872
11873 return VT.isScalarInteger() && !isa<ConstantSDNode>(Y);
11874}
11875
11878 MachineFunction &MF, unsigned Intrinsic) const {
11879 switch (Intrinsic) {
11880 default:
11881 return;
11882 case Intrinsic::loongarch_masked_atomicrmw_xchg_i32:
11883 case Intrinsic::loongarch_masked_atomicrmw_add_i32:
11884 case Intrinsic::loongarch_masked_atomicrmw_sub_i32:
11885 case Intrinsic::loongarch_masked_atomicrmw_nand_i32: {
11886 IntrinsicInfo Info;
11888 Info.memVT = MVT::i32;
11889 Info.ptrVal = I.getArgOperand(0);
11890 Info.offset = 0;
11891 Info.align = Align(4);
11894 Infos.push_back(Info);
11895 return;
11896 // TODO: Add more Intrinsics later.
11897 }
11898 }
11899}
11900
11901// When -mlamcas is enabled, MinCmpXchgSizeInBits will be set to 8,
11902// atomicrmw and/or/xor operations with operands less than 32 bits cannot be
11903// expanded to am{and/or/xor}[_db].w through AtomicExpandPass. To prevent
11904// regression, we need to implement it manually.
11907
11909 Op == AtomicRMWInst::And) &&
11910 "Unable to expand");
11911 unsigned MinWordSize = 4;
11912
11913 IRBuilder<> Builder(AI);
11914 LLVMContext &Ctx = Builder.getContext();
11915 const DataLayout &DL = AI->getDataLayout();
11916 Type *ValueType = AI->getType();
11917 Type *WordType = Type::getIntNTy(Ctx, MinWordSize * 8);
11918
11919 Value *Addr = AI->getPointerOperand();
11920 PointerType *PtrTy = cast<PointerType>(Addr->getType());
11921 IntegerType *IntTy = DL.getIndexType(Ctx, PtrTy->getAddressSpace());
11922
11923 Value *AlignedAddr = Builder.CreateIntrinsic(
11924 Intrinsic::ptrmask, {PtrTy, IntTy},
11925 {Addr, ConstantInt::get(IntTy, ~(uint64_t)(MinWordSize - 1))}, nullptr,
11926 "AlignedAddr");
11927
11928 Value *AddrInt = Builder.CreatePtrToInt(Addr, IntTy);
11929 Value *PtrLSB = Builder.CreateAnd(AddrInt, MinWordSize - 1, "PtrLSB");
11930 Value *ShiftAmt = Builder.CreateShl(PtrLSB, 3);
11931 ShiftAmt = Builder.CreateTrunc(ShiftAmt, WordType, "ShiftAmt");
11932 Value *Mask = Builder.CreateShl(
11933 ConstantInt::get(WordType,
11934 (1 << (DL.getTypeStoreSize(ValueType) * 8)) - 1),
11935 ShiftAmt, "Mask");
11936 Value *Inv_Mask = Builder.CreateNot(Mask, "Inv_Mask");
11937 Value *ValOperand_Shifted =
11938 Builder.CreateShl(Builder.CreateZExt(AI->getValOperand(), WordType),
11939 ShiftAmt, "ValOperand_Shifted");
11940 Value *NewOperand;
11941 if (Op == AtomicRMWInst::And)
11942 NewOperand = Builder.CreateOr(ValOperand_Shifted, Inv_Mask, "AndOperand");
11943 else
11944 NewOperand = ValOperand_Shifted;
11945
11946 AtomicRMWInst *NewAI =
11947 Builder.CreateAtomicRMW(Op, AlignedAddr, NewOperand, Align(MinWordSize),
11948 AI->getOrdering(), AI->getSyncScopeID());
11949
11950 Value *Shift = Builder.CreateLShr(NewAI, ShiftAmt, "shifted");
11951 Value *Trunc = Builder.CreateTrunc(Shift, ValueType, "extracted");
11952 Value *FinalOldResult = Builder.CreateBitCast(Trunc, ValueType);
11953 AI->replaceAllUsesWith(FinalOldResult);
11954 AI->eraseFromParent();
11955}
11956
11959 const AtomicRMWInst *AI) const {
11960 // TODO: Add more AtomicRMWInst that needs to be extended.
11961
11962 // Since floating-point operation requires a non-trivial set of data
11963 // operations, use CmpXChg to expand.
11964 if (AI->isFloatingPointOperation() ||
11970
11971 if (Subtarget.hasLAM_BH() && Subtarget.is64Bit() &&
11974 AI->getOperation() == AtomicRMWInst::Sub)) {
11976 }
11977
11978 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
11979 if (Subtarget.hasLAMCAS()) {
11980 if (Size < 32 && (AI->getOperation() == AtomicRMWInst::And ||
11984 if (AI->getOperation() == AtomicRMWInst::Nand || Size < 32)
11986 }
11987
11988 if (Size == 8 || Size == 16)
11991}
11992
11993static Intrinsic::ID
11995 AtomicRMWInst::BinOp BinOp) {
11996 if (GRLen == 64) {
11997 switch (BinOp) {
11998 default:
11999 llvm_unreachable("Unexpected AtomicRMW BinOp");
12001 return Intrinsic::loongarch_masked_atomicrmw_xchg_i64;
12002 case AtomicRMWInst::Add:
12003 return Intrinsic::loongarch_masked_atomicrmw_add_i64;
12004 case AtomicRMWInst::Sub:
12005 return Intrinsic::loongarch_masked_atomicrmw_sub_i64;
12007 return Intrinsic::loongarch_masked_atomicrmw_nand_i64;
12009 return Intrinsic::loongarch_masked_atomicrmw_umax_i64;
12011 return Intrinsic::loongarch_masked_atomicrmw_umin_i64;
12012 case AtomicRMWInst::Max:
12013 return Intrinsic::loongarch_masked_atomicrmw_max_i64;
12014 case AtomicRMWInst::Min:
12015 return Intrinsic::loongarch_masked_atomicrmw_min_i64;
12016 // TODO: support other AtomicRMWInst.
12017 }
12018 }
12019
12020 if (GRLen == 32) {
12021 switch (BinOp) {
12022 default:
12023 llvm_unreachable("Unexpected AtomicRMW BinOp");
12025 return Intrinsic::loongarch_masked_atomicrmw_xchg_i32;
12026 case AtomicRMWInst::Add:
12027 return Intrinsic::loongarch_masked_atomicrmw_add_i32;
12028 case AtomicRMWInst::Sub:
12029 return Intrinsic::loongarch_masked_atomicrmw_sub_i32;
12031 return Intrinsic::loongarch_masked_atomicrmw_nand_i32;
12033 return Intrinsic::loongarch_masked_atomicrmw_umax_i32;
12035 return Intrinsic::loongarch_masked_atomicrmw_umin_i32;
12036 case AtomicRMWInst::Max:
12037 return Intrinsic::loongarch_masked_atomicrmw_max_i32;
12038 case AtomicRMWInst::Min:
12039 return Intrinsic::loongarch_masked_atomicrmw_min_i32;
12040 // TODO: support other AtomicRMWInst.
12041 }
12042 }
12043
12044 llvm_unreachable("Unexpected GRLen\n");
12045}
12046
12049 const AtomicCmpXchgInst *CI) const {
12050
12051 if (Subtarget.hasLAMCAS())
12053
12055 if (Size == 8 || Size == 16)
12058}
12059
12061 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
12062 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
12063 unsigned GRLen = Subtarget.getGRLen();
12064 AtomicOrdering FailOrd = CI->getFailureOrdering();
12065 Value *FailureOrdering =
12066 Builder.getIntN(Subtarget.getGRLen(), static_cast<uint64_t>(FailOrd));
12067 Intrinsic::ID CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i32;
12068 if (GRLen == 64) {
12069 CmpXchgIntrID = Intrinsic::loongarch_masked_cmpxchg_i64;
12070 CmpVal = Builder.CreateSExt(CmpVal, Builder.getInt64Ty());
12071 NewVal = Builder.CreateSExt(NewVal, Builder.getInt64Ty());
12072 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
12073 }
12074 Type *Tys[] = {AlignedAddr->getType()};
12075 Value *Result = Builder.CreateIntrinsic(
12076 CmpXchgIntrID, Tys, {AlignedAddr, CmpVal, NewVal, Mask, FailureOrdering});
12077 if (GRLen == 64)
12078 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
12079 return Result;
12080}
12081
12083 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
12084 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
12085 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
12086 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
12087 // mask, as this produces better code than the LL/SC loop emitted by
12088 // int_loongarch_masked_atomicrmw_xchg.
12089 if (AI->getOperation() == AtomicRMWInst::Xchg &&
12092 if (CVal->isZero())
12093 return Builder.CreateAtomicRMW(AtomicRMWInst::And, AlignedAddr,
12094 Builder.CreateNot(Mask, "Inv_Mask"),
12095 AI->getAlign(), Ord);
12096 if (CVal->isMinusOne())
12097 return Builder.CreateAtomicRMW(AtomicRMWInst::Or, AlignedAddr, Mask,
12098 AI->getAlign(), Ord);
12099 }
12100
12101 unsigned GRLen = Subtarget.getGRLen();
12102 Value *Ordering =
12103 Builder.getIntN(GRLen, static_cast<uint64_t>(AI->getOrdering()));
12104 Type *Tys[] = {AlignedAddr->getType()};
12106 AI->getModule(),
12108
12109 if (GRLen == 64) {
12110 Incr = Builder.CreateSExt(Incr, Builder.getInt64Ty());
12111 Mask = Builder.CreateSExt(Mask, Builder.getInt64Ty());
12112 ShiftAmt = Builder.CreateSExt(ShiftAmt, Builder.getInt64Ty());
12113 }
12114
12115 Value *Result;
12116
12117 // Must pass the shift amount needed to sign extend the loaded value prior
12118 // to performing a signed comparison for min/max. ShiftAmt is the number of
12119 // bits to shift the value into position. Pass GRLen-ShiftAmt-ValWidth, which
12120 // is the number of bits to left+right shift the value in order to
12121 // sign-extend.
12122 if (AI->getOperation() == AtomicRMWInst::Min ||
12124 const DataLayout &DL = AI->getDataLayout();
12125 unsigned ValWidth =
12126 DL.getTypeStoreSizeInBits(AI->getValOperand()->getType());
12127 Value *SextShamt =
12128 Builder.CreateSub(Builder.getIntN(GRLen, GRLen - ValWidth), ShiftAmt);
12129 Result = Builder.CreateCall(LlwOpScwLoop,
12130 {AlignedAddr, Incr, Mask, SextShamt, Ordering});
12131 } else {
12132 Result =
12133 Builder.CreateCall(LlwOpScwLoop, {AlignedAddr, Incr, Mask, Ordering});
12134 }
12135
12136 if (GRLen == 64)
12137 Result = Builder.CreateTrunc(Result, Builder.getInt32Ty());
12138 return Result;
12139}
12140
12142 const MachineFunction &MF, EVT VT) const {
12143 VT = VT.getScalarType();
12144
12145 if (!VT.isSimple())
12146 return false;
12147
12148 switch (VT.getSimpleVT().SimpleTy) {
12149 case MVT::f32:
12150 case MVT::f64:
12151 return true;
12152 default:
12153 break;
12154 }
12155
12156 return false;
12157}
12158
12160 ExceptionHandling EH, const Constant *PersonalityFn) const {
12161 return LoongArch::R4;
12162}
12163
12165 ExceptionHandling EH, const Constant *PersonalityFn) const {
12166 return LoongArch::R5;
12167}
12168
12169//===----------------------------------------------------------------------===//
12170// Target Optimization Hooks
12171//===----------------------------------------------------------------------===//
12172
12175 // Feature FRECIPE instructions relative accuracy is 2^-14.
12176 // IEEE float has 23 digits and double has 52 digits.
12177 int RefinementSteps = VT.getScalarType() == MVT::f64 ? 2 : 1;
12178 return RefinementSteps;
12179}
12180
12181static bool
12183 assert(Subtarget.hasFrecipe() &&
12184 "Reciprocal estimate queried on unsupported target");
12185
12186 if (!VT.isSimple())
12187 return false;
12188
12189 switch (VT.getSimpleVT().SimpleTy) {
12190 case MVT::f32:
12191 // f32 is the base type for reciprocal estimate instructions.
12192 return true;
12193
12194 case MVT::f64:
12195 return Subtarget.hasBasicD();
12196
12197 case MVT::v4f32:
12198 case MVT::v2f64:
12199 return Subtarget.hasExtLSX();
12200
12201 case MVT::v8f32:
12202 case MVT::v4f64:
12203 return Subtarget.hasExtLASX();
12204
12205 default:
12206 return false;
12207 }
12208}
12209
12211 SelectionDAG &DAG, int Enabled,
12212 int &RefinementSteps,
12213 bool &UseOneConstNR,
12214 bool Reciprocal) const {
12216 "Enabled should never be Disabled here");
12217
12218 if (!Subtarget.hasFrecipe())
12219 return SDValue();
12220
12221 SDLoc DL(Operand);
12222 EVT VT = Operand.getValueType();
12223
12224 // Check supported types.
12225 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
12226 return SDValue();
12227
12228 // Handle refinement steps.
12229 if (RefinementSteps == ReciprocalEstimate::Unspecified)
12230 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
12231
12232 // LoongArch only has FRSQRTE which is 1.0 / sqrt(x).
12233 UseOneConstNR = false;
12234 SDValue Rsqrt = DAG.getNode(LoongArchISD::FRSQRTE, DL, VT, Operand);
12235
12236 // If the caller wants 1.0 / sqrt(x), or if further refinement steps
12237 // are needed (which rely on the reciprocal form), return the raw reciprocal
12238 // estimate.
12239 if (Reciprocal || RefinementSteps > 0)
12240 return Rsqrt;
12241
12242 // Otherwise, return sqrt(x) by multiplying with the operand.
12243 return DAG.getNode(ISD::FMUL, DL, VT, Operand, Rsqrt);
12244}
12245
12247 SelectionDAG &DAG,
12248 int Enabled,
12249 int &RefinementSteps) const {
12251 "Enabled should never be Disabled here");
12252
12253 if (!Subtarget.hasFrecipe())
12254 return SDValue();
12255
12256 SDLoc DL(Operand);
12257 EVT VT = Operand.getValueType();
12258
12259 // Check supported types.
12260 if (!isSupportedReciprocalEstimateType(VT, Subtarget))
12261 return SDValue();
12262
12263 if (RefinementSteps == ReciprocalEstimate::Unspecified)
12264 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
12265
12266 // FRECIPE computes 1.0 / x.
12267 return DAG.getNode(LoongArchISD::FRECIPE, DL, VT, Operand);
12268}
12269
12270//===----------------------------------------------------------------------===//
12271// LoongArch Inline Assembly Support
12272//===----------------------------------------------------------------------===//
12273
12275LoongArchTargetLowering::getConstraintType(StringRef Constraint) const {
12276 // LoongArch specific constraints in GCC: config/loongarch/constraints.md
12277 //
12278 // 'f': A floating-point register (if available).
12279 // 'k': A memory operand whose address is formed by a base register and
12280 // (optionally scaled) index register.
12281 // 'l': A signed 16-bit constant.
12282 // 'm': A memory operand whose address is formed by a base register and
12283 // offset that is suitable for use in instructions with the same
12284 // addressing mode as st.w and ld.w.
12285 // 'q': A general-purpose register except for $r0 and $r1 (for the csrxchg
12286 // instruction)
12287 // 'I': A signed 12-bit constant (for arithmetic instructions).
12288 // 'J': Integer zero.
12289 // 'K': An unsigned 12-bit constant (for logic instructions).
12290 // "ZB": An address that is held in a general-purpose register. The offset is
12291 // zero.
12292 // "ZC": A memory operand whose address is formed by a base register and
12293 // offset that is suitable for use in instructions with the same
12294 // addressing mode as ll.w and sc.w.
12295 if (Constraint.size() == 1) {
12296 switch (Constraint[0]) {
12297 default:
12298 break;
12299 case 'f':
12300 case 'q':
12301 return C_RegisterClass;
12302 case 'l':
12303 case 'I':
12304 case 'J':
12305 case 'K':
12306 return C_Immediate;
12307 case 'k':
12308 return C_Memory;
12309 }
12310 }
12311
12312 if (Constraint == "ZC" || Constraint == "ZB")
12313 return C_Memory;
12314
12315 // 'm' is handled here.
12316 return TargetLowering::getConstraintType(Constraint);
12317}
12318
12319InlineAsm::ConstraintCode LoongArchTargetLowering::getInlineAsmMemConstraint(
12320 StringRef ConstraintCode) const {
12321 return StringSwitch<InlineAsm::ConstraintCode>(ConstraintCode)
12325 .Default(TargetLowering::getInlineAsmMemConstraint(ConstraintCode));
12326}
12327
12328std::pair<unsigned, const TargetRegisterClass *>
12329LoongArchTargetLowering::getRegForInlineAsmConstraint(
12330 const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const {
12331 // First, see if this is a constraint that directly corresponds to a LoongArch
12332 // register class.
12333 if (Constraint.size() == 1) {
12334 switch (Constraint[0]) {
12335 case 'r':
12336 // TODO: Support fixed vectors up to GRLen?
12337 if (VT.isVector())
12338 break;
12339 return std::make_pair(0U, &LoongArch::GPRRegClass);
12340 case 'q':
12341 return std::make_pair(0U, &LoongArch::GPRNoR0R1RegClass);
12342 case 'f':
12343 if (Subtarget.hasBasicF() && VT == MVT::f32)
12344 return std::make_pair(0U, &LoongArch::FPR32RegClass);
12345 if (Subtarget.hasBasicD() && VT == MVT::f64)
12346 return std::make_pair(0U, &LoongArch::FPR64RegClass);
12347 if (Subtarget.hasExtLSX() &&
12348 TRI->isTypeLegalForClass(LoongArch::LSX128RegClass, VT))
12349 return std::make_pair(0U, &LoongArch::LSX128RegClass);
12350 if (Subtarget.hasExtLSX() && VT == MVT::i128)
12351 return std::make_pair(0U, &LoongArch::LSX128RegClass);
12352 if (Subtarget.hasExtLASX() &&
12353 TRI->isTypeLegalForClass(LoongArch::LASX256RegClass, VT))
12354 return std::make_pair(0U, &LoongArch::LASX256RegClass);
12355 break;
12356 default:
12357 break;
12358 }
12359 }
12360
12361 // TargetLowering::getRegForInlineAsmConstraint uses the name of the TableGen
12362 // record (e.g. the "R0" in `def R0`) to choose registers for InlineAsm
12363 // constraints while the official register name is prefixed with a '$'. So we
12364 // clip the '$' from the original constraint string (e.g. {$r0} to {r0}.)
12365 // before it being parsed. And TargetLowering::getRegForInlineAsmConstraint is
12366 // case insensitive, so no need to convert the constraint to upper case here.
12367 //
12368 // For now, no need to support ABI names (e.g. `$a0`) as clang will correctly
12369 // decode the usage of register name aliases into their official names. And
12370 // AFAIK, the not yet upstreamed `rustc` for LoongArch will always use
12371 // official register names.
12372 if (Constraint.starts_with("{$r") || Constraint.starts_with("{$f") ||
12373 Constraint.starts_with("{$vr") || Constraint.starts_with("{$xr")) {
12374 bool IsFP = Constraint[2] == 'f';
12375 std::pair<StringRef, StringRef> Temp = Constraint.split('$');
12376 std::pair<unsigned, const TargetRegisterClass *> R;
12378 TRI, join_items("", Temp.first, Temp.second), VT);
12379 // Match those names to the widest floating point register type available.
12380 if (IsFP) {
12381 unsigned RegNo = R.first;
12382 if (LoongArch::F0 <= RegNo && RegNo <= LoongArch::F31) {
12383 if (Subtarget.hasBasicD() && (VT == MVT::f64 || VT == MVT::Other)) {
12384 unsigned DReg = RegNo - LoongArch::F0 + LoongArch::F0_64;
12385 return std::make_pair(DReg, &LoongArch::FPR64RegClass);
12386 }
12387 }
12388 }
12389 return R;
12390 }
12391
12392 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
12393}
12394
12395void LoongArchTargetLowering::LowerAsmOperandForConstraint(
12396 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
12397 SelectionDAG &DAG) const {
12398 // Currently only support length 1 constraints.
12399 if (Constraint.size() == 1) {
12400 switch (Constraint[0]) {
12401 case 'l':
12402 // Validate & create a 16-bit signed immediate operand.
12403 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
12404 uint64_t CVal = C->getSExtValue();
12405 if (isInt<16>(CVal))
12406 Ops.push_back(DAG.getSignedTargetConstant(CVal, SDLoc(Op),
12407 Subtarget.getGRLenVT()));
12408 }
12409 return;
12410 case 'I':
12411 // Validate & create a 12-bit signed immediate operand.
12412 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
12413 uint64_t CVal = C->getSExtValue();
12414 if (isInt<12>(CVal))
12415 Ops.push_back(DAG.getSignedTargetConstant(CVal, SDLoc(Op),
12416 Subtarget.getGRLenVT()));
12417 }
12418 return;
12419 case 'J':
12420 // Validate & create an integer zero operand.
12421 if (auto *C = dyn_cast<ConstantSDNode>(Op))
12422 if (C->getZExtValue() == 0)
12423 Ops.push_back(
12424 DAG.getTargetConstant(0, SDLoc(Op), Subtarget.getGRLenVT()));
12425 return;
12426 case 'K':
12427 // Validate & create a 12-bit unsigned immediate operand.
12428 if (auto *C = dyn_cast<ConstantSDNode>(Op)) {
12429 uint64_t CVal = C->getZExtValue();
12430 if (isUInt<12>(CVal))
12431 Ops.push_back(
12432 DAG.getTargetConstant(CVal, SDLoc(Op), Subtarget.getGRLenVT()));
12433 }
12434 return;
12435 default:
12436 break;
12437 }
12438 }
12440}
12441
12442#define GET_REGISTER_MATCHER
12443#include "LoongArchGenAsmMatcher.inc"
12444
12447 const MachineFunction &MF) const {
12448 std::pair<StringRef, StringRef> Name = StringRef(RegName).split('$');
12449 std::string NewRegName = Name.second.str();
12450 Register Reg = MatchRegisterAltName(NewRegName);
12451 if (!Reg)
12452 Reg = MatchRegisterName(NewRegName);
12453 if (!Reg)
12454 return Reg;
12455 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
12456 if (!ReservedRegs.test(Reg))
12457 report_fatal_error(Twine("Trying to obtain non-reserved register \"" +
12458 StringRef(RegName) + "\"."));
12459 return Reg;
12460}
12461
12463 EVT VT, SDValue C) const {
12464 // TODO: Support vectors.
12465 if (!VT.isScalarInteger())
12466 return false;
12467
12468 // Omit the optimization if the data size exceeds GRLen.
12469 if (VT.getSizeInBits() > Subtarget.getGRLen())
12470 return false;
12471
12472 if (auto *ConstNode = dyn_cast<ConstantSDNode>(C.getNode())) {
12473 const APInt &Imm = ConstNode->getAPIntValue();
12474 // Break MUL into (SLLI + ADD/SUB) or ALSL.
12475 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
12476 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
12477 return true;
12478 // Break MUL into (ALSL x, (SLLI x, imm0), imm1).
12479 if (ConstNode->hasOneUse() &&
12480 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
12481 (Imm - 8).isPowerOf2() || (Imm - 16).isPowerOf2()))
12482 return true;
12483 // Break (MUL x, imm) into (ADD (SLLI x, s0), (SLLI x, s1)),
12484 // in which the immediate has two set bits. Or Break (MUL x, imm)
12485 // into (SUB (SLLI x, s0), (SLLI x, s1)), in which the immediate
12486 // equals to (1 << s0) - (1 << s1).
12487 if (ConstNode->hasOneUse() && !(Imm.sge(-2048) && Imm.sle(4095))) {
12488 unsigned Shifts = Imm.countr_zero();
12489 // Reject immediates which can be composed via a single LUI.
12490 if (Shifts >= 12)
12491 return false;
12492 // Reject multiplications can be optimized to
12493 // (SLLI (ALSL x, x, 1/2/3/4), s).
12494 APInt ImmPop = Imm.ashr(Shifts);
12495 if (ImmPop == 3 || ImmPop == 5 || ImmPop == 9 || ImmPop == 17)
12496 return false;
12497 // We do not consider the case `(-Imm - ImmSmall).isPowerOf2()`,
12498 // since it needs one more instruction than other 3 cases.
12499 APInt ImmSmall = APInt(Imm.getBitWidth(), 1ULL << Shifts, true);
12500 if ((Imm - ImmSmall).isPowerOf2() || (Imm + ImmSmall).isPowerOf2() ||
12501 (ImmSmall - Imm).isPowerOf2())
12502 return true;
12503 }
12504 }
12505
12506 return false;
12507}
12508
12510 const AddrMode &AM,
12511 Type *Ty, unsigned AS,
12512 Instruction *I) const {
12513 // LoongArch has four basic addressing modes:
12514 // 1. reg
12515 // 2. reg + 12-bit signed offset
12516 // 3. reg + 14-bit signed offset left-shifted by 2
12517 // 4. reg1 + reg2
12518 // TODO: Add more checks after support vector extension.
12519
12520 // No global is ever allowed as a base.
12521 if (AM.BaseGV)
12522 return false;
12523
12524 // Require a 12-bit signed offset or 14-bit signed offset left-shifted by 2
12525 // with `UAL` feature.
12526 if (!isInt<12>(AM.BaseOffs) &&
12527 !(isShiftedInt<14, 2>(AM.BaseOffs) && Subtarget.hasUAL()))
12528 return false;
12529
12530 switch (AM.Scale) {
12531 case 0:
12532 // "r+i" or just "i", depending on HasBaseReg.
12533 break;
12534 case 1:
12535 // "r+r+i" is not allowed.
12536 if (AM.HasBaseReg && AM.BaseOffs)
12537 return false;
12538 // Otherwise we have "r+r" or "r+i".
12539 break;
12540 case 2:
12541 // "2*r+r" or "2*r+i" is not allowed.
12542 if (AM.HasBaseReg || AM.BaseOffs)
12543 return false;
12544 // Allow "2*r" as "r+r".
12545 break;
12546 default:
12547 return false;
12548 }
12549
12550 return true;
12551}
12552
12554 return isInt<12>(Imm);
12555}
12556
12558 return isInt<12>(Imm);
12559}
12560
12562 // Zexts are free if they can be combined with a load.
12563 // Don't advertise i32->i64 zextload as being free for LA64. It interacts
12564 // poorly with type legalization of compares preferring sext.
12565 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
12566 EVT MemVT = LD->getMemoryVT();
12567 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
12568 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
12569 LD->getExtensionType() == ISD::ZEXTLOAD))
12570 return true;
12571 }
12572
12573 return TargetLowering::isZExtFree(Val, VT2);
12574}
12575
12577 EVT DstVT) const {
12578 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
12579}
12580
12582 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(32);
12583}
12584
12586 // TODO: Support vectors.
12587 if (Y.getValueType().isVector())
12588 return false;
12589
12590 return true;
12591}
12592
12594 // LAMCAS will use amcas[_DB].{b/h/w/d} which does not require extension.
12595 return Subtarget.hasLAMCAS() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
12596}
12597
12599 Type *Ty, bool IsSigned) const {
12600 if (Subtarget.is64Bit() && Ty->isIntegerTy(32))
12601 return true;
12602
12603 return IsSigned;
12604}
12605
12607 // Return false to suppress the unnecessary extensions if the LibCall
12608 // arguments or return value is a float narrower than GRLEN on a soft FP ABI.
12609 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
12610 Type.getSizeInBits() < Subtarget.getGRLen()))
12611 return false;
12612 return true;
12613}
12614
12615// memcpy, and other memory intrinsics, typically tries to use wider load/store
12616// if the source/dest is aligned and the copy size is large enough. We therefore
12617// want to align such objects passed to memory intrinsics.
12619 unsigned &MinSize,
12620 Align &PrefAlign) const {
12621 if (!isa<MemIntrinsic>(CI))
12622 return false;
12623
12624 if (Subtarget.is64Bit()) {
12625 MinSize = 8;
12626 PrefAlign = Align(8);
12627 } else {
12628 MinSize = 4;
12629 PrefAlign = Align(4);
12630 }
12631
12632 return true;
12633}
12634
12637 if (!VT.isScalableVector() && VT.getVectorNumElements() != 1 &&
12638 VT.getVectorElementType() != MVT::i1)
12639 return TypeWidenVector;
12640
12642}
12643
12644bool LoongArchTargetLowering::splitValueIntoRegisterParts(
12645 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
12646 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
12647 bool IsABIRegCopy = CC.has_value();
12648 EVT ValueVT = Val.getValueType();
12649
12650 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
12651 PartVT == MVT::f32) {
12652 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
12653 // nan, and cast to f32.
12654 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i16, Val);
12655 Val = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, Val);
12656 Val = DAG.getNode(ISD::OR, DL, MVT::i32, Val,
12657 DAG.getConstant(0xFFFF0000, DL, MVT::i32));
12658 Val = DAG.getNode(ISD::BITCAST, DL, MVT::f32, Val);
12659 Parts[0] = Val;
12660 return true;
12661 }
12662
12663 return false;
12664}
12665
12666SDValue LoongArchTargetLowering::joinRegisterPartsIntoValue(
12667 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
12668 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
12669 bool IsABIRegCopy = CC.has_value();
12670
12671 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
12672 PartVT == MVT::f32) {
12673 SDValue Val = Parts[0];
12674
12675 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
12676 Val = DAG.getNode(ISD::BITCAST, DL, MVT::i32, Val);
12677 Val = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, Val);
12678 Val = DAG.getNode(ISD::BITCAST, DL, ValueVT, Val);
12679 return Val;
12680 }
12681
12682 return SDValue();
12683}
12684
12685MVT LoongArchTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
12687 EVT VT) const {
12688 // Use f32 to pass f16.
12689 if (VT == MVT::f16 && Subtarget.hasBasicF())
12690 return MVT::f32;
12691
12693}
12694
12695unsigned LoongArchTargetLowering::getNumRegistersForCallingConv(
12697 // Use f32 to pass f16.
12698 if (VT == MVT::f16 && Subtarget.hasBasicF())
12699 return 1;
12700
12702}
12703
12705 const SDValue Op, KnownBits &Known, const APInt &DemandedElts,
12706 const SelectionDAG &DAG, unsigned Depth) const {
12707 unsigned Opc = Op.getOpcode();
12708 Known.resetAll();
12709 switch (Opc) {
12710 default:
12711 break;
12712 case LoongArchISD::VANYNONZERO:
12713 case LoongArchISD::VALLZERO:
12714 case LoongArchISD::VALL_ZERO:
12715 case LoongArchISD::VANY_ZERO:
12716 case LoongArchISD::VALL_NONZERO:
12717 case LoongArchISD::VANY_NONZERO: {
12718 // MOVCF2GR zero-extend the i1 cond to GPR.
12719 Known.Zero.setBitsFrom(1);
12720 break;
12721 }
12722 case LoongArchISD::VPICK_ZEXT_ELT: {
12723 assert(isa<VTSDNode>(Op->getOperand(2)) && "Unexpected operand!");
12724 EVT VT = cast<VTSDNode>(Op->getOperand(2))->getVT();
12725 unsigned VTBits = VT.getScalarSizeInBits();
12726 assert(Known.getBitWidth() >= VTBits && "Unexpected width!");
12727 Known.Zero.setBitsFrom(VTBits);
12728 break;
12729 }
12730 case LoongArchISD::BSTRINS: {
12731 // BSTRINS X, Y, msb, lsb: replace X[msb:lsb] with Y[msb-lsb:0].
12732 unsigned Msb = Op.getConstantOperandVal(2);
12733 unsigned Lsb = Op.getConstantOperandVal(3);
12734 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
12735 KnownBits Known2 = DAG.computeKnownBits(Op.getOperand(1), Depth + 1);
12736 Known.insertBits(Known2.trunc(Msb - Lsb + 1), Lsb);
12737 break;
12738 }
12739 case LoongArchISD::BSTRPICK: {
12740 // BSTRPICK X, msb, lsb: zero-extend X[msb:lsb].
12741 unsigned BitWidth = Known.getBitWidth();
12742 unsigned Msb = Op.getConstantOperandVal(1);
12743 unsigned Lsb = Op.getConstantOperandVal(2);
12744 Known = DAG.computeKnownBits(Op.getOperand(0), Depth + 1);
12745 Known = Known.extractBits(Msb - Lsb + 1, Lsb).zext(BitWidth);
12746 break;
12747 }
12748 }
12749}
12750
12752 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
12753 unsigned Depth) const {
12754 switch (Op.getOpcode()) {
12755 default:
12756 break;
12757 case LoongArchISD::BSTRINS: {
12758 unsigned BitWidth = Op.getScalarValueSizeInBits();
12759 unsigned Msb = Op.getConstantOperandVal(2);
12760 unsigned Lsb = Op.getConstantOperandVal(3);
12761 // The bits above msb come from X.
12762 if (Msb != BitWidth - 1) {
12763 unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(0), Depth + 1);
12764 return std::min(Tmp, BitWidth - 1 - Msb);
12765 }
12766 // The high bits come from Y shifted left by lsb.
12767 unsigned Tmp = DAG.ComputeNumSignBits(Op.getOperand(1), Depth + 1);
12768 return Tmp > Lsb ? Tmp - Lsb : 1;
12769 }
12770 }
12771 return 1;
12772}
12773
12775 SDValue Op, const APInt &OriginalDemandedBits,
12776 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
12777 unsigned Depth) const {
12778 EVT VT = Op.getValueType();
12779 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
12780 unsigned Opc = Op.getOpcode();
12781 switch (Opc) {
12782 default:
12783 break;
12784 case LoongArchISD::CRC_W_B_W:
12785 case LoongArchISD::CRC_W_H_W:
12786 case LoongArchISD::CRCC_W_B_W:
12787 case LoongArchISD::CRCC_W_H_W: {
12788 KnownBits KnownSrc;
12789 APInt DemandedSrcBits =
12790 APInt::getLowBitsSet(BitWidth, (Opc == LoongArchISD::CRC_W_B_W ||
12791 Opc == LoongArchISD::CRCC_W_B_W)
12792 ? 8
12793 : 16);
12794 return SimplifyDemandedBits(Op.getOperand(1), DemandedSrcBits,
12795 OriginalDemandedElts, KnownSrc, TLO, Depth + 1);
12796 }
12797 case LoongArchISD::VMSKLTZ:
12798 case LoongArchISD::XVMSKLTZ: {
12799 SDValue Src = Op.getOperand(0);
12800 MVT SrcVT = Src.getSimpleValueType();
12801 unsigned SrcBits = SrcVT.getScalarSizeInBits();
12802 unsigned NumElts = SrcVT.getVectorNumElements();
12803
12804 // If we don't need the sign bits at all just return zero.
12805 if (OriginalDemandedBits.countr_zero() >= NumElts)
12806 return TLO.CombineTo(Op, TLO.DAG.getConstant(0, SDLoc(Op), VT));
12807
12808 // Only demand the vector elements of the sign bits we need.
12809 APInt KnownUndef, KnownZero;
12810 APInt DemandedElts = OriginalDemandedBits.zextOrTrunc(NumElts);
12811 if (SimplifyDemandedVectorElts(Src, DemandedElts, KnownUndef, KnownZero,
12812 TLO, Depth + 1))
12813 return true;
12814
12815 Known.Zero = KnownZero.zext(BitWidth);
12816 Known.Zero.setHighBits(BitWidth - NumElts);
12817
12818 // [X]VMSKLTZ only uses the MSB from each vector element.
12819 KnownBits KnownSrc;
12820 APInt DemandedSrcBits = APInt::getSignMask(SrcBits);
12821 if (SimplifyDemandedBits(Src, DemandedSrcBits, DemandedElts, KnownSrc, TLO,
12822 Depth + 1))
12823 return true;
12824
12825 if (KnownSrc.One[SrcBits - 1])
12826 Known.One.setLowBits(NumElts);
12827 else if (KnownSrc.Zero[SrcBits - 1])
12828 Known.Zero.setLowBits(NumElts);
12829
12830 // Attempt to avoid multi-use ops if we don't need anything from it.
12832 Src, DemandedSrcBits, DemandedElts, TLO.DAG, Depth + 1))
12833 return TLO.CombineTo(Op, TLO.DAG.getNode(Opc, SDLoc(Op), VT, NewSrc));
12834 return false;
12835 }
12836 }
12837
12839 Op, OriginalDemandedBits, OriginalDemandedElts, Known, TLO, Depth);
12840}
12841
12843 unsigned Opc = VecOp.getOpcode();
12844
12845 // Assume target opcodes can't be scalarized.
12846 // TODO - do we have any exceptions?
12847 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opc))
12848 return false;
12849
12850 // If the vector op is not supported, try to convert to scalar.
12851 EVT VecVT = VecOp.getValueType();
12853 return true;
12854
12855 // If the vector op is supported, but the scalar op is not, the transform may
12856 // not be worthwhile.
12857 EVT ScalarVT = VecVT.getScalarType();
12858 return isOperationLegalOrCustomOrPromote(Opc, ScalarVT);
12859}
12860
12863 unsigned Index) const {
12866
12867 // Extract a 128-bit subvector from index 0 of a 256-bit vector is free.
12868 if (Index == 0)
12871}
12872
12874 unsigned Index) const {
12875 EVT EltVT = VT.getScalarType();
12876
12877 // Extract a scalar FP value from index 0 of a vector is free.
12878 return (EltVT == MVT::f32 || EltVT == MVT::f64) && Index == 0;
12879}
12880
12882 const MachineFunction &MF) const {
12883
12884 // If the function specifically requests inline stack probes, emit them.
12885 if (MF.getFunction().hasFnAttribute("probe-stack"))
12886 return MF.getFunction().getFnAttribute("probe-stack").getValueAsString() ==
12887 "inline-asm";
12888
12889 return false;
12890}
12891
12893 Align StackAlign) const {
12894 // The default stack probe size is 4096 if the function has no
12895 // stack-probe-size attribute.
12896 const Function &Fn = MF.getFunction();
12897 unsigned StackProbeSize =
12898 Fn.getFnAttributeAsParsedInteger("stack-probe-size", 4096);
12899 // Round down to the stack alignment.
12900 StackProbeSize = alignDown(StackProbeSize, StackAlign.value());
12901 return StackProbeSize ? StackProbeSize : StackAlign.value();
12902}
12903
12904SDValue
12905LoongArchTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
12906 SelectionDAG &DAG) const {
12908 if (!hasInlineStackProbe(MF))
12909 return SDValue();
12910
12911 const MVT GRLenVT = Subtarget.getGRLenVT();
12912 // Get the inputs.
12913 SDValue Chain = Op.getOperand(0);
12914 SDValue Size = Op.getOperand(1);
12915
12916 const MaybeAlign Align =
12917 cast<ConstantSDNode>(Op.getOperand(2))->getMaybeAlignValue();
12918 const SDLoc dl(Op);
12919 const EVT VT = Op.getValueType();
12920
12921 // Construct the new SP value in a GPR.
12922 SDValue SP = DAG.getCopyFromReg(Chain, dl, LoongArch::R3, GRLenVT);
12923 Chain = SP.getValue(1);
12924 SP = DAG.getNode(ISD::SUB, dl, GRLenVT, SP, Size);
12925 if (Align)
12926 SP = DAG.getNode(ISD::AND, dl, VT, SP.getValue(0),
12927 DAG.getSignedConstant(-Align->value(), dl, VT));
12928
12929 // Set the real SP to the new value with a probing loop.
12930 Chain = DAG.getNode(LoongArchISD::PROBED_ALLOCA, dl, MVT::Other, Chain, SP);
12931 return DAG.getMergeValues({SP, Chain}, dl);
12932}
12933
12936 MachineBasicBlock *MBB) const {
12937 MachineFunction &MF = *MBB->getParent();
12938 MachineBasicBlock::iterator MBBI = MI.getIterator();
12939 DebugLoc DL = MBB->findDebugLoc(MBBI);
12940 const Register TargetReg = MI.getOperand(0).getReg();
12941
12942 const LoongArchInstrInfo *TII = Subtarget.getInstrInfo();
12943 const bool IsLA64 = Subtarget.is64Bit();
12944 const Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
12945 const LoongArchTargetLowering *TLI = Subtarget.getTargetLowering();
12946 const uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
12947
12948 MachineFunction::iterator MBBInsertPoint = std::next(MBB->getIterator());
12949 MachineBasicBlock *const LoopTestMBB =
12950 MF.CreateMachineBasicBlock(MBB->getBasicBlock());
12951 MF.insert(MBBInsertPoint, LoopTestMBB);
12952 MachineBasicBlock *const ExitMBB =
12953 MF.CreateMachineBasicBlock(MBB->getBasicBlock());
12954 MF.insert(MBBInsertPoint, ExitMBB);
12955 const Register SPReg = LoongArch::R3;
12956 const Register ScratchReg =
12957 MF.getRegInfo().createVirtualRegister(&LoongArch::GPRRegClass);
12958
12959 // ScratchReg = ProbeSize
12960 TII->movImm(*MBB, MBBI, DL, ScratchReg, ProbeSize, MachineInstr::NoFlags);
12961
12962 // LoopTest:
12963 // sub.{w/d} $sp, $sp, ScratchReg
12964 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL,
12965 TII->get(IsLA64 ? LoongArch::SUB_D : LoongArch::SUB_W), SPReg)
12966 .addReg(SPReg)
12967 .addReg(ScratchReg);
12968
12969 // st.{w/d} $zero, $sp, 0
12970 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL,
12971 TII->get(IsLA64 ? LoongArch::ST_D : LoongArch::ST_W))
12972 .addReg(LoongArch::R0)
12973 .addReg(SPReg)
12974 .addImm(0);
12975
12976 // bltu TargetReg, $sp, LoopTest
12977 BuildMI(*LoopTestMBB, LoopTestMBB->end(), DL, TII->get(LoongArch::BLTU))
12978 .addReg(TargetReg)
12979 .addReg(SPReg)
12980 .addMBB(LoopTestMBB);
12981
12982 // move $sp, TargetReg
12983 BuildMI(*ExitMBB, ExitMBB->end(), DL, TII->get(LoongArch::OR), SPReg)
12984 .addReg(TargetReg)
12985 .addReg(LoongArch::R0);
12986
12987 ExitMBB->splice(ExitMBB->end(), MBB, std::next(MBBI), MBB->end());
12989
12990 LoopTestMBB->addSuccessor(ExitMBB);
12991 LoopTestMBB->addSuccessor(LoopTestMBB);
12992 MBB->addSuccessor(LoopTestMBB);
12993
12994 MI.eraseFromParent();
12995 MF.getInfo<LoongArchMachineFunctionInfo>()->setDynamicAllocation();
12996 return ExitMBB->begin()->getParent();
12997}
static MCRegister MatchRegisterName(StringRef Name)
static SDValue performSHLCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
If the operand is a bitwise AND with a constant RHS, and the shift has a constant RHS and is the only...
static bool checkValueWidth(SDValue V, unsigned width, ISD::LoadExtType &ExtType)
static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
unsigned NumParts
return SDValue()
static SDValue performANDCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI)
static SDValue performSELECT_CCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
static SDValue performSETCCCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, SelectionDAG &DAG)
const AArch64Subtarget & Subtarget
const TargetLowering & TLI
assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")
const AArch64InstrInfo & TII
static msgpack::DocNode getNode(msgpack::DocNode DN, msgpack::Type Type, MCValue Val)
uint64_t uint64_t
uint64_t Imm
MachineBasicBlock & MBB
MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL
MachineBasicBlock MachineBasicBlock::iterator MBBI
static MCRegister MatchRegisterAltName(StringRef Name)
Maps from the set of all alternative registernames to a register number.
Function Alias Analysis Results
static uint64_t getConstant(const Value *IndexValue)
static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty, SelectionDAG &DAG, unsigned Flags)
bpf unwind BPF Unwind Lower
#define X(NUM, ENUM, NAME)
Definition ELF.h:665
static GCRegistry::Add< ShadowStackGC > C("shadow-stack", "Very portable GC for uncooperative code generators")
static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")
static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val, const CCValAssign &VA, const SDLoc &DL)
static MachineBasicBlock * emitSelectPseudo(MachineInstr &MI, MachineBasicBlock *BB, unsigned Opcode)
static SDValue unpackFromRegLoc(const CSKYSubtarget &Subtarget, SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const SDLoc &DL)
#define clEnumValN(ENUMVAL, FLAGNAME, DESC)
static bool isSigned(unsigned Opcode)
IRTranslator LLVM IR MI
const size_t AbstractManglingParser< Derived, Alloc >::NumOps
const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]
#define RegName(no)
static SDValue performINTRINSIC_WO_CHAINCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performADDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
const MCPhysReg ArgFPR32s[]
static SDValue lower128BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Dispatching routine to lower various 128-bit LoongArch vector shuffles.
static SDValue lowerVECTOR_SHUFFLE_XVSHUF4I(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVSHUF4I (if possible).
const MCPhysReg ArgVRs[]
static SDValue lowerVECTOR_SHUFFLE_VPICKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPICKEV (if possible).
static SDValue combineSelectToBinOp(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVPICKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPICKOD (if possible).
static SDValue unpackF64OnLA32DSoftABI(SelectionDAG &DAG, SDValue Chain, const CCValAssign &VA, const CCValAssign &HiVA, const SDLoc &DL)
static bool fitsRegularPattern(typename SmallVectorImpl< ValType >::const_iterator Begin, unsigned CheckStride, typename SmallVectorImpl< ValType >::const_iterator End, ValType ExpectedIndex, unsigned ExpectedIndexStride)
Determine whether a range fits a regular pattern of values.
static SDValue lowerVECTOR_SHUFFLE_IsReverse(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE whose result is the reversed source vector.
static SDValue PromoteMaskArithmetic(SDValue N, const SDLoc &DL, EVT VT, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned Depth)
static SDValue performUINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performHorizWideningCombine(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static std::pair< SDValue, EVT > stripSignExtendInReg(SDValue V)
static SDValue emitIntrinsicErrorMessage(SDValue Op, StringRef ErrorMsg, SelectionDAG &DAG)
static SDValue ExtendSrcToDst(SDNode *N, SelectionDAG &DAG, unsigned ExtendOp)
static cl::opt< bool > ZeroDivCheck("loongarch-check-zero-division", cl::Hidden, cl::desc("Trap on integer division by zero."), cl::init(false))
static SDValue lowerVECTOR_SHUFFLE_XVPERMI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERMI (if possible).
static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VSHUF(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VSHUF.
static int getEstimateRefinementSteps(EVT VT, const LoongArchSubtarget &Subtarget)
static bool isSupportedReciprocalEstimateType(EVT VT, const LoongArchSubtarget &Subtarget)
static void emitErrorAndReplaceIntrinsicResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, StringRef ErrorMsg, bool WithChain=true)
static SDValue lowerVECTOR_SHUFFLEAsByteRotate(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE as byte rotate (if possible).
static SDValue checkIntrinsicImmArg(SDValue Op, unsigned ImmOp, SelectionDAG &DAG, bool IsSigned=false)
static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVINSVE0(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVINSVE0 (if possible).
static SDValue performMOVFR2GR_SCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VILVH(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VILVH (if possible).
static SDValue performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performDemandedBitsCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI)
static bool CC_LoongArch(const DataLayout &DL, LoongArchABI::ABI ABI, unsigned ValNo, MVT ValVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, CCState &State, bool IsRet, Type *OrigTy)
static std::tuple< unsigned, SDValue, EVT > matchBinOpWithSharedOperand(SDValue BinV, SDValue X)
static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG)
static SDValue performSPLIT_PAIR_F64Combine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performBITCASTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performSRLCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static MachineBasicBlock * emitSplitPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorBitSetImm(SDNode *Node, SelectionDAG &DAG)
static SDValue performSETCC_BITCASTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
@ NoMaterializeFPImm
@ MaterializeFPImm2Ins
@ MaterializeFPImm5Ins
@ MaterializeFPImm6Ins
@ MaterializeFPImm3Ins
@ MaterializeFPImm4Ins
static SDValue performEXTENDCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_XVPACKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPACKOD (if possible).
static bool buildVPERMIInfo(ArrayRef< int > Mask, SDValue V1, SDValue V2, SmallVectorImpl< SDValue > &SrcVec, unsigned &MaskImm)
static std::optional< bool > matchSetCC(SDValue LHS, SDValue RHS, ISD::CondCode CC, SDValue Val)
static SDValue combineAndNotIntoVANDN(SDNode *N, const SDLoc &DL, SelectionDAG &DAG)
Try to fold: (and (xor X, -1), Y) -> (vandn X, Y).
static SDValue lowerBUILD_VECTORAsBroadCastLoad(BuildVectorSDNode *BVOp, const SDLoc &DL, SelectionDAG &DAG)
#define CRC_CASE_EXT_BINARYOP(NAME, NODE)
static SDValue lowerVectorBitRevImm(SDNode *Node, SelectionDAG &DAG)
static bool checkBitcastSrcVectorSize(SDValue Src, unsigned Size, unsigned Depth)
static bool isConstantSplatVector(SDValue N, APInt &SplatValue, unsigned MinSizeInBits)
static SDValue lowerVECTOR_SHUFFLEAsShift(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, const APInt &Zeroable)
Lower VECTOR_SHUFFLE as shift (if possible).
static SDValue lowerVECTOR_SHUFFLE_VSHUF4I(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VSHUF4I (if possible).
static SDValue truncateVecElts(SDNode *Node, SelectionDAG &DAG)
static bool CC_LoongArch_GHC(unsigned ValNo, MVT ValVT, MVT LocVT, CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags, Type *OrigTy, CCState &State)
static MachineBasicBlock * insertDivByZeroTrap(MachineInstr &MI, MachineBasicBlock *MBB)
static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE_VEXTRINS(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VEXTRINS (if possible).
static SDValue lowerVectorBitClear(SDNode *Node, SelectionDAG &DAG)
static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VPACKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPACKEV (if possible).
static MachineBasicBlock * emitPseudoVMSKCOND(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue performSINT_TO_FPCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performVANDNCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
Do target-specific dag combines on LoongArchISD::VANDN nodes.
static void replaceVPICKVE2GRResults(SDNode *Node, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned ResOp)
static SDValue lowerVECTOR_SHUFFLEAsZeroOrAnyExtend(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const APInt &Zeroable)
Lower VECTOR_SHUFFLE as ZERO_EXTEND Or ANY_EXTEND (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVPERMIQ(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERMI_Q (if possible).
static SDValue legalizeIntrinsicImmArg(SDNode *Node, unsigned ImmOp, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, bool IsSigned=false)
static cl::opt< MaterializeFPImm > MaterializeFPImmInsNum("loongarch-materialize-float-imm", cl::Hidden, cl::desc("Maximum number of instructions used (including code sequence " "to generate the value and moving the value to FPR) when " "materializing floating-point immediates (default = 3)"), cl::init(MaterializeFPImm3Ins), cl::values(clEnumValN(NoMaterializeFPImm, "0", "Use constant pool"), clEnumValN(MaterializeFPImm2Ins, "2", "Materialize FP immediate within 2 instructions"), clEnumValN(MaterializeFPImm3Ins, "3", "Materialize FP immediate within 3 instructions"), clEnumValN(MaterializeFPImm4Ins, "4", "Materialize FP immediate within 4 instructions"), clEnumValN(MaterializeFPImm5Ins, "5", "Materialize FP immediate within 5 instructions"), clEnumValN(MaterializeFPImm6Ins, "6", "Materialize FP immediate within 6 instructions " "(behaves same as 5 on loongarch64)")))
static SDValue emitIntrinsicWithChainErrorMessage(SDValue Op, StringRef ErrorMsg, SelectionDAG &DAG)
const MCPhysReg ArgXRs[]
static bool CC_LoongArchAssign2GRLen(unsigned GRLen, CCState &State, CCValAssign VA1, ISD::ArgFlagsTy ArgFlags1, unsigned ValNo2, MVT ValVT2, MVT LocVT2, ISD::ArgFlagsTy ArgFlags2)
static unsigned getLoongArchWOpcode(unsigned Opcode)
const MCPhysReg ArgFPR64s[]
static MachineBasicBlock * emitPseudoCTPOP(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue performMOVGR2FR_WCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
#define IOCSRWR_CASE(NAME, NODE)
#define CRC_CASE_EXT_UNARYOP(NAME, NODE)
static SDValue lowerVECTOR_SHUFFLE_VPACKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPACKOD (if possible).
static SDValue signExtendBitcastSrcVector(SelectionDAG &DAG, EVT SExtVT, SDValue Src, const SDLoc &DL)
static SDValue isNOT(SDValue V, SelectionDAG &DAG)
static SDValue lower256BitShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Dispatching routine to lower various 256-bit LoongArch vector shuffles.
static SDValue lowerVECTOR_SHUFFLE_VREPLVEI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VREPLVEI (if possible).
static MachineBasicBlock * emitPseudoXVINSGR2VR(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
const MCPhysReg PreserveNoneArgGPRs[]
static void fillVector(ArrayRef< SDValue > Ops, SelectionDAG &DAG, SDLoc DL, const LoongArchSubtarget &Subtarget, SDValue &Vector, EVT ResTy)
static SDValue fillSubVectorFromBuildVector(BuildVectorSDNode *Node, SelectionDAG &DAG, SDLoc DL, const LoongArchSubtarget &Subtarget, EVT ResTy, unsigned first)
static bool isSelectPseudo(MachineInstr &MI)
static SDValue foldVMskZeroTest(SDValue LHS, SDValue RHS, ISD::CondCode CC, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorSplatImm(SDNode *Node, unsigned ImmOp, SelectionDAG &DAG, bool IsSigned=false)
const MCPhysReg ArgGPRs[]
static SDValue lowerVECTOR_SHUFFLE_XVPERM(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVPERM (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVILVL(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVILVL (if possible).
static SDValue performVNi128AddSubCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue performFP_TO_INTCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLE_VPERMI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into VPERMI (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVEXTRINS(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVEXTRINS (if possible).
static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG, int NumOp, unsigned ExtOpc=ISD::ANY_EXTEND)
static void replaceVecCondBranchResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget, unsigned ResOp)
#define ASRT_LE_GT_CASE(NAME)
static SDValue lowerVECTOR_SHUFFLE_XVPACKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPACKEV (if possible).
static SDValue matchDeinterleaveBuildVector(SDValue N, unsigned &StartIndex)
static SDValue performBR_CCCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static void computeZeroableShuffleElements(ArrayRef< int > Mask, SDValue V1, SDValue V2, APInt &KnownUndef, APInt &KnownZero)
Compute whether each element of a shuffle is zeroable.
static SDValue combineFP_ROUND(SDValue N, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
static bool isCheapVNi128VectorOperand(SDValue V, MVT PartVT, SelectionDAG &DAG)
static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue widenShuffleMask(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
static bool canonicalizeShuffleVectorByLane(const SDLoc &DL, MutableArrayRef< int > Mask, MVT VT, SDValue &V1, SDValue &V2, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Shuffle vectors by lane to generate more optimized instructions.
static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG)
static SDValue lowerVECTOR_SHUFFLE_XVILVH(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVILVH (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVSHUF(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVSHUF (if possible).
static void replaceCMP_XCHG_128Results(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG)
static SDValue lowerVectorPickVE2GR(SDNode *N, SelectionDAG &DAG, unsigned ResOp)
static SDValue performBITREV_WCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static bool isVectorRegisterVT(EVT VT, SelectionDAG &DAG)
static SDValue matchHalfOf128BitLanes(SDValue N, bool isLow)
#define IOCSRRD_CASE(NAME, NODE)
static int matchShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, ArrayRef< int > Mask)
Attempts to match vector shuffle as byte rotation.
static SDValue lowerVECTOR_SHUFFLE_XVREPLVE0(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVREPLVE0 (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVPICKEV(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into XVPICKEV (if possible).
static SDValue lowerVECTOR_SHUFFLE_XVREPLVEI(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
Lower VECTOR_SHUFFLE into XVREPLVEI (if possible).
static int matchShuffleAsShift(MVT &ShiftVT, unsigned &Opcode, unsigned ScalarSizeInBits, ArrayRef< int > Mask, int MaskOffset, const APInt &Zeroable)
Attempts to match a shuffle mask against the VBSLL, VBSRL, VSLLI and VSRLI instruction.
static SDValue lowerVECTOR_SHUFFLE_VILVL(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VILVL (if possible).
static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG, TargetLowering::DAGCombinerInfo &DCI, const LoongArchSubtarget &Subtarget)
static SDValue lowerVectorBitClearImm(SDNode *Node, SelectionDAG &DAG)
static MachineBasicBlock * emitBuildPairF64Pseudo(MachineInstr &MI, MachineBasicBlock *BB, const LoongArchSubtarget &Subtarget)
static SDValue lowerVECTOR_SHUFFLEAsLanePermuteAndShuffle(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE as lane permute and then shuffle (if possible).
static bool hasOnlyVectorUsers(LoadSDNode *Ld, SelectionDAG &DAG)
static void replaceINTRINSIC_WO_CHAINResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG, const LoongArchSubtarget &Subtarget)
#define CSR_CASE(ID)
static SDValue MergeBlocksConvert(SDNode *N, SelectionDAG &DAG, unsigned Opcode, unsigned BlockBits)
static SDValue lowerVECTOR_SHUFFLE_VPICKOD(const SDLoc &DL, ArrayRef< int > Mask, MVT VT, SDValue V1, SDValue V2, SelectionDAG &DAG)
Lower VECTOR_SHUFFLE into VPICKOD (if possible).
static Intrinsic::ID getIntrinsicForMaskedAtomicRMWBinOp(unsigned GRLen, AtomicRMWInst::BinOp BinOp)
static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS, ISD::CondCode &CC, SelectionDAG &DAG)
static Register allocateArgGPR(CCState &State)
static bool isRepeatedShuffleMask(unsigned LaneSizeInBits, MVT VT, ArrayRef< int > Mask, SmallVectorImpl< int > &RepeatedMask)
Test whether a shuffle mask is equivalent within each sub-lane.
static SDValue convertRMEncoding(SelectionDAG &DAG, const SDLoc &DL, MVT GRLenVT, SDValue RMValue)
#define F(x, y, z)
Definition MD5.cpp:54
#define I(x, y, z)
Definition MD5.cpp:57
Register Reg
Register const TargetRegisterInfo * TRI
Promote Memory to Register
Definition Mem2Reg.cpp:110
#define T
static CodeModel::Model getCodeModel(const PPCSubtarget &S, const TargetMachine &TM, const MachineOperand &MO)
static constexpr MCPhysReg SPReg
MachineBasicBlock * Folded
SmallVector< MachineOperand, 4 > Cond
RISCVCC::CondCode CC
MachineRegisterInfo & MRI
MachineInstr *& Lo
static bool isValid(const char C)
Returns true if C is a valid mangled character: <0-9a-zA-Z_>.
This file defines the SmallSet class.
This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...
#define STATISTIC(VARNAME, DESC)
Definition Statistic.h:171
This file contains some functions that are useful when dealing with strings.
#define LLVM_DEBUG(...)
Definition Debug.h:119
static bool inRange(const MCExpr *Expr, int64_t MinValue, int64_t MaxValue, bool AllowSymbol=false)
static TableGen::Emitter::Opt Y("gen-skeleton-entry", EmitSkeleton, "Generate example skeleton entry")
static bool isSequentialOrUndefInRange(ArrayRef< int > Mask, unsigned Pos, unsigned Size, int Low, int Step=1)
Return true if every element in Mask, beginning from position Pos and ending in Pos + Size,...
Value * RHS
Value * LHS
bool isZero() const
Definition APFloat.h:1579
APInt bitcastToAPInt() const
Definition APFloat.h:1475
Class for arbitrary precision integers.
Definition APInt.h:78
static APInt getAllOnes(unsigned numBits)
Return an APInt of a specified width with all bits set.
Definition APInt.h:230
LLVM_ABI APInt zext(unsigned width) const
Zero extend to a new width.
Definition APInt.cpp:1057
static APInt getSignMask(unsigned BitWidth)
Get the SignMask for a specific bit width.
Definition APInt.h:225
uint64_t getZExtValue() const
Get zero extended value.
Definition APInt.h:1560
LLVM_ABI APInt zextOrTrunc(unsigned width) const
Zero extend or truncate to width.
Definition APInt.cpp:1078
LLVM_ABI APInt trunc(unsigned width) const
Truncate to new width.
Definition APInt.cpp:970
void setBit(unsigned BitPosition)
Set the given bit to 1 whose position is given as "bitPosition".
Definition APInt.h:1350
bool isAllOnes() const
Determine if all bits are set. This is true for zero-width values.
Definition APInt.h:367
bool isZero() const
Determine if this value is zero, i.e. all bits are clear.
Definition APInt.h:376
LLVM_ABI APInt urem(const APInt &RHS) const
Unsigned remainder operation.
Definition APInt.cpp:1695
unsigned getBitWidth() const
Return the number of bits in the APInt.
Definition APInt.h:1508
unsigned countr_zero() const
Count the number of trailing zero bits.
Definition APInt.h:1659
bool isSignedIntN(unsigned N) const
Check if this APInt has an N-bits signed integer value.
Definition APInt.h:431
bool isSubsetOf(const APInt &RHS) const
This operation checks that all bits set in this APInt are also set in RHS.
Definition APInt.h:1261
static APInt getLowBitsSet(unsigned numBits, unsigned loBitsSet)
Constructs an APInt value that has the bottom loBitsSet bits set.
Definition APInt.h:302
static APInt getZero(unsigned numBits)
Get the '0' value for the specified bit-width.
Definition APInt.h:196
bool isOne() const
Determine if this is a value of 1.
Definition APInt.h:385
static APInt getBitsSetFrom(unsigned numBits, unsigned loBit)
Constructs an APInt value that has a contiguous range of bits set.
Definition APInt.h:282
APInt lshr(unsigned shiftAmt) const
Logical right-shift function.
Definition APInt.h:853
This class represents an incoming formal argument to a Function.
Definition Argument.h:32
unsigned getArgNo() const
Return the index of this formal argument in its containing function.
Definition Argument.h:50
Represent a constant reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:40
size_t size() const
Get the array size.
Definition ArrayRef.h:141
An instruction that atomically checks whether a specified value is in a memory location,...
AtomicOrdering getFailureOrdering() const
Returns the failure ordering constraint of this cmpxchg instruction.
an instruction that atomically reads a memory location, combines it with another value,...
Align getAlign() const
Return the alignment of the memory that is being allocated by the instruction.
BinOp
This enumeration lists the possible modifications atomicrmw can make.
@ Add
*p = old + v
@ USubCond
Subtract only if no unsigned overflow.
@ Min
*p = old <signed v ? old : v
@ Sub
*p = old - v
@ And
*p = old & v
@ Xor
*p = old ^ v
@ USubSat
*p = usub.sat(old, v) usub.sat matches the behavior of llvm.usub.sat.
@ UIncWrap
Increment one up to a maximum value.
@ Max
*p = old >signed v ? old : v
@ UMin
*p = old <unsigned v ? old : v
@ UMax
*p = old >unsigned v ? old : v
@ UDecWrap
Decrement one until a minimum value or zero.
@ Nand
*p = ~(old & v)
Value * getPointerOperand()
bool isFloatingPointOperation() const
BinOp getOperation() const
SyncScope::ID getSyncScopeID() const
Returns the synchronization scope ID of this rmw instruction.
AtomicOrdering getOrdering() const
Returns the ordering constraint of this rmw instruction.
LLVM_ABI StringRef getValueAsString() const
Return the attribute's value as a string.
LLVM Basic Block Representation.
Definition BasicBlock.h:62
bool test(unsigned Idx) const
Returns true if bit Idx is set.
Definition BitVector.h:483
size_type count() const
Returns the number of bits which are set.
Definition BitVector.h:182
A "pseudo-class" with methods for operating on BUILD_VECTORs.
LLVM_ABI bool isConstantSplat(APInt &SplatValue, APInt &SplatUndef, unsigned &SplatBitSize, bool &HasAnyUndefs, unsigned MinSplatBits=0, bool isBigEndian=false) const
Check if this is a constant splat, and if so, find the smallest element size that splats the vector.
CCState - This class holds information needed while lowering arguments and return values.
unsigned getFirstUnallocated(ArrayRef< MCPhysReg > Regs) const
getFirstUnallocated - Return the index of the first unallocated register in the set,...
LLVM_ABI void AnalyzeCallOperands(const SmallVectorImpl< ISD::OutputArg > &Outs, CCAssignFn Fn)
AnalyzeCallOperands - Analyze the outgoing arguments to a call, incorporating info about the passed v...
uint64_t getStackSize() const
Returns the size of the currently allocated portion of the stack.
LLVM_ABI void AnalyzeFormalArguments(const SmallVectorImpl< ISD::InputArg > &Ins, CCAssignFn Fn)
AnalyzeFormalArguments - Analyze an array of argument values, incorporating info about the formals in...
CCValAssign - Represent assignment of one arg/retval to a location.
static CCValAssign getPending(unsigned ValNo, MVT ValVT, MVT LocVT, LocInfo HTP, unsigned ExtraInfo=0)
Register getLocReg() const
LocInfo getLocInfo() const
static CCValAssign getReg(unsigned ValNo, MVT ValVT, MCRegister Reg, MVT LocVT, LocInfo HTP, bool IsCustom=false)
static CCValAssign getCustomReg(unsigned ValNo, MVT ValVT, MCRegister Reg, MVT LocVT, LocInfo HTP)
static CCValAssign getMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP, bool IsCustom=false)
bool needsCustom() const
int64_t getLocMemOffset() const
unsigned getValNo() const
static CCValAssign getCustomMem(unsigned ValNo, MVT ValVT, int64_t Offset, MVT LocVT, LocInfo HTP)
Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...
LLVM_ABI bool isMustTailCall() const
Tests if this call site must be tail call optimized.
iterator_range< User::op_iterator > args()
Iteration adapter for range-for loops.
This class represents a function call, abstracting a target machine's calling convention.
bool isTailCall() const
const APFloat & getValueAPF() const
This is the shared class of boolean and integer constants.
Definition Constants.h:87
bool isMinusOne() const
This function will return true iff every bit in this constant is set to true.
Definition Constants.h:231
bool isZero() const
This is just a convenience method to make client code smaller for a common code.
Definition Constants.h:219
uint64_t getZExtValue() const
int64_t getSExtValue() const
This is an important base class in LLVM.
Definition Constant.h:44
A parsed version of the target data layout string in and methods for querying it.
Definition DataLayout.h:64
unsigned getPointerSizeInBits(unsigned AS=0) const
The size in bits of the pointer representation in a given address space.
Definition DataLayout.h:504
LLVM_ABI Align getPrefTypeAlign(Type *Ty) const
Returns the preferred stack/global alignment for the specified type.
A debug info location.
Definition DebugLoc.h:126
FunctionType * getFunctionType() const
Returns the FunctionType for me.
Definition Function.h:212
iterator_range< arg_iterator > args()
Definition Function.h:886
Attribute getFnAttribute(Attribute::AttrKind Kind) const
Return the attribute for the given attribute kind.
Definition Function.cpp:766
uint64_t getFnAttributeAsParsedInteger(StringRef Kind, uint64_t Default=0) const
For a string attribute Kind, parse attribute as an integer.
Definition Function.cpp:778
CallingConv::ID getCallingConv() const
getCallingConv()/setCallingConv(CC) - These method get and set the calling convention of this functio...
Definition Function.h:273
LLVMContext & getContext() const
getContext - Return a reference to the LLVMContext associated with this function.
Definition Function.cpp:352
Argument * getArg(unsigned i) const
Definition Function.h:880
bool hasFnAttribute(Attribute::AttrKind Kind) const
Return true if the function has the attribute.
Definition Function.cpp:731
bool isDSOLocal() const
Common base class shared among various IRBuilders.
Definition IRBuilder.h:111
This provides a uniform API for creating instructions and inserting them into a basic block: either a...
Definition IRBuilder.h:2990
LLVM_ABI const Module * getModule() const
Return the module owning the function this instruction belongs to or nullptr it the function does not...
LLVM_ABI InstListType::iterator eraseFromParent()
This method unlinks 'this' from the containing basic block and deletes it.
LLVM_ABI const DataLayout & getDataLayout() const
Get the data layout of the module this instruction belongs to.
Class to represent integer types.
This is an important class for using LLVM in a threaded context.
Definition LLVMContext.h:76
LLVM_ABI void emitError(const Instruction *I, const Twine &ErrorStr)
emitError - Emit an error message to the currently installed error handler with optional location inf...
LLVM_ABI void diagnose(const DiagnosticInfo &DI)
Report a message to the currently installed diagnostic handler.
This class is used to represent ISD::LOAD nodes.
const SDValue & getBasePtr() const
ISD::LoadExtType getExtensionType() const
Return whether this is a plain node, or one of the varieties of value-extending loads.
LoongArchMachineFunctionInfo - This class is derived from MachineFunctionInfo and contains private Lo...
void setIncomingIndirectArg(unsigned ArgIndex, Register Reg)
Register getIncomingIndirectArg(unsigned ArgIndex) const
bool isUsedByReturnOnly(SDNode *N, SDValue &Chain) const override
Return true if result of the specified node is used by a return node only.
SDValue PerformDAGCombine(SDNode *N, DAGCombinerInfo &DCI) const override
This method will be invoked for all target nodes and for any target-independent nodes that the target...
Register getExceptionPointerRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception address on entry to an ...
SDValue getSqrtEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps, bool &UseOneConstNR, bool Reciprocal) const override
Hooks for building estimates in place of slower divisions and square roots.
unsigned ComputeNumSignBitsForTargetNode(SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
This method can be implemented by targets that want to expose additional information about sign bits ...
bool isLegalICmpImmediate(int64_t Imm) const override
Return true if the specified immediate is legal icmp immediate, that is the target has icmp instructi...
TargetLowering::AtomicExpansionKind shouldExpandAtomicCmpXchgInIR(const AtomicCmpXchgInst *CI) const override
Returns how the given atomic cmpxchg should be expanded by the IR-level AtomicExpand pass.
bool isFPImmCheapInGPR(const APFloat &Imm, EVT VT) const
Returns true if building Imm in a GPR and moving it to a float register is cheaper than loading it fr...
Value * emitMaskedAtomicCmpXchgIntrinsic(IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr, Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const override
Perform a masked cmpxchg using a target-specific intrinsic.
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override
Return the ValueType of the result of SETCC operations.
std::pair< bool, uint64_t > isImmVLDILegalForMode1(const APInt &SplatValue, const unsigned SplatBitSize) const
Check if a constant splat can be generated using [x]vldi, where imm[12] is 1.
void getTgtMemIntrinsic(SmallVectorImpl< IntrinsicInfo > &Infos, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override
Given an intrinsic, checks if on the target the intrinsic will need to map to a MemIntrinsicNode (tou...
bool isFMAFasterThanFMulAndFAdd(const MachineFunction &MF, EVT VT) const override
Return true if an FMA operation is faster than a pair of fmul and fadd instructions.
bool hasInlineStackProbe(const MachineFunction &MF) const override
True if stack clash protection is enabled for this function.
SDValue LowerCall(TargetLowering::CallLoweringInfo &CLI, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower calls into the specified DAG.
Register getExceptionSelectorRegister(ExceptionHandling EH, const Constant *PersonalityFn) const override
If a physical register, this returns the register that receives the exception typeid on entry to a la...
bool decomposeMulByConstant(LLVMContext &Context, EVT VT, SDValue C) const override
Return true if it is profitable to transform an integer multiplication-by-constant into simpler opera...
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override
Return true if extraction of a scalar element from the given vector type at the given index is cheap.
LegalizeTypeAction getPreferredVectorAction(MVT VT) const override
Return the preferred vector type legalization action.
bool isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const override
Return true if sign-extension from FromTy to ToTy is cheaper than zero-extension.
bool allowsMisalignedMemoryAccesses(EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const override
Determine if the target supports unaligned memory accesses.
bool isCheapToSpeculateCtlz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic ctlz.
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override
This callback is invoked for operations that are unsupported by the target, which are registered to u...
bool shouldAlignPointerArgs(CallInst *CI, unsigned &MinSize, Align &PrefAlign) const override
Return true if the pointer arguments to CI should be aligned by aligning the object whose address is ...
Value * emitMaskedAtomicRMWIntrinsic(IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr, Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const override
Perform a masked atomicrmw using a target-specific intrinsic.
bool isZExtFree(SDValue Val, EVT VT2) const override
Return true if zero-extending the specific node Val to type VT2 is free (either because it's implicit...
void computeKnownBitsForTargetNode(const SDValue Op, KnownBits &Known, const APInt &DemandedElts, const SelectionDAG &DAG, unsigned Depth) const override
Determine which of the bits specified in Mask are known to be either zero or one and return them in t...
bool signExtendConstant(const ConstantInt *CI) const override
Return true if this constant should be sign extended when promoting to a larger type.
ExtractSubvectorCost getExtractSubvectorCost(EVT ResVT, EVT SrcVT, unsigned Index) const override
Return the cost of extracting a subvector of type ResVT from a vector of type SrcVT,...
TargetLowering::AtomicExpansionKind shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const override
Returns how the IR-level AtomicExpand pass should expand the given AtomicRMW, if at all.
bool isLegalAddImmediate(int64_t Imm) const override
Return true if the specified immediate is legal add immediate, that is the target has add instruction...
bool isCheapToSpeculateCttz(Type *Ty) const override
Return true if it is cheap to speculate a call to intrinsic cttz.
bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, unsigned AS, Instruction *I=nullptr) const override
Return true if the addressing mode represented by AM is legal for this target, for a load/store of th...
MachineBasicBlock * emitDynamicProbedAlloc(MachineInstr &MI, MachineBasicBlock *MBB) const
bool shouldSignExtendTypeInLibCall(Type *Ty, bool IsSigned) const override
Returns true if arguments should be sign-extended in lib calls.
bool shouldScalarizeBinop(SDValue VecOp) const override
Try to convert an extract element of a vector binary operation into an extract element followed by a ...
bool isFPImmVLDILegal(const APFloat &Imm, EVT VT) const
bool shouldExtendTypeInLibCall(EVT Type) const override
Returns true if arguments should be extended in lib calls.
Register getRegisterByName(const char *RegName, LLT VT, const MachineFunction &MF) const override
Return the register ID of the name passed in.
bool hasAndNot(SDValue Y) const override
Return true if the target has a bitwise and-not operation: X = ~A & B This can be used to simplify se...
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override
Return true if folding a constant offset with the given GlobalAddress is legal.
unsigned getStackProbeSize(const MachineFunction &MF, Align StackAlign) const
void ReplaceNodeResults(SDNode *N, SmallVectorImpl< SDValue > &Results, SelectionDAG &DAG) const override
This callback is invoked when a node result type is illegal for the target, and the operation was reg...
bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth) const override
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
void emitExpandAtomicRMW(AtomicRMWInst *AI) const override
Perform a atomicrmw expansion using a target-specific way.
ISD::NodeType getExtendForAtomicCmpSwapArg() const override
Returns how the platform's atomic compare and swap expects its comparison value to be extended (ZERO_...
LoongArchTargetLowering(const TargetMachine &TM, const LoongArchSubtarget &STI)
SDValue LowerReturn(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, const SmallVectorImpl< SDValue > &OutVals, const SDLoc &DL, SelectionDAG &DAG) const override
This hook must be implemented to lower outgoing return values, described by the Outs array,...
bool hasAndNotCompare(SDValue Y) const override
Return true if the target should transform: (X & Y) == Y ---> (~X & Y) == 0 (X & Y) !...
SDValue getRecipEstimate(SDValue Operand, SelectionDAG &DAG, int Enabled, int &RefinementSteps) const override
Return a reciprocal estimate value for the input operand.
bool canMergeStoresTo(unsigned AddressSpace, EVT MemVT, const MachineFunction &MF) const override
Returns if it's reasonable to merge stores to MemVT size.
bool CanLowerReturn(CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg, const SmallVectorImpl< ISD::OutputArg > &Outs, LLVMContext &Context, const Type *RetTy) const override
This hook should be implemented to check whether the return values described by the Outs array can fi...
SDValue LowerFormalArguments(SDValue Chain, CallingConv::ID CallConv, bool IsVarArg, const SmallVectorImpl< ISD::InputArg > &Ins, const SDLoc &DL, SelectionDAG &DAG, SmallVectorImpl< SDValue > &InVals) const override
This hook must be implemented to lower the incoming (formal) arguments, described by the Ins array,...
bool mayBeEmittedAsTailCall(const CallInst *CI) const override
Return true if the target may be able emit the call instruction as a tail call.
Wrapper class representing physical registers. Should be passed by value.
Definition MCRegister.h:41
bool hasFeature(unsigned Feature) const
Machine Value Type.
static MVT getFloatingPointVT(unsigned BitWidth)
bool is128BitVector() const
Return true if this is a 128-bit vector type.
SimpleValueType SimpleTy
uint64_t getScalarSizeInBits() const
unsigned getVectorNumElements() const
bool isVector() const
Return true if this is a vector value type.
bool isInteger() const
Return true if this is an integer or a vector integer type.
bool isScalableVector() const
Return true if this is a vector value type where the runtime length is machine dependent.
TypeSize getSizeInBits() const
Returns the size of the specified MVT in bits.
static auto fixedlen_vector_valuetypes()
bool is256BitVector() const
Return true if this is a 256-bit vector type.
bool isScalarInteger() const
Return true if this is an integer, not including vectors.
static MVT getVectorVT(MVT VT, unsigned NumElements)
MVT getVectorElementType() const
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
static MVT getIntegerVT(unsigned BitWidth)
MVT getDoubleNumVectorElementsVT() const
MVT getHalfNumVectorElementsVT() const
Return a VT for a vector type with the same element type but half the number of elements.
MVT getScalarType() const
If this is a vector, return the element type, otherwise return this.
MVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
LLVM_ABI void transferSuccessorsAndUpdatePHIs(MachineBasicBlock *FromMBB)
Transfers all the successors, as in transferSuccessors, and update PHI operands in the successor bloc...
void push_back(MachineInstr *MI)
void setCallFrameSize(unsigned N)
Set the call frame size on entry to this basic block.
const BasicBlock * getBasicBlock() const
Return the LLVM basic block that this instance corresponded to originally.
LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())
Add Succ as a successor of this MachineBasicBlock.
const MachineFunction * getParent() const
Return the MachineFunction containing this basic block.
void splice(iterator Where, MachineBasicBlock *Other, iterator From)
Take an instruction from MBB 'Other' at the position From, and insert it into this MBB right before '...
MachineInstrBundleIterator< MachineInstr > iterator
The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.
LLVM_ABI int CreateFixedObject(uint64_t Size, int64_t SPOffset, bool IsImmutable, bool isAliased=false)
Create a new object at a fixed location on the stack.
LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)
Create a new statically sized stack object, returning a nonnegative identifier to represent it.
void setFrameAddressIsTaken(bool T)
void setHasTailCall(bool V=true)
void setReturnAddressIsTaken(bool s)
const TargetSubtargetInfo & getSubtarget() const
getSubtarget - Return the subtarget for which this machine code is being compiled.
MachineFrameInfo & getFrameInfo()
getFrameInfo - Return the frame info object for the current function.
MachineRegisterInfo & getRegInfo()
getRegInfo - Return information about the registers currently in use.
const DataLayout & getDataLayout() const
Return the DataLayout attached to the Module associated to this MF.
Function & getFunction()
Return the LLVM function that this machine code represents.
BasicBlockListType::iterator iterator
Ty * getInfo()
getInfo - Keep track of various per-function pieces of information for backends that would like to do...
Register addLiveIn(MCRegister PReg, const TargetRegisterClass *RC)
addLiveIn - Add the specified physical register as a live-in value and create a corresponding virtual...
MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags F, LLT MemTy, Align BaseAlignment, const MMOMetadata &Metadata=MMOMetadata(), SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)
getMachineMemOperand - Allocate a new MachineMemOperand.
MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)
CreateMachineInstr - Allocate a new MachineInstr.
void insert(iterator MBBI, MachineBasicBlock *MBB)
const MachineInstrBuilder & addReg(Register RegNo, RegState Flags={}, unsigned SubReg=0) const
Add a new virtual register operand.
const MachineInstrBuilder & addImm(int64_t Val) const
Add a new immediate operand.
const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const
Representation of each machine instruction.
LLVM_ABI void collectDebugValues(SmallVectorImpl< MachineInstr * > &DbgValues)
Scan instructions immediately following MI and collect any matching DBG_VALUEs.
const MachineOperand & getOperand(unsigned i) const
LLVM_ABI MachineInstrBundleIterator< MachineInstr > eraseFromParent()
Unlink 'this' from the containing basic block and delete it.
A description of a memory reference used in the backend.
Flags
Flags values. These may be or'd together.
@ MOVolatile
The memory access is volatile.
@ MODereferenceable
The memory access is dereferenceable (i.e., doesn't trap).
@ MOLoad
The memory access reads data.
@ MOInvariant
The memory access always returns the same value (or traps).
@ MOStore
The memory access writes data.
Flags getFlags() const
Return the raw flags of the source value,.
MachineOperand class - Representation of each machine instruction operand.
void setIsKill(bool Val=true)
void setIsUndef(bool Val=true)
Register getReg() const
getReg - Returns the register number.
static MachineOperand CreateReg(Register Reg, bool isDef, bool isImp=false, bool isKill=false, bool isDead=false, bool isUndef=false, bool isEarlyClobber=false, unsigned SubReg=0, bool isDebug=false, bool isInternalRead=false, bool isRenamable=false)
MachineRegisterInfo - Keep track of information for virtual and physical registers,...
LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")
createVirtualRegister - Create and return a new virtual register in the function with the specified r...
Align getAlign() const
MachineMemOperand * getMemOperand() const
Return the unique MachineMemOperand object describing the memory reference performed by operation.
const MachinePointerInfo & getPointerInfo() const
const SDValue & getChain() const
EVT getMemoryVT() const
Return the type of the in-memory value.
Represent a mutable reference to an array (0 or more elements consecutively in memory),...
Definition ArrayRef.h:294
Class to represent pointers.
unsigned getAddressSpace() const
Return the address space of the Pointer type.
Wrapper class representing virtual and physical registers.
Definition Register.h:20
constexpr bool isVirtual() const
Return true if the specified register number is in the virtual register namespace.
Definition Register.h:79
Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...
Represents one node in the SelectionDAG.
const APInt & getAsAPIntVal() const
Helper method returns the APInt value of a ConstantSDNode.
unsigned getOpcode() const
Return the SelectionDAG opcode value for this node.
bool hasOneUse() const
Return true if there is exactly one use of this node.
LLVM_ABI bool isOnlyUserOf(const SDNode *N) const
Return true if this node is the only use of N.
size_t use_size() const
Return the number of uses of this node.
MVT getSimpleValueType(unsigned ResNo) const
Return the type of a specified result as a simple type.
uint64_t getAsZExtVal() const
Helper method returns the zero-extended integer value of a ConstantSDNode.
unsigned getNumOperands() const
Return the number of values used by this operation.
const SDValue & getOperand(unsigned Num) const
EVT getValueType(unsigned ResNo) const
Return the type of a specified result.
bool isUndef() const
Returns true if the node type is UNDEF or POISON.
Represents a use of a SDNode.
Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.
bool isUndef() const
SDNode * getNode() const
get the SDNode which holds the desired result
bool hasOneUse() const
Return true if there is exactly one node using value ResNo of Node, in exactly one operand.
SDValue getValue(unsigned R) const
EVT getValueType() const
Return the ValueType of the referenced return value.
TypeSize getValueSizeInBits() const
Returns the size of the value in bits.
const SDValue & getOperand(unsigned i) const
ISD::CondCode getOperandCondCode(unsigned i) const
uint64_t getScalarValueSizeInBits() const
uint64_t getConstantOperandVal(unsigned i) const
MVT getSimpleValueType() const
Return the simple ValueType of the referenced return value.
unsigned getOpcode() const
This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...
SDValue getTargetGlobalAddress(const GlobalValue *GV, const SDLoc &DL, EVT VT, int64_t offset=0, unsigned TargetFlags=0)
LLVM_ABI SDValue FoldSetCC(EVT VT, SDValue N1, SDValue N2, ISD::CondCode Cond, const SDLoc &dl, SDNodeFlags Flags={})
Constant fold a setcc to true or false.
SDValue getCopyToReg(SDValue Chain, const SDLoc &dl, Register Reg, SDValue N)
LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)
Create a MERGE_VALUES node from the given operands.
LLVM_ABI SDVTList getVTList(EVT VT)
Return an SDVTList that represents the list of values specified.
LLVM_ABI SDValue getShiftAmountConstant(uint64_t Val, EVT VT, const SDLoc &DL)
LLVM_ABI SDValue getAllOnesConstant(const SDLoc &DL, EVT VT, bool IsTarget=false, bool IsOpaque=false)
LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)
These are used for target selectors to create a new node with specified return type(s),...
LLVM_ABI SDValue getFreeze(SDValue V)
Return a freeze using the SDLoc of the value operand.
bool isSafeToSpeculativelyExecute(unsigned Opcode) const
Some opcodes may create immediate undefined behavior when used with some values (integer division-by-...
SDValue getExtractSubvector(const SDLoc &DL, EVT VT, SDValue Vec, unsigned Idx)
Return the VT typed sub-vector of Vec at Idx.
LLVM_ABI SDValue getRegister(Register Reg, EVT VT)
LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())
Creates a MemIntrinsicNode that may produce a result and takes a list of operands.
SDValue getInsertSubvector(const SDLoc &DL, SDValue Vec, SDValue SubVec, unsigned Idx)
Insert SubVec at the Idx element of Vec.
SDValue getSetCC(const SDLoc &DL, EVT VT, SDValue LHS, SDValue RHS, ISD::CondCode Cond, SDValue Chain=SDValue(), bool IsSignaling=false, SDNodeFlags Flags={})
Helper function to make it easier to build SetCC's if you just have an ISD::CondCode instead of an SD...
void addNoMergeSiteInfo(const SDNode *Node, bool NoMerge)
Set NoMergeSiteInfo to be associated with Node if NoMerge is true.
LLVM_ABI SDValue getNOT(const SDLoc &DL, SDValue Val, EVT VT)
Create a bitwise NOT operation as (XOR Val, -1).
LLVM_ABI SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align DstAlign, Align SrcAlign, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional< bool > OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo=AAMDNodes(), BatchAAResults *BatchAA=nullptr)
const TargetLowering & getTargetLoweringInfo() const
static constexpr unsigned MaxRecursionDepth
SDValue getTargetJumpTable(int JTI, EVT VT, unsigned TargetFlags=0)
SDValue getUNDEF(EVT VT)
Return an UNDEF node. UNDEF does not have a useful SDLoc.
SDValue getCALLSEQ_END(SDValue Chain, SDValue Op1, SDValue Op2, SDValue InGlue, const SDLoc &DL)
Return a new CALLSEQ_END node, which always must have a glue result (to ensure it's not CSE'd).
SDValue getBuildVector(EVT VT, const SDLoc &DL, ArrayRef< SDValue > Ops)
Return an ISD::BUILD_VECTOR node.
LLVM_ABI bool isSplatValue(SDValue V, const APInt &DemandedElts, APInt &UndefElts, unsigned Depth=0) const
Test whether V has a splatted value for all the demanded elements.
LLVM_ABI SDValue getBitcast(EVT VT, SDValue V)
Return a bitcast using the SDLoc of the value operand, and casting to the provided type.
SDValue getCopyFromReg(SDValue Chain, const SDLoc &dl, Register Reg, EVT VT)
SDValue getSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS, SDValue RHS, SDNodeFlags Flags=SDNodeFlags())
Helper function to make it easier to build Select's if you just have operands and don't want to check...
LLVM_ABI SDValue getNegative(SDValue Val, const SDLoc &DL, EVT VT)
Create negative operation as (SUB 0, Val).
LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)
Mutate the specified machine node's memory references to the provided list.
LLVM_ABI SDValue getZeroExtendInReg(SDValue Op, const SDLoc &DL, EVT VT)
Return the expression required to zero extend the Op value assuming it was the smaller SrcTy value.
const DataLayout & getDataLayout() const
LLVM_ABI SDValue getStore(SDValue Chain, const SDLoc &dl, SDValue Val, SDValue Ptr, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Helper function to build ISD::STORE nodes.
LLVM_ABI SDValue getConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
Create a ConstantSDNode wrapping a constant value.
SDValue getSignedTargetConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI void ReplaceAllUsesWith(SDValue From, SDValue To)
Modify anything using 'From' to use 'To' instead.
LLVM_ABI SDValue getCommutedVectorShuffle(const ShuffleVectorSDNode &SV)
Returns an ISD::VECTOR_SHUFFLE node semantically equivalent to the shuffle node in input but with swa...
LLVM_ABI SDValue getExtLoad(ISD::LoadExtType ExtType, const SDLoc &dl, EVT VT, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, EVT MemVT, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
LLVM_ABI std::pair< SDValue, SDValue > SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the vector with EXTRACT_SUBVECTOR using the provided VTs and return the low/high part.
LLVM_ABI SDValue getSignedConstant(int64_t Val, const SDLoc &DL, EVT VT, bool isTarget=false, bool isOpaque=false)
SDValue getCALLSEQ_START(SDValue Chain, uint64_t InSize, uint64_t OutSize, const SDLoc &DL)
Return a new CALLSEQ_START node, that starts new call frame, in which InSize bytes are set up inside ...
LLVM_ABI bool SignBitIsZero(SDValue Op, unsigned Depth=0) const
Return true if the sign bit of Op is known to be zero.
LLVM_ABI SDValue getLoad(EVT VT, const SDLoc &dl, SDValue Chain, SDValue Ptr, MachinePointerInfo PtrInfo, MaybeAlign Alignment=MaybeAlign(), MachineMemOperand::Flags MMOFlags=MachineMemOperand::MONone, const MMOMetadata &Metadata=MMOMetadata())
Loads are not normal binary operators: their result type is not determined by their operands,...
LLVM_ABI SDValue FoldConstantArithmetic(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDValue > Ops, SDNodeFlags Flags=SDNodeFlags())
LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)
const TargetMachine & getTarget() const
LLVM_ABI SDValue WidenVector(const SDValue &N, const SDLoc &DL)
Widen the vector up to the next power of two using INSERT_SUBVECTOR.
LLVM_ABI SDValue getIntPtrConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI SDValue getValueType(EVT)
LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)
Gets or creates the specified node.
SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)
LLVM_ABI unsigned ComputeNumSignBits(SDValue Op, unsigned Depth=0) const
Return the number of times the sign bit of the register is replicated into the other bits.
SDValue getTargetBlockAddress(const BlockAddress *BA, EVT VT, int64_t Offset=0, unsigned TargetFlags=0)
LLVM_ABI bool isBaseWithConstantOffset(SDValue Op) const
Return true if the specified operand is an ISD::ADD with a ConstantSDNode on the right-hand side,...
LLVM_ABI SDValue getVectorIdxConstant(uint64_t Val, const SDLoc &DL, bool isTarget=false)
LLVM_ABI void ReplaceAllUsesOfValueWith(SDValue From, SDValue To)
Replace any uses of From with To, leaving uses of other values produced by From.getNode() alone.
MachineFunction & getMachineFunction() const
SDValue getSplatBuildVector(EVT VT, const SDLoc &DL, SDValue Op)
Return a splat ISD::BUILD_VECTOR node, consisting of Op splatted to all elements.
LLVM_ABI SDValue getFrameIndex(int FI, EVT VT, bool isTarget=false)
LLVM_ABI KnownBits computeKnownBits(SDValue Op, unsigned Depth=0) const
Determine which bits of Op are known to be either zero or one and return them in Known.
LLVM_ABI SDValue getRegisterMask(const uint32_t *RegMask)
LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)
Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...
LLVM_ABI SDValue getCondCode(ISD::CondCode Cond)
LLVM_ABI bool MaskedValueIsZero(SDValue Op, const APInt &Mask, unsigned Depth=0) const
Return true if 'Op & Mask' is known to be zero.
LLVMContext * getContext() const
LLVM_ABI SDValue getTargetExternalSymbol(const char *Sym, EVT VT, unsigned TargetFlags=0)
LLVM_ABI SDValue CreateStackTemporary(TypeSize Bytes, Align Alignment)
Create a stack temporary based on the size in bytes and the alignment.
SDValue getTargetConstantPool(const Constant *C, EVT VT, MaybeAlign Align=std::nullopt, int Offset=0, unsigned TargetFlags=0)
SDValue getEntryNode() const
Return the token chain corresponding to the entry of the function.
SDValue getSplat(EVT VT, const SDLoc &DL, SDValue Op)
Returns a node representing a splat of one value into all lanes of the provided vector type.
LLVM_ABI std::pair< SDValue, SDValue > SplitScalar(const SDValue &N, const SDLoc &DL, const EVT &LoVT, const EVT &HiVT)
Split the scalar node with EXTRACT_ELEMENT using the provided VTs and return the low/high part.
LLVM_ABI SDValue getVectorShuffle(EVT VT, const SDLoc &dl, SDValue N1, SDValue N2, ArrayRef< int > Mask)
Return an ISD::VECTOR_SHUFFLE node.
static LLVM_ABI bool isZeroEltSplatMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask chooses all elements with the same value as the first element of exa...
static LLVM_ABI bool isReverseMask(ArrayRef< int > Mask, int NumSrcElts)
Return true if this shuffle mask swaps the order of elements from exactly one source vector.
ArrayRef< int > getMask() const
SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...
Definition SmallSet.h:134
size_type count(const T &V) const
count - Return 1 if the element is in the set, 0 otherwise.
Definition SmallSet.h:176
std::pair< const_iterator, bool > insert(const T &V)
insert - Insert an element into the set if it isn't already there.
Definition SmallSet.h:184
This class consists of common code factored out of the SmallVector class to reduce code duplication b...
void assign(size_type NumElts, ValueParamT Elt)
reference emplace_back(ArgTypes &&... Args)
void reserve(size_type N)
typename SuperClass::const_iterator const_iterator
iterator insert(iterator I, T &&Elt)
void push_back(const T &Elt)
This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.
StackOffset holds a fixed and a scalable offset in bytes.
Definition TypeSize.h:31
Represent a constant reference to a string, i.e.
Definition StringRef.h:56
std::pair< StringRef, StringRef > split(char Separator) const
Split into two substrings around the first occurrence of a separator character.
Definition StringRef.h:736
bool starts_with(StringRef Prefix) const
Check if this string starts with the given Prefix.
Definition StringRef.h:258
constexpr size_t size() const
Get the string size.
Definition StringRef.h:144
TargetInstrInfo - Interface to description of machine instruction set.
void setBooleanVectorContents(BooleanContent Ty)
Specify how the target extends the result of a vector boolean value from a vector of i1 to a wider ty...
void setOperationAction(unsigned Op, MVT VT, LegalizeAction Action)
Indicate that the specified operation does not work with the specified type and indicate what to do a...
MachineBasicBlock * emitPatchPoint(MachineInstr &MI, MachineBasicBlock *MBB) const
Replace/modify any TargetFrameIndex operands with a targte-dependent sequence of memory operands that...
virtual const TargetRegisterClass * getRegClassFor(MVT VT, bool isDivergent=false) const
Return the register class that should be used for the specified value type.
const TargetMachine & getTargetMachine() const
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain targets require unusual breakdowns of certain types.
virtual bool isZExtFree(Type *FromTy, Type *ToTy) const
Return true if any actual instruction that defines a value of type FromTy implicitly zero-extends the...
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const
Certain combinations of ABIs, Targets and features require that types are legal for some operations a...
LegalizeTypeAction
This enum indicates whether a types are legal for a target, and if not, what action should be used to...
void setMaxBytesForAlignment(unsigned MaxBytes)
void setPrefLoopAlignment(Align Alignment)
Set the target's preferred loop alignment.
void setMaxAtomicSizeInBitsSupported(unsigned SizeInBits)
Set the maximum atomic operation size supported by the backend.
virtual TargetLoweringBase::LegalizeTypeAction getPreferredVectorAction(MVT VT) const
Return the preferred vector type legalization action.
void setMinFunctionAlignment(Align Alignment)
Set the target's minimum function alignment.
void setBooleanContents(BooleanContent Ty)
Specify how the target extends the result of integer and floating point boolean values from i1 to a w...
void computeRegisterProperties(const TargetRegisterInfo *TRI)
Once all of the register classes are added, this allows us to compute derived properties we expose.
virtual EVT getTypeToTransformTo(LLVMContext &Context, EVT VT) const
For types supported by the target, this is an identity function.
void addRegisterClass(MVT VT, const TargetRegisterClass *RC)
Add the specified register class as an available regclass for the specified value type.
bool isTypeLegal(EVT VT) const
Return true if the target has native support for the specified value type.
ExtractSubvectorCost
Enum that specifies how expensive lowering an EXTRACT_SUBVECTOR is.
virtual MVT getPointerTy(const DataLayout &DL, uint32_t AS=0) const
Return the pointer type for the given address space, defaults to the pointer type from the data layou...
void setPrefFunctionAlignment(Align Alignment)
Set the target's preferred function alignment.
bool isOperationLegal(unsigned Op, EVT VT) const
Return true if the specified operation is legal on this target.
FPImmAction
How the legalizer should materialize a floating-point immediate.
@ ConstantPool
Load the immediate from the constant pool.
@ Legal
The target can instruction select the immediate natively.
@ Bitcast
Build the immediate as a bitcast of an integer constant with the same bits.
void setTruncStoreAction(MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified truncating store does not work with the specified type and indicate what ...
bool isOperationLegalOrCustom(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
virtual bool allowsMemoryAccess(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const
Return true if the target supports a memory access of this type for the given address space and align...
virtual bool isBinOp(unsigned Opcode) const
Return true if the node is a math/logic binary operator.
void setMinCmpXchgSizeInBits(unsigned SizeInBits)
Sets the minimum cmpxchg or ll/sc size supported by the backend.
void setStackPointerRegisterToSaveRestore(Register R)
If set to a physical register, this specifies the register that llvm.savestack/llvm....
AtomicExpansionKind
Enum that specifies what an atomic load/AtomicRMWInst is expanded to, if at all.
void setCondCodeAction(ArrayRef< ISD::CondCode > CCs, MVT VT, LegalizeAction Action)
Indicate that the specified condition code is or isn't supported on the target and indicate what to d...
void setTargetDAGCombine(ArrayRef< ISD::NodeType > NTs)
Targets should invoke this method for each target independent node that they want to provide a custom...
void setLoadExtAction(unsigned ExtType, MVT ValVT, MVT MemVT, LegalizeAction Action)
Indicate that the specified load with extension does not work with the specified type and indicate wh...
LegalizeTypeAction getTypeAction(LLVMContext &Context, EVT VT) const
Return how we should legalize values of this type, either it is already legal (return 'Legal') or we ...
std::vector< ArgListEntry > ArgListTy
bool isOperationLegalOrCustomOrPromote(unsigned Op, EVT VT, bool LegalOnly=false) const
Return true if the specified operation is legal on this target or can be made legal with custom lower...
This class defines information used to lower LLVM code to legal SelectionDAG operators that the targe...
bool SimplifyDemandedVectorElts(SDValue Op, const APInt &DemandedEltMask, APInt &KnownUndef, APInt &KnownZero, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Vector Op.
virtual InlineAsm::ConstraintCode getInlineAsmMemConstraint(StringRef ConstraintCode) const
SDValue SimplifyMultipleUseDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, SelectionDAG &DAG, unsigned Depth=0) const
More limited version of SimplifyDemandedBits that can be used to "lookthrough" ops that don't contrib...
virtual ConstraintType getConstraintType(StringRef Constraint) const
Given a constraint, return the type of constraint it is for this target.
std::pair< SDValue, SDValue > LowerCallTo(CallLoweringInfo &CLI) const
This function lowers an abstract call to a function into an actual call.
virtual std::pair< unsigned, const TargetRegisterClass * > getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI, StringRef Constraint, MVT VT) const
Given a physical register constraint (e.g.
bool SimplifyDemandedBits(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0, bool AssumeSingleUse=false) const
Look at Op.
virtual bool SimplifyDemandedBitsForTargetNode(SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts, KnownBits &Known, TargetLoweringOpt &TLO, unsigned Depth=0) const
Attempt to simplify any target nodes based on the demanded bits/elts, returning true on success.
TargetLowering(const TargetLowering &)=delete
virtual void LowerAsmOperandForConstraint(SDValue Op, StringRef Constraint, std::vector< SDValue > &Ops, SelectionDAG &DAG) const
Lower the specified operand into the Ops vector.
std::pair< SDValue, SDValue > makeLibCall(SelectionDAG &DAG, RTLIB::LibcallImpl LibcallImpl, EVT RetVT, ArrayRef< SDValue > Ops, MakeLibCallOptions CallOptions, const SDLoc &dl, SDValue Chain=SDValue()) const
Returns a pair of (return value, chain).
Primary interface to the complete machine description for the target machine.
bool useTLSDESC() const
Returns true if this target uses TLS Descriptors.
bool useEmulatedTLS() const
Returns true if this target uses emulated TLS.
bool shouldAssumeDSOLocal(const GlobalValue *GV) const
CodeModel::Model getCodeModel() const
Returns the code model.
TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...
virtual const TargetInstrInfo * getInstrInfo() const
Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...
Definition Twine.h:82
The instances of the Type class are immutable: once they are created, they are never changed.
Definition Type.h:46
LLVM_ABI unsigned getIntegerBitWidth() const
LLVM_ABI TypeSize getPrimitiveSizeInBits() const LLVM_READONLY
Return the basic size of this type if it is a primitive type.
Definition Type.cpp:187
bool isIntegerTy() const
True if this is an instance of IntegerType.
Definition Type.h:252
static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)
Definition Type.cpp:298
Value * getOperand(unsigned i) const
Definition User.h:207
This class is used to represent EVT's, which are used to parameterize some operations.
LLVM Value Representation.
Definition Value.h:75
Type * getType() const
All values are typed, get the type of this value.
Definition Value.h:257
LLVM_ABI void replaceAllUsesWith(Value *V)
Change all uses of this to point to a new Value.
Definition Value.cpp:548
self_iterator getIterator()
Definition ilist_node.h:123
#define llvm_unreachable(msg)
Marks that the current location is not supposed to be reachable.
constexpr char Align[]
Key for Kernel::Arg::Metadata::mAlign.
constexpr char Args[]
Key for Kernel::Metadata::mArgs.
constexpr std::underlying_type_t< E > Mask()
Get a bitmask with 1s in all places up to the high-order bit of E's largest value.
unsigned ID
LLVM IR allows to use arbitrary numbers as calling convention identifiers.
Definition CallingConv.h:24
@ PreserveMost
Used for runtime calls that preserves most registers.
Definition CallingConv.h:63
@ GHC
Used by the Glasgow Haskell Compiler (GHC).
Definition CallingConv.h:50
@ Fast
Attempts to make calls as fast as possible (e.g.
Definition CallingConv.h:41
@ PreserveNone
Used for runtime calls that preserves none general registers.
Definition CallingConv.h:90
@ C
The default llvm calling convention, compatible with C.
Definition CallingConv.h:34
LLVM_ABI bool isConstantSplatVector(SDValue V, APInt &SplatValue)
Node predicates.
bool isNON_EXTLoad(const SDNode *N)
Returns true if the specified node is a non-extending load.
LLVM_ABI bool isConstantSplatVectorAllZeros(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are 0 ...
NodeType
ISD::NodeType enum - This enum defines the target-independent operators for a SelectionDAG.
Definition ISDOpcodes.h:43
@ SETCC
SetCC operator - This evaluates to a true value iff the condition is true.
Definition ISDOpcodes.h:851
@ STACKRESTORE
STACKRESTORE has two operands, an input chain and a pointer to restore to it returns an output chain.
@ STACKSAVE
STACKSAVE - STACKSAVE has one operand, an input chain.
@ STRICT_FSETCC
STRICT_FSETCC/STRICT_FSETCCS - Constrained versions of SETCC, used for floating-point operands only.
Definition ISDOpcodes.h:516
@ DELETED_NODE
DELETED_NODE - This is an illegal value that is used to catch errors.
Definition ISDOpcodes.h:47
@ POISON
POISON - A poison node.
Definition ISDOpcodes.h:238
@ SMUL_LOHI
SMUL_LOHI/UMUL_LOHI - Multiply two integers of type iN, producing a signed/unsigned value of type i[2...
Definition ISDOpcodes.h:277
@ INSERT_SUBVECTOR
INSERT_SUBVECTOR(VECTOR1, VECTOR2, IDX) - Returns a vector with VECTOR2 inserted into VECTOR1.
Definition ISDOpcodes.h:605
@ BSWAP
Byte Swap and Counting operators.
Definition ISDOpcodes.h:811
@ VAEND
VAEND, VASTART - VAEND and VASTART have three operands: an input chain, pointer, and a SRCVALUE.
@ ADD
Simple integer binary arithmetic operators.
Definition ISDOpcodes.h:266
@ LOAD
LOAD and STORE have token chains as their first operand, then the same operands as an LLVM load/store...
@ ANY_EXTEND
ANY_EXTEND - Used for integer types. The high bits are undefined.
Definition ISDOpcodes.h:885
@ FMA
FMA - Perform a * b + c with no intermediate rounding step.
Definition ISDOpcodes.h:523
@ INTRINSIC_VOID
OUTCHAIN = INTRINSIC_VOID(INCHAIN, INTRINSICID, arg1, arg2, ...) This node represents a target intrin...
Definition ISDOpcodes.h:222
@ GlobalAddress
Definition ISDOpcodes.h:90
@ SINT_TO_FP
[SU]INT_TO_FP - These operators convert integers (whose interpreted sign depends on the first letter)...
Definition ISDOpcodes.h:912
@ CONCAT_VECTORS
CONCAT_VECTORS(VECTOR0, VECTOR1, ...) - Given a number of values of vector type with the same length ...
Definition ISDOpcodes.h:589
@ FADD
Simple binary floating point operators.
Definition ISDOpcodes.h:420
@ ABS
ABS - Determine the unsigned absolute value of a signed integer value of the same bitwidth.
Definition ISDOpcodes.h:771
@ MEMBARRIER
MEMBARRIER - Compiler barrier only; generate a no-op.
@ ATOMIC_FENCE
OUTCHAIN = ATOMIC_FENCE(INCHAIN, ordering, scope) This corresponds to the fence instruction.
@ SIGN_EXTEND_VECTOR_INREG
SIGN_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register sign-extension of the low ...
Definition ISDOpcodes.h:942
@ FP16_TO_FP
FP16_TO_FP, FP_TO_FP16 - These operators are used to perform promotions and truncation for half-preci...
@ BITCAST
BITCAST - This operator converts between integer, vector and FP values, as if the value was stored to...
@ BUILD_PAIR
BUILD_PAIR - This is the opposite of EXTRACT_ELEMENT in some ways.
Definition ISDOpcodes.h:256
@ BUILTIN_OP_END
BUILTIN_OP_END - This must be the last enum value in this list.
@ GlobalTLSAddress
Definition ISDOpcodes.h:91
@ SET_ROUNDING
Set rounding mode.
@ SIGN_EXTEND
Conversion operators.
Definition ISDOpcodes.h:876
@ AVGCEILS
AVGCEILS/AVGCEILU - Rounding averaging add - Add two integers using an integer of type i[N+2],...
Definition ISDOpcodes.h:731
@ SCALAR_TO_VECTOR
SCALAR_TO_VECTOR(VAL) - This represents the operation of loading a scalar value into element 0 of the...
Definition ISDOpcodes.h:681
@ PREFETCH
PREFETCH - This corresponds to a prefetch intrinsic.
@ FSINCOS
FSINCOS - Compute both fsin and fcos as a single operation.
@ FNEG
Perform various unary floating-point operations inspired by libm.
@ BR_CC
BR_CC - Conditional branch.
@ BR_JT
BR_JT - Jumptable branch.
@ FCANONICALIZE
Returns platform specific canonical encoding of a floating point number.
Definition ISDOpcodes.h:546
@ IS_FPCLASS
Performs a check of floating point class property, defined by IEEE-754.
Definition ISDOpcodes.h:553
@ SSUBSAT
RESULT = [US]SUBSAT(LHS, RHS) - Perform saturation subtraction on 2 integers with the same bit width ...
Definition ISDOpcodes.h:377
@ SELECT
Select(COND, TRUEVAL, FALSEVAL).
Definition ISDOpcodes.h:828
@ UNDEF
UNDEF - An undefined node.
Definition ISDOpcodes.h:235
@ VACOPY
VACOPY - VACOPY has 5 operands: an input chain, a destination pointer, a source pointer,...
@ VECREDUCE_ADD
Integer reductions may have a result type larger than the vector element type.
@ GET_ROUNDING
Returns current rounding mode: -1 Undefined 0 Round to 0 1 Round to nearest, ties to even 2 Round to ...
@ MULHU
MULHU/MULHS - Multiply high - Multiply two integers of type iN, producing an unsigned/signed value of...
Definition ISDOpcodes.h:720
@ SHL
Shift and rotation operations.
Definition ISDOpcodes.h:793
@ VECTOR_SHUFFLE
VECTOR_SHUFFLE(VEC1, VEC2) - Returns a vector, of the same type as VEC1/VEC2.
Definition ISDOpcodes.h:659
@ EXTRACT_SUBVECTOR
EXTRACT_SUBVECTOR(VECTOR, IDX) - Returns a subvector from VECTOR.
Definition ISDOpcodes.h:619
@ FMINNUM_IEEE
FMINNUM_IEEE/FMAXNUM_IEEE - Same as FMINNUM/FMAXNUM, except that a signaling NaN operand deterministi...
@ READ_REGISTER
READ_REGISTER, WRITE_REGISTER - This node represents llvm.register on the DAG, which implements the n...
Definition ISDOpcodes.h:141
@ EXTRACT_VECTOR_ELT
EXTRACT_VECTOR_ELT(VECTOR, IDX) - Returns a single element from VECTOR identified by the (potentially...
Definition ISDOpcodes.h:581
@ CopyToReg
CopyToReg - This node has three operands: a chain, a register number to set to this value,...
Definition ISDOpcodes.h:226
@ ZERO_EXTEND
ZERO_EXTEND - Used for integer types, zeroing the new bits.
Definition ISDOpcodes.h:882
@ DEBUGTRAP
DEBUGTRAP - Trap intended to get the attention of a debugger.
@ SELECT_CC
Select with condition operator - This selects between a true value and a false value (ops #2 and #3) ...
Definition ISDOpcodes.h:843
@ ATOMIC_CMP_SWAP
Val, OUTCHAIN = ATOMIC_CMP_SWAP(INCHAIN, ptr, cmp, swap) For double-word atomic operations: ValLo,...
@ FMINNUM
FMINNUM/FMAXNUM - Same semantics as the llvm.minnum/maxnum intrinsics.
@ DYNAMIC_STACKALLOC
DYNAMIC_STACKALLOC - Allocate some number of bytes on the stack aligned to a specified boundary.
@ SIGN_EXTEND_INREG
SIGN_EXTEND_INREG - This operator atomically performs a SHL/SRA pair to sign extend a small value in ...
Definition ISDOpcodes.h:920
@ SMIN
[US]{MIN/MAX} - Binary minimum or maximum of signed or unsigned integers.
Definition ISDOpcodes.h:751
@ FP_EXTEND
X = FP_EXTEND(Y) - Extend a smaller FP type into a larger FP type.
@ VSELECT
Select with a vector condition (op #0) and two vector operands (ops #1 and #2), returning a vector re...
Definition ISDOpcodes.h:837
@ EH_DWARF_CFA
EH_DWARF_CFA - This node represents the pointer to the DWARF Canonical Frame Address (CFA),...
Definition ISDOpcodes.h:152
@ BF16_TO_FP
BF16_TO_FP, FP_TO_BF16 - These operators are used to perform promotions and truncation for bfloat16.
@ FRAMEADDR
FRAMEADDR, RETURNADDR - These nodes represent llvm.frameaddress and llvm.returnaddress on the DAG.
Definition ISDOpcodes.h:112
@ FP_TO_SINT
FP_TO_[US]INT - Convert a floating point value to a signed or unsigned integer.
Definition ISDOpcodes.h:958
@ AND
Bitwise operators - logical and, logical or, logical xor.
Definition ISDOpcodes.h:763
@ TRAP
TRAP - Trapping instruction.
@ INTRINSIC_WO_CHAIN
RESULT = INTRINSIC_WO_CHAIN(INTRINSICID, arg1, arg2, ...) This node represents a target intrinsic fun...
Definition ISDOpcodes.h:207
@ AVGFLOORS
AVGFLOORS/AVGFLOORU - Averaging add - Add two integers using an integer of type i[N+1],...
Definition ISDOpcodes.h:726
@ FREEZE
FREEZE - FREEZE(VAL) returns an arbitrary value if VAL is UNDEF (or is evaluated to UNDEF),...
Definition ISDOpcodes.h:243
@ INSERT_VECTOR_ELT
INSERT_VECTOR_ELT(VECTOR, VAL, IDX) - Returns VECTOR with the element at IDX replaced with VAL.
Definition ISDOpcodes.h:570
@ TokenFactor
TokenFactor - This node takes multiple tokens as input and produces a single token result.
Definition ISDOpcodes.h:55
@ FP_ROUND
X = FP_ROUND(Y, TRUNC) - Rounding 'Y' from a larger floating point type down to the precision of the ...
Definition ISDOpcodes.h:991
@ ZERO_EXTEND_VECTOR_INREG
ZERO_EXTEND_VECTOR_INREG(Vector) - This operator represents an in-register zero-extension of the low ...
Definition ISDOpcodes.h:953
@ FP_TO_SINT_SAT
FP_TO_[US]INT_SAT - Convert floating point value in operand 0 to a signed or unsigned scalar integer ...
Definition ISDOpcodes.h:977
@ TRUNCATE
TRUNCATE - Completely drop the high bits.
Definition ISDOpcodes.h:888
@ VAARG
VAARG - VAARG has four operands: an input chain, a pointer, a SRCVALUE, and the alignment.
@ BRCOND
BRCOND - Conditional branch.
@ SHL_PARTS
SHL_PARTS/SRA_PARTS/SRL_PARTS - These operators are used for expanded integer shift operations.
Definition ISDOpcodes.h:865
@ AssertSext
AssertSext, AssertZext - These nodes record if a register contains a value that has already been zero...
Definition ISDOpcodes.h:64
@ SADDSAT
RESULT = [US]ADDSAT(LHS, RHS) - Perform saturation addition on 2 integers with the same bit width (W)...
Definition ISDOpcodes.h:368
@ ABDS
ABDS/ABDU - Absolute difference - Return the absolute difference between two numbers interpreted as s...
Definition ISDOpcodes.h:746
@ INTRINSIC_W_CHAIN
RESULT,OUTCHAIN = INTRINSIC_W_CHAIN(INCHAIN, INTRINSICID, arg1, ...) This node represents a target in...
Definition ISDOpcodes.h:215
@ BUILD_VECTOR
BUILD_VECTOR(ELT0, ELT1, ELT2, ELT3,...) - Return a fixed-width vector with the specified,...
Definition ISDOpcodes.h:561
bool isExtVecInRegOpcode(unsigned Opcode)
LLVM_ABI bool isConstantSplatVectorAllOnes(SDValue V, bool BuildVectorOnly=false)
Return true if the specified value is a BUILD_VECTOR or SPLAT_VECTOR where all of the elements are ~0...
LLVM_ABI bool isFreezeUndef(SDValue V)
Return true if the specified value is FREEZE(UNDEF).
LLVM_ABI CondCode getSetCCInverse(CondCode Operation, EVT Type)
Return the operation corresponding to !(X op Y), where 'op' is a valid SetCC operation.
bool isBitwiseLogicOp(unsigned Opcode)
Whether this is bitwise logic opcode.
LLVM_ABI bool isBuildVectorAllOnes(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are ~0 or undef.
LLVM_ABI CondCode getSetCCSwappedOperands(CondCode Operation)
Return the operation corresponding to (Y op X) when given the operation for (X op Y).
LLVM_ABI bool isBuildVectorAllZeros(SDValue V)
Return true if the specified value is a BUILD_VECTOR where all of the elements are 0 or undef.
CondCode
ISD::CondCode enum - These are ordered carefully to make the bitfields below work out,...
LLVM_ABI bool isBuildVectorOfConstantSDNodes(SDValue V)
Return true if the specified value is a BUILD_VECTOR node of all ConstantSDNode or undef.
LLVM_ABI NodeType getVecReduceBaseOpcode(unsigned VecReduceOpcode)
Get underlying scalar opcode for VECREDUCE opcode.
LoadExtType
LoadExtType enum - This enum defines the three variants of LOADEXT (load with extension).
bool isNormalLoad(const SDNode *N)
Returns true if the specified node is a non-extending and unindexed load.
bool isIntEqualitySetCC(CondCode Code)
Return true if this is a setcc instruction that performs an equality comparison when used with intege...
This namespace contains an enum with a value for every intrinsic/builtin function known by LLVM.
LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > OverloadTys={})
Look up the Function declaration of the intrinsic id in the Module M.
ABI getTargetABI(StringRef ABIName)
InstSeq generateInstSeq(int64_t Val)
LLVM_ABI Libcall getSINTTOFP(EVT OpVT, EVT RetVT)
getSINTTOFP - Return the SINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getUINTTOFP(EVT OpVT, EVT RetVT)
getUINTTOFP - Return the UINTTOFP_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPTOSINT(EVT OpVT, EVT RetVT)
getFPTOSINT - Return the FPTOSINT_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
LLVM_ABI Libcall getFPROUND(EVT OpVT, EVT RetVT)
getFPROUND - Return the FPROUND_*_* value for the given types, or UNKNOWN_LIBCALL if there is none.
@ SingleThread
Synchronized with respect to signal handlers executing in the same thread.
Definition LLVMContext.h:63
ValuesClass values(OptsTy... Options)
Helper to build a ValuesClass by forwarding a variable number of arguments as an initializer list to ...
initializer< Ty > init(const Ty &Val)
Sequence
A sequence of states that a pointer may go through in which an objc_retain and objc_release are actua...
Definition PtrState.h:41
NodeAddr< NodeBase * > Node
Definition RDFGraph.h:381
This is an optimization pass for GlobalISel generic memory operations.
@ Low
Lower the current thread's priority such that it does not affect foreground tasks significantly.
Definition Threading.h:280
@ Offset
Definition DWP.cpp:577
bool all_of(R &&range, UnaryPredicate P)
Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1755
MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)
Builder interface. Specify how to create the initial instruction itself.
constexpr bool isInt(int64_t x)
Checks if an integer fits into the given bit width.
Definition MathExtras.h:166
LLVM_ABI bool isNullConstant(SDValue V)
Returns true if V is a constant integer zero.
@ Known
Known to have no common set bits.
@ Undef
Value of the register doesn't matter.
LLVM_ABI SDValue peekThroughBitcasts(SDValue V)
Return the non-bitcasted source operand of V if it exists.
constexpr RegState getKillRegState(bool B)
decltype(auto) dyn_cast(const From &Val)
dyn_cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:643
@ Load
The value being inserted comes from a load (InsertElement only).
@ Store
The extracted value is stored (ExtractElement only).
bool isIntOrFPConstant(SDValue V)
Return true if V is either a integer or FP constant.
int bit_width(T Value)
Returns the number of bits needed to represent Value if Value is nonzero.
Definition bit.h:325
constexpr T alignDown(U Value, V Align, W Skew=0)
Returns the largest unsigned integer less than or equal to Value and is Skew mod Align.
Definition MathExtras.h:546
constexpr bool isPowerOf2_64(uint64_t Value)
Return true if the argument is a power of two > 0 (64 bit edition.)
Definition MathExtras.h:285
LLVM_ABI bool widenShuffleMaskElts(int Scale, ArrayRef< int > Mask, SmallVectorImpl< int > &ScaledMask)
Try to transform a shuffle mask by replacing elements with the scaled index for an equivalent mask of...
unsigned Log2_64(uint64_t Value)
Return the floor log base 2 of the specified value, -1 if the value is zero.
Definition MathExtras.h:332
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Value
Definition InstrProf.h:143
constexpr bool isShiftedMask_64(uint64_t Value)
Return true if the argument contains a non-empty sequence of ones with the remainder zero (64 bit ver...
Definition MathExtras.h:274
bool any_of(R &&range, UnaryPredicate P)
Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.
Definition STLExtras.h:1762
MachineInstr * getImm(const MachineOperand &MO, const MachineRegisterInfo *MRI)
constexpr bool isPowerOf2_32(uint32_t Value)
Return true if the argument is a power of two > 0.
Definition MathExtras.h:280
decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)
LLVM_ABI raw_ostream & dbgs()
dbgs() - This returns a reference to a raw_ostream for debugging messages.
Definition Debug.cpp:209
LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)
Definition Error.cpp:163
constexpr bool isMask_64(uint64_t Value)
Return true if the argument is a non-empty sequence of ones starting at the least significant bit wit...
Definition MathExtras.h:262
constexpr bool isUInt(uint64_t x)
Checks if an unsigned integer fits into the given bit width.
Definition MathExtras.h:190
class LLVM_GSL_OWNER SmallVector
Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...
bool isa(const From &Val)
isa<X> - Return true if the parameter to the template is an instance of one of the template type argu...
Definition Casting.h:547
AtomicOrdering
Atomic ordering for LLVM's memory model.
@ Other
Any other memory.
Definition ModRef.h:68
@ First
Helpers to iterate all locations in the MemoryEffectsBase class.
Definition ModRef.h:74
uint16_t MCPhysReg
An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...
Definition MCRegister.h:21
@ Fast
Assign the register banks as fast as possible (default).
DWARFExpression::Operation Op
ArrayRef(const T &OneElt) -> ArrayRef< T >
constexpr bool isShiftedInt(int64_t x)
Checks if a signed integer is an N bit number shifted left by S.
Definition MathExtras.h:183
constexpr unsigned BitWidth
std::string join_items(Sep Separator, Args &&... Items)
Joins the strings in the parameter pack Items, adding Separator between the elements....
ExceptionHandling
Definition CodeGen.h:54
decltype(auto) cast(const From &Val)
cast<X> - Return the argument parameter cast to the specified type.
Definition Casting.h:559
LLVM_ABI bool isOneConstant(SDValue V)
Returns true if V is a constant integer one.
PointerUnion< const Value *, const PseudoSourceValue * > ValueType
RelativeUniformCounterPtr ValuesPtrExpr VTableAddr Next
Definition InstrProf.h:147
@ End
Run the machine verifier once, at the end of the pipeline.
Definition CodeGen.h:320
LLVM_ABI bool isAllOnesConstant(SDValue V)
Returns true if V is an integer constant with all bits set.
MCRegisterClass TargetRegisterClass
Definition FastISel.h:58
LLVM_ABI void reportFatalUsageError(Error Err)
Report a fatal error that does not indicate a bug in LLVM.
Definition Error.cpp:177
void swap(llvm::BitVector &LHS, llvm::BitVector &RHS)
Implement std::swap in terms of BitVector swap.
Definition BitVector.h:881
#define N
This struct is a compact representation of a valid (non-zero power of two) alignment.
Definition Alignment.h:39
constexpr uint64_t value() const
This is a hole in the type system and should not be abused.
Definition Alignment.h:82
Extended Value Type.
Definition ValueTypes.h:35
EVT changeVectorElementTypeToInteger() const
Return a vector with the same number of elements as this vector, but with the element type converted ...
Definition ValueTypes.h:90
TypeSize getStoreSize() const
Return the number of bytes overwritten by a store of the specified value type.
Definition ValueTypes.h:433
bool isSimple() const
Test if the given EVT is simple (as opposed to being extended).
Definition ValueTypes.h:160
static EVT getVectorVT(LLVMContext &Context, EVT VT, unsigned NumElements, bool IsScalable=false)
Returns the EVT that represents a vector NumElements in length, where each element is of type VT.
Definition ValueTypes.h:70
bool bitsGT(EVT VT) const
Return true if this has more bits than VT.
Definition ValueTypes.h:322
bool bitsLT(EVT VT) const
Return true if this has less bits than VT.
Definition ValueTypes.h:338
bool isFloatingPoint() const
Return true if this is a FP or a vector FP type.
Definition ValueTypes.h:170
TypeSize getSizeInBits() const
Return the size of the specified value type in bits.
Definition ValueTypes.h:411
uint64_t getScalarSizeInBits() const
Definition ValueTypes.h:423
MVT getSimpleVT() const
Return the SimpleValueType held in the specified simple EVT.
Definition ValueTypes.h:354
bool is128BitVector() const
Return true if this is a 128-bit vector type.
Definition ValueTypes.h:245
static EVT getIntegerVT(LLVMContext &Context, unsigned BitWidth)
Returns the EVT that represents an integer with the given number of bits.
Definition ValueTypes.h:61
uint64_t getFixedSizeInBits() const
Return the size of the specified fixed width value type in bits.
Definition ValueTypes.h:419
bool isFixedLengthVector() const
Definition ValueTypes.h:214
static EVT getFloatingPointVT(unsigned BitWidth)
Returns the EVT that represents a floating-point type with the given number of bits.
Definition ValueTypes.h:55
bool isVector() const
Return true if this is a vector value type.
Definition ValueTypes.h:191
EVT getScalarType() const
If this is a vector type, return the element type, otherwise return this.
Definition ValueTypes.h:361
bool is256BitVector() const
Return true if this is a 256-bit vector type.
Definition ValueTypes.h:250
LLVM_ABI Type * getTypeForEVT(LLVMContext &Context) const
This method returns an LLVM type corresponding to the specified EVT.
EVT getVectorElementType() const
Given a vector type, return the type of each element.
Definition ValueTypes.h:366
bool isScalarInteger() const
Return true if this is an integer, but not a vector.
Definition ValueTypes.h:180
unsigned getVectorNumElements() const
Given a vector type, return the number of elements it contains.
Definition ValueTypes.h:374
EVT getHalfNumVectorElementsVT(LLVMContext &Context) const
Definition ValueTypes.h:499
bool isInteger() const
Return true if this is an integer or a vector integer type.
Definition ValueTypes.h:175
Align getNonZeroOrigAlign() const
InputArg - This struct carries flags and type information about a single incoming (formal) argument o...
KnownBits trunc(unsigned BitWidth) const
Return known bits for a truncation of the value we're tracking.
Definition KnownBits.h:165
unsigned countMaxActiveBits() const
Returns the maximum number of bits needed to represent all possible unsigned values with these known ...
Definition KnownBits.h:310
Matching combinators.
This class contains a discriminated union of information about pointers in memory operands,...
static LLVM_ABI MachinePointerInfo getStack(MachineFunction &MF, int64_t Offset, uint8_t ID=0)
Stack pointer relative access.
static LLVM_ABI MachinePointerInfo getUnknownStack(MachineFunction &MF)
Stack memory without other information.
static LLVM_ABI MachinePointerInfo getGOT(MachineFunction &MF)
Return a MachinePointerInfo record that refers to a GOT entry.
static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)
Return a MachinePointerInfo record that refers to the specified FrameIndex.
This struct is a compact representation of a valid (power of two) or undefined (0) alignment.
Definition Alignment.h:111
This represents a list of ValueType's that has been intern'd by a SelectionDAG.
This represents an addressing mode of: BaseGV + BaseOffs + BaseReg + Scale*ScaleReg + ScalableOffset*...
This structure contains all information that is necessary for lowering calls.
SmallVector< ISD::InputArg, 32 > Ins
SmallVector< ISD::OutputArg, 32 > Outs
LLVM_ABI SDValue CombineTo(SDNode *N, ArrayRef< SDValue > To, bool AddTo=true)
This structure is used to pass arguments to makeLibCall function.
MakeLibCallOptions & setTypeListBeforeSoften(ArrayRef< EVT > OpsVT, EVT RetVT)
A convenience struct that encapsulates a DAG, and two SDValues for returning information from TargetL...