LCOV - code coverage report
Current view: top level - build-llvm/lib/Target/AArch64 - AArch64GenFastISel.inc (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 264 2998 8.8 %
Date: 2017-09-14 15:23:50 Functions: 28 180 15.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
       2             : |*                                                                            *|
       3             : |* "Fast" Instruction Selector for the AArch64 target                         *|
       4             : |*                                                                            *|
       5             : |* Automatically generated file, do not edit!                                 *|
       6             : |*                                                                            *|
       7             : \*===----------------------------------------------------------------------===*/
       8             : 
       9             : 
      10             : // FastEmit Immediate Predicate functions.
      11             : static bool Predicate_imm0_31(int64_t Imm) {
      12             : 
      13             :   return ((uint64_t)Imm) < 32;
      14             : 
      15             : }
      16             : static bool Predicate_imm0_63(int64_t Imm) {
      17             : 
      18             :   return ((uint64_t)Imm) < 64;
      19             : 
      20             : }
      21             : static bool Predicate_imm32_0_31(int64_t Imm) {
      22             : 
      23             :   return ((uint64_t)Imm) < 32;
      24             : 
      25             : }
      26             : static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
      27             : 
      28             :   return (((uint32_t)Imm) < 32);
      29             : 
      30             : }
      31             : static bool Predicate_tbz_imm32_63(int64_t Imm) {
      32             : 
      33             :   return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
      34             : 
      35             : }
      36             : static bool Predicate_VectorIndexD(int64_t Imm) {
      37             : 
      38             :   return ((uint64_t)Imm) < 2;
      39             : 
      40             : }
      41             : static bool Predicate_VectorIndexS(int64_t Imm) {
      42             : 
      43             :   return ((uint64_t)Imm) < 4;
      44             : 
      45             : }
      46             : static bool Predicate_VectorIndexH(int64_t Imm) {
      47             : 
      48             :   return ((uint64_t)Imm) < 8;
      49             : 
      50             : }
      51             : static bool Predicate_VectorIndexB(int64_t Imm) {
      52             : 
      53             :   return ((uint64_t)Imm) < 16;
      54             : 
      55             : }
      56             : static bool Predicate_imm0_255(int64_t Imm) {
      57             : 
      58         188 :   return ((uint32_t)Imm) < 256;
      59             : 
      60             : }
      61             : static bool Predicate_vecshiftL64(int64_t Imm) {
      62             : 
      63             :   return (((uint32_t)Imm) < 64);
      64             : 
      65             : }
      66             : static bool Predicate_vecshiftL32(int64_t Imm) {
      67             : 
      68             :   return (((uint32_t)Imm) < 32);
      69             : 
      70             : }
      71             : static bool Predicate_vecshiftR64(int64_t Imm) {
      72             : 
      73             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 65);
      74             : 
      75             : }
      76             : static bool Predicate_vecshiftL8(int64_t Imm) {
      77             : 
      78             :   return (((uint32_t)Imm) < 8);
      79             : 
      80             : }
      81             : static bool Predicate_vecshiftL16(int64_t Imm) {
      82             : 
      83             :   return (((uint32_t)Imm) < 16);
      84             : 
      85             : }
      86             : static bool Predicate_vecshiftR8(int64_t Imm) {
      87             : 
      88             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 9);
      89             : 
      90             : }
      91             : static bool Predicate_vecshiftR16(int64_t Imm) {
      92             : 
      93             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);
      94             : 
      95             : }
      96             : static bool Predicate_vecshiftR32(int64_t Imm) {
      97             : 
      98             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 33);
      99             : 
     100             : }
     101             : 
     102             : 
     103             : // FastEmit functions for AArch64ISD::THREAD_POINTER.
     104             : 
     105             : unsigned fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
     106           0 :   if (RetVT.SimpleTy != MVT::i64)
     107             :     return 0;
     108           0 :   return fastEmitInst_(AArch64::MOVbaseTLS, &AArch64::GPR64RegClass);
     109             : }
     110             : 
     111             : unsigned fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
     112           0 :   switch (VT.SimpleTy) {
     113           0 :   case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
     114             :   default: return 0;
     115             :   }
     116             : }
     117             : 
     118             : // Top-level FastEmit function.
     119             : 
     120           0 : unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
     121           0 :   switch (Opcode) {
     122           0 :   case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
     123             :   default: return 0;
     124             :   }
     125             : }
     126             : 
     127             : // FastEmit functions for AArch64ISD::CALL.
     128             : 
     129             : unsigned fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     130           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     131             :     return 0;
     132           0 :   return fastEmitInst_r(AArch64::BLR, &AArch64::GPR64RegClass, Op0, Op0IsKill);
     133             : }
     134             : 
     135             : unsigned fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     136           0 :   switch (VT.SimpleTy) {
     137           0 :   case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
     138             :   default: return 0;
     139             :   }
     140             : }
     141             : 
     142             : // FastEmit functions for AArch64ISD::CMEQz.
     143             : 
     144             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     145           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     146             :     return 0;
     147           0 :   if ((Subtarget->hasNEON())) {
     148           0 :     return fastEmitInst_r(AArch64::CMEQv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     149             :   }
     150             :   return 0;
     151             : }
     152             : 
     153             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     154           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     155             :     return 0;
     156           0 :   if ((Subtarget->hasNEON())) {
     157           0 :     return fastEmitInst_r(AArch64::CMEQv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     158             :   }
     159             :   return 0;
     160             : }
     161             : 
     162             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     163           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     164             :     return 0;
     165           0 :   if ((Subtarget->hasNEON())) {
     166           0 :     return fastEmitInst_r(AArch64::CMEQv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     167             :   }
     168             :   return 0;
     169             : }
     170             : 
     171             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     172           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     173             :     return 0;
     174           0 :   if ((Subtarget->hasNEON())) {
     175           0 :     return fastEmitInst_r(AArch64::CMEQv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     176             :   }
     177             :   return 0;
     178             : }
     179             : 
     180             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     181           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     182             :     return 0;
     183           0 :   if ((Subtarget->hasNEON())) {
     184           0 :     return fastEmitInst_r(AArch64::CMEQv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     185             :   }
     186             :   return 0;
     187             : }
     188             : 
     189             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     190           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     191             :     return 0;
     192           0 :   if ((Subtarget->hasNEON())) {
     193           0 :     return fastEmitInst_r(AArch64::CMEQv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     194             :   }
     195             :   return 0;
     196             : }
     197             : 
     198             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     199           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     200             :     return 0;
     201           0 :   if ((Subtarget->hasNEON())) {
     202           0 :     return fastEmitInst_r(AArch64::CMEQv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     203             :   }
     204             :   return 0;
     205             : }
     206             : 
     207             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     208           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     209             :     return 0;
     210           0 :   if ((Subtarget->hasNEON())) {
     211           0 :     return fastEmitInst_r(AArch64::CMEQv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     212             :   }
     213             :   return 0;
     214             : }
     215             : 
     216           0 : unsigned fastEmit_AArch64ISD_CMEQz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     217           0 :   switch (VT.SimpleTy) {
     218           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     219           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     220           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     221           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     222           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     223           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     224           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     225           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     226             :   default: return 0;
     227             :   }
     228             : }
     229             : 
     230             : // FastEmit functions for AArch64ISD::CMGEz.
     231             : 
     232             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     233           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     234             :     return 0;
     235           0 :   if ((Subtarget->hasNEON())) {
     236           0 :     return fastEmitInst_r(AArch64::CMGEv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     237             :   }
     238             :   return 0;
     239             : }
     240             : 
     241             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     242           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     243             :     return 0;
     244           0 :   if ((Subtarget->hasNEON())) {
     245           0 :     return fastEmitInst_r(AArch64::CMGEv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     246             :   }
     247             :   return 0;
     248             : }
     249             : 
     250             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     251           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     252             :     return 0;
     253           0 :   if ((Subtarget->hasNEON())) {
     254           0 :     return fastEmitInst_r(AArch64::CMGEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     255             :   }
     256             :   return 0;
     257             : }
     258             : 
     259             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     260           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     261             :     return 0;
     262           0 :   if ((Subtarget->hasNEON())) {
     263           0 :     return fastEmitInst_r(AArch64::CMGEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     264             :   }
     265             :   return 0;
     266             : }
     267             : 
     268             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     269           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     270             :     return 0;
     271           0 :   if ((Subtarget->hasNEON())) {
     272           0 :     return fastEmitInst_r(AArch64::CMGEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     273             :   }
     274             :   return 0;
     275             : }
     276             : 
     277             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     278           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     279             :     return 0;
     280           0 :   if ((Subtarget->hasNEON())) {
     281           0 :     return fastEmitInst_r(AArch64::CMGEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     282             :   }
     283             :   return 0;
     284             : }
     285             : 
     286             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     287           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     288             :     return 0;
     289           0 :   if ((Subtarget->hasNEON())) {
     290           0 :     return fastEmitInst_r(AArch64::CMGEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     291             :   }
     292             :   return 0;
     293             : }
     294             : 
     295             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     296           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     297             :     return 0;
     298           0 :   if ((Subtarget->hasNEON())) {
     299           0 :     return fastEmitInst_r(AArch64::CMGEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     300             :   }
     301             :   return 0;
     302             : }
     303             : 
     304           0 : unsigned fastEmit_AArch64ISD_CMGEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     305           0 :   switch (VT.SimpleTy) {
     306           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     307           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     308           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     309           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     310           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     311           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     312           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     313           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     314             :   default: return 0;
     315             :   }
     316             : }
     317             : 
     318             : // FastEmit functions for AArch64ISD::CMGTz.
     319             : 
     320             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     321           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     322             :     return 0;
     323           0 :   if ((Subtarget->hasNEON())) {
     324           0 :     return fastEmitInst_r(AArch64::CMGTv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     325             :   }
     326             :   return 0;
     327             : }
     328             : 
     329             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     330           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     331             :     return 0;
     332           0 :   if ((Subtarget->hasNEON())) {
     333           0 :     return fastEmitInst_r(AArch64::CMGTv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     334             :   }
     335             :   return 0;
     336             : }
     337             : 
     338             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     339           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     340             :     return 0;
     341           0 :   if ((Subtarget->hasNEON())) {
     342           0 :     return fastEmitInst_r(AArch64::CMGTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     343             :   }
     344             :   return 0;
     345             : }
     346             : 
     347             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     348           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     349             :     return 0;
     350           0 :   if ((Subtarget->hasNEON())) {
     351           0 :     return fastEmitInst_r(AArch64::CMGTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     352             :   }
     353             :   return 0;
     354             : }
     355             : 
     356             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     357           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     358             :     return 0;
     359           0 :   if ((Subtarget->hasNEON())) {
     360           0 :     return fastEmitInst_r(AArch64::CMGTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     361             :   }
     362             :   return 0;
     363             : }
     364             : 
     365             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     366           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     367             :     return 0;
     368           0 :   if ((Subtarget->hasNEON())) {
     369           0 :     return fastEmitInst_r(AArch64::CMGTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     370             :   }
     371             :   return 0;
     372             : }
     373             : 
     374             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     375           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     376             :     return 0;
     377           0 :   if ((Subtarget->hasNEON())) {
     378           0 :     return fastEmitInst_r(AArch64::CMGTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     379             :   }
     380             :   return 0;
     381             : }
     382             : 
     383             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     384           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     385             :     return 0;
     386           0 :   if ((Subtarget->hasNEON())) {
     387           0 :     return fastEmitInst_r(AArch64::CMGTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     388             :   }
     389             :   return 0;
     390             : }
     391             : 
     392           0 : unsigned fastEmit_AArch64ISD_CMGTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     393           0 :   switch (VT.SimpleTy) {
     394           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     395           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     396           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     397           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     398           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     399           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     400           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     401           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     402             :   default: return 0;
     403             :   }
     404             : }
     405             : 
     406             : // FastEmit functions for AArch64ISD::CMLEz.
     407             : 
     408             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     409           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     410             :     return 0;
     411           0 :   if ((Subtarget->hasNEON())) {
     412           0 :     return fastEmitInst_r(AArch64::CMLEv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     413             :   }
     414             :   return 0;
     415             : }
     416             : 
     417             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     418           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     419             :     return 0;
     420           0 :   if ((Subtarget->hasNEON())) {
     421           0 :     return fastEmitInst_r(AArch64::CMLEv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     422             :   }
     423             :   return 0;
     424             : }
     425             : 
     426             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     427           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     428             :     return 0;
     429           0 :   if ((Subtarget->hasNEON())) {
     430           0 :     return fastEmitInst_r(AArch64::CMLEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     431             :   }
     432             :   return 0;
     433             : }
     434             : 
     435             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     436           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     437             :     return 0;
     438           0 :   if ((Subtarget->hasNEON())) {
     439           0 :     return fastEmitInst_r(AArch64::CMLEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     440             :   }
     441             :   return 0;
     442             : }
     443             : 
     444             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     445           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     446             :     return 0;
     447           0 :   if ((Subtarget->hasNEON())) {
     448           0 :     return fastEmitInst_r(AArch64::CMLEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     449             :   }
     450             :   return 0;
     451             : }
     452             : 
     453             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     454           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     455             :     return 0;
     456           0 :   if ((Subtarget->hasNEON())) {
     457           0 :     return fastEmitInst_r(AArch64::CMLEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     458             :   }
     459             :   return 0;
     460             : }
     461             : 
     462             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     463           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     464             :     return 0;
     465           0 :   if ((Subtarget->hasNEON())) {
     466           0 :     return fastEmitInst_r(AArch64::CMLEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     467             :   }
     468             :   return 0;
     469             : }
     470             : 
     471             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     472           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     473             :     return 0;
     474           0 :   if ((Subtarget->hasNEON())) {
     475           0 :     return fastEmitInst_r(AArch64::CMLEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     476             :   }
     477             :   return 0;
     478             : }
     479             : 
     480           0 : unsigned fastEmit_AArch64ISD_CMLEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     481           0 :   switch (VT.SimpleTy) {
     482           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     483           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     484           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     485           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     486           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     487           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     488           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     489           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     490             :   default: return 0;
     491             :   }
     492             : }
     493             : 
     494             : // FastEmit functions for AArch64ISD::CMLTz.
     495             : 
     496             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     497           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     498             :     return 0;
     499           0 :   if ((Subtarget->hasNEON())) {
     500           0 :     return fastEmitInst_r(AArch64::CMLTv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     501             :   }
     502             :   return 0;
     503             : }
     504             : 
     505             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     506           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     507             :     return 0;
     508           0 :   if ((Subtarget->hasNEON())) {
     509           0 :     return fastEmitInst_r(AArch64::CMLTv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     510             :   }
     511             :   return 0;
     512             : }
     513             : 
     514             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     515           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     516             :     return 0;
     517           0 :   if ((Subtarget->hasNEON())) {
     518           0 :     return fastEmitInst_r(AArch64::CMLTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     519             :   }
     520             :   return 0;
     521             : }
     522             : 
     523             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     524           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     525             :     return 0;
     526           0 :   if ((Subtarget->hasNEON())) {
     527           0 :     return fastEmitInst_r(AArch64::CMLTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     528             :   }
     529             :   return 0;
     530             : }
     531             : 
     532             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     533           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     534             :     return 0;
     535           0 :   if ((Subtarget->hasNEON())) {
     536           0 :     return fastEmitInst_r(AArch64::CMLTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     537             :   }
     538             :   return 0;
     539             : }
     540             : 
     541             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     542           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     543             :     return 0;
     544           0 :   if ((Subtarget->hasNEON())) {
     545           0 :     return fastEmitInst_r(AArch64::CMLTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     546             :   }
     547             :   return 0;
     548             : }
     549             : 
     550             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     551           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     552             :     return 0;
     553           0 :   if ((Subtarget->hasNEON())) {
     554           0 :     return fastEmitInst_r(AArch64::CMLTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     555             :   }
     556             :   return 0;
     557             : }
     558             : 
     559             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     560           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     561             :     return 0;
     562           0 :   if ((Subtarget->hasNEON())) {
     563           0 :     return fastEmitInst_r(AArch64::CMLTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     564             :   }
     565             :   return 0;
     566             : }
     567             : 
     568           0 : unsigned fastEmit_AArch64ISD_CMLTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     569           0 :   switch (VT.SimpleTy) {
     570           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     571           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     572           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     573           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     574           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     575           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     576           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     577           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     578             :   default: return 0;
     579             :   }
     580             : }
     581             : 
     582             : // FastEmit functions for AArch64ISD::DUP.
     583             : 
     584             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
     585           0 :   if ((Subtarget->hasNEON())) {
     586           0 :     return fastEmitInst_r(AArch64::DUPv8i8gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     587             :   }
     588             :   return 0;
     589             : }
     590             : 
     591             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
     592           0 :   if ((Subtarget->hasNEON())) {
     593           0 :     return fastEmitInst_r(AArch64::DUPv16i8gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     594             :   }
     595             :   return 0;
     596             : }
     597             : 
     598             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
     599           0 :   if ((Subtarget->hasNEON())) {
     600           0 :     return fastEmitInst_r(AArch64::DUPv4i16gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     601             :   }
     602             :   return 0;
     603             : }
     604             : 
     605             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
     606           0 :   if ((Subtarget->hasNEON())) {
     607           0 :     return fastEmitInst_r(AArch64::DUPv8i16gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     608             :   }
     609             :   return 0;
     610             : }
     611             : 
     612             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
     613           0 :   if ((Subtarget->hasNEON())) {
     614           0 :     return fastEmitInst_r(AArch64::DUPv2i32gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     615             :   }
     616             :   return 0;
     617             : }
     618             : 
     619             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     620           0 :   if ((Subtarget->hasNEON())) {
     621           0 :     return fastEmitInst_r(AArch64::DUPv4i32gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     622             :   }
     623             :   return 0;
     624             : }
     625             : 
     626           0 : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     627           0 : switch (RetVT.SimpleTy) {
     628           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0, Op0IsKill);
     629           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0, Op0IsKill);
     630           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0, Op0IsKill);
     631           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0, Op0IsKill);
     632           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0, Op0IsKill);
     633           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
     634             :   default: return 0;
     635             : }
     636             : }
     637             : 
     638             : unsigned fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     639           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     640             :     return 0;
     641           0 :   if ((Subtarget->hasNEON())) {
     642           0 :     return fastEmitInst_r(AArch64::DUPv2i64gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     643             :   }
     644             :   return 0;
     645             : }
     646             : 
     647           0 : unsigned fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     648           0 :   switch (VT.SimpleTy) {
     649           0 :   case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     650           0 :   case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     651             :   default: return 0;
     652             :   }
     653             : }
     654             : 
     655             : // FastEmit functions for AArch64ISD::FCMEQz.
     656             : 
     657           0 : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     658           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     659             :     return 0;
     660           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     661           0 :     return fastEmitInst_r(AArch64::FCMEQv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     662             :   }
     663             :   return 0;
     664             : }
     665             : 
     666           0 : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     667           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     668             :     return 0;
     669           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     670           0 :     return fastEmitInst_r(AArch64::FCMEQv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     671             :   }
     672             :   return 0;
     673             : }
     674             : 
     675             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     676           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     677             :     return 0;
     678           0 :   if ((Subtarget->hasNEON())) {
     679           0 :     return fastEmitInst_r(AArch64::FCMEQv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     680             :   }
     681             :   return 0;
     682             : }
     683             : 
     684             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     685           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     686             :     return 0;
     687           0 :   if ((Subtarget->hasNEON())) {
     688           0 :     return fastEmitInst_r(AArch64::FCMEQv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     689             :   }
     690             :   return 0;
     691             : }
     692             : 
     693             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     694           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     695             :     return 0;
     696           0 :   if ((Subtarget->hasNEON())) {
     697           0 :     return fastEmitInst_r(AArch64::FCMEQv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     698             :   }
     699             :   return 0;
     700             : }
     701             : 
     702             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     703           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     704             :     return 0;
     705           0 :   if ((Subtarget->hasNEON())) {
     706           0 :     return fastEmitInst_r(AArch64::FCMEQv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     707             :   }
     708             :   return 0;
     709             : }
     710             : 
     711           0 : unsigned fastEmit_AArch64ISD_FCMEQz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     712           0 :   switch (VT.SimpleTy) {
     713           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     714           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     715           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     716           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     717           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     718           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     719             :   default: return 0;
     720             :   }
     721             : }
     722             : 
     723             : // FastEmit functions for AArch64ISD::FCMGEz.
     724             : 
     725           0 : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     726           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     727             :     return 0;
     728           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     729           0 :     return fastEmitInst_r(AArch64::FCMGEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     730             :   }
     731             :   return 0;
     732             : }
     733             : 
     734           0 : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     735           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     736             :     return 0;
     737           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     738           0 :     return fastEmitInst_r(AArch64::FCMGEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     739             :   }
     740             :   return 0;
     741             : }
     742             : 
     743             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     744           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     745             :     return 0;
     746           0 :   if ((Subtarget->hasNEON())) {
     747           0 :     return fastEmitInst_r(AArch64::FCMGEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     748             :   }
     749             :   return 0;
     750             : }
     751             : 
     752             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     753           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     754             :     return 0;
     755           0 :   if ((Subtarget->hasNEON())) {
     756           0 :     return fastEmitInst_r(AArch64::FCMGEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     757             :   }
     758             :   return 0;
     759             : }
     760             : 
     761             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     762           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     763             :     return 0;
     764           0 :   if ((Subtarget->hasNEON())) {
     765           0 :     return fastEmitInst_r(AArch64::FCMGEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     766             :   }
     767             :   return 0;
     768             : }
     769             : 
     770             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     771           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     772             :     return 0;
     773           0 :   if ((Subtarget->hasNEON())) {
     774           0 :     return fastEmitInst_r(AArch64::FCMGEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     775             :   }
     776             :   return 0;
     777             : }
     778             : 
     779           0 : unsigned fastEmit_AArch64ISD_FCMGEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     780           0 :   switch (VT.SimpleTy) {
     781           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     782           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     783           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     784           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     785           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     786           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     787             :   default: return 0;
     788             :   }
     789             : }
     790             : 
     791             : // FastEmit functions for AArch64ISD::FCMGTz.
     792             : 
     793           0 : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     794           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     795             :     return 0;
     796           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     797           0 :     return fastEmitInst_r(AArch64::FCMGTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     798             :   }
     799             :   return 0;
     800             : }
     801             : 
     802           0 : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     803           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     804             :     return 0;
     805           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     806           0 :     return fastEmitInst_r(AArch64::FCMGTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     807             :   }
     808             :   return 0;
     809             : }
     810             : 
     811             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     812           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     813             :     return 0;
     814           0 :   if ((Subtarget->hasNEON())) {
     815           0 :     return fastEmitInst_r(AArch64::FCMGTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     816             :   }
     817             :   return 0;
     818             : }
     819             : 
     820             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     821           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     822             :     return 0;
     823           0 :   if ((Subtarget->hasNEON())) {
     824           0 :     return fastEmitInst_r(AArch64::FCMGTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     825             :   }
     826             :   return 0;
     827             : }
     828             : 
     829             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     830           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     831             :     return 0;
     832           0 :   if ((Subtarget->hasNEON())) {
     833           0 :     return fastEmitInst_r(AArch64::FCMGTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     834             :   }
     835             :   return 0;
     836             : }
     837             : 
     838             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     839           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     840             :     return 0;
     841           0 :   if ((Subtarget->hasNEON())) {
     842           0 :     return fastEmitInst_r(AArch64::FCMGTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     843             :   }
     844             :   return 0;
     845             : }
     846             : 
     847           0 : unsigned fastEmit_AArch64ISD_FCMGTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     848           0 :   switch (VT.SimpleTy) {
     849           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     850           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     851           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     852           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     853           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     854           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     855             :   default: return 0;
     856             :   }
     857             : }
     858             : 
     859             : // FastEmit functions for AArch64ISD::FCMLEz.
     860             : 
     861           0 : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     862           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     863             :     return 0;
     864           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     865           0 :     return fastEmitInst_r(AArch64::FCMLEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     866             :   }
     867             :   return 0;
     868             : }
     869             : 
     870           0 : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     871           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     872             :     return 0;
     873           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     874           0 :     return fastEmitInst_r(AArch64::FCMLEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     875             :   }
     876             :   return 0;
     877             : }
     878             : 
     879             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     880           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     881             :     return 0;
     882           0 :   if ((Subtarget->hasNEON())) {
     883           0 :     return fastEmitInst_r(AArch64::FCMLEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     884             :   }
     885             :   return 0;
     886             : }
     887             : 
     888             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     889           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     890             :     return 0;
     891           0 :   if ((Subtarget->hasNEON())) {
     892           0 :     return fastEmitInst_r(AArch64::FCMLEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     893             :   }
     894             :   return 0;
     895             : }
     896             : 
     897             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     898           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     899             :     return 0;
     900           0 :   if ((Subtarget->hasNEON())) {
     901           0 :     return fastEmitInst_r(AArch64::FCMLEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     902             :   }
     903             :   return 0;
     904             : }
     905             : 
     906             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     907           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     908             :     return 0;
     909           0 :   if ((Subtarget->hasNEON())) {
     910           0 :     return fastEmitInst_r(AArch64::FCMLEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     911             :   }
     912             :   return 0;
     913             : }
     914             : 
     915           0 : unsigned fastEmit_AArch64ISD_FCMLEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     916           0 :   switch (VT.SimpleTy) {
     917           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     918           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     919           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     920           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     921           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     922           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     923             :   default: return 0;
     924             :   }
     925             : }
     926             : 
     927             : // FastEmit functions for AArch64ISD::FCMLTz.
     928             : 
     929           0 : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     930           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     931             :     return 0;
     932           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     933           0 :     return fastEmitInst_r(AArch64::FCMLTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     934             :   }
     935             :   return 0;
     936             : }
     937             : 
     938           0 : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     939           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     940             :     return 0;
     941           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     942           0 :     return fastEmitInst_r(AArch64::FCMLTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     943             :   }
     944             :   return 0;
     945             : }
     946             : 
     947             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     948           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     949             :     return 0;
     950           0 :   if ((Subtarget->hasNEON())) {
     951           0 :     return fastEmitInst_r(AArch64::FCMLTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     952             :   }
     953             :   return 0;
     954             : }
     955             : 
     956             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     957           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     958             :     return 0;
     959           0 :   if ((Subtarget->hasNEON())) {
     960           0 :     return fastEmitInst_r(AArch64::FCMLTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     961             :   }
     962             :   return 0;
     963             : }
     964             : 
     965             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     966           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     967             :     return 0;
     968           0 :   if ((Subtarget->hasNEON())) {
     969           0 :     return fastEmitInst_r(AArch64::FCMLTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     970             :   }
     971             :   return 0;
     972             : }
     973             : 
     974             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     975           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     976             :     return 0;
     977           0 :   if ((Subtarget->hasNEON())) {
     978           0 :     return fastEmitInst_r(AArch64::FCMLTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     979             :   }
     980             :   return 0;
     981             : }
     982             : 
     983           0 : unsigned fastEmit_AArch64ISD_FCMLTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     984           0 :   switch (VT.SimpleTy) {
     985           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     986           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     987           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     988           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     989           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     990           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     991             :   default: return 0;
     992             :   }
     993             : }
     994             : 
     995             : // FastEmit functions for AArch64ISD::FRECPE.
     996             : 
     997             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     998           0 :   if (RetVT.SimpleTy != MVT::f32)
     999             :     return 0;
    1000           0 :   return fastEmitInst_r(AArch64::FRECPEv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    1001             : }
    1002             : 
    1003             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1004           0 :   if (RetVT.SimpleTy != MVT::f64)
    1005             :     return 0;
    1006           0 :   return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1007             : }
    1008             : 
    1009             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1010           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    1011             :     return 0;
    1012           0 :   return fastEmitInst_r(AArch64::FRECPEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1013             : }
    1014             : 
    1015             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1016           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1017             :     return 0;
    1018           0 :   return fastEmitInst_r(AArch64::FRECPEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1019             : }
    1020             : 
    1021             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1022           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    1023             :     return 0;
    1024           0 :   return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1025             : }
    1026             : 
    1027             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1028           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1029             :     return 0;
    1030           0 :   return fastEmitInst_r(AArch64::FRECPEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1031             : }
    1032             : 
    1033           0 : unsigned fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1034           0 :   switch (VT.SimpleTy) {
    1035           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRECPE_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1036           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRECPE_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1037           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    1038           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1039           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
    1040           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1041             :   default: return 0;
    1042             :   }
    1043             : }
    1044             : 
    1045             : // FastEmit functions for AArch64ISD::FRSQRTE.
    1046             : 
    1047             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1048           0 :   if (RetVT.SimpleTy != MVT::f32)
    1049             :     return 0;
    1050           0 :   return fastEmitInst_r(AArch64::FRSQRTEv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    1051             : }
    1052             : 
    1053             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1054           0 :   if (RetVT.SimpleTy != MVT::f64)
    1055             :     return 0;
    1056           0 :   return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1057             : }
    1058             : 
    1059             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1060           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    1061             :     return 0;
    1062           0 :   return fastEmitInst_r(AArch64::FRSQRTEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1063             : }
    1064             : 
    1065             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1066           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1067             :     return 0;
    1068           0 :   return fastEmitInst_r(AArch64::FRSQRTEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1069             : }
    1070             : 
    1071             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1072           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    1073             :     return 0;
    1074           0 :   return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1075             : }
    1076             : 
    1077             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1078           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1079             :     return 0;
    1080           0 :   return fastEmitInst_r(AArch64::FRSQRTEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1081             : }
    1082             : 
    1083           0 : unsigned fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1084           0 :   switch (VT.SimpleTy) {
    1085           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1086           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1087           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    1088           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1089           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
    1090           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1091             :   default: return 0;
    1092             :   }
    1093             : }
    1094             : 
    1095             : // FastEmit functions for AArch64ISD::NEG.
    1096             : 
    1097             : unsigned fastEmit_AArch64ISD_NEG_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1098           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1099             :     return 0;
    1100           0 :   return fastEmitInst_r(AArch64::NEGv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1101             : }
    1102             : 
    1103             : unsigned fastEmit_AArch64ISD_NEG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1104           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1105             :     return 0;
    1106           0 :   return fastEmitInst_r(AArch64::NEGv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1107             : }
    1108             : 
    1109             : unsigned fastEmit_AArch64ISD_NEG_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1110           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1111             :     return 0;
    1112           0 :   return fastEmitInst_r(AArch64::NEGv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1113             : }
    1114             : 
    1115             : unsigned fastEmit_AArch64ISD_NEG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1116           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1117             :     return 0;
    1118           0 :   return fastEmitInst_r(AArch64::NEGv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1119             : }
    1120             : 
    1121             : unsigned fastEmit_AArch64ISD_NEG_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1122           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1123             :     return 0;
    1124           0 :   return fastEmitInst_r(AArch64::NEGv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1125             : }
    1126             : 
    1127             : unsigned fastEmit_AArch64ISD_NEG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1128           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1129             :     return 0;
    1130           0 :   return fastEmitInst_r(AArch64::NEGv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1131             : }
    1132             : 
    1133             : unsigned fastEmit_AArch64ISD_NEG_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1134           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    1135             :     return 0;
    1136           0 :   return fastEmitInst_r(AArch64::NEGv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1137             : }
    1138             : 
    1139             : unsigned fastEmit_AArch64ISD_NEG_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1140           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1141             :     return 0;
    1142           0 :   return fastEmitInst_r(AArch64::NEGv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1143             : }
    1144             : 
    1145           0 : unsigned fastEmit_AArch64ISD_NEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1146           0 :   switch (VT.SimpleTy) {
    1147           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_NEG_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1148           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_NEG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1149           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_NEG_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1150           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_NEG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1151           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_NEG_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1152           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_NEG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1153           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_NEG_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    1154           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_NEG_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1155             :   default: return 0;
    1156             :   }
    1157             : }
    1158             : 
    1159             : // FastEmit functions for AArch64ISD::NOT.
    1160             : 
    1161             : unsigned fastEmit_AArch64ISD_NOT_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1162           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1163             :     return 0;
    1164           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1165             : }
    1166             : 
    1167             : unsigned fastEmit_AArch64ISD_NOT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1168           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1169             :     return 0;
    1170           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1171             : }
    1172             : 
    1173             : unsigned fastEmit_AArch64ISD_NOT_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1174           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1175             :     return 0;
    1176           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1177             : }
    1178             : 
    1179             : unsigned fastEmit_AArch64ISD_NOT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1180           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1181             :     return 0;
    1182           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1183             : }
    1184             : 
    1185             : unsigned fastEmit_AArch64ISD_NOT_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1186           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1187             :     return 0;
    1188           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1189             : }
    1190             : 
    1191             : unsigned fastEmit_AArch64ISD_NOT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1192           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1193             :     return 0;
    1194           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1195             : }
    1196             : 
    1197             : unsigned fastEmit_AArch64ISD_NOT_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1198           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    1199             :     return 0;
    1200           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1201             : }
    1202             : 
    1203             : unsigned fastEmit_AArch64ISD_NOT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1204           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1205             :     return 0;
    1206           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1207             : }
    1208             : 
    1209           0 : unsigned fastEmit_AArch64ISD_NOT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1210           0 :   switch (VT.SimpleTy) {
    1211           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_NOT_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1212           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_NOT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1213           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_NOT_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1214           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_NOT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1215           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_NOT_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1216           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_NOT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1217           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_NOT_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    1218           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_NOT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1219             :   default: return 0;
    1220             :   }
    1221             : }
    1222             : 
    1223             : // FastEmit functions for AArch64ISD::REV16.
    1224             : 
    1225             : unsigned fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1226           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1227             :     return 0;
    1228           0 :   if ((Subtarget->hasNEON())) {
    1229           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1230             :   }
    1231             :   return 0;
    1232             : }
    1233             : 
    1234             : unsigned fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1235           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1236             :     return 0;
    1237           0 :   if ((Subtarget->hasNEON())) {
    1238           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1239             :   }
    1240             :   return 0;
    1241             : }
    1242             : 
    1243           0 : unsigned fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1244           0 :   switch (VT.SimpleTy) {
    1245           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1246           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1247             :   default: return 0;
    1248             :   }
    1249             : }
    1250             : 
    1251             : // FastEmit functions for AArch64ISD::REV32.
    1252             : 
    1253             : unsigned fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1254           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1255             :     return 0;
    1256           0 :   if ((Subtarget->hasNEON())) {
    1257           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1258             :   }
    1259             :   return 0;
    1260             : }
    1261             : 
    1262             : unsigned fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1263           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1264             :     return 0;
    1265           0 :   if ((Subtarget->hasNEON())) {
    1266           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1267             :   }
    1268             :   return 0;
    1269             : }
    1270             : 
    1271             : unsigned fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1272           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1273             :     return 0;
    1274           0 :   if ((Subtarget->hasNEON())) {
    1275           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1276             :   }
    1277             :   return 0;
    1278             : }
    1279             : 
    1280             : unsigned fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1281           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1282             :     return 0;
    1283           0 :   if ((Subtarget->hasNEON())) {
    1284           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1285             :   }
    1286             :   return 0;
    1287             : }
    1288             : 
    1289             : unsigned fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1290           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    1291             :     return 0;
    1292           0 :   return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1293             : }
    1294             : 
    1295             : unsigned fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1296           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    1297             :     return 0;
    1298           0 :   return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1299             : }
    1300             : 
    1301           0 : unsigned fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1302           0 :   switch (VT.SimpleTy) {
    1303           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1304           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1305           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1306           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1307           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    1308           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    1309             :   default: return 0;
    1310             :   }
    1311             : }
    1312             : 
    1313             : // FastEmit functions for AArch64ISD::REV64.
    1314             : 
    1315             : unsigned fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1316           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1317             :     return 0;
    1318           0 :   if ((Subtarget->hasNEON())) {
    1319           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1320             :   }
    1321             :   return 0;
    1322             : }
    1323             : 
    1324             : unsigned fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1325           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1326             :     return 0;
    1327           0 :   if ((Subtarget->hasNEON())) {
    1328           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1329             :   }
    1330             :   return 0;
    1331             : }
    1332             : 
    1333             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1334           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1335             :     return 0;
    1336           0 :   if ((Subtarget->hasNEON())) {
    1337           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1338             :   }
    1339             :   return 0;
    1340             : }
    1341             : 
    1342             : unsigned fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1343           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1344             :     return 0;
    1345           0 :   if ((Subtarget->hasNEON())) {
    1346           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1347             :   }
    1348             :   return 0;
    1349             : }
    1350             : 
    1351             : unsigned fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1352           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1353             :     return 0;
    1354           0 :   if ((Subtarget->hasNEON())) {
    1355           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1356             :   }
    1357             :   return 0;
    1358             : }
    1359             : 
    1360             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1361           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1362             :     return 0;
    1363           0 :   if ((Subtarget->hasNEON())) {
    1364           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1365             :   }
    1366             :   return 0;
    1367             : }
    1368             : 
    1369             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1370           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    1371             :     return 0;
    1372           0 :   return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1373             : }
    1374             : 
    1375             : unsigned fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1376           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    1377             :     return 0;
    1378           0 :   return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1379             : }
    1380             : 
    1381             : unsigned fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1382           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    1383             :     return 0;
    1384           0 :   return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1385             : }
    1386             : 
    1387             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1388           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1389             :     return 0;
    1390           0 :   return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1391             : }
    1392             : 
    1393           0 : unsigned fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1394           0 :   switch (VT.SimpleTy) {
    1395           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1396           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1397           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1398           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1399           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1400           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1401           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    1402           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    1403           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    1404           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1405             :   default: return 0;
    1406             :   }
    1407             : }
    1408             : 
    1409             : // FastEmit functions for AArch64ISD::SITOF.
    1410             : 
    1411           0 : unsigned fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1412           0 :   if (RetVT.SimpleTy != MVT::f16)
    1413             :     return 0;
    1414           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    1415           0 :     return fastEmitInst_r(AArch64::SCVTFv1i16, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    1416             :   }
    1417             :   return 0;
    1418             : }
    1419             : 
    1420             : unsigned fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1421           0 :   if (RetVT.SimpleTy != MVT::f32)
    1422             :     return 0;
    1423           0 :   if ((Subtarget->hasNEON())) {
    1424           0 :     return fastEmitInst_r(AArch64::SCVTFv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    1425             :   }
    1426             :   return 0;
    1427             : }
    1428             : 
    1429             : unsigned fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1430           0 :   if (RetVT.SimpleTy != MVT::f64)
    1431             :     return 0;
    1432           0 :   if ((Subtarget->hasNEON())) {
    1433           0 :     return fastEmitInst_r(AArch64::SCVTFv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1434             :   }
    1435             :   return 0;
    1436             : }
    1437             : 
    1438           0 : unsigned fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1439           0 :   switch (VT.SimpleTy) {
    1440           0 :   case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0, Op0IsKill);
    1441           0 :   case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1442           0 :   case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1443             :   default: return 0;
    1444             :   }
    1445             : }
    1446             : 
    1447             : // FastEmit functions for AArch64ISD::UITOF.
    1448             : 
    1449           0 : unsigned fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1450           0 :   if (RetVT.SimpleTy != MVT::f16)
    1451             :     return 0;
    1452           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    1453           0 :     return fastEmitInst_r(AArch64::UCVTFv1i16, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    1454             :   }
    1455             :   return 0;
    1456             : }
    1457             : 
    1458             : unsigned fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1459           0 :   if (RetVT.SimpleTy != MVT::f32)
    1460             :     return 0;
    1461           0 :   if ((Subtarget->hasNEON())) {
    1462           0 :     return fastEmitInst_r(AArch64::UCVTFv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    1463             :   }
    1464             :   return 0;
    1465             : }
    1466             : 
    1467             : unsigned fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1468           0 :   if (RetVT.SimpleTy != MVT::f64)
    1469             :     return 0;
    1470           0 :   if ((Subtarget->hasNEON())) {
    1471           0 :     return fastEmitInst_r(AArch64::UCVTFv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1472             :   }
    1473             :   return 0;
    1474             : }
    1475             : 
    1476           0 : unsigned fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1477           0 :   switch (VT.SimpleTy) {
    1478           0 :   case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0, Op0IsKill);
    1479           0 :   case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1480           0 :   case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1481             :   default: return 0;
    1482             :   }
    1483             : }
    1484             : 
    1485             : // FastEmit functions for ISD::ABS.
    1486             : 
    1487             : unsigned fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1488           0 :   if (RetVT.SimpleTy != MVT::i64)
    1489             :     return 0;
    1490           0 :   if ((Subtarget->hasNEON())) {
    1491           0 :     return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1492             :   }
    1493             :   return 0;
    1494             : }
    1495             : 
    1496             : unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1497           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1498             :     return 0;
    1499           0 :   if ((Subtarget->hasNEON())) {
    1500           0 :     return fastEmitInst_r(AArch64::ABSv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1501             :   }
    1502             :   return 0;
    1503             : }
    1504             : 
    1505             : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1506           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1507             :     return 0;
    1508           0 :   if ((Subtarget->hasNEON())) {
    1509           0 :     return fastEmitInst_r(AArch64::ABSv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1510             :   }
    1511             :   return 0;
    1512             : }
    1513             : 
    1514             : unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1515           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1516             :     return 0;
    1517           0 :   if ((Subtarget->hasNEON())) {
    1518           0 :     return fastEmitInst_r(AArch64::ABSv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1519             :   }
    1520             :   return 0;
    1521             : }
    1522             : 
    1523             : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1524           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1525             :     return 0;
    1526           0 :   if ((Subtarget->hasNEON())) {
    1527           0 :     return fastEmitInst_r(AArch64::ABSv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1528             :   }
    1529             :   return 0;
    1530             : }
    1531             : 
    1532             : unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1533           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1534             :     return 0;
    1535           0 :   if ((Subtarget->hasNEON())) {
    1536           0 :     return fastEmitInst_r(AArch64::ABSv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1537             :   }
    1538             :   return 0;
    1539             : }
    1540             : 
    1541             : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1542           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1543             :     return 0;
    1544           0 :   if ((Subtarget->hasNEON())) {
    1545           0 :     return fastEmitInst_r(AArch64::ABSv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1546             :   }
    1547             :   return 0;
    1548             : }
    1549             : 
    1550             : unsigned fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1551           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    1552             :     return 0;
    1553           0 :   if ((Subtarget->hasNEON())) {
    1554           0 :     return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1555             :   }
    1556             :   return 0;
    1557             : }
    1558             : 
    1559             : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1560           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1561             :     return 0;
    1562           0 :   if ((Subtarget->hasNEON())) {
    1563           0 :     return fastEmitInst_r(AArch64::ABSv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1564             :   }
    1565             :   return 0;
    1566             : }
    1567             : 
    1568           0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1569           0 :   switch (VT.SimpleTy) {
    1570           0 :   case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1571           0 :   case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1572           0 :   case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1573           0 :   case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1574           0 :   case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1575           0 :   case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1576           0 :   case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1577           0 :   case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    1578           0 :   case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1579             :   default: return 0;
    1580             :   }
    1581             : }
    1582             : 
    1583             : // FastEmit functions for ISD::BITCAST.
    1584             : 
    1585             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1586           0 :   if ((!Subtarget->isLittleEndian())) {
    1587           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1588             :   }
    1589             :   return 0;
    1590             : }
    1591             : 
    1592             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1593           0 :   if ((!Subtarget->isLittleEndian())) {
    1594           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1595             :   }
    1596             :   return 0;
    1597             : }
    1598             : 
    1599             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1600           0 :   if ((!Subtarget->isLittleEndian())) {
    1601           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1602             :   }
    1603             :   return 0;
    1604             : }
    1605             : 
    1606             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1607           0 :   if ((!Subtarget->isLittleEndian())) {
    1608           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1609             :   }
    1610             :   return 0;
    1611             : }
    1612             : 
    1613             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1614           0 :   if ((!Subtarget->isLittleEndian())) {
    1615           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1616             :   }
    1617             :   return 0;
    1618             : }
    1619             : 
    1620           4 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1621           4 : switch (RetVT.SimpleTy) {
    1622           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0, Op0IsKill);
    1623           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0, Op0IsKill);
    1624           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0, Op0IsKill);
    1625           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0, Op0IsKill);
    1626           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0, Op0IsKill);
    1627             :   default: return 0;
    1628             : }
    1629             : }
    1630             : 
    1631             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1632           2 :   if ((!Subtarget->isLittleEndian())) {
    1633           2 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1634             :   }
    1635             :   return 0;
    1636             : }
    1637             : 
    1638             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1639           0 :   if ((!Subtarget->isLittleEndian())) {
    1640           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1641             :   }
    1642             :   return 0;
    1643             : }
    1644             : 
    1645             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1646           0 :   if ((!Subtarget->isLittleEndian())) {
    1647           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1648             :   }
    1649             :   return 0;
    1650             : }
    1651             : 
    1652             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    1653           1 :   if ((!Subtarget->isLittleEndian())) {
    1654           1 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1655             :   }
    1656             :   return 0;
    1657             : }
    1658             : 
    1659             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1660           0 :   if ((!Subtarget->isLittleEndian())) {
    1661           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1662             :   }
    1663             :   return 0;
    1664             : }
    1665             : 
    1666             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1667           0 :   if ((!Subtarget->isLittleEndian())) {
    1668           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1669             :   }
    1670             :   return 0;
    1671             : }
    1672             : 
    1673             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    1674           0 :   if ((!Subtarget->isLittleEndian())) {
    1675           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1676             :   }
    1677             :   return 0;
    1678             : }
    1679             : 
    1680           5 : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1681           5 : switch (RetVT.SimpleTy) {
    1682           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0, Op0IsKill);
    1683           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0, Op0IsKill);
    1684           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0, Op0IsKill);
    1685           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0, Op0IsKill);
    1686           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0, Op0IsKill);
    1687           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0, Op0IsKill);
    1688           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0, Op0IsKill);
    1689             :   default: return 0;
    1690             : }
    1691             : }
    1692             : 
    1693             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1694           0 :   if ((!Subtarget->isLittleEndian())) {
    1695           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1696             :   }
    1697             :   return 0;
    1698             : }
    1699             : 
    1700             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1701           0 :   if ((!Subtarget->isLittleEndian())) {
    1702           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1703             :   }
    1704             :   return 0;
    1705             : }
    1706             : 
    1707             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1708           0 :   if ((!Subtarget->isLittleEndian())) {
    1709           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1710             :   }
    1711             :   return 0;
    1712             : }
    1713             : 
    1714             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    1715           0 :   if ((!Subtarget->isLittleEndian())) {
    1716           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1717             :   }
    1718             :   return 0;
    1719             : }
    1720             : 
    1721             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1722           0 :   if ((!Subtarget->isLittleEndian())) {
    1723           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1724             :   }
    1725             :   return 0;
    1726             : }
    1727             : 
    1728             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    1729           0 :   if ((!Subtarget->isLittleEndian())) {
    1730           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1731             :   }
    1732             :   return 0;
    1733             : }
    1734             : 
    1735           0 : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1736           0 : switch (RetVT.SimpleTy) {
    1737           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1738           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1739           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1740           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0, Op0IsKill);
    1741           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0, Op0IsKill);
    1742           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0, Op0IsKill);
    1743             :   default: return 0;
    1744             : }
    1745             : }
    1746             : 
    1747             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1748           2 :   if ((!Subtarget->isLittleEndian())) {
    1749           2 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1750             :   }
    1751             :   return 0;
    1752             : }
    1753             : 
    1754             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1755           0 :   if ((!Subtarget->isLittleEndian())) {
    1756           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1757             :   }
    1758             :   return 0;
    1759             : }
    1760             : 
    1761             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1762           0 :   if ((!Subtarget->isLittleEndian())) {
    1763           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1764             :   }
    1765             :   return 0;
    1766             : }
    1767             : 
    1768             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    1769           1 :   if ((!Subtarget->isLittleEndian())) {
    1770           1 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1771             :   }
    1772             :   return 0;
    1773             : }
    1774             : 
    1775             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1776           0 :   if ((!Subtarget->isLittleEndian())) {
    1777           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1778             :   }
    1779             :   return 0;
    1780             : }
    1781             : 
    1782             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1783           0 :   if ((!Subtarget->isLittleEndian())) {
    1784           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1785             :   }
    1786             :   return 0;
    1787             : }
    1788             : 
    1789             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    1790           0 :   if ((!Subtarget->isLittleEndian())) {
    1791           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1792             :   }
    1793             :   return 0;
    1794             : }
    1795             : 
    1796           5 : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1797           5 : switch (RetVT.SimpleTy) {
    1798           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0, Op0IsKill);
    1799           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0, Op0IsKill);
    1800           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0, Op0IsKill);
    1801           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0, Op0IsKill);
    1802           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v4f16_r(Op0, Op0IsKill);
    1803           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0, Op0IsKill);
    1804           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0, Op0IsKill);
    1805             :   default: return 0;
    1806             : }
    1807             : }
    1808             : 
    1809             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1810           0 :   if ((!Subtarget->isLittleEndian())) {
    1811           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1812             :   }
    1813             :   return 0;
    1814             : }
    1815             : 
    1816             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1817           0 :   if ((!Subtarget->isLittleEndian())) {
    1818           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1819             :   }
    1820             :   return 0;
    1821             : }
    1822             : 
    1823             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1824           0 :   if ((!Subtarget->isLittleEndian())) {
    1825           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1826             :   }
    1827             :   return 0;
    1828             : }
    1829             : 
    1830             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    1831           0 :   if ((!Subtarget->isLittleEndian())) {
    1832           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1833             :   }
    1834             :   return 0;
    1835             : }
    1836             : 
    1837             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1838           0 :   if ((!Subtarget->isLittleEndian())) {
    1839           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1840             :   }
    1841             :   return 0;
    1842             : }
    1843             : 
    1844             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    1845           0 :   if ((!Subtarget->isLittleEndian())) {
    1846           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1847             :   }
    1848             :   return 0;
    1849             : }
    1850             : 
    1851           0 : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1852           0 : switch (RetVT.SimpleTy) {
    1853           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0, Op0IsKill);
    1854           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    1855           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    1856           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v8f16_r(Op0, Op0IsKill);
    1857           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
    1858           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0, Op0IsKill);
    1859             :   default: return 0;
    1860             : }
    1861             : }
    1862             : 
    1863             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1864           2 :   if ((!Subtarget->isLittleEndian())) {
    1865           2 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1866             :   }
    1867             :   return 0;
    1868             : }
    1869             : 
    1870             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1871           0 :   if ((!Subtarget->isLittleEndian())) {
    1872           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1873             :   }
    1874             :   return 0;
    1875             : }
    1876             : 
    1877             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1878           0 :   if ((!Subtarget->isLittleEndian())) {
    1879           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1880             :   }
    1881             :   return 0;
    1882             : }
    1883             : 
    1884             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    1885           1 :   if ((!Subtarget->isLittleEndian())) {
    1886           1 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1887             :   }
    1888             :   return 0;
    1889             : }
    1890             : 
    1891             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1892           0 :   if ((!Subtarget->isLittleEndian())) {
    1893           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1894             :   }
    1895             :   return 0;
    1896             : }
    1897             : 
    1898             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    1899           0 :   if ((!Subtarget->isLittleEndian())) {
    1900           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1901             :   }
    1902             :   return 0;
    1903             : }
    1904             : 
    1905           5 : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1906           5 : switch (RetVT.SimpleTy) {
    1907           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0, Op0IsKill);
    1908           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0, Op0IsKill);
    1909           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0, Op0IsKill);
    1910           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0, Op0IsKill);
    1911           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0, Op0IsKill);
    1912           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0, Op0IsKill);
    1913             :   default: return 0;
    1914             : }
    1915             : }
    1916             : 
    1917             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1918           0 :   if ((!Subtarget->isLittleEndian())) {
    1919           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1920             :   }
    1921             :   return 0;
    1922             : }
    1923             : 
    1924             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1925           0 :   if ((!Subtarget->isLittleEndian())) {
    1926           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1927             :   }
    1928             :   return 0;
    1929             : }
    1930             : 
    1931             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1932           0 :   if ((!Subtarget->isLittleEndian())) {
    1933           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1934             :   }
    1935             :   return 0;
    1936             : }
    1937             : 
    1938             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    1939           0 :   if ((!Subtarget->isLittleEndian())) {
    1940           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1941             :   }
    1942             :   return 0;
    1943             : }
    1944             : 
    1945             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    1946           0 :   if ((!Subtarget->isLittleEndian())) {
    1947           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1948             :   }
    1949             :   return 0;
    1950             : }
    1951             : 
    1952           0 : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1953           0 : switch (RetVT.SimpleTy) {
    1954           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    1955           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    1956           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0, Op0IsKill);
    1957           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0, Op0IsKill);
    1958           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0, Op0IsKill);
    1959             :   default: return 0;
    1960             : }
    1961             : }
    1962             : 
    1963             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1964           0 :   if ((!Subtarget->isLittleEndian())) {
    1965           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1966             :   }
    1967             :   return 0;
    1968             : }
    1969             : 
    1970             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1971           0 :   if ((!Subtarget->isLittleEndian())) {
    1972           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1973             :   }
    1974             :   return 0;
    1975             : }
    1976             : 
    1977             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1978           0 :   if ((!Subtarget->isLittleEndian())) {
    1979           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1980             :   }
    1981             :   return 0;
    1982             : }
    1983             : 
    1984             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1985           0 :   if ((!Subtarget->isLittleEndian())) {
    1986           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1987             :   }
    1988             :   return 0;
    1989             : }
    1990             : 
    1991             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1992           0 :   if ((!Subtarget->isLittleEndian())) {
    1993           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1994             :   }
    1995             :   return 0;
    1996             : }
    1997             : 
    1998           4 : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1999           4 : switch (RetVT.SimpleTy) {
    2000           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0, Op0IsKill);
    2001           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0, Op0IsKill);
    2002           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0, Op0IsKill);
    2003           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0, Op0IsKill);
    2004           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0, Op0IsKill);
    2005             :   default: return 0;
    2006             : }
    2007             : }
    2008             : 
    2009             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2010           0 :   if ((!Subtarget->isLittleEndian())) {
    2011           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2012             :   }
    2013             :   return 0;
    2014             : }
    2015             : 
    2016             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2017           0 :   if ((!Subtarget->isLittleEndian())) {
    2018           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2019             :   }
    2020             :   return 0;
    2021             : }
    2022             : 
    2023             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2024           0 :   if ((!Subtarget->isLittleEndian())) {
    2025           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2026             :   }
    2027             :   return 0;
    2028             : }
    2029             : 
    2030             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    2031           0 :   if ((!Subtarget->isLittleEndian())) {
    2032           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2033             :   }
    2034             :   return 0;
    2035             : }
    2036             : 
    2037             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2038           0 :   if ((!Subtarget->isLittleEndian())) {
    2039           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2040             :   }
    2041             :   return 0;
    2042             : }
    2043             : 
    2044           0 : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2045           0 : switch (RetVT.SimpleTy) {
    2046           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    2047           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    2048           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    2049           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0, Op0IsKill);
    2050           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0, Op0IsKill);
    2051             :   default: return 0;
    2052             : }
    2053             : }
    2054             : 
    2055             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    2056           0 :   if ((!Subtarget->isLittleEndian())) {
    2057           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2058             :   }
    2059             :   return 0;
    2060             : }
    2061             : 
    2062             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    2063           0 :   if ((!Subtarget->isLittleEndian())) {
    2064           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2065             :   }
    2066             :   return 0;
    2067             : }
    2068             : 
    2069             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    2070           0 :   if ((!Subtarget->isLittleEndian())) {
    2071           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2072             :   }
    2073             :   return 0;
    2074             : }
    2075             : 
    2076             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    2077           0 :   if ((!Subtarget->isLittleEndian())) {
    2078           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2079             :   }
    2080             :   return 0;
    2081             : }
    2082             : 
    2083             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    2084           0 :   if ((!Subtarget->isLittleEndian())) {
    2085           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2086             :   }
    2087             :   return 0;
    2088             : }
    2089             : 
    2090             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    2091           0 :   if ((!Subtarget->isLittleEndian())) {
    2092           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2093             :   }
    2094             :   return 0;
    2095             : }
    2096             : 
    2097             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    2098           0 :   if ((!Subtarget->isLittleEndian())) {
    2099           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2100             :   }
    2101             :   return 0;
    2102             : }
    2103             : 
    2104           0 : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2105           0 : switch (RetVT.SimpleTy) {
    2106           0 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0, Op0IsKill);
    2107           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0, Op0IsKill);
    2108           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v4i16_r(Op0, Op0IsKill);
    2109           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0, Op0IsKill);
    2110           0 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0, Op0IsKill);
    2111           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0, Op0IsKill);
    2112           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0, Op0IsKill);
    2113             :   default: return 0;
    2114             : }
    2115             : }
    2116             : 
    2117             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2118           0 :   if ((!Subtarget->isLittleEndian())) {
    2119           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2120             :   }
    2121             :   return 0;
    2122             : }
    2123             : 
    2124             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2125           0 :   if ((!Subtarget->isLittleEndian())) {
    2126           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2127             :   }
    2128             :   return 0;
    2129             : }
    2130             : 
    2131             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2132           0 :   if ((!Subtarget->isLittleEndian())) {
    2133           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2134             :   }
    2135             :   return 0;
    2136             : }
    2137             : 
    2138             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2139           0 :   if ((!Subtarget->isLittleEndian())) {
    2140           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2141             :   }
    2142             :   return 0;
    2143             : }
    2144             : 
    2145             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2146           0 :   if ((!Subtarget->isLittleEndian())) {
    2147           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2148             :   }
    2149             :   return 0;
    2150             : }
    2151             : 
    2152             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    2153           0 :   if ((!Subtarget->isLittleEndian())) {
    2154           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2155             :   }
    2156             :   return 0;
    2157             : }
    2158             : 
    2159           0 : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2160           0 : switch (RetVT.SimpleTy) {
    2161           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0, Op0IsKill);
    2162           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v8i16_r(Op0, Op0IsKill);
    2163           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0, Op0IsKill);
    2164           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0, Op0IsKill);
    2165           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0, Op0IsKill);
    2166           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0, Op0IsKill);
    2167             :   default: return 0;
    2168             : }
    2169             : }
    2170             : 
    2171             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    2172           2 :   if ((!Subtarget->isLittleEndian())) {
    2173           2 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2174             :   }
    2175             :   return 0;
    2176             : }
    2177             : 
    2178             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    2179           0 :   if ((!Subtarget->isLittleEndian())) {
    2180           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2181             :   }
    2182             :   return 0;
    2183             : }
    2184             : 
    2185             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    2186           0 :   if ((!Subtarget->isLittleEndian())) {
    2187           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2188             :   }
    2189             :   return 0;
    2190             : }
    2191             : 
    2192             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    2193           1 :   if ((!Subtarget->isLittleEndian())) {
    2194           1 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2195             :   }
    2196             :   return 0;
    2197             : }
    2198             : 
    2199             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    2200           0 :   if ((!Subtarget->isLittleEndian())) {
    2201           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2202             :   }
    2203             :   return 0;
    2204             : }
    2205             : 
    2206             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    2207           0 :   if ((!Subtarget->isLittleEndian())) {
    2208           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2209             :   }
    2210             :   return 0;
    2211             : }
    2212             : 
    2213           5 : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2214           5 : switch (RetVT.SimpleTy) {
    2215           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0, Op0IsKill);
    2216           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0, Op0IsKill);
    2217           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0, Op0IsKill);
    2218           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0, Op0IsKill);
    2219           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0, Op0IsKill);
    2220           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0, Op0IsKill);
    2221             :   default: return 0;
    2222             : }
    2223             : }
    2224             : 
    2225             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2226           0 :   if ((!Subtarget->isLittleEndian())) {
    2227           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2228             :   }
    2229             :   return 0;
    2230             : }
    2231             : 
    2232             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2233           0 :   if ((!Subtarget->isLittleEndian())) {
    2234           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2235             :   }
    2236             :   return 0;
    2237             : }
    2238             : 
    2239             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2240           0 :   if ((!Subtarget->isLittleEndian())) {
    2241           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2242             :   }
    2243             :   return 0;
    2244             : }
    2245             : 
    2246             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    2247           0 :   if ((!Subtarget->isLittleEndian())) {
    2248           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2249             :   }
    2250             :   return 0;
    2251             : }
    2252             : 
    2253             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    2254           0 :   if ((!Subtarget->isLittleEndian())) {
    2255           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2256             :   }
    2257             :   return 0;
    2258             : }
    2259             : 
    2260           0 : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2261           0 : switch (RetVT.SimpleTy) {
    2262           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0, Op0IsKill);
    2263           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0, Op0IsKill);
    2264           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2265           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0, Op0IsKill);
    2266           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0, Op0IsKill);
    2267             :   default: return 0;
    2268             : }
    2269             : }
    2270             : 
    2271             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    2272           0 :   if ((!Subtarget->isLittleEndian())) {
    2273           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2274             :   }
    2275             :   return 0;
    2276             : }
    2277             : 
    2278             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    2279           0 :   if ((!Subtarget->isLittleEndian())) {
    2280           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2281             :   }
    2282             :   return 0;
    2283             : }
    2284             : 
    2285             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    2286           0 :   if ((!Subtarget->isLittleEndian())) {
    2287           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2288             :   }
    2289             :   return 0;
    2290             : }
    2291             : 
    2292             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    2293           0 :   if ((!Subtarget->isLittleEndian())) {
    2294           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2295             :   }
    2296             :   return 0;
    2297             : }
    2298             : 
    2299             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    2300           0 :   if ((!Subtarget->isLittleEndian())) {
    2301           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2302             :   }
    2303             :   return 0;
    2304             : }
    2305             : 
    2306           0 : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2307           0 : switch (RetVT.SimpleTy) {
    2308           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0, Op0IsKill);
    2309           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0, Op0IsKill);
    2310           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0, Op0IsKill);
    2311           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0, Op0IsKill);
    2312           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0, Op0IsKill);
    2313             :   default: return 0;
    2314             : }
    2315             : }
    2316             : 
    2317             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2318           0 :   if ((!Subtarget->isLittleEndian())) {
    2319           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2320             :   }
    2321             :   return 0;
    2322             : }
    2323             : 
    2324             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2325           0 :   if ((!Subtarget->isLittleEndian())) {
    2326           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2327             :   }
    2328             :   return 0;
    2329             : }
    2330             : 
    2331             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2332           0 :   if ((!Subtarget->isLittleEndian())) {
    2333           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2334             :   }
    2335             :   return 0;
    2336             : }
    2337             : 
    2338             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    2339           0 :   if ((!Subtarget->isLittleEndian())) {
    2340           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2341             :   }
    2342             :   return 0;
    2343             : }
    2344             : 
    2345             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2346           0 :   if ((!Subtarget->isLittleEndian())) {
    2347           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2348             :   }
    2349             :   return 0;
    2350             : }
    2351             : 
    2352           0 : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2353           0 : switch (RetVT.SimpleTy) {
    2354           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0, Op0IsKill);
    2355           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0, Op0IsKill);
    2356           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2357           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0, Op0IsKill);
    2358           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0, Op0IsKill);
    2359             :   default: return 0;
    2360             : }
    2361             : }
    2362             : 
    2363          34 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2364          34 :   switch (VT.SimpleTy) {
    2365           4 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2366           5 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    2367           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2368           5 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    2369           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2370           5 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    2371           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2372           4 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    2373           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2374           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2375           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2376           5 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2377           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2378           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
    2379           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2380             :   default: return 0;
    2381             :   }
    2382             : }
    2383             : 
    2384             : // FastEmit functions for ISD::BITREVERSE.
    2385             : 
    2386             : unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2387           0 :   if (RetVT.SimpleTy != MVT::i32)
    2388             :     return 0;
    2389           0 :   return fastEmitInst_r(AArch64::RBITWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    2390             : }
    2391             : 
    2392             : unsigned fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2393           0 :   if (RetVT.SimpleTy != MVT::i64)
    2394             :     return 0;
    2395           0 :   return fastEmitInst_r(AArch64::RBITXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2396             : }
    2397             : 
    2398           0 : unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2399           0 :   switch (VT.SimpleTy) {
    2400           0 :   case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2401           0 :   case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2402             :   default: return 0;
    2403             :   }
    2404             : }
    2405             : 
    2406             : // FastEmit functions for ISD::BRIND.
    2407             : 
    2408             : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2409           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2410             :     return 0;
    2411           0 :   return fastEmitInst_r(AArch64::BR, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2412             : }
    2413             : 
    2414             : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2415           0 :   switch (VT.SimpleTy) {
    2416           0 :   case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2417             :   default: return 0;
    2418             :   }
    2419             : }
    2420             : 
    2421             : // FastEmit functions for ISD::BSWAP.
    2422             : 
    2423             : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2424           0 :   if (RetVT.SimpleTy != MVT::i32)
    2425             :     return 0;
    2426           0 :   return fastEmitInst_r(AArch64::REVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    2427             : }
    2428             : 
    2429             : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2430           0 :   if (RetVT.SimpleTy != MVT::i64)
    2431             :     return 0;
    2432           0 :   return fastEmitInst_r(AArch64::REVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2433             : }
    2434             : 
    2435           0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2436           0 :   switch (VT.SimpleTy) {
    2437           0 :   case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2438           0 :   case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2439             :   default: return 0;
    2440             :   }
    2441             : }
    2442             : 
    2443             : // FastEmit functions for ISD::CTLZ.
    2444             : 
    2445             : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2446           0 :   if (RetVT.SimpleTy != MVT::i32)
    2447             :     return 0;
    2448           0 :   return fastEmitInst_r(AArch64::CLZWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    2449             : }
    2450             : 
    2451             : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2452           0 :   if (RetVT.SimpleTy != MVT::i64)
    2453             :     return 0;
    2454           0 :   return fastEmitInst_r(AArch64::CLZXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2455             : }
    2456             : 
    2457             : unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2458           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    2459             :     return 0;
    2460           0 :   if ((Subtarget->hasNEON())) {
    2461           0 :     return fastEmitInst_r(AArch64::CLZv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2462             :   }
    2463             :   return 0;
    2464             : }
    2465             : 
    2466             : unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2467           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2468             :     return 0;
    2469           0 :   if ((Subtarget->hasNEON())) {
    2470           0 :     return fastEmitInst_r(AArch64::CLZv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2471             :   }
    2472             :   return 0;
    2473             : }
    2474             : 
    2475             : unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2476           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    2477             :     return 0;
    2478           0 :   if ((Subtarget->hasNEON())) {
    2479           0 :     return fastEmitInst_r(AArch64::CLZv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2480             :   }
    2481             :   return 0;
    2482             : }
    2483             : 
    2484             : unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2485           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    2486             :     return 0;
    2487           0 :   if ((Subtarget->hasNEON())) {
    2488           0 :     return fastEmitInst_r(AArch64::CLZv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2489             :   }
    2490             :   return 0;
    2491             : }
    2492             : 
    2493             : unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2494           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    2495             :     return 0;
    2496           0 :   if ((Subtarget->hasNEON())) {
    2497           0 :     return fastEmitInst_r(AArch64::CLZv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2498             :   }
    2499             :   return 0;
    2500             : }
    2501             : 
    2502             : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2503           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2504             :     return 0;
    2505           0 :   if ((Subtarget->hasNEON())) {
    2506           0 :     return fastEmitInst_r(AArch64::CLZv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2507             :   }
    2508             :   return 0;
    2509             : }
    2510             : 
    2511           0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2512           0 :   switch (VT.SimpleTy) {
    2513           0 :   case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2514           0 :   case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2515           0 :   case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    2516           0 :   case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2517           0 :   case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    2518           0 :   case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2519           0 :   case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    2520           0 :   case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2521             :   default: return 0;
    2522             :   }
    2523             : }
    2524             : 
    2525             : // FastEmit functions for ISD::CTPOP.
    2526             : 
    2527             : unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2528           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    2529             :     return 0;
    2530           0 :   if ((Subtarget->hasNEON())) {
    2531           0 :     return fastEmitInst_r(AArch64::CNTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2532             :   }
    2533             :   return 0;
    2534             : }
    2535             : 
    2536             : unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2537           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2538             :     return 0;
    2539           0 :   if ((Subtarget->hasNEON())) {
    2540           0 :     return fastEmitInst_r(AArch64::CNTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2541             :   }
    2542             :   return 0;
    2543             : }
    2544             : 
    2545           0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2546           0 :   switch (VT.SimpleTy) {
    2547           0 :   case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    2548           0 :   case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2549             :   default: return 0;
    2550             :   }
    2551             : }
    2552             : 
    2553             : // FastEmit functions for ISD::FABS.
    2554             : 
    2555             : unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2556           0 :   if (RetVT.SimpleTy != MVT::f16)
    2557             :     return 0;
    2558           0 :   if ((Subtarget->hasFullFP16())) {
    2559           0 :     return fastEmitInst_r(AArch64::FABSHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2560             :   }
    2561             :   return 0;
    2562             : }
    2563             : 
    2564             : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2565           0 :   if (RetVT.SimpleTy != MVT::f32)
    2566             :     return 0;
    2567           0 :   if ((Subtarget->hasFPARMv8())) {
    2568           0 :     return fastEmitInst_r(AArch64::FABSSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2569             :   }
    2570             :   return 0;
    2571             : }
    2572             : 
    2573             : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2574           0 :   if (RetVT.SimpleTy != MVT::f64)
    2575             :     return 0;
    2576           0 :   if ((Subtarget->hasFPARMv8())) {
    2577           0 :     return fastEmitInst_r(AArch64::FABSDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2578             :   }
    2579             :   return 0;
    2580             : }
    2581             : 
    2582           0 : unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2583           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2584             :     return 0;
    2585           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2586           0 :     return fastEmitInst_r(AArch64::FABSv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2587             :   }
    2588             :   return 0;
    2589             : }
    2590             : 
    2591           0 : unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2592           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2593             :     return 0;
    2594           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2595           0 :     return fastEmitInst_r(AArch64::FABSv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2596             :   }
    2597             :   return 0;
    2598             : }
    2599             : 
    2600             : unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2601           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2602             :     return 0;
    2603           0 :   if ((Subtarget->hasNEON())) {
    2604           0 :     return fastEmitInst_r(AArch64::FABSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2605             :   }
    2606             :   return 0;
    2607             : }
    2608             : 
    2609             : unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2610           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2611             :     return 0;
    2612           0 :   if ((Subtarget->hasNEON())) {
    2613           0 :     return fastEmitInst_r(AArch64::FABSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2614             :   }
    2615             :   return 0;
    2616             : }
    2617             : 
    2618             : unsigned fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2619           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2620             :     return 0;
    2621           0 :   if ((Subtarget->hasNEON())) {
    2622           0 :     return fastEmitInst_r(AArch64::FABSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2623             :   }
    2624             :   return 0;
    2625             : }
    2626             : 
    2627           0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2628           0 :   switch (VT.SimpleTy) {
    2629           0 :   case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2630           0 :   case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2631           0 :   case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2632           0 :   case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2633           0 :   case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2634           0 :   case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2635           0 :   case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2636           0 :   case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2637             :   default: return 0;
    2638             :   }
    2639             : }
    2640             : 
    2641             : // FastEmit functions for ISD::FCEIL.
    2642             : 
    2643             : unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2644           0 :   if (RetVT.SimpleTy != MVT::f16)
    2645             :     return 0;
    2646           0 :   if ((Subtarget->hasFullFP16())) {
    2647           0 :     return fastEmitInst_r(AArch64::FRINTPHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2648             :   }
    2649             :   return 0;
    2650             : }
    2651             : 
    2652             : unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2653           0 :   if (RetVT.SimpleTy != MVT::f32)
    2654             :     return 0;
    2655           0 :   if ((Subtarget->hasFPARMv8())) {
    2656           0 :     return fastEmitInst_r(AArch64::FRINTPSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2657             :   }
    2658             :   return 0;
    2659             : }
    2660             : 
    2661             : unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2662           0 :   if (RetVT.SimpleTy != MVT::f64)
    2663             :     return 0;
    2664           0 :   if ((Subtarget->hasFPARMv8())) {
    2665           0 :     return fastEmitInst_r(AArch64::FRINTPDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2666             :   }
    2667             :   return 0;
    2668             : }
    2669             : 
    2670           0 : unsigned fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2671           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2672             :     return 0;
    2673           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2674           0 :     return fastEmitInst_r(AArch64::FRINTPv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2675             :   }
    2676             :   return 0;
    2677             : }
    2678             : 
    2679           0 : unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2680           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2681             :     return 0;
    2682           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2683           0 :     return fastEmitInst_r(AArch64::FRINTPv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2684             :   }
    2685             :   return 0;
    2686             : }
    2687             : 
    2688             : unsigned fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2689           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2690             :     return 0;
    2691           0 :   if ((Subtarget->hasNEON())) {
    2692           0 :     return fastEmitInst_r(AArch64::FRINTPv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2693             :   }
    2694             :   return 0;
    2695             : }
    2696             : 
    2697             : unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2698           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2699             :     return 0;
    2700           0 :   if ((Subtarget->hasNEON())) {
    2701           0 :     return fastEmitInst_r(AArch64::FRINTPv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2702             :   }
    2703             :   return 0;
    2704             : }
    2705             : 
    2706             : unsigned fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2707           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2708             :     return 0;
    2709           0 :   if ((Subtarget->hasNEON())) {
    2710           0 :     return fastEmitInst_r(AArch64::FRINTPv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2711             :   }
    2712             :   return 0;
    2713             : }
    2714             : 
    2715           0 : unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2716           0 :   switch (VT.SimpleTy) {
    2717           0 :   case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2718           0 :   case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2719           0 :   case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2720           0 :   case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2721           0 :   case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2722           0 :   case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2723           0 :   case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2724           0 :   case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2725             :   default: return 0;
    2726             :   }
    2727             : }
    2728             : 
    2729             : // FastEmit functions for ISD::FFLOOR.
    2730             : 
    2731             : unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2732           0 :   if (RetVT.SimpleTy != MVT::f16)
    2733             :     return 0;
    2734           0 :   if ((Subtarget->hasFullFP16())) {
    2735           0 :     return fastEmitInst_r(AArch64::FRINTMHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2736             :   }
    2737             :   return 0;
    2738             : }
    2739             : 
    2740             : unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2741           0 :   if (RetVT.SimpleTy != MVT::f32)
    2742             :     return 0;
    2743           0 :   if ((Subtarget->hasFPARMv8())) {
    2744           0 :     return fastEmitInst_r(AArch64::FRINTMSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2745             :   }
    2746             :   return 0;
    2747             : }
    2748             : 
    2749             : unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2750           0 :   if (RetVT.SimpleTy != MVT::f64)
    2751             :     return 0;
    2752           0 :   if ((Subtarget->hasFPARMv8())) {
    2753           0 :     return fastEmitInst_r(AArch64::FRINTMDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2754             :   }
    2755             :   return 0;
    2756             : }
    2757             : 
    2758           0 : unsigned fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2759           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2760             :     return 0;
    2761           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2762           0 :     return fastEmitInst_r(AArch64::FRINTMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2763             :   }
    2764             :   return 0;
    2765             : }
    2766             : 
    2767           0 : unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2768           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2769             :     return 0;
    2770           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2771           0 :     return fastEmitInst_r(AArch64::FRINTMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2772             :   }
    2773             :   return 0;
    2774             : }
    2775             : 
    2776             : unsigned fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2777           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2778             :     return 0;
    2779           0 :   if ((Subtarget->hasNEON())) {
    2780           0 :     return fastEmitInst_r(AArch64::FRINTMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2781             :   }
    2782             :   return 0;
    2783             : }
    2784             : 
    2785             : unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2786           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2787             :     return 0;
    2788           0 :   if ((Subtarget->hasNEON())) {
    2789           0 :     return fastEmitInst_r(AArch64::FRINTMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2790             :   }
    2791             :   return 0;
    2792             : }
    2793             : 
    2794             : unsigned fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2795           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2796             :     return 0;
    2797           0 :   if ((Subtarget->hasNEON())) {
    2798           0 :     return fastEmitInst_r(AArch64::FRINTMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2799             :   }
    2800             :   return 0;
    2801             : }
    2802             : 
    2803           0 : unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2804           0 :   switch (VT.SimpleTy) {
    2805           0 :   case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2806           0 :   case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2807           0 :   case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2808           0 :   case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2809           0 :   case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2810           0 :   case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2811           0 :   case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2812           0 :   case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2813             :   default: return 0;
    2814             :   }
    2815             : }
    2816             : 
    2817             : // FastEmit functions for ISD::FNEARBYINT.
    2818             : 
    2819             : unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2820           0 :   if (RetVT.SimpleTy != MVT::f16)
    2821             :     return 0;
    2822           0 :   if ((Subtarget->hasFullFP16())) {
    2823           0 :     return fastEmitInst_r(AArch64::FRINTIHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2824             :   }
    2825             :   return 0;
    2826             : }
    2827             : 
    2828             : unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2829           0 :   if (RetVT.SimpleTy != MVT::f32)
    2830             :     return 0;
    2831           0 :   if ((Subtarget->hasFPARMv8())) {
    2832           0 :     return fastEmitInst_r(AArch64::FRINTISr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2833             :   }
    2834             :   return 0;
    2835             : }
    2836             : 
    2837             : unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2838           0 :   if (RetVT.SimpleTy != MVT::f64)
    2839             :     return 0;
    2840           0 :   if ((Subtarget->hasFPARMv8())) {
    2841           0 :     return fastEmitInst_r(AArch64::FRINTIDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2842             :   }
    2843             :   return 0;
    2844             : }
    2845             : 
    2846           0 : unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2847           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2848             :     return 0;
    2849           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2850           0 :     return fastEmitInst_r(AArch64::FRINTIv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2851             :   }
    2852             :   return 0;
    2853             : }
    2854             : 
    2855           0 : unsigned fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2856           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2857             :     return 0;
    2858           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2859           0 :     return fastEmitInst_r(AArch64::FRINTIv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2860             :   }
    2861             :   return 0;
    2862             : }
    2863             : 
    2864             : unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2865           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2866             :     return 0;
    2867           0 :   if ((Subtarget->hasNEON())) {
    2868           0 :     return fastEmitInst_r(AArch64::FRINTIv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2869             :   }
    2870             :   return 0;
    2871             : }
    2872             : 
    2873             : unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2874           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2875             :     return 0;
    2876           0 :   if ((Subtarget->hasNEON())) {
    2877           0 :     return fastEmitInst_r(AArch64::FRINTIv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2878             :   }
    2879             :   return 0;
    2880             : }
    2881             : 
    2882             : unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2883           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2884             :     return 0;
    2885           0 :   if ((Subtarget->hasNEON())) {
    2886           0 :     return fastEmitInst_r(AArch64::FRINTIv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2887             :   }
    2888             :   return 0;
    2889             : }
    2890             : 
    2891           0 : unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2892           0 :   switch (VT.SimpleTy) {
    2893           0 :   case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2894           0 :   case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2895           0 :   case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2896           0 :   case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2897           0 :   case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2898           0 :   case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2899           0 :   case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2900           0 :   case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2901             :   default: return 0;
    2902             :   }
    2903             : }
    2904             : 
    2905             : // FastEmit functions for ISD::FNEG.
    2906             : 
    2907             : unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2908           0 :   if (RetVT.SimpleTy != MVT::f16)
    2909             :     return 0;
    2910           0 :   if ((Subtarget->hasFullFP16())) {
    2911           0 :     return fastEmitInst_r(AArch64::FNEGHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2912             :   }
    2913             :   return 0;
    2914             : }
    2915             : 
    2916             : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2917           0 :   if (RetVT.SimpleTy != MVT::f32)
    2918             :     return 0;
    2919           0 :   if ((Subtarget->hasFPARMv8())) {
    2920           0 :     return fastEmitInst_r(AArch64::FNEGSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2921             :   }
    2922             :   return 0;
    2923             : }
    2924             : 
    2925             : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2926           0 :   if (RetVT.SimpleTy != MVT::f64)
    2927             :     return 0;
    2928           0 :   if ((Subtarget->hasFPARMv8())) {
    2929           0 :     return fastEmitInst_r(AArch64::FNEGDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2930             :   }
    2931             :   return 0;
    2932             : }
    2933             : 
    2934           0 : unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2935           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2936             :     return 0;
    2937           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2938           0 :     return fastEmitInst_r(AArch64::FNEGv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2939             :   }
    2940             :   return 0;
    2941             : }
    2942             : 
    2943           0 : unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2944           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2945             :     return 0;
    2946           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2947           0 :     return fastEmitInst_r(AArch64::FNEGv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2948             :   }
    2949             :   return 0;
    2950             : }
    2951             : 
    2952             : unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2953           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2954             :     return 0;
    2955           0 :   if ((Subtarget->hasNEON())) {
    2956           0 :     return fastEmitInst_r(AArch64::FNEGv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2957             :   }
    2958             :   return 0;
    2959             : }
    2960             : 
    2961             : unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2962           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2963             :     return 0;
    2964           0 :   if ((Subtarget->hasNEON())) {
    2965           0 :     return fastEmitInst_r(AArch64::FNEGv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2966             :   }
    2967             :   return 0;
    2968             : }
    2969             : 
    2970             : unsigned fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2971           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2972             :     return 0;
    2973           0 :   if ((Subtarget->hasNEON())) {
    2974           0 :     return fastEmitInst_r(AArch64::FNEGv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2975             :   }
    2976             :   return 0;
    2977             : }
    2978             : 
    2979           0 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2980           0 :   switch (VT.SimpleTy) {
    2981           0 :   case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2982           0 :   case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2983           0 :   case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2984           0 :   case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2985           0 :   case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2986           0 :   case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2987           0 :   case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2988           0 :   case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2989             :   default: return 0;
    2990             :   }
    2991             : }
    2992             : 
    2993             : // FastEmit functions for ISD::FP_EXTEND.
    2994             : 
    2995             : unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    2996           0 :   if ((Subtarget->hasFPARMv8())) {
    2997           0 :     return fastEmitInst_r(AArch64::FCVTSHr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2998             :   }
    2999             :   return 0;
    3000             : }
    3001             : 
    3002             : unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3003           0 :   if ((Subtarget->hasFPARMv8())) {
    3004           0 :     return fastEmitInst_r(AArch64::FCVTDHr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3005             :   }
    3006             :   return 0;
    3007             : }
    3008             : 
    3009           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3010           0 : switch (RetVT.SimpleTy) {
    3011           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0, Op0IsKill);
    3012           0 :   case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0, Op0IsKill);
    3013             :   default: return 0;
    3014             : }
    3015             : }
    3016             : 
    3017             : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3018           0 :   if (RetVT.SimpleTy != MVT::f64)
    3019             :     return 0;
    3020           0 :   if ((Subtarget->hasFPARMv8())) {
    3021           0 :     return fastEmitInst_r(AArch64::FCVTDSr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3022             :   }
    3023             :   return 0;
    3024             : }
    3025             : 
    3026             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3027           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3028             :     return 0;
    3029           0 :   return fastEmitInst_r(AArch64::FCVTLv4i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3030             : }
    3031             : 
    3032             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3033           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3034             :     return 0;
    3035           0 :   return fastEmitInst_r(AArch64::FCVTLv2i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3036             : }
    3037             : 
    3038           0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3039           0 :   switch (VT.SimpleTy) {
    3040           0 :   case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3041           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3042           0 :   case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3043           0 :   case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3044             :   default: return 0;
    3045             :   }
    3046             : }
    3047             : 
    3048             : // FastEmit functions for ISD::FP_ROUND.
    3049             : 
    3050             : unsigned fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3051           0 :   if (RetVT.SimpleTy != MVT::f16)
    3052             :     return 0;
    3053           0 :   if ((Subtarget->hasFPARMv8())) {
    3054           0 :     return fastEmitInst_r(AArch64::FCVTHSr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3055             :   }
    3056             :   return 0;
    3057             : }
    3058             : 
    3059             : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3060           0 :   if ((Subtarget->hasFPARMv8())) {
    3061           0 :     return fastEmitInst_r(AArch64::FCVTHDr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3062             :   }
    3063             :   return 0;
    3064             : }
    3065             : 
    3066             : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3067           0 :   if ((Subtarget->hasFPARMv8())) {
    3068           0 :     return fastEmitInst_r(AArch64::FCVTSDr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3069             :   }
    3070             :   return 0;
    3071             : }
    3072             : 
    3073           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3074           0 : switch (RetVT.SimpleTy) {
    3075           0 :   case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0, Op0IsKill);
    3076           0 :   case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0, Op0IsKill);
    3077             :   default: return 0;
    3078             : }
    3079             : }
    3080             : 
    3081             : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3082           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3083             :     return 0;
    3084           0 :   return fastEmitInst_r(AArch64::FCVTNv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3085             : }
    3086             : 
    3087             : unsigned fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3088           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3089             :     return 0;
    3090           0 :   return fastEmitInst_r(AArch64::FCVTNv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3091             : }
    3092             : 
    3093           0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3094           0 :   switch (VT.SimpleTy) {
    3095           0 :   case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3096           0 :   case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3097           0 :   case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3098           0 :   case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3099             :   default: return 0;
    3100             :   }
    3101             : }
    3102             : 
    3103             : // FastEmit functions for ISD::FP_TO_SINT.
    3104             : 
    3105             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3106           1 :   if ((Subtarget->hasFullFP16())) {
    3107           0 :     return fastEmitInst_r(AArch64::FCVTZSUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3108             :   }
    3109             :   return 0;
    3110             : }
    3111             : 
    3112             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3113           0 :   if ((Subtarget->hasFullFP16())) {
    3114           0 :     return fastEmitInst_r(AArch64::FCVTZSUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3115             :   }
    3116             :   return 0;
    3117             : }
    3118             : 
    3119           1 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3120           1 : switch (RetVT.SimpleTy) {
    3121           1 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
    3122           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
    3123             :   default: return 0;
    3124             : }
    3125             : }
    3126             : 
    3127             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3128           3 :   if ((Subtarget->hasFPARMv8())) {
    3129           3 :     return fastEmitInst_r(AArch64::FCVTZSUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3130             :   }
    3131             :   return 0;
    3132             : }
    3133             : 
    3134             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3135           2 :   if ((Subtarget->hasFPARMv8())) {
    3136           2 :     return fastEmitInst_r(AArch64::FCVTZSUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3137             :   }
    3138             :   return 0;
    3139             : }
    3140             : 
    3141           5 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3142           5 : switch (RetVT.SimpleTy) {
    3143           3 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    3144           2 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    3145             :   default: return 0;
    3146             : }
    3147             : }
    3148             : 
    3149             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3150           3 :   if ((Subtarget->hasFPARMv8())) {
    3151           3 :     return fastEmitInst_r(AArch64::FCVTZSUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3152             :   }
    3153             :   return 0;
    3154             : }
    3155             : 
    3156             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3157           2 :   if ((Subtarget->hasFPARMv8())) {
    3158           2 :     return fastEmitInst_r(AArch64::FCVTZSUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3159             :   }
    3160             :   return 0;
    3161             : }
    3162             : 
    3163           5 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3164           5 : switch (RetVT.SimpleTy) {
    3165           3 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    3166           2 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    3167             :   default: return 0;
    3168             : }
    3169             : }
    3170             : 
    3171           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3172           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    3173             :     return 0;
    3174           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3175           0 :     return fastEmitInst_r(AArch64::FCVTZSv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3176             :   }
    3177             :   return 0;
    3178             : }
    3179             : 
    3180           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3181           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3182             :     return 0;
    3183           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3184           0 :     return fastEmitInst_r(AArch64::FCVTZSv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3185             :   }
    3186             :   return 0;
    3187             : }
    3188             : 
    3189             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3190           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    3191             :     return 0;
    3192           0 :   if ((Subtarget->hasNEON())) {
    3193           0 :     return fastEmitInst_r(AArch64::FCVTZSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3194             :   }
    3195             :   return 0;
    3196             : }
    3197             : 
    3198             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3199           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3200             :     return 0;
    3201           0 :   if ((Subtarget->hasNEON())) {
    3202           0 :     return fastEmitInst_r(AArch64::FCVTZSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3203             :   }
    3204             :   return 0;
    3205             : }
    3206             : 
    3207             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3208           1 :   if (RetVT.SimpleTy != MVT::v2i64)
    3209             :     return 0;
    3210           1 :   if ((Subtarget->hasNEON())) {
    3211           1 :     return fastEmitInst_r(AArch64::FCVTZSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3212             :   }
    3213             :   return 0;
    3214             : }
    3215             : 
    3216          12 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3217          12 :   switch (VT.SimpleTy) {
    3218           1 :   case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3219           5 :   case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3220           5 :   case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3221           0 :   case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3222           0 :   case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3223           0 :   case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3224           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3225           1 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3226             :   default: return 0;
    3227             :   }
    3228             : }
    3229             : 
    3230             : // FastEmit functions for ISD::FP_TO_UINT.
    3231             : 
    3232             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3233           0 :   if ((Subtarget->hasFullFP16())) {
    3234           0 :     return fastEmitInst_r(AArch64::FCVTZUUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3235             :   }
    3236             :   return 0;
    3237             : }
    3238             : 
    3239             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3240           0 :   if ((Subtarget->hasFullFP16())) {
    3241           0 :     return fastEmitInst_r(AArch64::FCVTZUUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3242             :   }
    3243             :   return 0;
    3244             : }
    3245             : 
    3246           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3247           0 : switch (RetVT.SimpleTy) {
    3248           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
    3249           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
    3250             :   default: return 0;
    3251             : }
    3252             : }
    3253             : 
    3254             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3255           0 :   if ((Subtarget->hasFPARMv8())) {
    3256           0 :     return fastEmitInst_r(AArch64::FCVTZUUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3257             :   }
    3258             :   return 0;
    3259             : }
    3260             : 
    3261             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3262           0 :   if ((Subtarget->hasFPARMv8())) {
    3263           0 :     return fastEmitInst_r(AArch64::FCVTZUUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3264             :   }
    3265             :   return 0;
    3266             : }
    3267             : 
    3268           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3269           0 : switch (RetVT.SimpleTy) {
    3270           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    3271           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    3272             :   default: return 0;
    3273             : }
    3274             : }
    3275             : 
    3276             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3277           0 :   if ((Subtarget->hasFPARMv8())) {
    3278           0 :     return fastEmitInst_r(AArch64::FCVTZUUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3279             :   }
    3280             :   return 0;
    3281             : }
    3282             : 
    3283             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3284           0 :   if ((Subtarget->hasFPARMv8())) {
    3285           0 :     return fastEmitInst_r(AArch64::FCVTZUUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3286             :   }
    3287             :   return 0;
    3288             : }
    3289             : 
    3290           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3291           0 : switch (RetVT.SimpleTy) {
    3292           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    3293           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    3294             :   default: return 0;
    3295             : }
    3296             : }
    3297             : 
    3298           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3299           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    3300             :     return 0;
    3301           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3302           0 :     return fastEmitInst_r(AArch64::FCVTZUv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3303             :   }
    3304             :   return 0;
    3305             : }
    3306             : 
    3307           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3308           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3309             :     return 0;
    3310           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3311           0 :     return fastEmitInst_r(AArch64::FCVTZUv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3312             :   }
    3313             :   return 0;
    3314             : }
    3315             : 
    3316             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3317           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    3318             :     return 0;
    3319           0 :   if ((Subtarget->hasNEON())) {
    3320           0 :     return fastEmitInst_r(AArch64::FCVTZUv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3321             :   }
    3322             :   return 0;
    3323             : }
    3324             : 
    3325             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3326           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3327             :     return 0;
    3328           0 :   if ((Subtarget->hasNEON())) {
    3329           0 :     return fastEmitInst_r(AArch64::FCVTZUv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3330             :   }
    3331             :   return 0;
    3332             : }
    3333             : 
    3334             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3335           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3336             :     return 0;
    3337           0 :   if ((Subtarget->hasNEON())) {
    3338           0 :     return fastEmitInst_r(AArch64::FCVTZUv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3339             :   }
    3340             :   return 0;
    3341             : }
    3342             : 
    3343           0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3344           0 :   switch (VT.SimpleTy) {
    3345           0 :   case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3346           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3347           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3348           0 :   case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3349           0 :   case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3350           0 :   case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3351           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3352           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3353             :   default: return 0;
    3354             :   }
    3355             : }
    3356             : 
    3357             : // FastEmit functions for ISD::FRINT.
    3358             : 
    3359             : unsigned fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3360           0 :   if (RetVT.SimpleTy != MVT::f16)
    3361             :     return 0;
    3362           0 :   if ((Subtarget->hasFullFP16())) {
    3363           0 :     return fastEmitInst_r(AArch64::FRINTXHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3364             :   }
    3365             :   return 0;
    3366             : }
    3367             : 
    3368             : unsigned fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3369           0 :   if (RetVT.SimpleTy != MVT::f32)
    3370             :     return 0;
    3371           0 :   if ((Subtarget->hasFPARMv8())) {
    3372           0 :     return fastEmitInst_r(AArch64::FRINTXSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3373             :   }
    3374             :   return 0;
    3375             : }
    3376             : 
    3377             : unsigned fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3378           0 :   if (RetVT.SimpleTy != MVT::f64)
    3379             :     return 0;
    3380           0 :   if ((Subtarget->hasFPARMv8())) {
    3381           0 :     return fastEmitInst_r(AArch64::FRINTXDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3382             :   }
    3383             :   return 0;
    3384             : }
    3385             : 
    3386           0 : unsigned fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3387           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3388             :     return 0;
    3389           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3390           0 :     return fastEmitInst_r(AArch64::FRINTXv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3391             :   }
    3392             :   return 0;
    3393             : }
    3394             : 
    3395           0 : unsigned fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3396           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3397             :     return 0;
    3398           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3399           0 :     return fastEmitInst_r(AArch64::FRINTXv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3400             :   }
    3401             :   return 0;
    3402             : }
    3403             : 
    3404             : unsigned fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3405           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3406             :     return 0;
    3407           0 :   if ((Subtarget->hasNEON())) {
    3408           0 :     return fastEmitInst_r(AArch64::FRINTXv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3409             :   }
    3410             :   return 0;
    3411             : }
    3412             : 
    3413             : unsigned fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3414           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3415             :     return 0;
    3416           0 :   if ((Subtarget->hasNEON())) {
    3417           0 :     return fastEmitInst_r(AArch64::FRINTXv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3418             :   }
    3419             :   return 0;
    3420             : }
    3421             : 
    3422             : unsigned fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3423           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3424             :     return 0;
    3425           0 :   if ((Subtarget->hasNEON())) {
    3426           0 :     return fastEmitInst_r(AArch64::FRINTXv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3427             :   }
    3428             :   return 0;
    3429             : }
    3430             : 
    3431           0 : unsigned fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3432           0 :   switch (VT.SimpleTy) {
    3433           0 :   case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3434           0 :   case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3435           0 :   case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3436           0 :   case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3437           0 :   case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3438           0 :   case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3439           0 :   case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3440           0 :   case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3441             :   default: return 0;
    3442             :   }
    3443             : }
    3444             : 
    3445             : // FastEmit functions for ISD::FROUND.
    3446             : 
    3447             : unsigned fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3448           0 :   if (RetVT.SimpleTy != MVT::f16)
    3449             :     return 0;
    3450           0 :   if ((Subtarget->hasFullFP16())) {
    3451           0 :     return fastEmitInst_r(AArch64::FRINTAHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3452             :   }
    3453             :   return 0;
    3454             : }
    3455             : 
    3456             : unsigned fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3457           0 :   if (RetVT.SimpleTy != MVT::f32)
    3458             :     return 0;
    3459           0 :   if ((Subtarget->hasFPARMv8())) {
    3460           0 :     return fastEmitInst_r(AArch64::FRINTASr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3461             :   }
    3462             :   return 0;
    3463             : }
    3464             : 
    3465             : unsigned fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3466           0 :   if (RetVT.SimpleTy != MVT::f64)
    3467             :     return 0;
    3468           0 :   if ((Subtarget->hasFPARMv8())) {
    3469           0 :     return fastEmitInst_r(AArch64::FRINTADr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3470             :   }
    3471             :   return 0;
    3472             : }
    3473             : 
    3474           0 : unsigned fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3475           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3476             :     return 0;
    3477           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3478           0 :     return fastEmitInst_r(AArch64::FRINTAv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3479             :   }
    3480             :   return 0;
    3481             : }
    3482             : 
    3483           0 : unsigned fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3484           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3485             :     return 0;
    3486           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3487           0 :     return fastEmitInst_r(AArch64::FRINTAv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3488             :   }
    3489             :   return 0;
    3490             : }
    3491             : 
    3492             : unsigned fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3493           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3494             :     return 0;
    3495           0 :   if ((Subtarget->hasNEON())) {
    3496           0 :     return fastEmitInst_r(AArch64::FRINTAv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3497             :   }
    3498             :   return 0;
    3499             : }
    3500             : 
    3501             : unsigned fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3502           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3503             :     return 0;
    3504           0 :   if ((Subtarget->hasNEON())) {
    3505           0 :     return fastEmitInst_r(AArch64::FRINTAv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3506             :   }
    3507             :   return 0;
    3508             : }
    3509             : 
    3510             : unsigned fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3511           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3512             :     return 0;
    3513           0 :   if ((Subtarget->hasNEON())) {
    3514           0 :     return fastEmitInst_r(AArch64::FRINTAv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3515             :   }
    3516             :   return 0;
    3517             : }
    3518             : 
    3519           0 : unsigned fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3520           0 :   switch (VT.SimpleTy) {
    3521           0 :   case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3522           0 :   case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3523           0 :   case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3524           0 :   case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3525           0 :   case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3526           0 :   case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3527           0 :   case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3528           0 :   case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3529             :   default: return 0;
    3530             :   }
    3531             : }
    3532             : 
    3533             : // FastEmit functions for ISD::FSQRT.
    3534             : 
    3535             : unsigned fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3536           0 :   if (RetVT.SimpleTy != MVT::f16)
    3537             :     return 0;
    3538           0 :   if ((Subtarget->hasFullFP16())) {
    3539           0 :     return fastEmitInst_r(AArch64::FSQRTHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3540             :   }
    3541             :   return 0;
    3542             : }
    3543             : 
    3544             : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3545           1 :   if (RetVT.SimpleTy != MVT::f32)
    3546             :     return 0;
    3547           1 :   if ((Subtarget->hasFPARMv8())) {
    3548           1 :     return fastEmitInst_r(AArch64::FSQRTSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3549             :   }
    3550             :   return 0;
    3551             : }
    3552             : 
    3553             : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3554           1 :   if (RetVT.SimpleTy != MVT::f64)
    3555             :     return 0;
    3556           1 :   if ((Subtarget->hasFPARMv8())) {
    3557           1 :     return fastEmitInst_r(AArch64::FSQRTDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3558             :   }
    3559             :   return 0;
    3560             : }
    3561             : 
    3562           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3563           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3564             :     return 0;
    3565           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3566           0 :     return fastEmitInst_r(AArch64::FSQRTv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3567             :   }
    3568             :   return 0;
    3569             : }
    3570             : 
    3571           0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3572           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3573             :     return 0;
    3574           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3575           0 :     return fastEmitInst_r(AArch64::FSQRTv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3576             :   }
    3577             :   return 0;
    3578             : }
    3579             : 
    3580             : unsigned fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3581           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3582             :     return 0;
    3583           0 :   if ((Subtarget->hasNEON())) {
    3584           0 :     return fastEmitInst_r(AArch64::FSQRTv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3585             :   }
    3586             :   return 0;
    3587             : }
    3588             : 
    3589             : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3590           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3591             :     return 0;
    3592           0 :   if ((Subtarget->hasNEON())) {
    3593           0 :     return fastEmitInst_r(AArch64::FSQRTv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3594             :   }
    3595             :   return 0;
    3596             : }
    3597             : 
    3598             : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3599           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3600             :     return 0;
    3601           0 :   if ((Subtarget->hasNEON())) {
    3602           0 :     return fastEmitInst_r(AArch64::FSQRTv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3603             :   }
    3604             :   return 0;
    3605             : }
    3606             : 
    3607           2 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3608           2 :   switch (VT.SimpleTy) {
    3609           0 :   case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3610           1 :   case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3611           1 :   case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3612           0 :   case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3613           0 :   case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3614           0 :   case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3615           0 :   case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3616           0 :   case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3617             :   default: return 0;
    3618             :   }
    3619             : }
    3620             : 
    3621             : // FastEmit functions for ISD::FTRUNC.
    3622             : 
    3623             : unsigned fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3624           0 :   if (RetVT.SimpleTy != MVT::f16)
    3625             :     return 0;
    3626           0 :   if ((Subtarget->hasFullFP16())) {
    3627           0 :     return fastEmitInst_r(AArch64::FRINTZHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3628             :   }
    3629             :   return 0;
    3630             : }
    3631             : 
    3632             : unsigned fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3633           0 :   if (RetVT.SimpleTy != MVT::f32)
    3634             :     return 0;
    3635           0 :   if ((Subtarget->hasFPARMv8())) {
    3636           0 :     return fastEmitInst_r(AArch64::FRINTZSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3637             :   }
    3638             :   return 0;
    3639             : }
    3640             : 
    3641             : unsigned fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3642           0 :   if (RetVT.SimpleTy != MVT::f64)
    3643             :     return 0;
    3644           0 :   if ((Subtarget->hasFPARMv8())) {
    3645           0 :     return fastEmitInst_r(AArch64::FRINTZDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3646             :   }
    3647             :   return 0;
    3648             : }
    3649             : 
    3650           0 : unsigned fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3651           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3652             :     return 0;
    3653           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3654           0 :     return fastEmitInst_r(AArch64::FRINTZv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3655             :   }
    3656             :   return 0;
    3657             : }
    3658             : 
    3659           0 : unsigned fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3660           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3661             :     return 0;
    3662           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3663           0 :     return fastEmitInst_r(AArch64::FRINTZv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3664             :   }
    3665             :   return 0;
    3666             : }
    3667             : 
    3668             : unsigned fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3669           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3670             :     return 0;
    3671           0 :   if ((Subtarget->hasNEON())) {
    3672           0 :     return fastEmitInst_r(AArch64::FRINTZv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3673             :   }
    3674             :   return 0;
    3675             : }
    3676             : 
    3677             : unsigned fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3678           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3679             :     return 0;
    3680           0 :   if ((Subtarget->hasNEON())) {
    3681           0 :     return fastEmitInst_r(AArch64::FRINTZv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3682             :   }
    3683             :   return 0;
    3684             : }
    3685             : 
    3686             : unsigned fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3687           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3688             :     return 0;
    3689           0 :   if ((Subtarget->hasNEON())) {
    3690           0 :     return fastEmitInst_r(AArch64::FRINTZv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3691             :   }
    3692             :   return 0;
    3693             : }
    3694             : 
    3695           0 : unsigned fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3696           0 :   switch (VT.SimpleTy) {
    3697           0 :   case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3698           0 :   case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3699           0 :   case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3700           0 :   case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3701           0 :   case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3702           0 :   case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3703           0 :   case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3704           0 :   case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3705             :   default: return 0;
    3706             :   }
    3707             : }
    3708             : 
    3709             : // FastEmit functions for ISD::SINT_TO_FP.
    3710             : 
    3711             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3712           1 :   if ((Subtarget->hasFullFP16())) {
    3713           0 :     return fastEmitInst_r(AArch64::SCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3714             :   }
    3715             :   return 0;
    3716             : }
    3717             : 
    3718             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3719           5 :   if ((Subtarget->hasFPARMv8())) {
    3720           5 :     return fastEmitInst_r(AArch64::SCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3721             :   }
    3722             :   return 0;
    3723             : }
    3724             : 
    3725             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3726           5 :   if ((Subtarget->hasFPARMv8())) {
    3727           5 :     return fastEmitInst_r(AArch64::SCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3728             :   }
    3729             :   return 0;
    3730             : }
    3731             : 
    3732          11 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3733          11 : switch (RetVT.SimpleTy) {
    3734           1 :   case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
    3735           5 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    3736           5 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    3737             :   default: return 0;
    3738             : }
    3739             : }
    3740             : 
    3741             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3742           1 :   if ((Subtarget->hasFullFP16())) {
    3743           0 :     return fastEmitInst_r(AArch64::SCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3744             :   }
    3745             :   return 0;
    3746             : }
    3747             : 
    3748             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3749           3 :   if ((Subtarget->hasFPARMv8())) {
    3750           3 :     return fastEmitInst_r(AArch64::SCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3751             :   }
    3752             :   return 0;
    3753             : }
    3754             : 
    3755             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3756           3 :   if ((Subtarget->hasFPARMv8())) {
    3757           3 :     return fastEmitInst_r(AArch64::SCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3758             :   }
    3759             :   return 0;
    3760             : }
    3761             : 
    3762           9 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3763           9 : switch (RetVT.SimpleTy) {
    3764           1 :   case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
    3765           3 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    3766           3 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    3767             :   default: return 0;
    3768             : }
    3769             : }
    3770             : 
    3771           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3772           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3773             :     return 0;
    3774           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3775           0 :     return fastEmitInst_r(AArch64::SCVTFv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3776             :   }
    3777             :   return 0;
    3778             : }
    3779             : 
    3780           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3781           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3782             :     return 0;
    3783           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3784           0 :     return fastEmitInst_r(AArch64::SCVTFv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3785             :   }
    3786             :   return 0;
    3787             : }
    3788             : 
    3789             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3790           1 :   if (RetVT.SimpleTy != MVT::v2f32)
    3791             :     return 0;
    3792           0 :   if ((Subtarget->hasNEON())) {
    3793           0 :     return fastEmitInst_r(AArch64::SCVTFv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3794             :   }
    3795             :   return 0;
    3796             : }
    3797             : 
    3798             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3799           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3800             :     return 0;
    3801           0 :   if ((Subtarget->hasNEON())) {
    3802           0 :     return fastEmitInst_r(AArch64::SCVTFv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3803             :   }
    3804             :   return 0;
    3805             : }
    3806             : 
    3807             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3808           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3809             :     return 0;
    3810           0 :   if ((Subtarget->hasNEON())) {
    3811           0 :     return fastEmitInst_r(AArch64::SCVTFv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3812             :   }
    3813             :   return 0;
    3814             : }
    3815             : 
    3816          21 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3817          21 :   switch (VT.SimpleTy) {
    3818          11 :   case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3819           9 :   case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3820           0 :   case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    3821           0 :   case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3822           1 :   case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    3823           0 :   case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3824           0 :   case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3825             :   default: return 0;
    3826             :   }
    3827             : }
    3828             : 
    3829             : // FastEmit functions for ISD::TRUNCATE.
    3830             : 
    3831             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3832          12 :   if (RetVT.SimpleTy != MVT::i32)
    3833             :     return 0;
    3834          12 :   return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, AArch64::sub_32);
    3835             : }
    3836             : 
    3837             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3838           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    3839             :     return 0;
    3840           0 :   if ((Subtarget->hasNEON())) {
    3841           0 :     return fastEmitInst_r(AArch64::XTNv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3842             :   }
    3843             :   return 0;
    3844             : }
    3845             : 
    3846             : unsigned fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3847           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    3848             :     return 0;
    3849           0 :   if ((Subtarget->hasNEON())) {
    3850           0 :     return fastEmitInst_r(AArch64::XTNv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3851             :   }
    3852             :   return 0;
    3853             : }
    3854             : 
    3855             : unsigned fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3856           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    3857             :     return 0;
    3858           0 :   if ((Subtarget->hasNEON())) {
    3859           0 :     return fastEmitInst_r(AArch64::XTNv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3860             :   }
    3861             :   return 0;
    3862             : }
    3863             : 
    3864          12 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3865          12 :   switch (VT.SimpleTy) {
    3866          12 :   case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3867           0 :   case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3868           0 :   case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3869           0 :   case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3870             :   default: return 0;
    3871             :   }
    3872             : }
    3873             : 
    3874             : // FastEmit functions for ISD::UINT_TO_FP.
    3875             : 
    3876             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3877           0 :   if ((Subtarget->hasFullFP16())) {
    3878           0 :     return fastEmitInst_r(AArch64::UCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3879             :   }
    3880             :   return 0;
    3881             : }
    3882             : 
    3883             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3884           0 :   if ((Subtarget->hasFPARMv8())) {
    3885           0 :     return fastEmitInst_r(AArch64::UCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3886             :   }
    3887             :   return 0;
    3888             : }
    3889             : 
    3890             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3891           0 :   if ((Subtarget->hasFPARMv8())) {
    3892           0 :     return fastEmitInst_r(AArch64::UCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3893             :   }
    3894             :   return 0;
    3895             : }
    3896             : 
    3897           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3898           0 : switch (RetVT.SimpleTy) {
    3899           0 :   case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
    3900           0 :   case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    3901           0 :   case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    3902             :   default: return 0;
    3903             : }
    3904             : }
    3905             : 
    3906             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3907           0 :   if ((Subtarget->hasFullFP16())) {
    3908           0 :     return fastEmitInst_r(AArch64::UCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3909             :   }
    3910             :   return 0;
    3911             : }
    3912             : 
    3913             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3914           0 :   if ((Subtarget->hasFPARMv8())) {
    3915           0 :     return fastEmitInst_r(AArch64::UCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3916             :   }
    3917             :   return 0;
    3918             : }
    3919             : 
    3920             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3921           0 :   if ((Subtarget->hasFPARMv8())) {
    3922           0 :     return fastEmitInst_r(AArch64::UCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3923             :   }
    3924             :   return 0;
    3925             : }
    3926             : 
    3927           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3928           0 : switch (RetVT.SimpleTy) {
    3929           0 :   case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
    3930           0 :   case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    3931           0 :   case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    3932             :   default: return 0;
    3933             : }
    3934             : }
    3935             : 
    3936           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3937           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3938             :     return 0;
    3939           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3940           0 :     return fastEmitInst_r(AArch64::UCVTFv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3941             :   }
    3942             :   return 0;
    3943             : }
    3944             : 
    3945           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3946           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3947             :     return 0;
    3948           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3949           0 :     return fastEmitInst_r(AArch64::UCVTFv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3950             :   }
    3951             :   return 0;
    3952             : }
    3953             : 
    3954             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3955           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3956             :     return 0;
    3957           0 :   if ((Subtarget->hasNEON())) {
    3958           0 :     return fastEmitInst_r(AArch64::UCVTFv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3959             :   }
    3960             :   return 0;
    3961             : }
    3962             : 
    3963             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3964           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3965             :     return 0;
    3966           0 :   if ((Subtarget->hasNEON())) {
    3967           0 :     return fastEmitInst_r(AArch64::UCVTFv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3968             :   }
    3969             :   return 0;
    3970             : }
    3971             : 
    3972             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3973           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3974             :     return 0;
    3975           0 :   if ((Subtarget->hasNEON())) {
    3976           0 :     return fastEmitInst_r(AArch64::UCVTFv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3977             :   }
    3978             :   return 0;
    3979             : }
    3980             : 
    3981           0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3982           0 :   switch (VT.SimpleTy) {
    3983           0 :   case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3984           0 :   case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3985           0 :   case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    3986           0 :   case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3987           0 :   case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    3988           0 :   case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3989           0 :   case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3990             :   default: return 0;
    3991             :   }
    3992             : }
    3993             : 
    3994             : // Top-level FastEmit function.
    3995             : 
    3996          81 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
    3997          81 :   switch (Opcode) {
    3998           0 :   case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
    3999           0 :   case AArch64ISD::CMEQz: return fastEmit_AArch64ISD_CMEQz_r(VT, RetVT, Op0, Op0IsKill);
    4000           0 :   case AArch64ISD::CMGEz: return fastEmit_AArch64ISD_CMGEz_r(VT, RetVT, Op0, Op0IsKill);
    4001           0 :   case AArch64ISD::CMGTz: return fastEmit_AArch64ISD_CMGTz_r(VT, RetVT, Op0, Op0IsKill);
    4002           0 :   case AArch64ISD::CMLEz: return fastEmit_AArch64ISD_CMLEz_r(VT, RetVT, Op0, Op0IsKill);
    4003           0 :   case AArch64ISD::CMLTz: return fastEmit_AArch64ISD_CMLTz_r(VT, RetVT, Op0, Op0IsKill);
    4004           0 :   case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0, Op0IsKill);
    4005           0 :   case AArch64ISD::FCMEQz: return fastEmit_AArch64ISD_FCMEQz_r(VT, RetVT, Op0, Op0IsKill);
    4006           0 :   case AArch64ISD::FCMGEz: return fastEmit_AArch64ISD_FCMGEz_r(VT, RetVT, Op0, Op0IsKill);
    4007           0 :   case AArch64ISD::FCMGTz: return fastEmit_AArch64ISD_FCMGTz_r(VT, RetVT, Op0, Op0IsKill);
    4008           0 :   case AArch64ISD::FCMLEz: return fastEmit_AArch64ISD_FCMLEz_r(VT, RetVT, Op0, Op0IsKill);
    4009           0 :   case AArch64ISD::FCMLTz: return fastEmit_AArch64ISD_FCMLTz_r(VT, RetVT, Op0, Op0IsKill);
    4010           0 :   case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0, Op0IsKill);
    4011           0 :   case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0, Op0IsKill);
    4012           0 :   case AArch64ISD::NEG: return fastEmit_AArch64ISD_NEG_r(VT, RetVT, Op0, Op0IsKill);
    4013           0 :   case AArch64ISD::NOT: return fastEmit_AArch64ISD_NOT_r(VT, RetVT, Op0, Op0IsKill);
    4014           0 :   case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0, Op0IsKill);
    4015           0 :   case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0, Op0IsKill);
    4016           0 :   case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0, Op0IsKill);
    4017           0 :   case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0, Op0IsKill);
    4018           0 :   case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0, Op0IsKill);
    4019           0 :   case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
    4020          34 :   case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
    4021           0 :   case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0, Op0IsKill);
    4022           0 :   case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    4023           0 :   case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
    4024           0 :   case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
    4025           0 :   case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
    4026           0 :   case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
    4027           0 :   case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0, Op0IsKill);
    4028           0 :   case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0, Op0IsKill);
    4029           0 :   case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0, Op0IsKill);
    4030           0 :   case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
    4031           0 :   case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    4032           0 :   case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
    4033          12 :   case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
    4034           0 :   case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
    4035           0 :   case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0, Op0IsKill);
    4036           0 :   case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0, Op0IsKill);
    4037           2 :   case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
    4038           0 :   case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0, Op0IsKill);
    4039          21 :   case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    4040          12 :   case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
    4041           0 :   case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    4042             :   default: return 0;
    4043             :   }
    4044             : }
    4045             : 
    4046             : // FastEmit functions for AArch64ISD::CMEQ.
    4047             : 
    4048             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4049           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4050             :     return 0;
    4051           0 :   if ((Subtarget->hasNEON())) {
    4052           0 :     return fastEmitInst_rr(AArch64::CMEQv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4053             :   }
    4054             :   return 0;
    4055             : }
    4056             : 
    4057             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4058           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4059             :     return 0;
    4060           0 :   if ((Subtarget->hasNEON())) {
    4061           0 :     return fastEmitInst_rr(AArch64::CMEQv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4062             :   }
    4063             :   return 0;
    4064             : }
    4065             : 
    4066             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4067           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4068             :     return 0;
    4069           0 :   if ((Subtarget->hasNEON())) {
    4070           0 :     return fastEmitInst_rr(AArch64::CMEQv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4071             :   }
    4072             :   return 0;
    4073             : }
    4074             : 
    4075             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4076           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4077             :     return 0;
    4078           0 :   if ((Subtarget->hasNEON())) {
    4079           0 :     return fastEmitInst_rr(AArch64::CMEQv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4080             :   }
    4081             :   return 0;
    4082             : }
    4083             : 
    4084             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4085           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4086             :     return 0;
    4087           0 :   if ((Subtarget->hasNEON())) {
    4088           0 :     return fastEmitInst_rr(AArch64::CMEQv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4089             :   }
    4090             :   return 0;
    4091             : }
    4092             : 
    4093             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4094           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4095             :     return 0;
    4096           0 :   if ((Subtarget->hasNEON())) {
    4097           0 :     return fastEmitInst_rr(AArch64::CMEQv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4098             :   }
    4099             :   return 0;
    4100             : }
    4101             : 
    4102             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4103           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4104             :     return 0;
    4105           0 :   if ((Subtarget->hasNEON())) {
    4106           0 :     return fastEmitInst_rr(AArch64::CMEQv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4107             :   }
    4108             :   return 0;
    4109             : }
    4110             : 
    4111             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4112           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4113             :     return 0;
    4114           0 :   if ((Subtarget->hasNEON())) {
    4115           0 :     return fastEmitInst_rr(AArch64::CMEQv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4116             :   }
    4117             :   return 0;
    4118             : }
    4119             : 
    4120           0 : unsigned fastEmit_AArch64ISD_CMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4121           0 :   switch (VT.SimpleTy) {
    4122           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4123           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4124           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4125           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4126           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4127           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4128           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4129           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4130             :   default: return 0;
    4131             :   }
    4132             : }
    4133             : 
    4134             : // FastEmit functions for AArch64ISD::CMGE.
    4135             : 
    4136             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4137           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4138             :     return 0;
    4139           0 :   if ((Subtarget->hasNEON())) {
    4140           0 :     return fastEmitInst_rr(AArch64::CMGEv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4141             :   }
    4142             :   return 0;
    4143             : }
    4144             : 
    4145             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4146           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4147             :     return 0;
    4148           0 :   if ((Subtarget->hasNEON())) {
    4149           0 :     return fastEmitInst_rr(AArch64::CMGEv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4150             :   }
    4151             :   return 0;
    4152             : }
    4153             : 
    4154             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4155           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4156             :     return 0;
    4157           0 :   if ((Subtarget->hasNEON())) {
    4158           0 :     return fastEmitInst_rr(AArch64::CMGEv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4159             :   }
    4160             :   return 0;
    4161             : }
    4162             : 
    4163             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4164           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4165             :     return 0;
    4166           0 :   if ((Subtarget->hasNEON())) {
    4167           0 :     return fastEmitInst_rr(AArch64::CMGEv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4168             :   }
    4169             :   return 0;
    4170             : }
    4171             : 
    4172             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4173           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4174             :     return 0;
    4175           0 :   if ((Subtarget->hasNEON())) {
    4176           0 :     return fastEmitInst_rr(AArch64::CMGEv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4177             :   }
    4178             :   return 0;
    4179             : }
    4180             : 
    4181             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4182           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4183             :     return 0;
    4184           0 :   if ((Subtarget->hasNEON())) {
    4185           0 :     return fastEmitInst_rr(AArch64::CMGEv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4186             :   }
    4187             :   return 0;
    4188             : }
    4189             : 
    4190             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4191           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4192             :     return 0;
    4193           0 :   if ((Subtarget->hasNEON())) {
    4194           0 :     return fastEmitInst_rr(AArch64::CMGEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4195             :   }
    4196             :   return 0;
    4197             : }
    4198             : 
    4199             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4200           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4201             :     return 0;
    4202           0 :   if ((Subtarget->hasNEON())) {
    4203           0 :     return fastEmitInst_rr(AArch64::CMGEv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4204             :   }
    4205             :   return 0;
    4206             : }
    4207             : 
    4208           0 : unsigned fastEmit_AArch64ISD_CMGE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4209           0 :   switch (VT.SimpleTy) {
    4210           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4211           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4212           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4213           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4214           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4215           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4216           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4217           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4218             :   default: return 0;
    4219             :   }
    4220             : }
    4221             : 
    4222             : // FastEmit functions for AArch64ISD::CMGT.
    4223             : 
    4224             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4225           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4226             :     return 0;
    4227           0 :   if ((Subtarget->hasNEON())) {
    4228           0 :     return fastEmitInst_rr(AArch64::CMGTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4229             :   }
    4230             :   return 0;
    4231             : }
    4232             : 
    4233             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4234           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4235             :     return 0;
    4236           0 :   if ((Subtarget->hasNEON())) {
    4237           0 :     return fastEmitInst_rr(AArch64::CMGTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4238             :   }
    4239             :   return 0;
    4240             : }
    4241             : 
    4242             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4243           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4244             :     return 0;
    4245           0 :   if ((Subtarget->hasNEON())) {
    4246           0 :     return fastEmitInst_rr(AArch64::CMGTv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4247             :   }
    4248             :   return 0;
    4249             : }
    4250             : 
    4251             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4252           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4253             :     return 0;
    4254           0 :   if ((Subtarget->hasNEON())) {
    4255           0 :     return fastEmitInst_rr(AArch64::CMGTv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4256             :   }
    4257             :   return 0;
    4258             : }
    4259             : 
    4260             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4261           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4262             :     return 0;
    4263           0 :   if ((Subtarget->hasNEON())) {
    4264           0 :     return fastEmitInst_rr(AArch64::CMGTv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4265             :   }
    4266             :   return 0;
    4267             : }
    4268             : 
    4269             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4270           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4271             :     return 0;
    4272           0 :   if ((Subtarget->hasNEON())) {
    4273           0 :     return fastEmitInst_rr(AArch64::CMGTv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4274             :   }
    4275             :   return 0;
    4276             : }
    4277             : 
    4278             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4279           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4280             :     return 0;
    4281           0 :   if ((Subtarget->hasNEON())) {
    4282           0 :     return fastEmitInst_rr(AArch64::CMGTv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4283             :   }
    4284             :   return 0;
    4285             : }
    4286             : 
    4287             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4288           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4289             :     return 0;
    4290           0 :   if ((Subtarget->hasNEON())) {
    4291           0 :     return fastEmitInst_rr(AArch64::CMGTv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4292             :   }
    4293             :   return 0;
    4294             : }
    4295             : 
    4296           0 : unsigned fastEmit_AArch64ISD_CMGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4297           0 :   switch (VT.SimpleTy) {
    4298           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4299           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4300           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4301           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4302           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4303           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4304           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4305           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4306             :   default: return 0;
    4307             :   }
    4308             : }
    4309             : 
    4310             : // FastEmit functions for AArch64ISD::CMHI.
    4311             : 
    4312             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4313           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4314             :     return 0;
    4315           0 :   if ((Subtarget->hasNEON())) {
    4316           0 :     return fastEmitInst_rr(AArch64::CMHIv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4317             :   }
    4318             :   return 0;
    4319             : }
    4320             : 
    4321             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4322           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4323             :     return 0;
    4324           0 :   if ((Subtarget->hasNEON())) {
    4325           0 :     return fastEmitInst_rr(AArch64::CMHIv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4326             :   }
    4327             :   return 0;
    4328             : }
    4329             : 
    4330             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4331           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4332             :     return 0;
    4333           0 :   if ((Subtarget->hasNEON())) {
    4334           0 :     return fastEmitInst_rr(AArch64::CMHIv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4335             :   }
    4336             :   return 0;
    4337             : }
    4338             : 
    4339             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4340           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4341             :     return 0;
    4342           0 :   if ((Subtarget->hasNEON())) {
    4343           0 :     return fastEmitInst_rr(AArch64::CMHIv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4344             :   }
    4345             :   return 0;
    4346             : }
    4347             : 
    4348             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4349           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4350             :     return 0;
    4351           0 :   if ((Subtarget->hasNEON())) {
    4352           0 :     return fastEmitInst_rr(AArch64::CMHIv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4353             :   }
    4354             :   return 0;
    4355             : }
    4356             : 
    4357             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4358           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4359             :     return 0;
    4360           0 :   if ((Subtarget->hasNEON())) {
    4361           0 :     return fastEmitInst_rr(AArch64::CMHIv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4362             :   }
    4363             :   return 0;
    4364             : }
    4365             : 
    4366             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4367           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4368             :     return 0;
    4369           0 :   if ((Subtarget->hasNEON())) {
    4370           0 :     return fastEmitInst_rr(AArch64::CMHIv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4371             :   }
    4372             :   return 0;
    4373             : }
    4374             : 
    4375             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4376           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4377             :     return 0;
    4378           0 :   if ((Subtarget->hasNEON())) {
    4379           0 :     return fastEmitInst_rr(AArch64::CMHIv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4380             :   }
    4381             :   return 0;
    4382             : }
    4383             : 
    4384           0 : unsigned fastEmit_AArch64ISD_CMHI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4385           0 :   switch (VT.SimpleTy) {
    4386           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4387           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4388           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4389           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4390           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4391           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4392           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4393           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4394             :   default: return 0;
    4395             :   }
    4396             : }
    4397             : 
    4398             : // FastEmit functions for AArch64ISD::CMHS.
    4399             : 
    4400             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4401           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4402             :     return 0;
    4403           0 :   if ((Subtarget->hasNEON())) {
    4404           0 :     return fastEmitInst_rr(AArch64::CMHSv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4405             :   }
    4406             :   return 0;
    4407             : }
    4408             : 
    4409             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4410           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4411             :     return 0;
    4412           0 :   if ((Subtarget->hasNEON())) {
    4413           0 :     return fastEmitInst_rr(AArch64::CMHSv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4414             :   }
    4415             :   return 0;
    4416             : }
    4417             : 
    4418             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4419           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4420             :     return 0;
    4421           0 :   if ((Subtarget->hasNEON())) {
    4422           0 :     return fastEmitInst_rr(AArch64::CMHSv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4423             :   }
    4424             :   return 0;
    4425             : }
    4426             : 
    4427             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4428           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4429             :     return 0;
    4430           0 :   if ((Subtarget->hasNEON())) {
    4431           0 :     return fastEmitInst_rr(AArch64::CMHSv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4432             :   }
    4433             :   return 0;
    4434             : }
    4435             : 
    4436             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4437           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4438             :     return 0;
    4439           0 :   if ((Subtarget->hasNEON())) {
    4440           0 :     return fastEmitInst_rr(AArch64::CMHSv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4441             :   }
    4442             :   return 0;
    4443             : }
    4444             : 
    4445             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4446           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4447             :     return 0;
    4448           0 :   if ((Subtarget->hasNEON())) {
    4449           0 :     return fastEmitInst_rr(AArch64::CMHSv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4450             :   }
    4451             :   return 0;
    4452             : }
    4453             : 
    4454             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4455           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4456             :     return 0;
    4457           0 :   if ((Subtarget->hasNEON())) {
    4458           0 :     return fastEmitInst_rr(AArch64::CMHSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4459             :   }
    4460             :   return 0;
    4461             : }
    4462             : 
    4463             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4464           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4465             :     return 0;
    4466           0 :   if ((Subtarget->hasNEON())) {
    4467           0 :     return fastEmitInst_rr(AArch64::CMHSv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4468             :   }
    4469             :   return 0;
    4470             : }
    4471             : 
    4472           0 : unsigned fastEmit_AArch64ISD_CMHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4473           0 :   switch (VT.SimpleTy) {
    4474           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4475           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4476           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4477           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4478           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4479           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4480           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4481           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4482             :   default: return 0;
    4483             :   }
    4484             : }
    4485             : 
    4486             : // FastEmit functions for AArch64ISD::FCMEQ.
    4487             : 
    4488             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4489           0 :   if (RetVT.SimpleTy != MVT::i32)
    4490             :     return 0;
    4491           0 :   if ((Subtarget->hasNEON())) {
    4492           0 :     return fastEmitInst_rr(AArch64::FCMEQ32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4493             :   }
    4494             :   return 0;
    4495             : }
    4496             : 
    4497             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4498           0 :   if (RetVT.SimpleTy != MVT::i64)
    4499             :     return 0;
    4500           0 :   if ((Subtarget->hasNEON())) {
    4501           0 :     return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4502             :   }
    4503             :   return 0;
    4504             : }
    4505             : 
    4506           0 : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4507           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4508             :     return 0;
    4509           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4510           0 :     return fastEmitInst_rr(AArch64::FCMEQv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4511             :   }
    4512             :   return 0;
    4513             : }
    4514             : 
    4515           0 : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4516           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4517             :     return 0;
    4518           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4519           0 :     return fastEmitInst_rr(AArch64::FCMEQv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4520             :   }
    4521             :   return 0;
    4522             : }
    4523             : 
    4524             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4525           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4526             :     return 0;
    4527           0 :   if ((Subtarget->hasNEON())) {
    4528           0 :     return fastEmitInst_rr(AArch64::FCMEQv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4529             :   }
    4530             :   return 0;
    4531             : }
    4532             : 
    4533             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4534           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4535             :     return 0;
    4536           0 :   if ((Subtarget->hasNEON())) {
    4537           0 :     return fastEmitInst_rr(AArch64::FCMEQv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4538             :   }
    4539             :   return 0;
    4540             : }
    4541             : 
    4542             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4543           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4544             :     return 0;
    4545           0 :   if ((Subtarget->hasNEON())) {
    4546           0 :     return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4547             :   }
    4548             :   return 0;
    4549             : }
    4550             : 
    4551             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4552           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4553             :     return 0;
    4554           0 :   if ((Subtarget->hasNEON())) {
    4555           0 :     return fastEmitInst_rr(AArch64::FCMEQv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4556             :   }
    4557             :   return 0;
    4558             : }
    4559             : 
    4560           0 : unsigned fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4561           0 :   switch (VT.SimpleTy) {
    4562           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4563           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4564           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4565           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4566           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4567           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4568           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4569           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4570             :   default: return 0;
    4571             :   }
    4572             : }
    4573             : 
    4574             : // FastEmit functions for AArch64ISD::FCMGE.
    4575             : 
    4576             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4577           0 :   if (RetVT.SimpleTy != MVT::i32)
    4578             :     return 0;
    4579           0 :   if ((Subtarget->hasNEON())) {
    4580           0 :     return fastEmitInst_rr(AArch64::FCMGE32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4581             :   }
    4582             :   return 0;
    4583             : }
    4584             : 
    4585             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4586           0 :   if (RetVT.SimpleTy != MVT::i64)
    4587             :     return 0;
    4588           0 :   if ((Subtarget->hasNEON())) {
    4589           0 :     return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4590             :   }
    4591             :   return 0;
    4592             : }
    4593             : 
    4594           0 : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4595           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4596             :     return 0;
    4597           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4598           0 :     return fastEmitInst_rr(AArch64::FCMGEv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4599             :   }
    4600             :   return 0;
    4601             : }
    4602             : 
    4603           0 : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4604           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4605             :     return 0;
    4606           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4607           0 :     return fastEmitInst_rr(AArch64::FCMGEv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4608             :   }
    4609             :   return 0;
    4610             : }
    4611             : 
    4612             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4613           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4614             :     return 0;
    4615           0 :   if ((Subtarget->hasNEON())) {
    4616           0 :     return fastEmitInst_rr(AArch64::FCMGEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4617             :   }
    4618             :   return 0;
    4619             : }
    4620             : 
    4621             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4622           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4623             :     return 0;
    4624           0 :   if ((Subtarget->hasNEON())) {
    4625           0 :     return fastEmitInst_rr(AArch64::FCMGEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4626             :   }
    4627             :   return 0;
    4628             : }
    4629             : 
    4630             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4631           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4632             :     return 0;
    4633           0 :   if ((Subtarget->hasNEON())) {
    4634           0 :     return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4635             :   }
    4636             :   return 0;
    4637             : }
    4638             : 
    4639             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4640           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4641             :     return 0;
    4642           0 :   if ((Subtarget->hasNEON())) {
    4643           0 :     return fastEmitInst_rr(AArch64::FCMGEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4644             :   }
    4645             :   return 0;
    4646             : }
    4647             : 
    4648           0 : unsigned fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4649           0 :   switch (VT.SimpleTy) {
    4650           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4651           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4652           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4653           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4654           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4655           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4656           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4657           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4658             :   default: return 0;
    4659             :   }
    4660             : }
    4661             : 
    4662             : // FastEmit functions for AArch64ISD::FCMGT.
    4663             : 
    4664             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4665           0 :   if (RetVT.SimpleTy != MVT::i32)
    4666             :     return 0;
    4667           0 :   if ((Subtarget->hasNEON())) {
    4668           0 :     return fastEmitInst_rr(AArch64::FCMGT32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4669             :   }
    4670             :   return 0;
    4671             : }
    4672             : 
    4673             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4674           0 :   if (RetVT.SimpleTy != MVT::i64)
    4675             :     return 0;
    4676           0 :   if ((Subtarget->hasNEON())) {
    4677           0 :     return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4678             :   }
    4679             :   return 0;
    4680             : }
    4681             : 
    4682           0 : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4683           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4684             :     return 0;
    4685           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4686           0 :     return fastEmitInst_rr(AArch64::FCMGTv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4687             :   }
    4688             :   return 0;
    4689             : }
    4690             : 
    4691           0 : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4692           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4693             :     return 0;
    4694           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4695           0 :     return fastEmitInst_rr(AArch64::FCMGTv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4696             :   }
    4697             :   return 0;
    4698             : }
    4699             : 
    4700             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4701           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4702             :     return 0;
    4703           0 :   if ((Subtarget->hasNEON())) {
    4704           0 :     return fastEmitInst_rr(AArch64::FCMGTv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4705             :   }
    4706             :   return 0;
    4707             : }
    4708             : 
    4709             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4710           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4711             :     return 0;
    4712           0 :   if ((Subtarget->hasNEON())) {
    4713           0 :     return fastEmitInst_rr(AArch64::FCMGTv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4714             :   }
    4715             :   return 0;
    4716             : }
    4717             : 
    4718             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4719           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4720             :     return 0;
    4721           0 :   if ((Subtarget->hasNEON())) {
    4722           0 :     return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4723             :   }
    4724             :   return 0;
    4725             : }
    4726             : 
    4727             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4728           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4729             :     return 0;
    4730           0 :   if ((Subtarget->hasNEON())) {
    4731           0 :     return fastEmitInst_rr(AArch64::FCMGTv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4732             :   }
    4733             :   return 0;
    4734             : }
    4735             : 
    4736           0 : unsigned fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4737           0 :   switch (VT.SimpleTy) {
    4738           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4739           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4740           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4741           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4742           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4743           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4744           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4745           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4746             :   default: return 0;
    4747             :   }
    4748             : }
    4749             : 
    4750             : // FastEmit functions for AArch64ISD::FCMP.
    4751             : 
    4752             : unsigned fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4753           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4754             :     return 0;
    4755           0 :   if ((Subtarget->hasFullFP16())) {
    4756           0 :     return fastEmitInst_rr(AArch64::FCMPHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4757             :   }
    4758             :   return 0;
    4759             : }
    4760             : 
    4761             : unsigned fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4762           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4763             :     return 0;
    4764           0 :   if ((Subtarget->hasFPARMv8())) {
    4765           0 :     return fastEmitInst_rr(AArch64::FCMPSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4766             :   }
    4767             :   return 0;
    4768             : }
    4769             : 
    4770             : unsigned fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4771           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4772             :     return 0;
    4773           0 :   if ((Subtarget->hasFPARMv8())) {
    4774           0 :     return fastEmitInst_rr(AArch64::FCMPDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4775             :   }
    4776             :   return 0;
    4777             : }
    4778             : 
    4779           0 : unsigned fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4780           0 :   switch (VT.SimpleTy) {
    4781           0 :   case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4782           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4783           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4784             :   default: return 0;
    4785             :   }
    4786             : }
    4787             : 
    4788             : // FastEmit functions for AArch64ISD::FRECPS.
    4789             : 
    4790             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4791           0 :   if (RetVT.SimpleTy != MVT::f32)
    4792             :     return 0;
    4793           0 :   return fastEmitInst_rr(AArch64::FRECPS32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4794             : }
    4795             : 
    4796             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4797           0 :   if (RetVT.SimpleTy != MVT::f64)
    4798             :     return 0;
    4799           0 :   return fastEmitInst_rr(AArch64::FRECPS64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4800             : }
    4801             : 
    4802             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4803           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    4804             :     return 0;
    4805           0 :   return fastEmitInst_rr(AArch64::FRECPSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4806             : }
    4807             : 
    4808             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4809           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4810             :     return 0;
    4811           0 :   return fastEmitInst_rr(AArch64::FRECPSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4812             : }
    4813             : 
    4814             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4815           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4816             :     return 0;
    4817           0 :   return fastEmitInst_rr(AArch64::FRECPSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4818             : }
    4819             : 
    4820           0 : unsigned fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4821           0 :   switch (VT.SimpleTy) {
    4822           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4823           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4824           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4825           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4826           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4827             :   default: return 0;
    4828             :   }
    4829             : }
    4830             : 
    4831             : // FastEmit functions for AArch64ISD::FRSQRTS.
    4832             : 
    4833             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4834           0 :   if (RetVT.SimpleTy != MVT::f32)
    4835             :     return 0;
    4836           0 :   return fastEmitInst_rr(AArch64::FRSQRTS32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4837             : }
    4838             : 
    4839             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4840           0 :   if (RetVT.SimpleTy != MVT::f64)
    4841             :     return 0;
    4842           0 :   return fastEmitInst_rr(AArch64::FRSQRTS64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4843             : }
    4844             : 
    4845             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4846           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    4847             :     return 0;
    4848           0 :   return fastEmitInst_rr(AArch64::FRSQRTSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4849             : }
    4850             : 
    4851             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4852           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4853             :     return 0;
    4854           0 :   return fastEmitInst_rr(AArch64::FRSQRTSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4855             : }
    4856             : 
    4857             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4858           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4859             :     return 0;
    4860           0 :   return fastEmitInst_rr(AArch64::FRSQRTSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4861             : }
    4862             : 
    4863           0 : unsigned fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4864           0 :   switch (VT.SimpleTy) {
    4865           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4866           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4867           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4868           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4869           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4870             :   default: return 0;
    4871             :   }
    4872             : }
    4873             : 
    4874             : // FastEmit functions for AArch64ISD::SMULL.
    4875             : 
    4876             : unsigned fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4877           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4878             :     return 0;
    4879           0 :   return fastEmitInst_rr(AArch64::SMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4880             : }
    4881             : 
    4882             : unsigned fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4883           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4884             :     return 0;
    4885           0 :   return fastEmitInst_rr(AArch64::SMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4886             : }
    4887             : 
    4888             : unsigned fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4889           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4890             :     return 0;
    4891           0 :   return fastEmitInst_rr(AArch64::SMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4892             : }
    4893             : 
    4894           0 : unsigned fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4895           0 :   switch (VT.SimpleTy) {
    4896           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4897           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4898           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4899             :   default: return 0;
    4900             :   }
    4901             : }
    4902             : 
    4903             : // FastEmit functions for AArch64ISD::TRN1.
    4904             : 
    4905             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4906           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4907             :     return 0;
    4908           0 :   if ((Subtarget->hasNEON())) {
    4909           0 :     return fastEmitInst_rr(AArch64::TRN1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4910             :   }
    4911             :   return 0;
    4912             : }
    4913             : 
    4914             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4915           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4916             :     return 0;
    4917           0 :   if ((Subtarget->hasNEON())) {
    4918           0 :     return fastEmitInst_rr(AArch64::TRN1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4919             :   }
    4920             :   return 0;
    4921             : }
    4922             : 
    4923             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4924           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4925             :     return 0;
    4926           0 :   if ((Subtarget->hasNEON())) {
    4927           0 :     return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4928             :   }
    4929             :   return 0;
    4930             : }
    4931             : 
    4932             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4933           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4934             :     return 0;
    4935           0 :   if ((Subtarget->hasNEON())) {
    4936           0 :     return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4937             :   }
    4938             :   return 0;
    4939             : }
    4940             : 
    4941             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4942           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4943             :     return 0;
    4944           0 :   if ((Subtarget->hasNEON())) {
    4945           0 :     return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4946             :   }
    4947             :   return 0;
    4948             : }
    4949             : 
    4950             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4951           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4952             :     return 0;
    4953           0 :   if ((Subtarget->hasNEON())) {
    4954           0 :     return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4955             :   }
    4956             :   return 0;
    4957             : }
    4958             : 
    4959             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4960           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4961             :     return 0;
    4962           0 :   if ((Subtarget->hasNEON())) {
    4963           0 :     return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4964             :   }
    4965             :   return 0;
    4966             : }
    4967             : 
    4968             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4969           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    4970             :     return 0;
    4971           0 :   if ((Subtarget->hasNEON())) {
    4972           0 :     return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4973             :   }
    4974             :   return 0;
    4975             : }
    4976             : 
    4977             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4978           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    4979             :     return 0;
    4980           0 :   if ((Subtarget->hasNEON())) {
    4981           0 :     return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4982             :   }
    4983             :   return 0;
    4984             : }
    4985             : 
    4986             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4987           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    4988             :     return 0;
    4989           0 :   if ((Subtarget->hasNEON())) {
    4990           0 :     return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4991             :   }
    4992             :   return 0;
    4993             : }
    4994             : 
    4995             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4996           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4997             :     return 0;
    4998           0 :   if ((Subtarget->hasNEON())) {
    4999           0 :     return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5000             :   }
    5001             :   return 0;
    5002             : }
    5003             : 
    5004             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5005           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5006             :     return 0;
    5007           0 :   if ((Subtarget->hasNEON())) {
    5008           0 :     return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5009             :   }
    5010             :   return 0;
    5011             : }
    5012             : 
    5013           0 : unsigned fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5014           0 :   switch (VT.SimpleTy) {
    5015           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5016           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5017           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5018           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5019           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5020           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5021           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5022           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5023           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5024           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5025           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5026           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5027             :   default: return 0;
    5028             :   }
    5029             : }
    5030             : 
    5031             : // FastEmit functions for AArch64ISD::TRN2.
    5032             : 
    5033             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5034           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5035             :     return 0;
    5036           0 :   if ((Subtarget->hasNEON())) {
    5037           0 :     return fastEmitInst_rr(AArch64::TRN2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5038             :   }
    5039             :   return 0;
    5040             : }
    5041             : 
    5042             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5043           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5044             :     return 0;
    5045           0 :   if ((Subtarget->hasNEON())) {
    5046           0 :     return fastEmitInst_rr(AArch64::TRN2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5047             :   }
    5048             :   return 0;
    5049             : }
    5050             : 
    5051             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5052           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5053             :     return 0;
    5054           0 :   if ((Subtarget->hasNEON())) {
    5055           0 :     return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5056             :   }
    5057             :   return 0;
    5058             : }
    5059             : 
    5060             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5061           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5062             :     return 0;
    5063           0 :   if ((Subtarget->hasNEON())) {
    5064           0 :     return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5065             :   }
    5066             :   return 0;
    5067             : }
    5068             : 
    5069             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5070           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5071             :     return 0;
    5072           0 :   if ((Subtarget->hasNEON())) {
    5073           0 :     return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5074             :   }
    5075             :   return 0;
    5076             : }
    5077             : 
    5078             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5079           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5080             :     return 0;
    5081           0 :   if ((Subtarget->hasNEON())) {
    5082           0 :     return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5083             :   }
    5084             :   return 0;
    5085             : }
    5086             : 
    5087             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5088           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5089             :     return 0;
    5090           0 :   if ((Subtarget->hasNEON())) {
    5091           0 :     return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5092             :   }
    5093             :   return 0;
    5094             : }
    5095             : 
    5096             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5097           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5098             :     return 0;
    5099           0 :   if ((Subtarget->hasNEON())) {
    5100           0 :     return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5101             :   }
    5102             :   return 0;
    5103             : }
    5104             : 
    5105             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5106           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5107             :     return 0;
    5108           0 :   if ((Subtarget->hasNEON())) {
    5109           0 :     return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5110             :   }
    5111             :   return 0;
    5112             : }
    5113             : 
    5114             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5115           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5116             :     return 0;
    5117           0 :   if ((Subtarget->hasNEON())) {
    5118           0 :     return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5119             :   }
    5120             :   return 0;
    5121             : }
    5122             : 
    5123             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5124           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5125             :     return 0;
    5126           0 :   if ((Subtarget->hasNEON())) {
    5127           0 :     return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5128             :   }
    5129             :   return 0;
    5130             : }
    5131             : 
    5132             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5133           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5134             :     return 0;
    5135           0 :   if ((Subtarget->hasNEON())) {
    5136           0 :     return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5137             :   }
    5138             :   return 0;
    5139             : }
    5140             : 
    5141           0 : unsigned fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5142           0 :   switch (VT.SimpleTy) {
    5143           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5144           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5145           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5146           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5147           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5148           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5149           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5150           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5151           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5152           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5153           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5154           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5155             :   default: return 0;
    5156             :   }
    5157             : }
    5158             : 
    5159             : // FastEmit functions for AArch64ISD::UMULL.
    5160             : 
    5161             : unsigned fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5162           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5163             :     return 0;
    5164           0 :   return fastEmitInst_rr(AArch64::UMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5165             : }
    5166             : 
    5167             : unsigned fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5168           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5169             :     return 0;
    5170           0 :   return fastEmitInst_rr(AArch64::UMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5171             : }
    5172             : 
    5173             : unsigned fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5174           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5175             :     return 0;
    5176           0 :   return fastEmitInst_rr(AArch64::UMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5177             : }
    5178             : 
    5179           0 : unsigned fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5180           0 :   switch (VT.SimpleTy) {
    5181           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5182           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5183           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5184             :   default: return 0;
    5185             :   }
    5186             : }
    5187             : 
    5188             : // FastEmit functions for AArch64ISD::UZP1.
    5189             : 
    5190             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5191           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5192             :     return 0;
    5193           0 :   if ((Subtarget->hasNEON())) {
    5194           0 :     return fastEmitInst_rr(AArch64::UZP1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5195             :   }
    5196             :   return 0;
    5197             : }
    5198             : 
    5199             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5200           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5201             :     return 0;
    5202           0 :   if ((Subtarget->hasNEON())) {
    5203           0 :     return fastEmitInst_rr(AArch64::UZP1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5204             :   }
    5205             :   return 0;
    5206             : }
    5207             : 
    5208             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5209           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5210             :     return 0;
    5211           0 :   if ((Subtarget->hasNEON())) {
    5212           0 :     return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5213             :   }
    5214             :   return 0;
    5215             : }
    5216             : 
    5217             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5218           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5219             :     return 0;
    5220           0 :   if ((Subtarget->hasNEON())) {
    5221           0 :     return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5222             :   }
    5223             :   return 0;
    5224             : }
    5225             : 
    5226             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5227           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5228             :     return 0;
    5229           0 :   if ((Subtarget->hasNEON())) {
    5230           0 :     return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5231             :   }
    5232             :   return 0;
    5233             : }
    5234             : 
    5235             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5236           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5237             :     return 0;
    5238           0 :   if ((Subtarget->hasNEON())) {
    5239           0 :     return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5240             :   }
    5241             :   return 0;
    5242             : }
    5243             : 
    5244             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5245           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5246             :     return 0;
    5247           0 :   if ((Subtarget->hasNEON())) {
    5248           0 :     return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5249             :   }
    5250             :   return 0;
    5251             : }
    5252             : 
    5253             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5254           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5255             :     return 0;
    5256           0 :   if ((Subtarget->hasNEON())) {
    5257           0 :     return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5258             :   }
    5259             :   return 0;
    5260             : }
    5261             : 
    5262             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5263           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5264             :     return 0;
    5265           0 :   if ((Subtarget->hasNEON())) {
    5266           0 :     return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5267             :   }
    5268             :   return 0;
    5269             : }
    5270             : 
    5271             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5272           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5273             :     return 0;
    5274           0 :   if ((Subtarget->hasNEON())) {
    5275           0 :     return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5276             :   }
    5277             :   return 0;
    5278             : }
    5279             : 
    5280             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5281           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5282             :     return 0;
    5283           0 :   if ((Subtarget->hasNEON())) {
    5284           0 :     return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5285             :   }
    5286             :   return 0;
    5287             : }
    5288             : 
    5289             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5290           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5291             :     return 0;
    5292           0 :   if ((Subtarget->hasNEON())) {
    5293           0 :     return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5294             :   }
    5295             :   return 0;
    5296             : }
    5297             : 
    5298           0 : unsigned fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5299           0 :   switch (VT.SimpleTy) {
    5300           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5301           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5302           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5303           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5304           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5305           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5306           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5307           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5308           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5309           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5310           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5311           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5312             :   default: return 0;
    5313             :   }
    5314             : }
    5315             : 
    5316             : // FastEmit functions for AArch64ISD::UZP2.
    5317             : 
    5318             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5319           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5320             :     return 0;
    5321           0 :   if ((Subtarget->hasNEON())) {
    5322           0 :     return fastEmitInst_rr(AArch64::UZP2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5323             :   }
    5324             :   return 0;
    5325             : }
    5326             : 
    5327             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5328           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5329             :     return 0;
    5330           0 :   if ((Subtarget->hasNEON())) {
    5331           0 :     return fastEmitInst_rr(AArch64::UZP2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5332             :   }
    5333             :   return 0;
    5334             : }
    5335             : 
    5336             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5337           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5338             :     return 0;
    5339           0 :   if ((Subtarget->hasNEON())) {
    5340           0 :     return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5341             :   }
    5342             :   return 0;
    5343             : }
    5344             : 
    5345             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5346           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5347             :     return 0;
    5348           0 :   if ((Subtarget->hasNEON())) {
    5349           0 :     return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5350             :   }
    5351             :   return 0;
    5352             : }
    5353             : 
    5354             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5355           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5356             :     return 0;
    5357           0 :   if ((Subtarget->hasNEON())) {
    5358           0 :     return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5359             :   }
    5360             :   return 0;
    5361             : }
    5362             : 
    5363             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5364           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5365             :     return 0;
    5366           0 :   if ((Subtarget->hasNEON())) {
    5367           0 :     return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5368             :   }
    5369             :   return 0;
    5370             : }
    5371             : 
    5372             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5373           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5374             :     return 0;
    5375           0 :   if ((Subtarget->hasNEON())) {
    5376           0 :     return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5377             :   }
    5378             :   return 0;
    5379             : }
    5380             : 
    5381             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5382           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5383             :     return 0;
    5384           0 :   if ((Subtarget->hasNEON())) {
    5385           0 :     return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5386             :   }
    5387             :   return 0;
    5388             : }
    5389             : 
    5390             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5391           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5392             :     return 0;
    5393           0 :   if ((Subtarget->hasNEON())) {
    5394           0 :     return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5395             :   }
    5396             :   return 0;
    5397             : }
    5398             : 
    5399             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5400           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5401             :     return 0;
    5402           0 :   if ((Subtarget->hasNEON())) {
    5403           0 :     return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5404             :   }
    5405             :   return 0;
    5406             : }
    5407             : 
    5408             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5409           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5410             :     return 0;
    5411           0 :   if ((Subtarget->hasNEON())) {
    5412           0 :     return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5413             :   }
    5414             :   return 0;
    5415             : }
    5416             : 
    5417             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5418           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5419             :     return 0;
    5420           0 :   if ((Subtarget->hasNEON())) {
    5421           0 :     return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5422             :   }
    5423             :   return 0;
    5424             : }
    5425             : 
    5426           0 : unsigned fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5427           0 :   switch (VT.SimpleTy) {
    5428           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5429           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5430           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5431           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5432           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5433           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5434           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5435           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5436           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5437           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5438           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5439           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5440             :   default: return 0;
    5441             :   }
    5442             : }
    5443             : 
    5444             : // FastEmit functions for AArch64ISD::ZIP1.
    5445             : 
    5446             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5447           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5448             :     return 0;
    5449           0 :   if ((Subtarget->hasNEON())) {
    5450           0 :     return fastEmitInst_rr(AArch64::ZIP1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5451             :   }
    5452             :   return 0;
    5453             : }
    5454             : 
    5455             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5456           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5457             :     return 0;
    5458           0 :   if ((Subtarget->hasNEON())) {
    5459           0 :     return fastEmitInst_rr(AArch64::ZIP1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5460             :   }
    5461             :   return 0;
    5462             : }
    5463             : 
    5464             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5465           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5466             :     return 0;
    5467           0 :   if ((Subtarget->hasNEON())) {
    5468           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5469             :   }
    5470             :   return 0;
    5471             : }
    5472             : 
    5473             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5474           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5475             :     return 0;
    5476           0 :   if ((Subtarget->hasNEON())) {
    5477           0 :     return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5478             :   }
    5479             :   return 0;
    5480             : }
    5481             : 
    5482             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5483           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5484             :     return 0;
    5485           0 :   if ((Subtarget->hasNEON())) {
    5486           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5487             :   }
    5488             :   return 0;
    5489             : }
    5490             : 
    5491             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5492           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5493             :     return 0;
    5494           0 :   if ((Subtarget->hasNEON())) {
    5495           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5496             :   }
    5497             :   return 0;
    5498             : }
    5499             : 
    5500             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5501           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5502             :     return 0;
    5503           0 :   if ((Subtarget->hasNEON())) {
    5504           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5505             :   }
    5506             :   return 0;
    5507             : }
    5508             : 
    5509             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5510           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5511             :     return 0;
    5512           0 :   if ((Subtarget->hasNEON())) {
    5513           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5514             :   }
    5515             :   return 0;
    5516             : }
    5517             : 
    5518             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5519           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5520             :     return 0;
    5521           0 :   if ((Subtarget->hasNEON())) {
    5522           0 :     return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5523             :   }
    5524             :   return 0;
    5525             : }
    5526             : 
    5527             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5528           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5529             :     return 0;
    5530           0 :   if ((Subtarget->hasNEON())) {
    5531           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5532             :   }
    5533             :   return 0;
    5534             : }
    5535             : 
    5536             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5537           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5538             :     return 0;
    5539           0 :   if ((Subtarget->hasNEON())) {
    5540           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5541             :   }
    5542             :   return 0;
    5543             : }
    5544             : 
    5545             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5546           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5547             :     return 0;
    5548           0 :   if ((Subtarget->hasNEON())) {
    5549           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5550             :   }
    5551             :   return 0;
    5552             : }
    5553             : 
    5554           0 : unsigned fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5555           0 :   switch (VT.SimpleTy) {
    5556           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5557           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5558           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5559           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5560           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5561           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5562           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5563           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5564           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5565           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5566           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5567           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5568             :   default: return 0;
    5569             :   }
    5570             : }
    5571             : 
    5572             : // FastEmit functions for AArch64ISD::ZIP2.
    5573             : 
    5574             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5575           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5576             :     return 0;
    5577           0 :   if ((Subtarget->hasNEON())) {
    5578           0 :     return fastEmitInst_rr(AArch64::ZIP2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5579             :   }
    5580             :   return 0;
    5581             : }
    5582             : 
    5583             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5584           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5585             :     return 0;
    5586           0 :   if ((Subtarget->hasNEON())) {
    5587           0 :     return fastEmitInst_rr(AArch64::ZIP2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5588             :   }
    5589             :   return 0;
    5590             : }
    5591             : 
    5592             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5593           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5594             :     return 0;
    5595           0 :   if ((Subtarget->hasNEON())) {
    5596           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5597             :   }
    5598             :   return 0;
    5599             : }
    5600             : 
    5601             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5602           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5603             :     return 0;
    5604           0 :   if ((Subtarget->hasNEON())) {
    5605           0 :     return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5606             :   }
    5607             :   return 0;
    5608             : }
    5609             : 
    5610             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5611           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5612             :     return 0;
    5613           0 :   if ((Subtarget->hasNEON())) {
    5614           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5615             :   }
    5616             :   return 0;
    5617             : }
    5618             : 
    5619             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5620           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5621             :     return 0;
    5622           0 :   if ((Subtarget->hasNEON())) {
    5623           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5624             :   }
    5625             :   return 0;
    5626             : }
    5627             : 
    5628             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5629           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5630             :     return 0;
    5631           0 :   if ((Subtarget->hasNEON())) {
    5632           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5633             :   }
    5634             :   return 0;
    5635             : }
    5636             : 
    5637             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5638           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5639             :     return 0;
    5640           0 :   if ((Subtarget->hasNEON())) {
    5641           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5642             :   }
    5643             :   return 0;
    5644             : }
    5645             : 
    5646             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5647           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5648             :     return 0;
    5649           0 :   if ((Subtarget->hasNEON())) {
    5650           0 :     return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5651             :   }
    5652             :   return 0;
    5653             : }
    5654             : 
    5655             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5656           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5657             :     return 0;
    5658           0 :   if ((Subtarget->hasNEON())) {
    5659           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5660             :   }
    5661             :   return 0;
    5662             : }
    5663             : 
    5664             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5665           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5666             :     return 0;
    5667           0 :   if ((Subtarget->hasNEON())) {
    5668           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5669             :   }
    5670             :   return 0;
    5671             : }
    5672             : 
    5673             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5674           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5675             :     return 0;
    5676           0 :   if ((Subtarget->hasNEON())) {
    5677           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5678             :   }
    5679             :   return 0;
    5680             : }
    5681             : 
    5682           0 : unsigned fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5683           0 :   switch (VT.SimpleTy) {
    5684           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5685           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5686           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5687           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5688           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5689           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5690           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5691           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5692           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5693           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5694           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5695           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5696             :   default: return 0;
    5697             :   }
    5698             : }
    5699             : 
    5700             : // FastEmit functions for ISD::ADD.
    5701             : 
    5702             : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5703           0 :   if (RetVT.SimpleTy != MVT::i32)
    5704             :     return 0;
    5705           0 :   return fastEmitInst_rr(AArch64::ADDWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5706             : }
    5707             : 
    5708             : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5709           7 :   if (RetVT.SimpleTy != MVT::i64)
    5710             :     return 0;
    5711           7 :   return fastEmitInst_rr(AArch64::ADDXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5712             : }
    5713             : 
    5714             : unsigned fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5715          11 :   if (RetVT.SimpleTy != MVT::v8i8)
    5716             :     return 0;
    5717          11 :   if ((Subtarget->hasNEON())) {
    5718          11 :     return fastEmitInst_rr(AArch64::ADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5719             :   }
    5720             :   return 0;
    5721             : }
    5722             : 
    5723             : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5724           7 :   if (RetVT.SimpleTy != MVT::v16i8)
    5725             :     return 0;
    5726           7 :   if ((Subtarget->hasNEON())) {
    5727           7 :     return fastEmitInst_rr(AArch64::ADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5728             :   }
    5729             :   return 0;
    5730             : }
    5731             : 
    5732             : unsigned fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5733          10 :   if (RetVT.SimpleTy != MVT::v4i16)
    5734             :     return 0;
    5735          10 :   if ((Subtarget->hasNEON())) {
    5736          10 :     return fastEmitInst_rr(AArch64::ADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5737             :   }
    5738             :   return 0;
    5739             : }
    5740             : 
    5741             : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5742           7 :   if (RetVT.SimpleTy != MVT::v8i16)
    5743             :     return 0;
    5744           7 :   if ((Subtarget->hasNEON())) {
    5745           7 :     return fastEmitInst_rr(AArch64::ADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5746             :   }
    5747             :   return 0;
    5748             : }
    5749             : 
    5750             : unsigned fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5751          10 :   if (RetVT.SimpleTy != MVT::v2i32)
    5752             :     return 0;
    5753          10 :   if ((Subtarget->hasNEON())) {
    5754          10 :     return fastEmitInst_rr(AArch64::ADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5755             :   }
    5756             :   return 0;
    5757             : }
    5758             : 
    5759             : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5760           7 :   if (RetVT.SimpleTy != MVT::v4i32)
    5761             :     return 0;
    5762           7 :   if ((Subtarget->hasNEON())) {
    5763           7 :     return fastEmitInst_rr(AArch64::ADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5764             :   }
    5765             :   return 0;
    5766             : }
    5767             : 
    5768             : unsigned fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5769          12 :   if (RetVT.SimpleTy != MVT::v1i64)
    5770             :     return 0;
    5771          12 :   if ((Subtarget->hasNEON())) {
    5772          12 :     return fastEmitInst_rr(AArch64::ADDv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5773             :   }
    5774             :   return 0;
    5775             : }
    5776             : 
    5777             : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5778           7 :   if (RetVT.SimpleTy != MVT::v2i64)
    5779             :     return 0;
    5780           7 :   if ((Subtarget->hasNEON())) {
    5781           7 :     return fastEmitInst_rr(AArch64::ADDv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5782             :   }
    5783             :   return 0;
    5784             : }
    5785             : 
    5786          78 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5787          78 :   switch (VT.SimpleTy) {
    5788           0 :   case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5789           7 :   case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5790          11 :   case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5791           7 :   case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5792          10 :   case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5793           7 :   case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5794          10 :   case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5795           7 :   case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5796          12 :   case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5797           7 :   case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5798             :   default: return 0;
    5799             :   }
    5800             : }
    5801             : 
    5802             : // FastEmit functions for ISD::AND.
    5803             : 
    5804             : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5805          30 :   if (RetVT.SimpleTy != MVT::i32)
    5806             :     return 0;
    5807          30 :   return fastEmitInst_rr(AArch64::ANDWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5808             : }
    5809             : 
    5810             : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5811           4 :   if (RetVT.SimpleTy != MVT::i64)
    5812             :     return 0;
    5813           4 :   return fastEmitInst_rr(AArch64::ANDXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5814             : }
    5815             : 
    5816             : unsigned fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5817           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5818             :     return 0;
    5819           0 :   if ((Subtarget->hasNEON())) {
    5820           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5821             :   }
    5822             :   return 0;
    5823             : }
    5824             : 
    5825             : unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5826           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5827             :     return 0;
    5828           0 :   if ((Subtarget->hasNEON())) {
    5829           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5830             :   }
    5831             :   return 0;
    5832             : }
    5833             : 
    5834             : unsigned fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5835           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5836             :     return 0;
    5837           0 :   if ((Subtarget->hasNEON())) {
    5838           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5839             :   }
    5840             :   return 0;
    5841             : }
    5842             : 
    5843             : unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5844           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5845             :     return 0;
    5846           0 :   if ((Subtarget->hasNEON())) {
    5847           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5848             :   }
    5849             :   return 0;
    5850             : }
    5851             : 
    5852             : unsigned fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5853           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5854             :     return 0;
    5855           0 :   if ((Subtarget->hasNEON())) {
    5856           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5857             :   }
    5858             :   return 0;
    5859             : }
    5860             : 
    5861             : unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5862           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5863             :     return 0;
    5864           0 :   if ((Subtarget->hasNEON())) {
    5865           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5866             :   }
    5867             :   return 0;
    5868             : }
    5869             : 
    5870             : unsigned fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5871           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    5872             :     return 0;
    5873           0 :   if ((Subtarget->hasNEON())) {
    5874           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5875             :   }
    5876             :   return 0;
    5877             : }
    5878             : 
    5879             : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5880           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5881             :     return 0;
    5882           0 :   if ((Subtarget->hasNEON())) {
    5883           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5884             :   }
    5885             :   return 0;
    5886             : }
    5887             : 
    5888          34 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5889          34 :   switch (VT.SimpleTy) {
    5890          30 :   case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5891           4 :   case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5892           0 :   case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5893           0 :   case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5894           0 :   case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5895           0 :   case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5896           0 :   case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5897           0 :   case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5898           0 :   case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5899           0 :   case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5900             :   default: return 0;
    5901             :   }
    5902             : }
    5903             : 
    5904             : // FastEmit functions for ISD::FADD.
    5905             : 
    5906             : unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5907           0 :   if (RetVT.SimpleTy != MVT::f16)
    5908             :     return 0;
    5909           0 :   if ((Subtarget->hasFullFP16())) {
    5910           0 :     return fastEmitInst_rr(AArch64::FADDHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5911             :   }
    5912             :   return 0;
    5913             : }
    5914             : 
    5915             : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5916          29 :   if (RetVT.SimpleTy != MVT::f32)
    5917             :     return 0;
    5918          29 :   if ((Subtarget->hasFPARMv8())) {
    5919          29 :     return fastEmitInst_rr(AArch64::FADDSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5920             :   }
    5921             :   return 0;
    5922             : }
    5923             : 
    5924             : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5925          56 :   if (RetVT.SimpleTy != MVT::f64)
    5926             :     return 0;
    5927          56 :   if ((Subtarget->hasFPARMv8())) {
    5928          56 :     return fastEmitInst_rr(AArch64::FADDDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5929             :   }
    5930             :   return 0;
    5931             : }
    5932             : 
    5933           0 : unsigned fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5934           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5935             :     return 0;
    5936           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    5937           0 :     return fastEmitInst_rr(AArch64::FADDv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5938             :   }
    5939             :   return 0;
    5940             : }
    5941             : 
    5942           0 : unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5943           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5944             :     return 0;
    5945           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    5946           0 :     return fastEmitInst_rr(AArch64::FADDv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5947             :   }
    5948             :   return 0;
    5949             : }
    5950             : 
    5951             : unsigned fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5952          10 :   if (RetVT.SimpleTy != MVT::v2f32)
    5953             :     return 0;
    5954          10 :   if ((Subtarget->hasNEON())) {
    5955          10 :     return fastEmitInst_rr(AArch64::FADDv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5956             :   }
    5957             :   return 0;
    5958             : }
    5959             : 
    5960             : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5961          10 :   if (RetVT.SimpleTy != MVT::v4f32)
    5962             :     return 0;
    5963          10 :   if ((Subtarget->hasNEON())) {
    5964          10 :     return fastEmitInst_rr(AArch64::FADDv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5965             :   }
    5966             :   return 0;
    5967             : }
    5968             : 
    5969             : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5970           7 :   if (RetVT.SimpleTy != MVT::v2f64)
    5971             :     return 0;
    5972           7 :   if ((Subtarget->hasNEON())) {
    5973           7 :     return fastEmitInst_rr(AArch64::FADDv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5974             :   }
    5975             :   return 0;
    5976             : }
    5977             : 
    5978         118 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5979         118 :   switch (VT.SimpleTy) {
    5980           0 :   case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5981          29 :   case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5982          56 :   case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5983           0 :   case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5984           0 :   case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5985          10 :   case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5986          10 :   case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5987           7 :   case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5988             :   default: return 0;
    5989             :   }
    5990             : }
    5991             : 
    5992             : // FastEmit functions for ISD::FDIV.
    5993             : 
    5994             : unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5995           0 :   if (RetVT.SimpleTy != MVT::f16)
    5996             :     return 0;
    5997           0 :   if ((Subtarget->hasFullFP16())) {
    5998           0 :     return fastEmitInst_rr(AArch64::FDIVHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5999             :   }
    6000             :   return 0;
    6001             : }
    6002             : 
    6003             : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6004           8 :   if (RetVT.SimpleTy != MVT::f32)
    6005             :     return 0;
    6006           8 :   if ((Subtarget->hasFPARMv8())) {
    6007           8 :     return fastEmitInst_rr(AArch64::FDIVSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6008             :   }
    6009             :   return 0;
    6010             : }
    6011             : 
    6012             : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6013           8 :   if (RetVT.SimpleTy != MVT::f64)
    6014             :     return 0;
    6015           8 :   if ((Subtarget->hasFPARMv8())) {
    6016           8 :     return fastEmitInst_rr(AArch64::FDIVDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6017             :   }
    6018             :   return 0;
    6019             : }
    6020             : 
    6021           0 : unsigned fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6022           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6023             :     return 0;
    6024           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6025           0 :     return fastEmitInst_rr(AArch64::FDIVv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6026             :   }
    6027             :   return 0;
    6028             : }
    6029             : 
    6030           0 : unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6031           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6032             :     return 0;
    6033           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6034           0 :     return fastEmitInst_rr(AArch64::FDIVv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6035             :   }
    6036             :   return 0;
    6037             : }
    6038             : 
    6039             : unsigned fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6040           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6041             :     return 0;
    6042           0 :   if ((Subtarget->hasNEON())) {
    6043           0 :     return fastEmitInst_rr(AArch64::FDIVv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6044             :   }
    6045             :   return 0;
    6046             : }
    6047             : 
    6048             : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6049           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6050             :     return 0;
    6051           0 :   if ((Subtarget->hasNEON())) {
    6052           0 :     return fastEmitInst_rr(AArch64::FDIVv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6053             :   }
    6054             :   return 0;
    6055             : }
    6056             : 
    6057             : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6058           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6059             :     return 0;
    6060           0 :   if ((Subtarget->hasNEON())) {
    6061           0 :     return fastEmitInst_rr(AArch64::FDIVv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6062             :   }
    6063             :   return 0;
    6064             : }
    6065             : 
    6066          16 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6067          16 :   switch (VT.SimpleTy) {
    6068           0 :   case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6069           8 :   case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6070           8 :   case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6071           0 :   case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6072           0 :   case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6073           0 :   case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6074           0 :   case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6075           0 :   case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6076             :   default: return 0;
    6077             :   }
    6078             : }
    6079             : 
    6080             : // FastEmit functions for ISD::FMAXNAN.
    6081             : 
    6082             : unsigned fastEmit_ISD_FMAXNAN_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6083           0 :   if (RetVT.SimpleTy != MVT::f16)
    6084             :     return 0;
    6085           0 :   if ((Subtarget->hasFullFP16())) {
    6086           0 :     return fastEmitInst_rr(AArch64::FMAXHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6087             :   }
    6088             :   return 0;
    6089             : }
    6090             : 
    6091             : unsigned fastEmit_ISD_FMAXNAN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6092           0 :   if (RetVT.SimpleTy != MVT::f32)
    6093             :     return 0;
    6094           0 :   if ((Subtarget->hasFPARMv8())) {
    6095           0 :     return fastEmitInst_rr(AArch64::FMAXSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6096             :   }
    6097             :   return 0;
    6098             : }
    6099             : 
    6100             : unsigned fastEmit_ISD_FMAXNAN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6101           0 :   if (RetVT.SimpleTy != MVT::f64)
    6102             :     return 0;
    6103           0 :   if ((Subtarget->hasFPARMv8())) {
    6104           0 :     return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6105             :   }
    6106             :   return 0;
    6107             : }
    6108             : 
    6109           0 : unsigned fastEmit_ISD_FMAXNAN_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6110           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6111             :     return 0;
    6112           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6113           0 :     return fastEmitInst_rr(AArch64::FMAXv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6114             :   }
    6115             :   return 0;
    6116             : }
    6117             : 
    6118           0 : unsigned fastEmit_ISD_FMAXNAN_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6119           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6120             :     return 0;
    6121           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6122           0 :     return fastEmitInst_rr(AArch64::FMAXv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6123             :   }
    6124             :   return 0;
    6125             : }
    6126             : 
    6127             : unsigned fastEmit_ISD_FMAXNAN_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6128           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6129             :     return 0;
    6130           0 :   if ((Subtarget->hasNEON())) {
    6131           0 :     return fastEmitInst_rr(AArch64::FMAXv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6132             :   }
    6133             :   return 0;
    6134             : }
    6135             : 
    6136             : unsigned fastEmit_ISD_FMAXNAN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6137           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6138             :     return 0;
    6139           0 :   if ((Subtarget->hasNEON())) {
    6140           0 :     return fastEmitInst_rr(AArch64::FMAXv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6141             :   }
    6142             :   return 0;
    6143             : }
    6144             : 
    6145             : unsigned fastEmit_ISD_FMAXNAN_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6146           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6147             :     return 0;
    6148           0 :   return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6149             : }
    6150             : 
    6151             : unsigned fastEmit_ISD_FMAXNAN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6152           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6153             :     return 0;
    6154           0 :   if ((Subtarget->hasNEON())) {
    6155           0 :     return fastEmitInst_rr(AArch64::FMAXv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6156             :   }
    6157             :   return 0;
    6158             : }
    6159             : 
    6160           0 : unsigned fastEmit_ISD_FMAXNAN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6161           0 :   switch (VT.SimpleTy) {
    6162           0 :   case MVT::f16: return fastEmit_ISD_FMAXNAN_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6163           0 :   case MVT::f32: return fastEmit_ISD_FMAXNAN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6164           0 :   case MVT::f64: return fastEmit_ISD_FMAXNAN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6165           0 :   case MVT::v4f16: return fastEmit_ISD_FMAXNAN_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6166           0 :   case MVT::v8f16: return fastEmit_ISD_FMAXNAN_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6167           0 :   case MVT::v2f32: return fastEmit_ISD_FMAXNAN_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6168           0 :   case MVT::v4f32: return fastEmit_ISD_FMAXNAN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6169           0 :   case MVT::v1f64: return fastEmit_ISD_FMAXNAN_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6170           0 :   case MVT::v2f64: return fastEmit_ISD_FMAXNAN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6171             :   default: return 0;
    6172             :   }
    6173             : }
    6174             : 
    6175             : // FastEmit functions for ISD::FMAXNUM.
    6176             : 
    6177             : unsigned fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6178           0 :   if (RetVT.SimpleTy != MVT::f16)
    6179             :     return 0;
    6180           0 :   if ((Subtarget->hasFullFP16())) {
    6181           0 :     return fastEmitInst_rr(AArch64::FMAXNMHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6182             :   }
    6183             :   return 0;
    6184             : }
    6185             : 
    6186             : unsigned fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6187           0 :   if (RetVT.SimpleTy != MVT::f32)
    6188             :     return 0;
    6189           0 :   if ((Subtarget->hasFPARMv8())) {
    6190           0 :     return fastEmitInst_rr(AArch64::FMAXNMSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6191             :   }
    6192             :   return 0;
    6193             : }
    6194             : 
    6195             : unsigned fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6196           0 :   if (RetVT.SimpleTy != MVT::f64)
    6197             :     return 0;
    6198           0 :   if ((Subtarget->hasFPARMv8())) {
    6199           0 :     return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6200             :   }
    6201             :   return 0;
    6202             : }
    6203             : 
    6204           0 : unsigned fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6205           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6206             :     return 0;
    6207           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6208           0 :     return fastEmitInst_rr(AArch64::FMAXNMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6209             :   }
    6210             :   return 0;
    6211             : }
    6212             : 
    6213           0 : unsigned fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6214           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6215             :     return 0;
    6216           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6217           0 :     return fastEmitInst_rr(AArch64::FMAXNMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6218             :   }
    6219             :   return 0;
    6220             : }
    6221             : 
    6222             : unsigned fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6223           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6224             :     return 0;
    6225           0 :   if ((Subtarget->hasNEON())) {
    6226           0 :     return fastEmitInst_rr(AArch64::FMAXNMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6227             :   }
    6228             :   return 0;
    6229             : }
    6230             : 
    6231             : unsigned fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6232           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6233             :     return 0;
    6234           0 :   if ((Subtarget->hasNEON())) {
    6235           0 :     return fastEmitInst_rr(AArch64::FMAXNMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6236             :   }
    6237             :   return 0;
    6238             : }
    6239             : 
    6240             : unsigned fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6241           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6242             :     return 0;
    6243           0 :   return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6244             : }
    6245             : 
    6246             : unsigned fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6247           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6248             :     return 0;
    6249           0 :   if ((Subtarget->hasNEON())) {
    6250           0 :     return fastEmitInst_rr(AArch64::FMAXNMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6251             :   }
    6252             :   return 0;
    6253             : }
    6254             : 
    6255           0 : unsigned fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6256           0 :   switch (VT.SimpleTy) {
    6257           0 :   case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6258           0 :   case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6259           0 :   case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6260           0 :   case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6261           0 :   case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6262           0 :   case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6263           0 :   case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6264           0 :   case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6265           0 :   case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6266             :   default: return 0;
    6267             :   }
    6268             : }
    6269             : 
    6270             : // FastEmit functions for ISD::FMINNAN.
    6271             : 
    6272             : unsigned fastEmit_ISD_FMINNAN_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6273           0 :   if (RetVT.SimpleTy != MVT::f16)
    6274             :     return 0;
    6275           0 :   if ((Subtarget->hasFullFP16())) {
    6276           0 :     return fastEmitInst_rr(AArch64::FMINHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6277             :   }
    6278             :   return 0;
    6279             : }
    6280             : 
    6281             : unsigned fastEmit_ISD_FMINNAN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6282           0 :   if (RetVT.SimpleTy != MVT::f32)
    6283             :     return 0;
    6284           0 :   if ((Subtarget->hasFPARMv8())) {
    6285           0 :     return fastEmitInst_rr(AArch64::FMINSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6286             :   }
    6287             :   return 0;
    6288             : }
    6289             : 
    6290             : unsigned fastEmit_ISD_FMINNAN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6291           0 :   if (RetVT.SimpleTy != MVT::f64)
    6292             :     return 0;
    6293           0 :   if ((Subtarget->hasFPARMv8())) {
    6294           0 :     return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6295             :   }
    6296             :   return 0;
    6297             : }
    6298             : 
    6299           0 : unsigned fastEmit_ISD_FMINNAN_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6300           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6301             :     return 0;
    6302           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6303           0 :     return fastEmitInst_rr(AArch64::FMINv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6304             :   }
    6305             :   return 0;
    6306             : }
    6307             : 
    6308           0 : unsigned fastEmit_ISD_FMINNAN_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6309           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6310             :     return 0;
    6311           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6312           0 :     return fastEmitInst_rr(AArch64::FMINv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6313             :   }
    6314             :   return 0;
    6315             : }
    6316             : 
    6317             : unsigned fastEmit_ISD_FMINNAN_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6318           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6319             :     return 0;
    6320           0 :   if ((Subtarget->hasNEON())) {
    6321           0 :     return fastEmitInst_rr(AArch64::FMINv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6322             :   }
    6323             :   return 0;
    6324             : }
    6325             : 
    6326             : unsigned fastEmit_ISD_FMINNAN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6327           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6328             :     return 0;
    6329           0 :   if ((Subtarget->hasNEON())) {
    6330           0 :     return fastEmitInst_rr(AArch64::FMINv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6331             :   }
    6332             :   return 0;
    6333             : }
    6334             : 
    6335             : unsigned fastEmit_ISD_FMINNAN_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6336           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6337             :     return 0;
    6338           0 :   return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6339             : }
    6340             : 
    6341             : unsigned fastEmit_ISD_FMINNAN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6342           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6343             :     return 0;
    6344           0 :   if ((Subtarget->hasNEON())) {
    6345           0 :     return fastEmitInst_rr(AArch64::FMINv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6346             :   }
    6347             :   return 0;
    6348             : }
    6349             : 
    6350           0 : unsigned fastEmit_ISD_FMINNAN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6351           0 :   switch (VT.SimpleTy) {
    6352           0 :   case MVT::f16: return fastEmit_ISD_FMINNAN_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6353           0 :   case MVT::f32: return fastEmit_ISD_FMINNAN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6354           0 :   case MVT::f64: return fastEmit_ISD_FMINNAN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6355           0 :   case MVT::v4f16: return fastEmit_ISD_FMINNAN_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6356           0 :   case MVT::v8f16: return fastEmit_ISD_FMINNAN_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6357           0 :   case MVT::v2f32: return fastEmit_ISD_FMINNAN_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6358           0 :   case MVT::v4f32: return fastEmit_ISD_FMINNAN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6359           0 :   case MVT::v1f64: return fastEmit_ISD_FMINNAN_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6360           0 :   case MVT::v2f64: return fastEmit_ISD_FMINNAN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6361             :   default: return 0;
    6362             :   }
    6363             : }
    6364             : 
    6365             : // FastEmit functions for ISD::FMINNUM.
    6366             : 
    6367             : unsigned fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6368           0 :   if (RetVT.SimpleTy != MVT::f16)
    6369             :     return 0;
    6370           0 :   if ((Subtarget->hasFullFP16())) {
    6371           0 :     return fastEmitInst_rr(AArch64::FMINNMHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6372             :   }
    6373             :   return 0;
    6374             : }
    6375             : 
    6376             : unsigned fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6377           0 :   if (RetVT.SimpleTy != MVT::f32)
    6378             :     return 0;
    6379           0 :   if ((Subtarget->hasFPARMv8())) {
    6380           0 :     return fastEmitInst_rr(AArch64::FMINNMSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6381             :   }
    6382             :   return 0;
    6383             : }
    6384             : 
    6385             : unsigned fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6386           0 :   if (RetVT.SimpleTy != MVT::f64)
    6387             :     return 0;
    6388           0 :   if ((Subtarget->hasFPARMv8())) {
    6389           0 :     return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6390             :   }
    6391             :   return 0;
    6392             : }
    6393             : 
    6394           0 : unsigned fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6395           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6396             :     return 0;
    6397           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6398           0 :     return fastEmitInst_rr(AArch64::FMINNMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6399             :   }
    6400             :   return 0;
    6401             : }
    6402             : 
    6403           0 : unsigned fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6404           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6405             :     return 0;
    6406           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6407           0 :     return fastEmitInst_rr(AArch64::FMINNMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6408             :   }
    6409             :   return 0;
    6410             : }
    6411             : 
    6412             : unsigned fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6413           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6414             :     return 0;
    6415           0 :   if ((Subtarget->hasNEON())) {
    6416           0 :     return fastEmitInst_rr(AArch64::FMINNMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6417             :   }
    6418             :   return 0;
    6419             : }
    6420             : 
    6421             : unsigned fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6422           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6423             :     return 0;
    6424           0 :   if ((Subtarget->hasNEON())) {
    6425           0 :     return fastEmitInst_rr(AArch64::FMINNMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6426             :   }
    6427             :   return 0;
    6428             : }
    6429             : 
    6430             : unsigned fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6431           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6432             :     return 0;
    6433           0 :   return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6434             : }
    6435             : 
    6436             : unsigned fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6437           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6438             :     return 0;
    6439           0 :   if ((Subtarget->hasNEON())) {
    6440           0 :     return fastEmitInst_rr(AArch64::FMINNMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6441             :   }
    6442             :   return 0;
    6443             : }
    6444             : 
    6445           0 : unsigned fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6446           0 :   switch (VT.SimpleTy) {
    6447           0 :   case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6448           0 :   case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6449           0 :   case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6450           0 :   case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6451           0 :   case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6452           0 :   case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6453           0 :   case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6454           0 :   case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6455           0 :   case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6456             :   default: return 0;
    6457             :   }
    6458             : }
    6459             : 
    6460             : // FastEmit functions for ISD::FMUL.
    6461             : 
    6462             : unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6463           0 :   if (RetVT.SimpleTy != MVT::f16)
    6464             :     return 0;
    6465           0 :   if ((Subtarget->hasFullFP16())) {
    6466           0 :     return fastEmitInst_rr(AArch64::FMULHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6467             :   }
    6468             :   return 0;
    6469             : }
    6470             : 
    6471             : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6472           8 :   if (RetVT.SimpleTy != MVT::f32)
    6473             :     return 0;
    6474           8 :   if ((Subtarget->hasFPARMv8())) {
    6475           8 :     return fastEmitInst_rr(AArch64::FMULSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6476             :   }
    6477             :   return 0;
    6478             : }
    6479             : 
    6480             : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6481           8 :   if (RetVT.SimpleTy != MVT::f64)
    6482             :     return 0;
    6483           8 :   if ((Subtarget->hasFPARMv8())) {
    6484           8 :     return fastEmitInst_rr(AArch64::FMULDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6485             :   }
    6486             :   return 0;
    6487             : }
    6488             : 
    6489           0 : unsigned fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6490           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6491             :     return 0;
    6492           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6493           0 :     return fastEmitInst_rr(AArch64::FMULv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6494             :   }
    6495             :   return 0;
    6496             : }
    6497             : 
    6498           0 : unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6499           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6500             :     return 0;
    6501           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6502           0 :     return fastEmitInst_rr(AArch64::FMULv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6503             :   }
    6504             :   return 0;
    6505             : }
    6506             : 
    6507             : unsigned fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6508           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6509             :     return 0;
    6510           0 :   if ((Subtarget->hasNEON())) {
    6511           0 :     return fastEmitInst_rr(AArch64::FMULv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6512             :   }
    6513             :   return 0;
    6514             : }
    6515             : 
    6516             : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6517           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6518             :     return 0;
    6519           0 :   if ((Subtarget->hasNEON())) {
    6520           0 :     return fastEmitInst_rr(AArch64::FMULv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6521             :   }
    6522             :   return 0;
    6523             : }
    6524             : 
    6525             : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6526           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6527             :     return 0;
    6528           0 :   if ((Subtarget->hasNEON())) {
    6529           0 :     return fastEmitInst_rr(AArch64::FMULv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6530             :   }
    6531             :   return 0;
    6532             : }
    6533             : 
    6534          16 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6535          16 :   switch (VT.SimpleTy) {
    6536           0 :   case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6537           8 :   case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6538           8 :   case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6539           0 :   case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6540           0 :   case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6541           0 :   case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6542           0 :   case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6543           0 :   case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6544             :   default: return 0;
    6545             :   }
    6546             : }
    6547             : 
    6548             : // FastEmit functions for ISD::FSUB.
    6549             : 
    6550             : unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6551           0 :   if (RetVT.SimpleTy != MVT::f16)
    6552             :     return 0;
    6553           0 :   if ((Subtarget->hasFullFP16())) {
    6554           0 :     return fastEmitInst_rr(AArch64::FSUBHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6555             :   }
    6556             :   return 0;
    6557             : }
    6558             : 
    6559             : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6560           0 :   if (RetVT.SimpleTy != MVT::f32)
    6561             :     return 0;
    6562           0 :   if ((Subtarget->hasFPARMv8())) {
    6563           0 :     return fastEmitInst_rr(AArch64::FSUBSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6564             :   }
    6565             :   return 0;
    6566             : }
    6567             : 
    6568             : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6569           0 :   if (RetVT.SimpleTy != MVT::f64)
    6570             :     return 0;
    6571           0 :   if ((Subtarget->hasFPARMv8())) {
    6572           0 :     return fastEmitInst_rr(AArch64::FSUBDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6573             :   }
    6574             :   return 0;
    6575             : }
    6576             : 
    6577           0 : unsigned fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6578           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6579             :     return 0;
    6580           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6581           0 :     return fastEmitInst_rr(AArch64::FSUBv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6582             :   }
    6583             :   return 0;
    6584             : }
    6585             : 
    6586           0 : unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6587           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6588             :     return 0;
    6589           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6590           0 :     return fastEmitInst_rr(AArch64::FSUBv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6591             :   }
    6592             :   return 0;
    6593             : }
    6594             : 
    6595             : unsigned fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6596           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6597             :     return 0;
    6598           0 :   if ((Subtarget->hasNEON())) {
    6599           0 :     return fastEmitInst_rr(AArch64::FSUBv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6600             :   }
    6601             :   return 0;
    6602             : }
    6603             : 
    6604             : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6605           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6606             :     return 0;
    6607           0 :   if ((Subtarget->hasNEON())) {
    6608           0 :     return fastEmitInst_rr(AArch64::FSUBv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6609             :   }
    6610             :   return 0;
    6611             : }
    6612             : 
    6613             : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6614           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6615             :     return 0;
    6616           0 :   if ((Subtarget->hasNEON())) {
    6617           0 :     return fastEmitInst_rr(AArch64::FSUBv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6618             :   }
    6619             :   return 0;
    6620             : }
    6621             : 
    6622           0 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6623           0 :   switch (VT.SimpleTy) {
    6624           0 :   case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6625           0 :   case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6626           0 :   case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6627           0 :   case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6628           0 :   case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6629           0 :   case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6630           0 :   case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6631           0 :   case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6632             :   default: return 0;
    6633             :   }
    6634             : }
    6635             : 
    6636             : // FastEmit functions for ISD::MUL.
    6637             : 
    6638             : unsigned fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6639           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6640             :     return 0;
    6641           0 :   if ((Subtarget->hasNEON())) {
    6642           0 :     return fastEmitInst_rr(AArch64::MULv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6643             :   }
    6644             :   return 0;
    6645             : }
    6646             : 
    6647             : unsigned fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6648           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6649             :     return 0;
    6650           0 :   if ((Subtarget->hasNEON())) {
    6651           0 :     return fastEmitInst_rr(AArch64::MULv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6652             :   }
    6653             :   return 0;
    6654             : }
    6655             : 
    6656             : unsigned fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6657           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6658             :     return 0;
    6659           0 :   if ((Subtarget->hasNEON())) {
    6660           0 :     return fastEmitInst_rr(AArch64::MULv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6661             :   }
    6662             :   return 0;
    6663             : }
    6664             : 
    6665             : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6666           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6667             :     return 0;
    6668           0 :   if ((Subtarget->hasNEON())) {
    6669           0 :     return fastEmitInst_rr(AArch64::MULv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6670             :   }
    6671             :   return 0;
    6672             : }
    6673             : 
    6674             : unsigned fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6675           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6676             :     return 0;
    6677           0 :   if ((Subtarget->hasNEON())) {
    6678           0 :     return fastEmitInst_rr(AArch64::MULv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6679             :   }
    6680             :   return 0;
    6681             : }
    6682             : 
    6683             : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6684           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6685             :     return 0;
    6686           0 :   if ((Subtarget->hasNEON())) {
    6687           0 :     return fastEmitInst_rr(AArch64::MULv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6688             :   }
    6689             :   return 0;
    6690             : }
    6691             : 
    6692           0 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6693           0 :   switch (VT.SimpleTy) {
    6694           0 :   case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6695           0 :   case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6696           0 :   case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6697           0 :   case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6698           0 :   case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6699           0 :   case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6700             :   default: return 0;
    6701             :   }
    6702             : }
    6703             : 
    6704             : // FastEmit functions for ISD::MULHS.
    6705             : 
    6706             : unsigned fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6707           3 :   if (RetVT.SimpleTy != MVT::i64)
    6708             :     return 0;
    6709           3 :   return fastEmitInst_rr(AArch64::SMULHrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6710             : }
    6711             : 
    6712             : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6713           3 :   switch (VT.SimpleTy) {
    6714           3 :   case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6715             :   default: return 0;
    6716             :   }
    6717             : }
    6718             : 
    6719             : // FastEmit functions for ISD::MULHU.
    6720             : 
    6721             : unsigned fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6722           4 :   if (RetVT.SimpleTy != MVT::i64)
    6723             :     return 0;
    6724           4 :   return fastEmitInst_rr(AArch64::UMULHrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6725             : }
    6726             : 
    6727             : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6728           4 :   switch (VT.SimpleTy) {
    6729           4 :   case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6730             :   default: return 0;
    6731             :   }
    6732             : }
    6733             : 
    6734             : // FastEmit functions for ISD::OR.
    6735             : 
    6736             : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6737           8 :   if (RetVT.SimpleTy != MVT::i32)
    6738             :     return 0;
    6739           8 :   return fastEmitInst_rr(AArch64::ORRWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6740             : }
    6741             : 
    6742             : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6743           2 :   if (RetVT.SimpleTy != MVT::i64)
    6744             :     return 0;
    6745           2 :   return fastEmitInst_rr(AArch64::ORRXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6746             : }
    6747             : 
    6748             : unsigned fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6749           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6750             :     return 0;
    6751           0 :   if ((Subtarget->hasNEON())) {
    6752           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6753             :   }
    6754             :   return 0;
    6755             : }
    6756             : 
    6757             : unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6758           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6759             :     return 0;
    6760           0 :   if ((Subtarget->hasNEON())) {
    6761           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6762             :   }
    6763             :   return 0;
    6764             : }
    6765             : 
    6766             : unsigned fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6767           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6768             :     return 0;
    6769           0 :   if ((Subtarget->hasNEON())) {
    6770           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6771             :   }
    6772             :   return 0;
    6773             : }
    6774             : 
    6775             : unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6776           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6777             :     return 0;
    6778           0 :   if ((Subtarget->hasNEON())) {
    6779           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6780             :   }
    6781             :   return 0;
    6782             : }
    6783             : 
    6784             : unsigned fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6785           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6786             :     return 0;
    6787           0 :   if ((Subtarget->hasNEON())) {
    6788           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6789             :   }
    6790             :   return 0;
    6791             : }
    6792             : 
    6793             : unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6794           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6795             :     return 0;
    6796           0 :   if ((Subtarget->hasNEON())) {
    6797           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6798             :   }
    6799             :   return 0;
    6800             : }
    6801             : 
    6802             : unsigned fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6803           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    6804             :     return 0;
    6805           0 :   if ((Subtarget->hasNEON())) {
    6806           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6807             :   }
    6808             :   return 0;
    6809             : }
    6810             : 
    6811             : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6812           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6813             :     return 0;
    6814           0 :   if ((Subtarget->hasNEON())) {
    6815           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6816             :   }
    6817             :   return 0;
    6818             : }
    6819             : 
    6820          10 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6821          10 :   switch (VT.SimpleTy) {
    6822           8 :   case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6823           2 :   case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6824           0 :   case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6825           0 :   case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6826           0 :   case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6827           0 :   case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6828           0 :   case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6829           0 :   case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6830           0 :   case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6831           0 :   case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6832             :   default: return 0;
    6833             :   }
    6834             : }
    6835             : 
    6836             : // FastEmit functions for ISD::ROTR.
    6837             : 
    6838             : unsigned fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6839           0 :   if (RetVT.SimpleTy != MVT::i64)
    6840             :     return 0;
    6841           0 :   return fastEmitInst_rr(AArch64::RORVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6842             : }
    6843             : 
    6844             : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6845           0 :   switch (VT.SimpleTy) {
    6846           0 :   case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6847             :   default: return 0;
    6848             :   }
    6849             : }
    6850             : 
    6851             : // FastEmit functions for ISD::SDIV.
    6852             : 
    6853             : unsigned fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6854           0 :   if (RetVT.SimpleTy != MVT::i32)
    6855             :     return 0;
    6856           0 :   return fastEmitInst_rr(AArch64::SDIVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6857             : }
    6858             : 
    6859             : unsigned fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6860           0 :   if (RetVT.SimpleTy != MVT::i64)
    6861             :     return 0;
    6862           0 :   return fastEmitInst_rr(AArch64::SDIVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6863             : }
    6864             : 
    6865           0 : unsigned fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6866           0 :   switch (VT.SimpleTy) {
    6867           0 :   case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6868           0 :   case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6869             :   default: return 0;
    6870             :   }
    6871             : }
    6872             : 
    6873             : // FastEmit functions for ISD::SHL.
    6874             : 
    6875             : unsigned fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6876           0 :   if (RetVT.SimpleTy != MVT::i64)
    6877             :     return 0;
    6878           0 :   return fastEmitInst_rr(AArch64::LSLVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6879             : }
    6880             : 
    6881             : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6882           0 :   switch (VT.SimpleTy) {
    6883           0 :   case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6884             :   default: return 0;
    6885             :   }
    6886             : }
    6887             : 
    6888             : // FastEmit functions for ISD::SMAX.
    6889             : 
    6890             : unsigned fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6891           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6892             :     return 0;
    6893           0 :   if ((Subtarget->hasNEON())) {
    6894           0 :     return fastEmitInst_rr(AArch64::SMAXv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6895             :   }
    6896             :   return 0;
    6897             : }
    6898             : 
    6899             : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6900           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6901             :     return 0;
    6902           0 :   if ((Subtarget->hasNEON())) {
    6903           0 :     return fastEmitInst_rr(AArch64::SMAXv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6904             :   }
    6905             :   return 0;
    6906             : }
    6907             : 
    6908             : unsigned fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6909           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6910             :     return 0;
    6911           0 :   if ((Subtarget->hasNEON())) {
    6912           0 :     return fastEmitInst_rr(AArch64::SMAXv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6913             :   }
    6914             :   return 0;
    6915             : }
    6916             : 
    6917             : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6918           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6919             :     return 0;
    6920           0 :   if ((Subtarget->hasNEON())) {
    6921           0 :     return fastEmitInst_rr(AArch64::SMAXv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6922             :   }
    6923             :   return 0;
    6924             : }
    6925             : 
    6926             : unsigned fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6927           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6928             :     return 0;
    6929           0 :   if ((Subtarget->hasNEON())) {
    6930           0 :     return fastEmitInst_rr(AArch64::SMAXv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6931             :   }
    6932             :   return 0;
    6933             : }
    6934             : 
    6935             : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6936           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6937             :     return 0;
    6938           0 :   if ((Subtarget->hasNEON())) {
    6939           0 :     return fastEmitInst_rr(AArch64::SMAXv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6940             :   }
    6941             :   return 0;
    6942             : }
    6943             : 
    6944           0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6945           0 :   switch (VT.SimpleTy) {
    6946           0 :   case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6947           0 :   case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6948           0 :   case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6949           0 :   case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6950           0 :   case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6951           0 :   case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6952             :   default: return 0;
    6953             :   }
    6954             : }
    6955             : 
    6956             : // FastEmit functions for ISD::SMIN.
    6957             : 
    6958             : unsigned fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6959           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6960             :     return 0;
    6961           0 :   if ((Subtarget->hasNEON())) {
    6962           0 :     return fastEmitInst_rr(AArch64::SMINv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6963             :   }
    6964             :   return 0;
    6965             : }
    6966             : 
    6967             : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6968           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6969             :     return 0;
    6970           0 :   if ((Subtarget->hasNEON())) {
    6971           0 :     return fastEmitInst_rr(AArch64::SMINv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6972             :   }
    6973             :   return 0;
    6974             : }
    6975             : 
    6976             : unsigned fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6977           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6978             :     return 0;
    6979           0 :   if ((Subtarget->hasNEON())) {
    6980           0 :     return fastEmitInst_rr(AArch64::SMINv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6981             :   }
    6982             :   return 0;
    6983             : }
    6984             : 
    6985             : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6986           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6987             :     return 0;
    6988           0 :   if ((Subtarget->hasNEON())) {
    6989           0 :     return fastEmitInst_rr(AArch64::SMINv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6990             :   }
    6991             :   return 0;
    6992             : }
    6993             : 
    6994             : unsigned fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6995           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6996             :     return 0;
    6997           0 :   if ((Subtarget->hasNEON())) {
    6998           0 :     return fastEmitInst_rr(AArch64::SMINv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6999             :   }
    7000             :   return 0;
    7001             : }
    7002             : 
    7003             : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7004           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7005             :     return 0;
    7006           0 :   if ((Subtarget->hasNEON())) {
    7007           0 :     return fastEmitInst_rr(AArch64::SMINv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7008             :   }
    7009             :   return 0;
    7010             : }
    7011             : 
    7012           0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7013           0 :   switch (VT.SimpleTy) {
    7014           0 :   case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7015           0 :   case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7016           0 :   case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7017           0 :   case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7018           0 :   case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7019           0 :   case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7020             :   default: return 0;
    7021             :   }
    7022             : }
    7023             : 
    7024             : // FastEmit functions for ISD::SRA.
    7025             : 
    7026             : unsigned fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7027           0 :   if (RetVT.SimpleTy != MVT::i64)
    7028             :     return 0;
    7029           0 :   return fastEmitInst_rr(AArch64::ASRVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7030             : }
    7031             : 
    7032             : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7033           0 :   switch (VT.SimpleTy) {
    7034           0 :   case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7035             :   default: return 0;
    7036             :   }
    7037             : }
    7038             : 
    7039             : // FastEmit functions for ISD::SRL.
    7040             : 
    7041             : unsigned fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7042           3 :   if (RetVT.SimpleTy != MVT::i64)
    7043             :     return 0;
    7044           3 :   return fastEmitInst_rr(AArch64::LSRVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7045             : }
    7046             : 
    7047             : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7048           3 :   switch (VT.SimpleTy) {
    7049           3 :   case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7050             :   default: return 0;
    7051             :   }
    7052             : }
    7053             : 
    7054             : // FastEmit functions for ISD::SUB.
    7055             : 
    7056             : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7057           0 :   if (RetVT.SimpleTy != MVT::i32)
    7058             :     return 0;
    7059           0 :   return fastEmitInst_rr(AArch64::SUBSWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7060             : }
    7061             : 
    7062             : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7063           0 :   if (RetVT.SimpleTy != MVT::i64)
    7064             :     return 0;
    7065           0 :   return fastEmitInst_rr(AArch64::SUBSXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7066             : }
    7067             : 
    7068             : unsigned fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7069           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7070             :     return 0;
    7071           0 :   if ((Subtarget->hasNEON())) {
    7072           0 :     return fastEmitInst_rr(AArch64::SUBv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7073             :   }
    7074             :   return 0;
    7075             : }
    7076             : 
    7077             : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7078           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7079             :     return 0;
    7080           0 :   if ((Subtarget->hasNEON())) {
    7081           0 :     return fastEmitInst_rr(AArch64::SUBv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7082             :   }
    7083             :   return 0;
    7084             : }
    7085             : 
    7086             : unsigned fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7087           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7088             :     return 0;
    7089           0 :   if ((Subtarget->hasNEON())) {
    7090           0 :     return fastEmitInst_rr(AArch64::SUBv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7091             :   }
    7092             :   return 0;
    7093             : }
    7094             : 
    7095             : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7096           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7097             :     return 0;
    7098           0 :   if ((Subtarget->hasNEON())) {
    7099           0 :     return fastEmitInst_rr(AArch64::SUBv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7100             :   }
    7101             :   return 0;
    7102             : }
    7103             : 
    7104             : unsigned fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7105           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7106             :     return 0;
    7107           0 :   if ((Subtarget->hasNEON())) {
    7108           0 :     return fastEmitInst_rr(AArch64::SUBv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7109             :   }
    7110             :   return 0;
    7111             : }
    7112             : 
    7113             : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7114           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7115             :     return 0;
    7116           0 :   if ((Subtarget->hasNEON())) {
    7117           0 :     return fastEmitInst_rr(AArch64::SUBv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7118             :   }
    7119             :   return 0;
    7120             : }
    7121             : 
    7122             : unsigned fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7123           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    7124             :     return 0;
    7125           0 :   if ((Subtarget->hasNEON())) {
    7126           0 :     return fastEmitInst_rr(AArch64::SUBv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7127             :   }
    7128             :   return 0;
    7129             : }
    7130             : 
    7131             : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7132           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7133             :     return 0;
    7134           0 :   if ((Subtarget->hasNEON())) {
    7135           0 :     return fastEmitInst_rr(AArch64::SUBv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7136             :   }
    7137             :   return 0;
    7138             : }
    7139             : 
    7140           0 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7141           0 :   switch (VT.SimpleTy) {
    7142           0 :   case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7143           0 :   case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7144           0 :   case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7145           0 :   case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7146           0 :   case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7147           0 :   case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7148           0 :   case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7149           0 :   case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7150           0 :   case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7151           0 :   case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7152             :   default: return 0;
    7153             :   }
    7154             : }
    7155             : 
    7156             : // FastEmit functions for ISD::UDIV.
    7157             : 
    7158             : unsigned fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7159           0 :   if (RetVT.SimpleTy != MVT::i32)
    7160             :     return 0;
    7161           0 :   return fastEmitInst_rr(AArch64::UDIVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7162             : }
    7163             : 
    7164             : unsigned fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7165           0 :   if (RetVT.SimpleTy != MVT::i64)
    7166             :     return 0;
    7167           0 :   return fastEmitInst_rr(AArch64::UDIVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7168             : }
    7169             : 
    7170           0 : unsigned fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7171           0 :   switch (VT.SimpleTy) {
    7172           0 :   case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7173           0 :   case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7174             :   default: return 0;
    7175             :   }
    7176             : }
    7177             : 
    7178             : // FastEmit functions for ISD::UMAX.
    7179             : 
    7180             : unsigned fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7181           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7182             :     return 0;
    7183           0 :   if ((Subtarget->hasNEON())) {
    7184           0 :     return fastEmitInst_rr(AArch64::UMAXv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7185             :   }
    7186             :   return 0;
    7187             : }
    7188             : 
    7189             : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7190           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7191             :     return 0;
    7192           0 :   if ((Subtarget->hasNEON())) {
    7193           0 :     return fastEmitInst_rr(AArch64::UMAXv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7194             :   }
    7195             :   return 0;
    7196             : }
    7197             : 
    7198             : unsigned fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7199           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7200             :     return 0;
    7201           0 :   if ((Subtarget->hasNEON())) {
    7202           0 :     return fastEmitInst_rr(AArch64::UMAXv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7203             :   }
    7204             :   return 0;
    7205             : }
    7206             : 
    7207             : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7208           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7209             :     return 0;
    7210           0 :   if ((Subtarget->hasNEON())) {
    7211           0 :     return fastEmitInst_rr(AArch64::UMAXv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7212             :   }
    7213             :   return 0;
    7214             : }
    7215             : 
    7216             : unsigned fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7217           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7218             :     return 0;
    7219           0 :   if ((Subtarget->hasNEON())) {
    7220           0 :     return fastEmitInst_rr(AArch64::UMAXv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7221             :   }
    7222             :   return 0;
    7223             : }
    7224             : 
    7225             : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7226           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7227             :     return 0;
    7228           0 :   if ((Subtarget->hasNEON())) {
    7229           0 :     return fastEmitInst_rr(AArch64::UMAXv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7230             :   }
    7231             :   return 0;
    7232             : }
    7233             : 
    7234           0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7235           0 :   switch (VT.SimpleTy) {
    7236           0 :   case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7237           0 :   case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7238           0 :   case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7239           0 :   case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7240           0 :   case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7241           0 :   case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7242             :   default: return 0;
    7243             :   }
    7244             : }
    7245             : 
    7246             : // FastEmit functions for ISD::UMIN.
    7247             : 
    7248             : unsigned fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7249           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7250             :     return 0;
    7251           0 :   if ((Subtarget->hasNEON())) {
    7252           0 :     return fastEmitInst_rr(AArch64::UMINv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7253             :   }
    7254             :   return 0;
    7255             : }
    7256             : 
    7257             : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7258           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7259             :     return 0;
    7260           0 :   if ((Subtarget->hasNEON())) {
    7261           0 :     return fastEmitInst_rr(AArch64::UMINv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7262             :   }
    7263             :   return 0;
    7264             : }
    7265             : 
    7266             : unsigned fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7267           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7268             :     return 0;
    7269           0 :   if ((Subtarget->hasNEON())) {
    7270           0 :     return fastEmitInst_rr(AArch64::UMINv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7271             :   }
    7272             :   return 0;
    7273             : }
    7274             : 
    7275             : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7276           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7277             :     return 0;
    7278           0 :   if ((Subtarget->hasNEON())) {
    7279           0 :     return fastEmitInst_rr(AArch64::UMINv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7280             :   }
    7281             :   return 0;
    7282             : }
    7283             : 
    7284             : unsigned fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7285           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7286             :     return 0;
    7287           0 :   if ((Subtarget->hasNEON())) {
    7288           0 :     return fastEmitInst_rr(AArch64::UMINv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7289             :   }
    7290             :   return 0;
    7291             : }
    7292             : 
    7293             : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7294           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7295             :     return 0;
    7296           0 :   if ((Subtarget->hasNEON())) {
    7297           0 :     return fastEmitInst_rr(AArch64::UMINv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7298             :   }
    7299             :   return 0;
    7300             : }
    7301             : 
    7302           0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7303           0 :   switch (VT.SimpleTy) {
    7304           0 :   case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7305           0 :   case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7306           0 :   case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7307           0 :   case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7308           0 :   case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7309           0 :   case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7310             :   default: return 0;
    7311             :   }
    7312             : }
    7313             : 
    7314             : // FastEmit functions for ISD::XOR.
    7315             : 
    7316             : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7317           7 :   if (RetVT.SimpleTy != MVT::i32)
    7318             :     return 0;
    7319           7 :   return fastEmitInst_rr(AArch64::EORWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7320             : }
    7321             : 
    7322             : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7323           2 :   if (RetVT.SimpleTy != MVT::i64)
    7324             :     return 0;
    7325           2 :   return fastEmitInst_rr(AArch64::EORXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7326             : }
    7327             : 
    7328             : unsigned fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7329           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7330             :     return 0;
    7331           0 :   if ((Subtarget->hasNEON())) {
    7332           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7333             :   }
    7334             :   return 0;
    7335             : }
    7336             : 
    7337             : unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7338           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7339             :     return 0;
    7340           0 :   if ((Subtarget->hasNEON())) {
    7341           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7342             :   }
    7343             :   return 0;
    7344             : }
    7345             : 
    7346             : unsigned fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7347           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7348             :     return 0;
    7349           0 :   if ((Subtarget->hasNEON())) {
    7350           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7351             :   }
    7352             :   return 0;
    7353             : }
    7354             : 
    7355             : unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7356           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7357             :     return 0;
    7358           0 :   if ((Subtarget->hasNEON())) {
    7359           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7360             :   }
    7361             :   return 0;
    7362             : }
    7363             : 
    7364             : unsigned fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7365           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7366             :     return 0;
    7367           0 :   if ((Subtarget->hasNEON())) {
    7368           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7369             :   }
    7370             :   return 0;
    7371             : }
    7372             : 
    7373             : unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7374           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7375             :     return 0;
    7376           0 :   if ((Subtarget->hasNEON())) {
    7377           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7378             :   }
    7379             :   return 0;
    7380             : }
    7381             : 
    7382             : unsigned fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7383           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    7384             :     return 0;
    7385           0 :   if ((Subtarget->hasNEON())) {
    7386           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7387             :   }
    7388             :   return 0;
    7389             : }
    7390             : 
    7391             : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7392           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7393             :     return 0;
    7394           0 :   if ((Subtarget->hasNEON())) {
    7395           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7396             :   }
    7397             :   return 0;
    7398             : }
    7399             : 
    7400           9 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7401           9 :   switch (VT.SimpleTy) {
    7402           7 :   case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7403           2 :   case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7404           0 :   case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7405           0 :   case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7406           0 :   case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7407           0 :   case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7408           0 :   case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7409           0 :   case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7410           0 :   case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7411           0 :   case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7412             :   default: return 0;
    7413             :   }
    7414             : }
    7415             : 
    7416             : // Top-level FastEmit function.
    7417             : 
    7418         299 : unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) override {
    7419         299 :   switch (Opcode) {
    7420           0 :   case AArch64ISD::CMEQ: return fastEmit_AArch64ISD_CMEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7421           0 :   case AArch64ISD::CMGE: return fastEmit_AArch64ISD_CMGE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7422           0 :   case AArch64ISD::CMGT: return fastEmit_AArch64ISD_CMGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7423           0 :   case AArch64ISD::CMHI: return fastEmit_AArch64ISD_CMHI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7424           0 :   case AArch64ISD::CMHS: return fastEmit_AArch64ISD_CMHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7425           0 :   case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7426           0 :   case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7427           0 :   case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7428           0 :   case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7429           0 :   case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7430           0 :   case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7431           0 :   case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7432           0 :   case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7433           0 :   case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7434           0 :   case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7435           0 :   case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7436           0 :   case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7437           0 :   case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7438           0 :   case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7439          78 :   case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7440          34 :   case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7441         118 :   case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7442          16 :   case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7443           0 :   case ISD::FMAXNAN: return fastEmit_ISD_FMAXNAN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7444           0 :   case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7445           0 :   case ISD::FMINNAN: return fastEmit_ISD_FMINNAN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7446           0 :   case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7447          16 :   case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7448           0 :   case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7449           0 :   case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7450           3 :   case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7451           4 :   case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7452          10 :   case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7453           0 :   case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7454           0 :   case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7455           0 :   case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7456           0 :   case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7457           0 :   case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7458           0 :   case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7459           3 :   case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7460           0 :   case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7461           0 :   case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7462           0 :   case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7463           0 :   case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7464           9 :   case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7465             :   default: return 0;
    7466             :   }
    7467             : }
    7468             : 
    7469             : // FastEmit functions for AArch64ISD::DUPLANE64.
    7470             : 
    7471             : unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7472             :   if (RetVT.SimpleTy != MVT::v2i64)
    7473             :     return 0;
    7474             :   if ((Subtarget->hasNEON())) {
    7475             :     return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7476             :   }
    7477             :   return 0;
    7478             : }
    7479             : 
    7480             : unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7481             :   if (RetVT.SimpleTy != MVT::v2f64)
    7482             :     return 0;
    7483             :   return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7484             : }
    7485             : 
    7486             : unsigned fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7487             :   switch (VT.SimpleTy) {
    7488             :   case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7489             :   case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7490             :   default: return 0;
    7491             :   }
    7492             : }
    7493             : 
    7494             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7495             : 
    7496             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7497             :   if (RetVT.SimpleTy != MVT::i64)
    7498             :     return 0;
    7499             :   if ((Subtarget->hasNEON())) {
    7500             :     return fastEmitInst_ri(AArch64::UMOVvi64, &AArch64::GPR64RegClass, Op0, Op0IsKill, imm1);
    7501             :   }
    7502             :   return 0;
    7503             : }
    7504             : 
    7505             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7506             :   if (RetVT.SimpleTy != MVT::f64)
    7507             :     return 0;
    7508             :   return fastEmitInst_ri(AArch64::CPYi64, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7509             : }
    7510             : 
    7511             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7512             :   switch (VT.SimpleTy) {
    7513             :   case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7514             :   case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7515             :   default: return 0;
    7516             :   }
    7517             : }
    7518             : 
    7519             : // Top-level FastEmit function.
    7520             : 
    7521             : unsigned fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7522             :   switch (Opcode) {
    7523             :   case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, Op0IsKill, imm1);
    7524             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, Op0IsKill, imm1);
    7525             :   default: return 0;
    7526             :   }
    7527             : }
    7528             : 
    7529             : // FastEmit functions for AArch64ISD::DUPLANE32.
    7530             : 
    7531             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7532             :   if ((Subtarget->hasNEON())) {
    7533             :     return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7534             :   }
    7535             :   return 0;
    7536             : }
    7537             : 
    7538             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7539             :   if ((Subtarget->hasNEON())) {
    7540             :     return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7541             :   }
    7542             :   return 0;
    7543             : }
    7544             : 
    7545             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7546             : switch (RetVT.SimpleTy) {
    7547             :   case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7548             :   case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7549             :   default: return 0;
    7550             : }
    7551             : }
    7552             : 
    7553             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7554             :   return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7555             : }
    7556             : 
    7557             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7558             :   return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7559             : }
    7560             : 
    7561             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7562             : switch (RetVT.SimpleTy) {
    7563             :   case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7564             :   case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7565             :   default: return 0;
    7566             : }
    7567             : }
    7568             : 
    7569             : unsigned fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7570             :   switch (VT.SimpleTy) {
    7571             :   case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7572             :   case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7573             :   default: return 0;
    7574             :   }
    7575             : }
    7576             : 
    7577             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7578             : 
    7579             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7580             :   if (RetVT.SimpleTy != MVT::i32)
    7581             :     return 0;
    7582             :   if ((Subtarget->hasNEON())) {
    7583             :     return fastEmitInst_ri(AArch64::UMOVvi32, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
    7584             :   }
    7585             :   return 0;
    7586             : }
    7587             : 
    7588             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7589             :   if (RetVT.SimpleTy != MVT::f32)
    7590             :     return 0;
    7591             :   return fastEmitInst_ri(AArch64::CPYi32, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7592             : }
    7593             : 
    7594             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7595             :   switch (VT.SimpleTy) {
    7596             :   case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7597             :   case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7598             :   default: return 0;
    7599             :   }
    7600             : }
    7601             : 
    7602             : // Top-level FastEmit function.
    7603             : 
    7604             : unsigned fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7605             :   switch (Opcode) {
    7606             :   case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, Op0IsKill, imm1);
    7607             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, Op0IsKill, imm1);
    7608             :   default: return 0;
    7609             :   }
    7610             : }
    7611             : 
    7612             : // FastEmit functions for AArch64ISD::DUPLANE16.
    7613             : 
    7614             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7615             :   if ((Subtarget->hasNEON())) {
    7616             :     return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7617             :   }
    7618             :   return 0;
    7619             : }
    7620             : 
    7621             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7622             :   if ((Subtarget->hasNEON())) {
    7623             :     return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7624             :   }
    7625             :   return 0;
    7626             : }
    7627             : 
    7628             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7629             : switch (RetVT.SimpleTy) {
    7630             :   case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7631             :   case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7632             :   default: return 0;
    7633             : }
    7634             : }
    7635             : 
    7636             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7637             :   return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7638             : }
    7639             : 
    7640             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7641             :   return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7642             : }
    7643             : 
    7644             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7645             : switch (RetVT.SimpleTy) {
    7646             :   case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7647             :   case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7648             :   default: return 0;
    7649             : }
    7650             : }
    7651             : 
    7652             : unsigned fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7653             :   switch (VT.SimpleTy) {
    7654             :   case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7655             :   case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7656             :   default: return 0;
    7657             :   }
    7658             : }
    7659             : 
    7660             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7661             : 
    7662             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7663             :   if (RetVT.SimpleTy != MVT::i32)
    7664             :     return 0;
    7665             :   if ((Subtarget->hasNEON())) {
    7666             :     return fastEmitInst_ri(AArch64::UMOVvi16, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
    7667             :   }
    7668             :   return 0;
    7669             : }
    7670             : 
    7671             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7672             :   if (RetVT.SimpleTy != MVT::f16)
    7673             :     return 0;
    7674             :   return fastEmitInst_ri(AArch64::CPYi16, &AArch64::FPR16RegClass, Op0, Op0IsKill, imm1);
    7675             : }
    7676             : 
    7677             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7678             :   switch (VT.SimpleTy) {
    7679             :   case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7680             :   case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7681             :   default: return 0;
    7682             :   }
    7683             : }
    7684             : 
    7685             : // Top-level FastEmit function.
    7686             : 
    7687             : unsigned fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7688             :   switch (Opcode) {
    7689             :   case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, Op0IsKill, imm1);
    7690             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, Op0IsKill, imm1);
    7691             :   default: return 0;
    7692             :   }
    7693             : }
    7694             : 
    7695             : // FastEmit functions for AArch64ISD::DUPLANE8.
    7696             : 
    7697             : unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7698             :   if ((Subtarget->hasNEON())) {
    7699             :     return fastEmitInst_ri(AArch64::DUPv8i8lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7700             :   }
    7701             :   return 0;
    7702             : }
    7703             : 
    7704             : unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7705             :   if ((Subtarget->hasNEON())) {
    7706             :     return fastEmitInst_ri(AArch64::DUPv16i8lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7707             :   }
    7708             :   return 0;
    7709             : }
    7710             : 
    7711             : unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7712             : switch (RetVT.SimpleTy) {
    7713             :   case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, Op0IsKill, imm1);
    7714             :   case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, Op0IsKill, imm1);
    7715             :   default: return 0;
    7716             : }
    7717             : }
    7718             : 
    7719             : unsigned fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7720             :   switch (VT.SimpleTy) {
    7721             :   case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, Op0IsKill, imm1);
    7722             :   default: return 0;
    7723             :   }
    7724             : }
    7725             : 
    7726             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7727             : 
    7728             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7729             :   if (RetVT.SimpleTy != MVT::i32)
    7730             :     return 0;
    7731             :   if ((Subtarget->hasNEON())) {
    7732             :     return fastEmitInst_ri(AArch64::UMOVvi8, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
    7733             :   }
    7734             :   return 0;
    7735             : }
    7736             : 
    7737             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7738             :   switch (VT.SimpleTy) {
    7739             :   case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, Op0IsKill, imm1);
    7740             :   default: return 0;
    7741             :   }
    7742             : }
    7743             : 
    7744             : // Top-level FastEmit function.
    7745             : 
    7746             : unsigned fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7747             :   switch (Opcode) {
    7748             :   case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, Op0IsKill, imm1);
    7749             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, Op0IsKill, imm1);
    7750             :   default: return 0;
    7751             :   }
    7752             : }
    7753             : 
    7754             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    7755             : 
    7756             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7757             :   if (RetVT.SimpleTy != MVT::i64)
    7758             :     return 0;
    7759             :   if ((Subtarget->hasNEON())) {
    7760             :     return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7761             :   }
    7762             :   return 0;
    7763             : }
    7764             : 
    7765             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7766             :   if (RetVT.SimpleTy != MVT::v1i64)
    7767             :     return 0;
    7768             :   if ((Subtarget->hasNEON())) {
    7769             :     return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7770             :   }
    7771             :   return 0;
    7772             : }
    7773             : 
    7774             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7775             :   if (RetVT.SimpleTy != MVT::v2i64)
    7776             :     return 0;
    7777             :   if ((Subtarget->hasNEON())) {
    7778             :     return fastEmitInst_ri(AArch64::SQSHLUv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7779             :   }
    7780             :   return 0;
    7781             : }
    7782             : 
    7783             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7784             :   switch (VT.SimpleTy) {
    7785             :   case MVT::i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7786             :   case MVT::v1i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7787             :   case MVT::v2i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7788             :   default: return 0;
    7789             :   }
    7790             : }
    7791             : 
    7792             : // FastEmit functions for AArch64ISD::SQSHL_I.
    7793             : 
    7794             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7795             :   if (RetVT.SimpleTy != MVT::i64)
    7796             :     return 0;
    7797             :   if ((Subtarget->hasNEON())) {
    7798             :     return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7799             :   }
    7800             :   return 0;
    7801             : }
    7802             : 
    7803             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7804             :   if (RetVT.SimpleTy != MVT::v1i64)
    7805             :     return 0;
    7806             :   if ((Subtarget->hasNEON())) {
    7807             :     return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7808             :   }
    7809             :   return 0;
    7810             : }
    7811             : 
    7812             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7813             :   if (RetVT.SimpleTy != MVT::v2i64)
    7814             :     return 0;
    7815             :   if ((Subtarget->hasNEON())) {
    7816             :     return fastEmitInst_ri(AArch64::SQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7817             :   }
    7818             :   return 0;
    7819             : }
    7820             : 
    7821             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7822             :   switch (VT.SimpleTy) {
    7823             :   case MVT::i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7824             :   case MVT::v1i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7825             :   case MVT::v2i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7826             :   default: return 0;
    7827             :   }
    7828             : }
    7829             : 
    7830             : // FastEmit functions for AArch64ISD::UQSHL_I.
    7831             : 
    7832             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7833             :   if (RetVT.SimpleTy != MVT::i64)
    7834             :     return 0;
    7835             :   if ((Subtarget->hasNEON())) {
    7836             :     return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7837             :   }
    7838             :   return 0;
    7839             : }
    7840             : 
    7841             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7842             :   if (RetVT.SimpleTy != MVT::v1i64)
    7843             :     return 0;
    7844             :   if ((Subtarget->hasNEON())) {
    7845             :     return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7846             :   }
    7847             :   return 0;
    7848             : }
    7849             : 
    7850             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7851             :   if (RetVT.SimpleTy != MVT::v2i64)
    7852             :     return 0;
    7853             :   if ((Subtarget->hasNEON())) {
    7854             :     return fastEmitInst_ri(AArch64::UQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7855             :   }
    7856             :   return 0;
    7857             : }
    7858             : 
    7859             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7860             :   switch (VT.SimpleTy) {
    7861             :   case MVT::i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7862             :   case MVT::v1i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7863             :   case MVT::v2i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7864             :   default: return 0;
    7865             :   }
    7866             : }
    7867             : 
    7868             : // FastEmit functions for AArch64ISD::VSHL.
    7869             : 
    7870             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7871             :   if (RetVT.SimpleTy != MVT::v1i64)
    7872             :     return 0;
    7873             :   if ((Subtarget->hasNEON())) {
    7874             :     return fastEmitInst_ri(AArch64::SHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7875             :   }
    7876             :   return 0;
    7877             : }
    7878             : 
    7879             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7880             :   if (RetVT.SimpleTy != MVT::v2i64)
    7881             :     return 0;
    7882             :   if ((Subtarget->hasNEON())) {
    7883             :     return fastEmitInst_ri(AArch64::SHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7884             :   }
    7885             :   return 0;
    7886             : }
    7887             : 
    7888             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7889             :   switch (VT.SimpleTy) {
    7890             :   case MVT::v1i64: return fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7891             :   case MVT::v2i64: return fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7892             :   default: return 0;
    7893             :   }
    7894             : }
    7895             : 
    7896             : // Top-level FastEmit function.
    7897             : 
    7898             : unsigned fastEmit_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7899             :   switch (Opcode) {
    7900             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7901             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7902             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7903             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7904             :   default: return 0;
    7905             :   }
    7906             : }
    7907             : 
    7908             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    7909             : 
    7910             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7911             :   if (RetVT.SimpleTy != MVT::i32)
    7912             :     return 0;
    7913             :   if ((Subtarget->hasNEON())) {
    7914             :     return fastEmitInst_ri(AArch64::SQSHLUs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7915             :   }
    7916             :   return 0;
    7917             : }
    7918             : 
    7919             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7920             :   if (RetVT.SimpleTy != MVT::v2i32)
    7921             :     return 0;
    7922             :   if ((Subtarget->hasNEON())) {
    7923             :     return fastEmitInst_ri(AArch64::SQSHLUv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7924             :   }
    7925             :   return 0;
    7926             : }
    7927             : 
    7928             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7929             :   if (RetVT.SimpleTy != MVT::v4i32)
    7930             :     return 0;
    7931             :   if ((Subtarget->hasNEON())) {
    7932             :     return fastEmitInst_ri(AArch64::SQSHLUv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7933             :   }
    7934             :   return 0;
    7935             : }
    7936             : 
    7937             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7938             :   switch (VT.SimpleTy) {
    7939             :   case MVT::i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7940             :   case MVT::v2i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7941             :   case MVT::v4i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7942             :   default: return 0;
    7943             :   }
    7944             : }
    7945             : 
    7946             : // FastEmit functions for AArch64ISD::SQSHL_I.
    7947             : 
    7948             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7949             :   if (RetVT.SimpleTy != MVT::i32)
    7950             :     return 0;
    7951             :   if ((Subtarget->hasNEON())) {
    7952             :     return fastEmitInst_ri(AArch64::SQSHLs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7953             :   }
    7954             :   return 0;
    7955             : }
    7956             : 
    7957             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7958             :   if (RetVT.SimpleTy != MVT::v2i32)
    7959             :     return 0;
    7960             :   if ((Subtarget->hasNEON())) {
    7961             :     return fastEmitInst_ri(AArch64::SQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7962             :   }
    7963             :   return 0;
    7964             : }
    7965             : 
    7966             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7967             :   if (RetVT.SimpleTy != MVT::v4i32)
    7968             :     return 0;
    7969             :   if ((Subtarget->hasNEON())) {
    7970             :     return fastEmitInst_ri(AArch64::SQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7971             :   }
    7972             :   return 0;
    7973             : }
    7974             : 
    7975             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7976             :   switch (VT.SimpleTy) {
    7977             :   case MVT::i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7978             :   case MVT::v2i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7979             :   case MVT::v4i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7980             :   default: return 0;
    7981             :   }
    7982             : }
    7983             : 
    7984             : // FastEmit functions for AArch64ISD::UQSHL_I.
    7985             : 
    7986             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7987             :   if (RetVT.SimpleTy != MVT::i32)
    7988             :     return 0;
    7989             :   if ((Subtarget->hasNEON())) {
    7990             :     return fastEmitInst_ri(AArch64::UQSHLs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7991             :   }
    7992             :   return 0;
    7993             : }
    7994             : 
    7995             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7996             :   if (RetVT.SimpleTy != MVT::v2i32)
    7997             :     return 0;
    7998             :   if ((Subtarget->hasNEON())) {
    7999             :     return fastEmitInst_ri(AArch64::UQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8000             :   }
    8001             :   return 0;
    8002             : }
    8003             : 
    8004             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8005             :   if (RetVT.SimpleTy != MVT::v4i32)
    8006             :     return 0;
    8007             :   if ((Subtarget->hasNEON())) {
    8008             :     return fastEmitInst_ri(AArch64::UQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8009             :   }
    8010             :   return 0;
    8011             : }
    8012             : 
    8013             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8014             :   switch (VT.SimpleTy) {
    8015             :   case MVT::i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    8016             :   case MVT::v2i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    8017             :   case MVT::v4i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    8018             :   default: return 0;
    8019             :   }
    8020             : }
    8021             : 
    8022             : // FastEmit functions for AArch64ISD::VSHL.
    8023             : 
    8024             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8025             :   if (RetVT.SimpleTy != MVT::v2i32)
    8026             :     return 0;
    8027             :   if ((Subtarget->hasNEON())) {
    8028             :     return fastEmitInst_ri(AArch64::SHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8029             :   }
    8030             :   return 0;
    8031             : }
    8032             : 
    8033             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8034             :   if (RetVT.SimpleTy != MVT::v4i32)
    8035             :     return 0;
    8036             :   if ((Subtarget->hasNEON())) {
    8037             :     return fastEmitInst_ri(AArch64::SHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8038             :   }
    8039             :   return 0;
    8040             : }
    8041             : 
    8042             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8043             :   switch (VT.SimpleTy) {
    8044             :   case MVT::v2i32: return fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    8045             :   case MVT::v4i32: return fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    8046             :   default: return 0;
    8047             :   }
    8048             : }
    8049             : 
    8050             : // Top-level FastEmit function.
    8051             : 
    8052             : unsigned fastEmit_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8053             :   switch (Opcode) {
    8054             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8055             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8056             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8057             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8058             :   default: return 0;
    8059             :   }
    8060             : }
    8061             : 
    8062             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8063             : 
    8064             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8065             :   if (RetVT.SimpleTy != MVT::i64)
    8066             :     return 0;
    8067             :   if ((Subtarget->hasNEON())) {
    8068             :     return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8069             :   }
    8070             :   return 0;
    8071             : }
    8072             : 
    8073             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8074             :   if (RetVT.SimpleTy != MVT::v1i64)
    8075             :     return 0;
    8076             :   if ((Subtarget->hasNEON())) {
    8077             :     return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8078             :   }
    8079             :   return 0;
    8080             : }
    8081             : 
    8082             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8083             :   if (RetVT.SimpleTy != MVT::v2i64)
    8084             :     return 0;
    8085             :   if ((Subtarget->hasNEON())) {
    8086             :     return fastEmitInst_ri(AArch64::SRSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8087             :   }
    8088             :   return 0;
    8089             : }
    8090             : 
    8091             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8092             :   switch (VT.SimpleTy) {
    8093             :   case MVT::i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8094             :   case MVT::v1i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8095             :   case MVT::v2i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8096             :   default: return 0;
    8097             :   }
    8098             : }
    8099             : 
    8100             : // FastEmit functions for AArch64ISD::URSHR_I.
    8101             : 
    8102             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8103             :   if (RetVT.SimpleTy != MVT::i64)
    8104             :     return 0;
    8105             :   if ((Subtarget->hasNEON())) {
    8106             :     return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8107             :   }
    8108             :   return 0;
    8109             : }
    8110             : 
    8111             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8112             :   if (RetVT.SimpleTy != MVT::v1i64)
    8113             :     return 0;
    8114             :   if ((Subtarget->hasNEON())) {
    8115             :     return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8116             :   }
    8117             :   return 0;
    8118             : }
    8119             : 
    8120             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8121             :   if (RetVT.SimpleTy != MVT::v2i64)
    8122             :     return 0;
    8123             :   if ((Subtarget->hasNEON())) {
    8124             :     return fastEmitInst_ri(AArch64::URSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8125             :   }
    8126             :   return 0;
    8127             : }
    8128             : 
    8129             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8130             :   switch (VT.SimpleTy) {
    8131             :   case MVT::i64: return fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8132             :   case MVT::v1i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8133             :   case MVT::v2i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8134             :   default: return 0;
    8135             :   }
    8136             : }
    8137             : 
    8138             : // FastEmit functions for AArch64ISD::VASHR.
    8139             : 
    8140             : unsigned fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8141             :   if (RetVT.SimpleTy != MVT::i64)
    8142             :     return 0;
    8143             :   if ((Subtarget->hasNEON())) {
    8144             :     return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8145             :   }
    8146             :   return 0;
    8147             : }
    8148             : 
    8149             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8150             :   if (RetVT.SimpleTy != MVT::v1i64)
    8151             :     return 0;
    8152             :   if ((Subtarget->hasNEON())) {
    8153             :     return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8154             :   }
    8155             :   return 0;
    8156             : }
    8157             : 
    8158             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8159             :   if (RetVT.SimpleTy != MVT::v2i64)
    8160             :     return 0;
    8161             :   if ((Subtarget->hasNEON())) {
    8162             :     return fastEmitInst_ri(AArch64::SSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8163             :   }
    8164             :   return 0;
    8165             : }
    8166             : 
    8167             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8168             :   switch (VT.SimpleTy) {
    8169             :   case MVT::i64: return fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8170             :   case MVT::v1i64: return fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8171             :   case MVT::v2i64: return fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8172             :   default: return 0;
    8173             :   }
    8174             : }
    8175             : 
    8176             : // FastEmit functions for AArch64ISD::VLSHR.
    8177             : 
    8178             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8179             :   if (RetVT.SimpleTy != MVT::i64)
    8180             :     return 0;
    8181             :   if ((Subtarget->hasNEON())) {
    8182             :     return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8183             :   }
    8184             :   return 0;
    8185             : }
    8186             : 
    8187             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8188             :   if (RetVT.SimpleTy != MVT::v1i64)
    8189             :     return 0;
    8190             :   if ((Subtarget->hasNEON())) {
    8191             :     return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8192             :   }
    8193             :   return 0;
    8194             : }
    8195             : 
    8196             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8197             :   if (RetVT.SimpleTy != MVT::v2i64)
    8198             :     return 0;
    8199             :   if ((Subtarget->hasNEON())) {
    8200             :     return fastEmitInst_ri(AArch64::USHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8201             :   }
    8202             :   return 0;
    8203             : }
    8204             : 
    8205             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8206             :   switch (VT.SimpleTy) {
    8207             :   case MVT::i64: return fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8208             :   case MVT::v1i64: return fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8209             :   case MVT::v2i64: return fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8210             :   default: return 0;
    8211             :   }
    8212             : }
    8213             : 
    8214             : // Top-level FastEmit function.
    8215             : 
    8216             : unsigned fastEmit_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8217             :   switch (Opcode) {
    8218             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8219             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8220             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8221             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8222             :   default: return 0;
    8223             :   }
    8224             : }
    8225             : 
    8226             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    8227             : 
    8228             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8229             :   if (RetVT.SimpleTy != MVT::v8i8)
    8230             :     return 0;
    8231             :   if ((Subtarget->hasNEON())) {
    8232             :     return fastEmitInst_ri(AArch64::SQSHLUv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8233             :   }
    8234             :   return 0;
    8235             : }
    8236             : 
    8237             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8238             :   if (RetVT.SimpleTy != MVT::v16i8)
    8239             :     return 0;
    8240             :   if ((Subtarget->hasNEON())) {
    8241             :     return fastEmitInst_ri(AArch64::SQSHLUv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8242             :   }
    8243             :   return 0;
    8244             : }
    8245             : 
    8246             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8247             :   switch (VT.SimpleTy) {
    8248             :   case MVT::v8i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8249             :   case MVT::v16i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8250             :   default: return 0;
    8251             :   }
    8252             : }
    8253             : 
    8254             : // FastEmit functions for AArch64ISD::SQSHL_I.
    8255             : 
    8256             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8257             :   if (RetVT.SimpleTy != MVT::v8i8)
    8258             :     return 0;
    8259             :   if ((Subtarget->hasNEON())) {
    8260             :     return fastEmitInst_ri(AArch64::SQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8261             :   }
    8262             :   return 0;
    8263             : }
    8264             : 
    8265             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8266             :   if (RetVT.SimpleTy != MVT::v16i8)
    8267             :     return 0;
    8268             :   if ((Subtarget->hasNEON())) {
    8269             :     return fastEmitInst_ri(AArch64::SQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8270             :   }
    8271             :   return 0;
    8272             : }
    8273             : 
    8274             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8275             :   switch (VT.SimpleTy) {
    8276             :   case MVT::v8i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8277             :   case MVT::v16i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8278             :   default: return 0;
    8279             :   }
    8280             : }
    8281             : 
    8282             : // FastEmit functions for AArch64ISD::UQSHL_I.
    8283             : 
    8284             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8285             :   if (RetVT.SimpleTy != MVT::v8i8)
    8286             :     return 0;
    8287             :   if ((Subtarget->hasNEON())) {
    8288             :     return fastEmitInst_ri(AArch64::UQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8289             :   }
    8290             :   return 0;
    8291             : }
    8292             : 
    8293             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8294             :   if (RetVT.SimpleTy != MVT::v16i8)
    8295             :     return 0;
    8296             :   if ((Subtarget->hasNEON())) {
    8297             :     return fastEmitInst_ri(AArch64::UQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8298             :   }
    8299             :   return 0;
    8300             : }
    8301             : 
    8302             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8303             :   switch (VT.SimpleTy) {
    8304             :   case MVT::v8i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8305             :   case MVT::v16i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8306             :   default: return 0;
    8307             :   }
    8308             : }
    8309             : 
    8310             : // FastEmit functions for AArch64ISD::VSHL.
    8311             : 
    8312             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8313             :   if (RetVT.SimpleTy != MVT::v8i8)
    8314             :     return 0;
    8315             :   if ((Subtarget->hasNEON())) {
    8316             :     return fastEmitInst_ri(AArch64::SHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8317             :   }
    8318             :   return 0;
    8319             : }
    8320             : 
    8321             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8322             :   if (RetVT.SimpleTy != MVT::v16i8)
    8323             :     return 0;
    8324             :   if ((Subtarget->hasNEON())) {
    8325             :     return fastEmitInst_ri(AArch64::SHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8326             :   }
    8327             :   return 0;
    8328             : }
    8329             : 
    8330             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8331             :   switch (VT.SimpleTy) {
    8332             :   case MVT::v8i8: return fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8333             :   case MVT::v16i8: return fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8334             :   default: return 0;
    8335             :   }
    8336             : }
    8337             : 
    8338             : // Top-level FastEmit function.
    8339             : 
    8340             : unsigned fastEmit_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8341             :   switch (Opcode) {
    8342             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8343             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8344             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8345             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8346             :   default: return 0;
    8347             :   }
    8348             : }
    8349             : 
    8350             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    8351             : 
    8352             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8353             :   if (RetVT.SimpleTy != MVT::v4i16)
    8354             :     return 0;
    8355             :   if ((Subtarget->hasNEON())) {
    8356             :     return fastEmitInst_ri(AArch64::SQSHLUv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8357             :   }
    8358             :   return 0;
    8359             : }
    8360             : 
    8361             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8362             :   if (RetVT.SimpleTy != MVT::v8i16)
    8363             :     return 0;
    8364             :   if ((Subtarget->hasNEON())) {
    8365             :     return fastEmitInst_ri(AArch64::SQSHLUv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8366             :   }
    8367             :   return 0;
    8368             : }
    8369             : 
    8370             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8371             :   switch (VT.SimpleTy) {
    8372             :   case MVT::v4i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8373             :   case MVT::v8i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8374             :   default: return 0;
    8375             :   }
    8376             : }
    8377             : 
    8378             : // FastEmit functions for AArch64ISD::SQSHL_I.
    8379             : 
    8380             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8381             :   if (RetVT.SimpleTy != MVT::v4i16)
    8382             :     return 0;
    8383             :   if ((Subtarget->hasNEON())) {
    8384             :     return fastEmitInst_ri(AArch64::SQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8385             :   }
    8386             :   return 0;
    8387             : }
    8388             : 
    8389             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8390             :   if (RetVT.SimpleTy != MVT::v8i16)
    8391             :     return 0;
    8392             :   if ((Subtarget->hasNEON())) {
    8393             :     return fastEmitInst_ri(AArch64::SQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8394             :   }
    8395             :   return 0;
    8396             : }
    8397             : 
    8398             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8399             :   switch (VT.SimpleTy) {
    8400             :   case MVT::v4i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8401             :   case MVT::v8i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8402             :   default: return 0;
    8403             :   }
    8404             : }
    8405             : 
    8406             : // FastEmit functions for AArch64ISD::UQSHL_I.
    8407             : 
    8408             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8409             :   if (RetVT.SimpleTy != MVT::v4i16)
    8410             :     return 0;
    8411             :   if ((Subtarget->hasNEON())) {
    8412             :     return fastEmitInst_ri(AArch64::UQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8413             :   }
    8414             :   return 0;
    8415             : }
    8416             : 
    8417             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8418             :   if (RetVT.SimpleTy != MVT::v8i16)
    8419             :     return 0;
    8420             :   if ((Subtarget->hasNEON())) {
    8421             :     return fastEmitInst_ri(AArch64::UQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8422             :   }
    8423             :   return 0;
    8424             : }
    8425             : 
    8426             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8427             :   switch (VT.SimpleTy) {
    8428             :   case MVT::v4i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8429             :   case MVT::v8i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8430             :   default: return 0;
    8431             :   }
    8432             : }
    8433             : 
    8434             : // FastEmit functions for AArch64ISD::VSHL.
    8435             : 
    8436             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8437             :   if (RetVT.SimpleTy != MVT::v4i16)
    8438             :     return 0;
    8439             :   if ((Subtarget->hasNEON())) {
    8440             :     return fastEmitInst_ri(AArch64::SHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8441             :   }
    8442             :   return 0;
    8443             : }
    8444             : 
    8445             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8446             :   if (RetVT.SimpleTy != MVT::v8i16)
    8447             :     return 0;
    8448             :   if ((Subtarget->hasNEON())) {
    8449             :     return fastEmitInst_ri(AArch64::SHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8450             :   }
    8451             :   return 0;
    8452             : }
    8453             : 
    8454             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8455             :   switch (VT.SimpleTy) {
    8456             :   case MVT::v4i16: return fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8457             :   case MVT::v8i16: return fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8458             :   default: return 0;
    8459             :   }
    8460             : }
    8461             : 
    8462             : // Top-level FastEmit function.
    8463             : 
    8464             : unsigned fastEmit_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8465             :   switch (Opcode) {
    8466             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8467             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8468             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8469             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8470             :   default: return 0;
    8471             :   }
    8472             : }
    8473             : 
    8474             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8475             : 
    8476             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8477             :   if (RetVT.SimpleTy != MVT::v8i8)
    8478             :     return 0;
    8479             :   if ((Subtarget->hasNEON())) {
    8480             :     return fastEmitInst_ri(AArch64::SRSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8481             :   }
    8482             :   return 0;
    8483             : }
    8484             : 
    8485             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8486             :   if (RetVT.SimpleTy != MVT::v16i8)
    8487             :     return 0;
    8488             :   if ((Subtarget->hasNEON())) {
    8489             :     return fastEmitInst_ri(AArch64::SRSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8490             :   }
    8491             :   return 0;
    8492             : }
    8493             : 
    8494             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8495             :   switch (VT.SimpleTy) {
    8496             :   case MVT::v8i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8497             :   case MVT::v16i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8498             :   default: return 0;
    8499             :   }
    8500             : }
    8501             : 
    8502             : // FastEmit functions for AArch64ISD::URSHR_I.
    8503             : 
    8504             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8505             :   if (RetVT.SimpleTy != MVT::v8i8)
    8506             :     return 0;
    8507             :   if ((Subtarget->hasNEON())) {
    8508             :     return fastEmitInst_ri(AArch64::URSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8509             :   }
    8510             :   return 0;
    8511             : }
    8512             : 
    8513             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8514             :   if (RetVT.SimpleTy != MVT::v16i8)
    8515             :     return 0;
    8516             :   if ((Subtarget->hasNEON())) {
    8517             :     return fastEmitInst_ri(AArch64::URSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8518             :   }
    8519             :   return 0;
    8520             : }
    8521             : 
    8522             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8523             :   switch (VT.SimpleTy) {
    8524             :   case MVT::v8i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8525             :   case MVT::v16i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8526             :   default: return 0;
    8527             :   }
    8528             : }
    8529             : 
    8530             : // FastEmit functions for AArch64ISD::VASHR.
    8531             : 
    8532             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8533             :   if (RetVT.SimpleTy != MVT::v8i8)
    8534             :     return 0;
    8535             :   if ((Subtarget->hasNEON())) {
    8536             :     return fastEmitInst_ri(AArch64::SSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8537             :   }
    8538             :   return 0;
    8539             : }
    8540             : 
    8541             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8542             :   if (RetVT.SimpleTy != MVT::v16i8)
    8543             :     return 0;
    8544             :   if ((Subtarget->hasNEON())) {
    8545             :     return fastEmitInst_ri(AArch64::SSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8546             :   }
    8547             :   return 0;
    8548             : }
    8549             : 
    8550             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8551             :   switch (VT.SimpleTy) {
    8552             :   case MVT::v8i8: return fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8553             :   case MVT::v16i8: return fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8554             :   default: return 0;
    8555             :   }
    8556             : }
    8557             : 
    8558             : // FastEmit functions for AArch64ISD::VLSHR.
    8559             : 
    8560             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8561             :   if (RetVT.SimpleTy != MVT::v8i8)
    8562             :     return 0;
    8563             :   if ((Subtarget->hasNEON())) {
    8564             :     return fastEmitInst_ri(AArch64::USHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8565             :   }
    8566             :   return 0;
    8567             : }
    8568             : 
    8569             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8570             :   if (RetVT.SimpleTy != MVT::v16i8)
    8571             :     return 0;
    8572             :   if ((Subtarget->hasNEON())) {
    8573             :     return fastEmitInst_ri(AArch64::USHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8574             :   }
    8575             :   return 0;
    8576             : }
    8577             : 
    8578             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8579             :   switch (VT.SimpleTy) {
    8580             :   case MVT::v8i8: return fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8581             :   case MVT::v16i8: return fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8582             :   default: return 0;
    8583             :   }
    8584             : }
    8585             : 
    8586             : // Top-level FastEmit function.
    8587             : 
    8588             : unsigned fastEmit_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8589             :   switch (Opcode) {
    8590             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8591             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8592             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8593             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8594             :   default: return 0;
    8595             :   }
    8596             : }
    8597             : 
    8598             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8599             : 
    8600             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8601             :   if (RetVT.SimpleTy != MVT::v4i16)
    8602             :     return 0;
    8603             :   if ((Subtarget->hasNEON())) {
    8604             :     return fastEmitInst_ri(AArch64::SRSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8605             :   }
    8606             :   return 0;
    8607             : }
    8608             : 
    8609             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8610             :   if (RetVT.SimpleTy != MVT::v8i16)
    8611             :     return 0;
    8612             :   if ((Subtarget->hasNEON())) {
    8613             :     return fastEmitInst_ri(AArch64::SRSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8614             :   }
    8615             :   return 0;
    8616             : }
    8617             : 
    8618             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8619             :   switch (VT.SimpleTy) {
    8620             :   case MVT::v4i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8621             :   case MVT::v8i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8622             :   default: return 0;
    8623             :   }
    8624             : }
    8625             : 
    8626             : // FastEmit functions for AArch64ISD::URSHR_I.
    8627             : 
    8628             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8629             :   if (RetVT.SimpleTy != MVT::v4i16)
    8630             :     return 0;
    8631             :   if ((Subtarget->hasNEON())) {
    8632             :     return fastEmitInst_ri(AArch64::URSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8633             :   }
    8634             :   return 0;
    8635             : }
    8636             : 
    8637             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8638             :   if (RetVT.SimpleTy != MVT::v8i16)
    8639             :     return 0;
    8640             :   if ((Subtarget->hasNEON())) {
    8641             :     return fastEmitInst_ri(AArch64::URSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8642             :   }
    8643             :   return 0;
    8644             : }
    8645             : 
    8646             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8647             :   switch (VT.SimpleTy) {
    8648             :   case MVT::v4i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8649             :   case MVT::v8i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8650             :   default: return 0;
    8651             :   }
    8652             : }
    8653             : 
    8654             : // FastEmit functions for AArch64ISD::VASHR.
    8655             : 
    8656             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8657             :   if (RetVT.SimpleTy != MVT::v4i16)
    8658             :     return 0;
    8659             :   if ((Subtarget->hasNEON())) {
    8660             :     return fastEmitInst_ri(AArch64::SSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8661             :   }
    8662             :   return 0;
    8663             : }
    8664             : 
    8665             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8666             :   if (RetVT.SimpleTy != MVT::v8i16)
    8667             :     return 0;
    8668             :   if ((Subtarget->hasNEON())) {
    8669             :     return fastEmitInst_ri(AArch64::SSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8670             :   }
    8671             :   return 0;
    8672             : }
    8673             : 
    8674             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8675             :   switch (VT.SimpleTy) {
    8676             :   case MVT::v4i16: return fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8677             :   case MVT::v8i16: return fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8678             :   default: return 0;
    8679             :   }
    8680             : }
    8681             : 
    8682             : // FastEmit functions for AArch64ISD::VLSHR.
    8683             : 
    8684             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8685             :   if (RetVT.SimpleTy != MVT::v4i16)
    8686             :     return 0;
    8687             :   if ((Subtarget->hasNEON())) {
    8688             :     return fastEmitInst_ri(AArch64::USHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8689             :   }
    8690             :   return 0;
    8691             : }
    8692             : 
    8693             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8694             :   if (RetVT.SimpleTy != MVT::v8i16)
    8695             :     return 0;
    8696             :   if ((Subtarget->hasNEON())) {
    8697             :     return fastEmitInst_ri(AArch64::USHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8698             :   }
    8699             :   return 0;
    8700             : }
    8701             : 
    8702             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8703             :   switch (VT.SimpleTy) {
    8704             :   case MVT::v4i16: return fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8705             :   case MVT::v8i16: return fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8706             :   default: return 0;
    8707             :   }
    8708             : }
    8709             : 
    8710             : // Top-level FastEmit function.
    8711             : 
    8712             : unsigned fastEmit_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8713             :   switch (Opcode) {
    8714             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8715             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8716             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8717             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8718             :   default: return 0;
    8719             :   }
    8720             : }
    8721             : 
    8722             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8723             : 
    8724             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8725             :   if (RetVT.SimpleTy != MVT::v2i32)
    8726             :     return 0;
    8727             :   if ((Subtarget->hasNEON())) {
    8728             :     return fastEmitInst_ri(AArch64::SRSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8729             :   }
    8730             :   return 0;
    8731             : }
    8732             : 
    8733             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8734             :   if (RetVT.SimpleTy != MVT::v4i32)
    8735             :     return 0;
    8736             :   if ((Subtarget->hasNEON())) {
    8737             :     return fastEmitInst_ri(AArch64::SRSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8738             :   }
    8739             :   return 0;
    8740             : }
    8741             : 
    8742             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8743             :   switch (VT.SimpleTy) {
    8744             :   case MVT::v2i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8745             :   case MVT::v4i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8746             :   default: return 0;
    8747             :   }
    8748             : }
    8749             : 
    8750             : // FastEmit functions for AArch64ISD::URSHR_I.
    8751             : 
    8752             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8753             :   if (RetVT.SimpleTy != MVT::v2i32)
    8754             :     return 0;
    8755             :   if ((Subtarget->hasNEON())) {
    8756             :     return fastEmitInst_ri(AArch64::URSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8757             :   }
    8758             :   return 0;
    8759             : }
    8760             : 
    8761             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8762             :   if (RetVT.SimpleTy != MVT::v4i32)
    8763             :     return 0;
    8764             :   if ((Subtarget->hasNEON())) {
    8765             :     return fastEmitInst_ri(AArch64::URSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8766             :   }
    8767             :   return 0;
    8768             : }
    8769             : 
    8770             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8771             :   switch (VT.SimpleTy) {
    8772             :   case MVT::v2i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8773             :   case MVT::v4i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8774             :   default: return 0;
    8775             :   }
    8776             : }
    8777             : 
    8778             : // FastEmit functions for AArch64ISD::VASHR.
    8779             : 
    8780             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8781             :   if (RetVT.SimpleTy != MVT::v2i32)
    8782             :     return 0;
    8783             :   if ((Subtarget->hasNEON())) {
    8784             :     return fastEmitInst_ri(AArch64::SSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8785             :   }
    8786             :   return 0;
    8787             : }
    8788             : 
    8789             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8790             :   if (RetVT.SimpleTy != MVT::v4i32)
    8791             :     return 0;
    8792             :   if ((Subtarget->hasNEON())) {
    8793             :     return fastEmitInst_ri(AArch64::SSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8794             :   }
    8795             :   return 0;
    8796             : }
    8797             : 
    8798             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8799             :   switch (VT.SimpleTy) {
    8800             :   case MVT::v2i32: return fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8801             :   case MVT::v4i32: return fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8802             :   default: return 0;
    8803             :   }
    8804             : }
    8805             : 
    8806             : // FastEmit functions for AArch64ISD::VLSHR.
    8807             : 
    8808             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8809             :   if (RetVT.SimpleTy != MVT::v2i32)
    8810             :     return 0;
    8811             :   if ((Subtarget->hasNEON())) {
    8812             :     return fastEmitInst_ri(AArch64::USHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8813             :   }
    8814             :   return 0;
    8815             : }
    8816             : 
    8817             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8818             :   if (RetVT.SimpleTy != MVT::v4i32)
    8819             :     return 0;
    8820             :   if ((Subtarget->hasNEON())) {
    8821             :     return fastEmitInst_ri(AArch64::USHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8822             :   }
    8823             :   return 0;
    8824             : }
    8825             : 
    8826             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8827             :   switch (VT.SimpleTy) {
    8828             :   case MVT::v2i32: return fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8829             :   case MVT::v4i32: return fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8830             :   default: return 0;
    8831             :   }
    8832             : }
    8833             : 
    8834             : // Top-level FastEmit function.
    8835             : 
    8836             : unsigned fastEmit_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8837             :   switch (Opcode) {
    8838             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8839             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8840             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8841             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8842             :   default: return 0;
    8843             :   }
    8844             : }
    8845             : 
    8846             : // FastEmit functions for ISD::Constant.
    8847             : 
    8848             : unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
    8849         188 :   if (RetVT.SimpleTy != MVT::i32)
    8850             :     return 0;
    8851         188 :   return fastEmitInst_i(AArch64::MOVi32imm, &AArch64::GPR32RegClass, imm0);
    8852             : }
    8853             : 
    8854             : unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
    8855          75 :   if (RetVT.SimpleTy != MVT::i64)
    8856             :     return 0;
    8857          75 :   return fastEmitInst_i(AArch64::MOVi64imm, &AArch64::GPR64RegClass, imm0);
    8858             : }
    8859             : 
    8860         348 : unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
    8861         348 :   switch (VT.SimpleTy) {
    8862         188 :   case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
    8863          75 :   case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
    8864             :   default: return 0;
    8865             :   }
    8866             : }
    8867             : 
    8868             : // Top-level FastEmit function.
    8869             : 
    8870         348 : unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
    8871         884 :   if (VT == MVT::i32 && Predicate_imm0_255(imm0))
    8872         179 :     if (unsigned Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
    8873             :       return Reg;
    8874             : 
    8875         348 :   switch (Opcode) {
    8876         348 :   case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
    8877             :   default: return 0;
    8878             :   }
    8879             : }
    8880             : 
    8881             : // FastEmit functions for AArch64ISD::FMOV.
    8882             : 
    8883           0 : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
    8884           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    8885           0 :     return fastEmitInst_i(AArch64::FMOVv4f16_ns, &AArch64::FPR64RegClass, imm0);
    8886             :   }
    8887             :   return 0;
    8888             : }
    8889             : 
    8890           0 : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
    8891           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    8892           0 :     return fastEmitInst_i(AArch64::FMOVv8f16_ns, &AArch64::FPR128RegClass, imm0);
    8893             :   }
    8894             :   return 0;
    8895             : }
    8896             : 
    8897             : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
    8898           0 :   if ((Subtarget->hasNEON())) {
    8899           0 :     return fastEmitInst_i(AArch64::FMOVv2f32_ns, &AArch64::FPR64RegClass, imm0);
    8900             :   }
    8901             :   return 0;
    8902             : }
    8903             : 
    8904             : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
    8905           0 :   if ((Subtarget->hasNEON())) {
    8906           0 :     return fastEmitInst_i(AArch64::FMOVv4f32_ns, &AArch64::FPR128RegClass, imm0);
    8907             :   }
    8908             :   return 0;
    8909             : }
    8910             : 
    8911             : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
    8912           0 :   if ((Subtarget->hasNEON())) {
    8913           0 :     return fastEmitInst_i(AArch64::FMOVv2f64_ns, &AArch64::FPR128RegClass, imm0);
    8914             :   }
    8915             :   return 0;
    8916             : }
    8917             : 
    8918           0 : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
    8919           0 : switch (RetVT.SimpleTy) {
    8920           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
    8921           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
    8922           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
    8923           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
    8924           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
    8925             :   default: return 0;
    8926             : }
    8927             : }
    8928             : 
    8929             : unsigned fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
    8930           0 :   switch (VT.SimpleTy) {
    8931           0 :   case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
    8932             :   default: return 0;
    8933             :   }
    8934             : }
    8935             : 
    8936             : // FastEmit functions for AArch64ISD::MOVI.
    8937             : 
    8938             : unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
    8939           0 :   if ((Subtarget->hasNEON())) {
    8940           0 :     return fastEmitInst_i(AArch64::MOVIv8b_ns, &AArch64::FPR64RegClass, imm0);
    8941             :   }
    8942             :   return 0;
    8943             : }
    8944             : 
    8945             : unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
    8946           0 :   if ((Subtarget->hasNEON())) {
    8947           0 :     return fastEmitInst_i(AArch64::MOVIv16b_ns, &AArch64::FPR128RegClass, imm0);
    8948             :   }
    8949             :   return 0;
    8950             : }
    8951             : 
    8952           0 : unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
    8953           0 : switch (RetVT.SimpleTy) {
    8954           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
    8955           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
    8956             :   default: return 0;
    8957             : }
    8958             : }
    8959             : 
    8960             : unsigned fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
    8961           0 :   switch (VT.SimpleTy) {
    8962           0 :   case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
    8963             :   default: return 0;
    8964             :   }
    8965             : }
    8966             : 
    8967             : // FastEmit functions for AArch64ISD::MOVIedit.
    8968             : 
    8969             : unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
    8970           0 :   return fastEmitInst_i(AArch64::MOVID, &AArch64::FPR64RegClass, imm0);
    8971             : }
    8972             : 
    8973             : unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
    8974           0 :   if ((Subtarget->hasNEON())) {
    8975           0 :     return fastEmitInst_i(AArch64::MOVIv2d_ns, &AArch64::FPR128RegClass, imm0);
    8976             :   }
    8977             :   return 0;
    8978             : }
    8979             : 
    8980           0 : unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
    8981           0 : switch (RetVT.SimpleTy) {
    8982           0 :   case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
    8983           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
    8984             :   default: return 0;
    8985             : }
    8986             : }
    8987             : 
    8988             : unsigned fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
    8989           0 :   switch (VT.SimpleTy) {
    8990           0 :   case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
    8991             :   default: return 0;
    8992             :   }
    8993             : }
    8994             : 
    8995             : // Top-level FastEmit function.
    8996             : 
    8997         179 : unsigned fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
    8998         179 :   switch (Opcode) {
    8999           0 :   case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
    9000           0 :   case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
    9001           0 :   case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
    9002             :   default: return 0;
    9003             :   }
    9004             : }
    9005             : 

Generated by: LCOV version 1.13