LCOV - code coverage report
Current view: top level - build-llvm/lib/Target/AArch64 - AArch64GenFastISel.inc (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 248 2980 8.3 %
Date: 2018-06-17 00:07:59 Functions: 29 180 16.1 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
       2             : |*                                                                            *|
       3             : |* "Fast" Instruction Selector for the AArch64 target                         *|
       4             : |*                                                                            *|
       5             : |* Automatically generated file, do not edit!                                 *|
       6             : |*                                                                            *|
       7             : \*===----------------------------------------------------------------------===*/
       8             : 
       9             : 
      10             : // FastEmit Immediate Predicate functions.
      11             : static bool Predicate_imm0_31(int64_t Imm) {
      12             : 
      13             :   return ((uint64_t)Imm) < 32;
      14             : 
      15             : }
      16             : static bool Predicate_imm0_63(int64_t Imm) {
      17             : 
      18             :   return ((uint64_t)Imm) < 64;
      19             : 
      20             : }
      21             : static bool Predicate_imm32_0_31(int64_t Imm) {
      22             : 
      23             :   return ((uint64_t)Imm) < 32;
      24             : 
      25             : }
      26             : static bool Predicate_tbz_imm0_31_diag(int64_t Imm) {
      27             : 
      28             :   return (((uint32_t)Imm) < 32);
      29             : 
      30             : }
      31             : static bool Predicate_tbz_imm32_63(int64_t Imm) {
      32             : 
      33             :   return (((uint32_t)Imm) > 31) && (((uint32_t)Imm) < 64);
      34             : 
      35             : }
      36             : static bool Predicate_VectorIndexD(int64_t Imm) {
      37             :  return ((uint64_t)Imm) < 2; 
      38             : }
      39             : static bool Predicate_VectorIndexS(int64_t Imm) {
      40             :  return ((uint64_t)Imm) < 4; 
      41             : }
      42             : static bool Predicate_VectorIndexH(int64_t Imm) {
      43             :  return ((uint64_t)Imm) < 8; 
      44             : }
      45             : static bool Predicate_VectorIndexB(int64_t Imm) {
      46             :  return ((uint64_t)Imm) < 16; 
      47             : }
      48             : static bool Predicate_imm0_255(int64_t Imm) {
      49             : 
      50         180 :   return ((uint32_t)Imm) < 256;
      51             : 
      52             : }
      53             : static bool Predicate_vecshiftL64(int64_t Imm) {
      54             : 
      55             :   return (((uint32_t)Imm) < 64);
      56             : 
      57             : }
      58             : static bool Predicate_vecshiftL32(int64_t Imm) {
      59             : 
      60             :   return (((uint32_t)Imm) < 32);
      61             : 
      62             : }
      63             : static bool Predicate_vecshiftR64(int64_t Imm) {
      64             : 
      65             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 65);
      66             : 
      67             : }
      68             : static bool Predicate_vecshiftL8(int64_t Imm) {
      69             : 
      70             :   return (((uint32_t)Imm) < 8);
      71             : 
      72             : }
      73             : static bool Predicate_vecshiftL16(int64_t Imm) {
      74             : 
      75             :   return (((uint32_t)Imm) < 16);
      76             : 
      77             : }
      78             : static bool Predicate_vecshiftR8(int64_t Imm) {
      79             : 
      80             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 9);
      81             : 
      82             : }
      83             : static bool Predicate_vecshiftR16(int64_t Imm) {
      84             : 
      85             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 17);
      86             : 
      87             : }
      88             : static bool Predicate_vecshiftR32(int64_t Imm) {
      89             : 
      90             :   return (((uint32_t)Imm) > 0) && (((uint32_t)Imm) < 33);
      91             : 
      92             : }
      93             : 
      94             : 
      95             : // FastEmit functions for AArch64ISD::THREAD_POINTER.
      96             : 
      97             : unsigned fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(MVT RetVT) {
      98           0 :   if (RetVT.SimpleTy != MVT::i64)
      99             :     return 0;
     100           0 :   return fastEmitInst_(AArch64::MOVbaseTLS, &AArch64::GPR64RegClass);
     101             : }
     102             : 
     103             : unsigned fastEmit_AArch64ISD_THREAD_POINTER_(MVT VT, MVT RetVT) {
     104           0 :   switch (VT.SimpleTy) {
     105           0 :   case MVT::i64: return fastEmit_AArch64ISD_THREAD_POINTER_MVT_i64_(RetVT);
     106             :   default: return 0;
     107             :   }
     108             : }
     109             : 
     110             : // Top-level FastEmit function.
     111             : 
     112           9 : unsigned fastEmit_(MVT VT, MVT RetVT, unsigned Opcode) override {
     113           9 :   switch (Opcode) {
     114           0 :   case AArch64ISD::THREAD_POINTER: return fastEmit_AArch64ISD_THREAD_POINTER_(VT, RetVT);
     115             :   default: return 0;
     116             :   }
     117             : }
     118             : 
     119             : // FastEmit functions for AArch64ISD::CALL.
     120             : 
     121             : unsigned fastEmit_AArch64ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     122           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     123             :     return 0;
     124           0 :   return fastEmitInst_r(AArch64::BLR, &AArch64::GPR64RegClass, Op0, Op0IsKill);
     125             : }
     126             : 
     127             : unsigned fastEmit_AArch64ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     128           0 :   switch (VT.SimpleTy) {
     129           0 :   case MVT::i64: return fastEmit_AArch64ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
     130             :   default: return 0;
     131             :   }
     132             : }
     133             : 
     134             : // FastEmit functions for AArch64ISD::CMEQz.
     135             : 
     136             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     137           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     138             :     return 0;
     139           0 :   if ((Subtarget->hasNEON())) {
     140           0 :     return fastEmitInst_r(AArch64::CMEQv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     141             :   }
     142             :   return 0;
     143             : }
     144             : 
     145             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     146           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     147             :     return 0;
     148           0 :   if ((Subtarget->hasNEON())) {
     149           0 :     return fastEmitInst_r(AArch64::CMEQv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     150             :   }
     151             :   return 0;
     152             : }
     153             : 
     154             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     155           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     156             :     return 0;
     157           0 :   if ((Subtarget->hasNEON())) {
     158           0 :     return fastEmitInst_r(AArch64::CMEQv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     159             :   }
     160             :   return 0;
     161             : }
     162             : 
     163             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     164           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     165             :     return 0;
     166           0 :   if ((Subtarget->hasNEON())) {
     167           0 :     return fastEmitInst_r(AArch64::CMEQv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     168             :   }
     169             :   return 0;
     170             : }
     171             : 
     172             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     173           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     174             :     return 0;
     175           0 :   if ((Subtarget->hasNEON())) {
     176           0 :     return fastEmitInst_r(AArch64::CMEQv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     177             :   }
     178             :   return 0;
     179             : }
     180             : 
     181             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     182           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     183             :     return 0;
     184           0 :   if ((Subtarget->hasNEON())) {
     185           0 :     return fastEmitInst_r(AArch64::CMEQv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     186             :   }
     187             :   return 0;
     188             : }
     189             : 
     190             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     191           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     192             :     return 0;
     193           0 :   if ((Subtarget->hasNEON())) {
     194           0 :     return fastEmitInst_r(AArch64::CMEQv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     195             :   }
     196             :   return 0;
     197             : }
     198             : 
     199             : unsigned fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     200           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     201             :     return 0;
     202           0 :   if ((Subtarget->hasNEON())) {
     203           0 :     return fastEmitInst_r(AArch64::CMEQv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     204             :   }
     205             :   return 0;
     206             : }
     207             : 
     208           0 : unsigned fastEmit_AArch64ISD_CMEQz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     209           0 :   switch (VT.SimpleTy) {
     210           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMEQz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     211           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMEQz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     212           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMEQz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     213           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMEQz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     214           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMEQz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     215           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMEQz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     216           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMEQz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     217           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMEQz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     218             :   default: return 0;
     219             :   }
     220             : }
     221             : 
     222             : // FastEmit functions for AArch64ISD::CMGEz.
     223             : 
     224             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     225           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     226             :     return 0;
     227           0 :   if ((Subtarget->hasNEON())) {
     228           0 :     return fastEmitInst_r(AArch64::CMGEv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     229             :   }
     230             :   return 0;
     231             : }
     232             : 
     233             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     234           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     235             :     return 0;
     236           0 :   if ((Subtarget->hasNEON())) {
     237           0 :     return fastEmitInst_r(AArch64::CMGEv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     238             :   }
     239             :   return 0;
     240             : }
     241             : 
     242             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     243           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     244             :     return 0;
     245           0 :   if ((Subtarget->hasNEON())) {
     246           0 :     return fastEmitInst_r(AArch64::CMGEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     247             :   }
     248             :   return 0;
     249             : }
     250             : 
     251             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     252           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     253             :     return 0;
     254           0 :   if ((Subtarget->hasNEON())) {
     255           0 :     return fastEmitInst_r(AArch64::CMGEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     256             :   }
     257             :   return 0;
     258             : }
     259             : 
     260             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     261           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     262             :     return 0;
     263           0 :   if ((Subtarget->hasNEON())) {
     264           0 :     return fastEmitInst_r(AArch64::CMGEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     265             :   }
     266             :   return 0;
     267             : }
     268             : 
     269             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     270           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     271             :     return 0;
     272           0 :   if ((Subtarget->hasNEON())) {
     273           0 :     return fastEmitInst_r(AArch64::CMGEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     274             :   }
     275             :   return 0;
     276             : }
     277             : 
     278             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     279           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     280             :     return 0;
     281           0 :   if ((Subtarget->hasNEON())) {
     282           0 :     return fastEmitInst_r(AArch64::CMGEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     283             :   }
     284             :   return 0;
     285             : }
     286             : 
     287             : unsigned fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     288           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     289             :     return 0;
     290           0 :   if ((Subtarget->hasNEON())) {
     291           0 :     return fastEmitInst_r(AArch64::CMGEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     292             :   }
     293             :   return 0;
     294             : }
     295             : 
     296           0 : unsigned fastEmit_AArch64ISD_CMGEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     297           0 :   switch (VT.SimpleTy) {
     298           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGEz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     299           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGEz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     300           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGEz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     301           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGEz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     302           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGEz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     303           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGEz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     304           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGEz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     305           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGEz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     306             :   default: return 0;
     307             :   }
     308             : }
     309             : 
     310             : // FastEmit functions for AArch64ISD::CMGTz.
     311             : 
     312             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     313           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     314             :     return 0;
     315           0 :   if ((Subtarget->hasNEON())) {
     316           0 :     return fastEmitInst_r(AArch64::CMGTv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     317             :   }
     318             :   return 0;
     319             : }
     320             : 
     321             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     322           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     323             :     return 0;
     324           0 :   if ((Subtarget->hasNEON())) {
     325           0 :     return fastEmitInst_r(AArch64::CMGTv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     326             :   }
     327             :   return 0;
     328             : }
     329             : 
     330             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     331           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     332             :     return 0;
     333           0 :   if ((Subtarget->hasNEON())) {
     334           0 :     return fastEmitInst_r(AArch64::CMGTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     335             :   }
     336             :   return 0;
     337             : }
     338             : 
     339             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     340           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     341             :     return 0;
     342           0 :   if ((Subtarget->hasNEON())) {
     343           0 :     return fastEmitInst_r(AArch64::CMGTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     344             :   }
     345             :   return 0;
     346             : }
     347             : 
     348             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     349           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     350             :     return 0;
     351           0 :   if ((Subtarget->hasNEON())) {
     352           0 :     return fastEmitInst_r(AArch64::CMGTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     353             :   }
     354             :   return 0;
     355             : }
     356             : 
     357             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     358           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     359             :     return 0;
     360           0 :   if ((Subtarget->hasNEON())) {
     361           0 :     return fastEmitInst_r(AArch64::CMGTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     362             :   }
     363             :   return 0;
     364             : }
     365             : 
     366             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     367           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     368             :     return 0;
     369           0 :   if ((Subtarget->hasNEON())) {
     370           0 :     return fastEmitInst_r(AArch64::CMGTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     371             :   }
     372             :   return 0;
     373             : }
     374             : 
     375             : unsigned fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     376           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     377             :     return 0;
     378           0 :   if ((Subtarget->hasNEON())) {
     379           0 :     return fastEmitInst_r(AArch64::CMGTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     380             :   }
     381             :   return 0;
     382             : }
     383             : 
     384           0 : unsigned fastEmit_AArch64ISD_CMGTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     385           0 :   switch (VT.SimpleTy) {
     386           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGTz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     387           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGTz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     388           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGTz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     389           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGTz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     390           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGTz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     391           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGTz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     392           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGTz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     393           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGTz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     394             :   default: return 0;
     395             :   }
     396             : }
     397             : 
     398             : // FastEmit functions for AArch64ISD::CMLEz.
     399             : 
     400             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     401           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     402             :     return 0;
     403           0 :   if ((Subtarget->hasNEON())) {
     404           0 :     return fastEmitInst_r(AArch64::CMLEv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     405             :   }
     406             :   return 0;
     407             : }
     408             : 
     409             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     410           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     411             :     return 0;
     412           0 :   if ((Subtarget->hasNEON())) {
     413           0 :     return fastEmitInst_r(AArch64::CMLEv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     414             :   }
     415             :   return 0;
     416             : }
     417             : 
     418             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     419           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     420             :     return 0;
     421           0 :   if ((Subtarget->hasNEON())) {
     422           0 :     return fastEmitInst_r(AArch64::CMLEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     423             :   }
     424             :   return 0;
     425             : }
     426             : 
     427             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     428           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     429             :     return 0;
     430           0 :   if ((Subtarget->hasNEON())) {
     431           0 :     return fastEmitInst_r(AArch64::CMLEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     432             :   }
     433             :   return 0;
     434             : }
     435             : 
     436             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     437           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     438             :     return 0;
     439           0 :   if ((Subtarget->hasNEON())) {
     440           0 :     return fastEmitInst_r(AArch64::CMLEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     441             :   }
     442             :   return 0;
     443             : }
     444             : 
     445             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     446           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     447             :     return 0;
     448           0 :   if ((Subtarget->hasNEON())) {
     449           0 :     return fastEmitInst_r(AArch64::CMLEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     450             :   }
     451             :   return 0;
     452             : }
     453             : 
     454             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     455           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     456             :     return 0;
     457           0 :   if ((Subtarget->hasNEON())) {
     458           0 :     return fastEmitInst_r(AArch64::CMLEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     459             :   }
     460             :   return 0;
     461             : }
     462             : 
     463             : unsigned fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     464           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     465             :     return 0;
     466           0 :   if ((Subtarget->hasNEON())) {
     467           0 :     return fastEmitInst_r(AArch64::CMLEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     468             :   }
     469             :   return 0;
     470             : }
     471             : 
     472           0 : unsigned fastEmit_AArch64ISD_CMLEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     473           0 :   switch (VT.SimpleTy) {
     474           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMLEz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     475           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMLEz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     476           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMLEz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     477           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMLEz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     478           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMLEz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     479           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMLEz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     480           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMLEz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     481           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMLEz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     482             :   default: return 0;
     483             :   }
     484             : }
     485             : 
     486             : // FastEmit functions for AArch64ISD::CMLTz.
     487             : 
     488             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     489           0 :   if (RetVT.SimpleTy != MVT::v8i8)
     490             :     return 0;
     491           0 :   if ((Subtarget->hasNEON())) {
     492           0 :     return fastEmitInst_r(AArch64::CMLTv8i8rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     493             :   }
     494             :   return 0;
     495             : }
     496             : 
     497             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     498           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     499             :     return 0;
     500           0 :   if ((Subtarget->hasNEON())) {
     501           0 :     return fastEmitInst_r(AArch64::CMLTv16i8rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     502             :   }
     503             :   return 0;
     504             : }
     505             : 
     506             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     507           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     508             :     return 0;
     509           0 :   if ((Subtarget->hasNEON())) {
     510           0 :     return fastEmitInst_r(AArch64::CMLTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     511             :   }
     512             :   return 0;
     513             : }
     514             : 
     515             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     516           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     517             :     return 0;
     518           0 :   if ((Subtarget->hasNEON())) {
     519           0 :     return fastEmitInst_r(AArch64::CMLTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     520             :   }
     521             :   return 0;
     522             : }
     523             : 
     524             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     525           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     526             :     return 0;
     527           0 :   if ((Subtarget->hasNEON())) {
     528           0 :     return fastEmitInst_r(AArch64::CMLTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     529             :   }
     530             :   return 0;
     531             : }
     532             : 
     533             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     534           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     535             :     return 0;
     536           0 :   if ((Subtarget->hasNEON())) {
     537           0 :     return fastEmitInst_r(AArch64::CMLTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     538             :   }
     539             :   return 0;
     540             : }
     541             : 
     542             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     543           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     544             :     return 0;
     545           0 :   if ((Subtarget->hasNEON())) {
     546           0 :     return fastEmitInst_r(AArch64::CMLTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     547             :   }
     548             :   return 0;
     549             : }
     550             : 
     551             : unsigned fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     552           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     553             :     return 0;
     554           0 :   if ((Subtarget->hasNEON())) {
     555           0 :     return fastEmitInst_r(AArch64::CMLTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     556             :   }
     557             :   return 0;
     558             : }
     559             : 
     560           0 : unsigned fastEmit_AArch64ISD_CMLTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     561           0 :   switch (VT.SimpleTy) {
     562           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMLTz_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
     563           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMLTz_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     564           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMLTz_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
     565           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMLTz_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     566           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMLTz_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
     567           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMLTz_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     568           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMLTz_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
     569           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMLTz_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     570             :   default: return 0;
     571             :   }
     572             : }
     573             : 
     574             : // FastEmit functions for AArch64ISD::DUP.
     575             : 
     576             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
     577           0 :   if ((Subtarget->hasNEON())) {
     578           0 :     return fastEmitInst_r(AArch64::DUPv8i8gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     579             :   }
     580             :   return 0;
     581             : }
     582             : 
     583             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
     584           0 :   if ((Subtarget->hasNEON())) {
     585           0 :     return fastEmitInst_r(AArch64::DUPv16i8gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     586             :   }
     587             :   return 0;
     588             : }
     589             : 
     590             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
     591           0 :   if ((Subtarget->hasNEON())) {
     592           0 :     return fastEmitInst_r(AArch64::DUPv4i16gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     593             :   }
     594             :   return 0;
     595             : }
     596             : 
     597             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
     598           0 :   if ((Subtarget->hasNEON())) {
     599           0 :     return fastEmitInst_r(AArch64::DUPv8i16gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     600             :   }
     601             :   return 0;
     602             : }
     603             : 
     604             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
     605           0 :   if ((Subtarget->hasNEON())) {
     606           0 :     return fastEmitInst_r(AArch64::DUPv2i32gpr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     607             :   }
     608             :   return 0;
     609             : }
     610             : 
     611             : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     612           0 :   if ((Subtarget->hasNEON())) {
     613           0 :     return fastEmitInst_r(AArch64::DUPv4i32gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     614             :   }
     615             :   return 0;
     616             : }
     617             : 
     618           0 : unsigned fastEmit_AArch64ISD_DUP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     619           0 : switch (RetVT.SimpleTy) {
     620           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i8_r(Op0, Op0IsKill);
     621           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v16i8_r(Op0, Op0IsKill);
     622           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i16_r(Op0, Op0IsKill);
     623           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v8i16_r(Op0, Op0IsKill);
     624           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v2i32_r(Op0, Op0IsKill);
     625           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_DUP_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
     626             :   default: return 0;
     627             : }
     628             : }
     629             : 
     630             : unsigned fastEmit_AArch64ISD_DUP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     631           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     632             :     return 0;
     633           0 :   if ((Subtarget->hasNEON())) {
     634           0 :     return fastEmitInst_r(AArch64::DUPv2i64gpr, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     635             :   }
     636             :   return 0;
     637             : }
     638             : 
     639           0 : unsigned fastEmit_AArch64ISD_DUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     640           0 :   switch (VT.SimpleTy) {
     641           0 :   case MVT::i32: return fastEmit_AArch64ISD_DUP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     642           0 :   case MVT::i64: return fastEmit_AArch64ISD_DUP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     643             :   default: return 0;
     644             :   }
     645             : }
     646             : 
     647             : // FastEmit functions for AArch64ISD::FCMEQz.
     648             : 
     649           0 : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     650           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     651             :     return 0;
     652           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     653           0 :     return fastEmitInst_r(AArch64::FCMEQv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     654             :   }
     655             :   return 0;
     656             : }
     657             : 
     658           0 : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     659           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     660             :     return 0;
     661           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     662           0 :     return fastEmitInst_r(AArch64::FCMEQv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     663             :   }
     664             :   return 0;
     665             : }
     666             : 
     667             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     668           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     669             :     return 0;
     670           0 :   if ((Subtarget->hasNEON())) {
     671           0 :     return fastEmitInst_r(AArch64::FCMEQv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     672             :   }
     673             :   return 0;
     674             : }
     675             : 
     676             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     677           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     678             :     return 0;
     679           0 :   if ((Subtarget->hasNEON())) {
     680           0 :     return fastEmitInst_r(AArch64::FCMEQv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     681             :   }
     682             :   return 0;
     683             : }
     684             : 
     685             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     686           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     687             :     return 0;
     688           0 :   if ((Subtarget->hasNEON())) {
     689           0 :     return fastEmitInst_r(AArch64::FCMEQv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     690             :   }
     691             :   return 0;
     692             : }
     693             : 
     694             : unsigned fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     695           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     696             :     return 0;
     697           0 :   if ((Subtarget->hasNEON())) {
     698           0 :     return fastEmitInst_r(AArch64::FCMEQv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     699             :   }
     700             :   return 0;
     701             : }
     702             : 
     703           0 : unsigned fastEmit_AArch64ISD_FCMEQz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     704           0 :   switch (VT.SimpleTy) {
     705           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     706           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     707           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     708           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     709           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     710           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     711             :   default: return 0;
     712             :   }
     713             : }
     714             : 
     715             : // FastEmit functions for AArch64ISD::FCMGEz.
     716             : 
     717           0 : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     718           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     719             :     return 0;
     720           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     721           0 :     return fastEmitInst_r(AArch64::FCMGEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     722             :   }
     723             :   return 0;
     724             : }
     725             : 
     726           0 : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     727           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     728             :     return 0;
     729           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     730           0 :     return fastEmitInst_r(AArch64::FCMGEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     731             :   }
     732             :   return 0;
     733             : }
     734             : 
     735             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     736           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     737             :     return 0;
     738           0 :   if ((Subtarget->hasNEON())) {
     739           0 :     return fastEmitInst_r(AArch64::FCMGEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     740             :   }
     741             :   return 0;
     742             : }
     743             : 
     744             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     745           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     746             :     return 0;
     747           0 :   if ((Subtarget->hasNEON())) {
     748           0 :     return fastEmitInst_r(AArch64::FCMGEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     749             :   }
     750             :   return 0;
     751             : }
     752             : 
     753             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     754           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     755             :     return 0;
     756           0 :   if ((Subtarget->hasNEON())) {
     757           0 :     return fastEmitInst_r(AArch64::FCMGEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     758             :   }
     759             :   return 0;
     760             : }
     761             : 
     762             : unsigned fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     763           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     764             :     return 0;
     765           0 :   if ((Subtarget->hasNEON())) {
     766           0 :     return fastEmitInst_r(AArch64::FCMGEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     767             :   }
     768             :   return 0;
     769             : }
     770             : 
     771           0 : unsigned fastEmit_AArch64ISD_FCMGEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     772           0 :   switch (VT.SimpleTy) {
     773           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     774           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGEz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     775           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     776           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGEz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     777           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     778           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGEz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     779             :   default: return 0;
     780             :   }
     781             : }
     782             : 
     783             : // FastEmit functions for AArch64ISD::FCMGTz.
     784             : 
     785           0 : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     786           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     787             :     return 0;
     788           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     789           0 :     return fastEmitInst_r(AArch64::FCMGTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     790             :   }
     791             :   return 0;
     792             : }
     793             : 
     794           0 : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     795           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     796             :     return 0;
     797           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     798           0 :     return fastEmitInst_r(AArch64::FCMGTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     799             :   }
     800             :   return 0;
     801             : }
     802             : 
     803             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     804           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     805             :     return 0;
     806           0 :   if ((Subtarget->hasNEON())) {
     807           0 :     return fastEmitInst_r(AArch64::FCMGTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     808             :   }
     809             :   return 0;
     810             : }
     811             : 
     812             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     813           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     814             :     return 0;
     815           0 :   if ((Subtarget->hasNEON())) {
     816           0 :     return fastEmitInst_r(AArch64::FCMGTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     817             :   }
     818             :   return 0;
     819             : }
     820             : 
     821             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     822           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     823             :     return 0;
     824           0 :   if ((Subtarget->hasNEON())) {
     825           0 :     return fastEmitInst_r(AArch64::FCMGTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     826             :   }
     827             :   return 0;
     828             : }
     829             : 
     830             : unsigned fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     831           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     832             :     return 0;
     833           0 :   if ((Subtarget->hasNEON())) {
     834           0 :     return fastEmitInst_r(AArch64::FCMGTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     835             :   }
     836             :   return 0;
     837             : }
     838             : 
     839           0 : unsigned fastEmit_AArch64ISD_FCMGTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     840           0 :   switch (VT.SimpleTy) {
     841           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     842           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGTz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     843           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     844           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGTz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     845           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     846           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGTz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     847             :   default: return 0;
     848             :   }
     849             : }
     850             : 
     851             : // FastEmit functions for AArch64ISD::FCMLEz.
     852             : 
     853           0 : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     854           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     855             :     return 0;
     856           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     857           0 :     return fastEmitInst_r(AArch64::FCMLEv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     858             :   }
     859             :   return 0;
     860             : }
     861             : 
     862           0 : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     863           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     864             :     return 0;
     865           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     866           0 :     return fastEmitInst_r(AArch64::FCMLEv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     867             :   }
     868             :   return 0;
     869             : }
     870             : 
     871             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     872           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     873             :     return 0;
     874           0 :   if ((Subtarget->hasNEON())) {
     875           0 :     return fastEmitInst_r(AArch64::FCMLEv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     876             :   }
     877             :   return 0;
     878             : }
     879             : 
     880             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     881           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     882             :     return 0;
     883           0 :   if ((Subtarget->hasNEON())) {
     884           0 :     return fastEmitInst_r(AArch64::FCMLEv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     885             :   }
     886             :   return 0;
     887             : }
     888             : 
     889             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     890           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     891             :     return 0;
     892           0 :   if ((Subtarget->hasNEON())) {
     893           0 :     return fastEmitInst_r(AArch64::FCMLEv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     894             :   }
     895             :   return 0;
     896             : }
     897             : 
     898             : unsigned fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     899           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     900             :     return 0;
     901           0 :   if ((Subtarget->hasNEON())) {
     902           0 :     return fastEmitInst_r(AArch64::FCMLEv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     903             :   }
     904             :   return 0;
     905             : }
     906             : 
     907           0 : unsigned fastEmit_AArch64ISD_FCMLEz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     908           0 :   switch (VT.SimpleTy) {
     909           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     910           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMLEz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     911           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     912           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMLEz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     913           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     914           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMLEz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     915             :   default: return 0;
     916             :   }
     917             : }
     918             : 
     919             : // FastEmit functions for AArch64ISD::FCMLTz.
     920             : 
     921           0 : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     922           0 :   if (RetVT.SimpleTy != MVT::v4i16)
     923             :     return 0;
     924           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     925           0 :     return fastEmitInst_r(AArch64::FCMLTv4i16rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     926             :   }
     927             :   return 0;
     928             : }
     929             : 
     930           0 : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     931           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     932             :     return 0;
     933           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
     934           0 :     return fastEmitInst_r(AArch64::FCMLTv8i16rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     935             :   }
     936             :   return 0;
     937             : }
     938             : 
     939             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     940           0 :   if (RetVT.SimpleTy != MVT::v2i32)
     941             :     return 0;
     942           0 :   if ((Subtarget->hasNEON())) {
     943           0 :     return fastEmitInst_r(AArch64::FCMLTv2i32rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     944             :   }
     945             :   return 0;
     946             : }
     947             : 
     948             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     949           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     950             :     return 0;
     951           0 :   if ((Subtarget->hasNEON())) {
     952           0 :     return fastEmitInst_r(AArch64::FCMLTv4i32rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     953             :   }
     954             :   return 0;
     955             : }
     956             : 
     957             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     958           0 :   if (RetVT.SimpleTy != MVT::v1i64)
     959             :     return 0;
     960           0 :   if ((Subtarget->hasNEON())) {
     961           0 :     return fastEmitInst_r(AArch64::FCMLTv1i64rz, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     962             :   }
     963             :   return 0;
     964             : }
     965             : 
     966             : unsigned fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     967           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     968             :     return 0;
     969           0 :   if ((Subtarget->hasNEON())) {
     970           0 :     return fastEmitInst_r(AArch64::FCMLTv2i64rz, &AArch64::FPR128RegClass, Op0, Op0IsKill);
     971             :   }
     972             :   return 0;
     973             : }
     974             : 
     975           0 : unsigned fastEmit_AArch64ISD_FCMLTz_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     976           0 :   switch (VT.SimpleTy) {
     977           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
     978           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMLTz_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
     979           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
     980           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMLTz_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     981           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
     982           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMLTz_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
     983             :   default: return 0;
     984             :   }
     985             : }
     986             : 
     987             : // FastEmit functions for AArch64ISD::FRECPE.
     988             : 
     989             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     990           0 :   if (RetVT.SimpleTy != MVT::f32)
     991             :     return 0;
     992           0 :   return fastEmitInst_r(AArch64::FRECPEv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
     993             : }
     994             : 
     995             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     996           0 :   if (RetVT.SimpleTy != MVT::f64)
     997             :     return 0;
     998           0 :   return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
     999             : }
    1000             : 
    1001             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1002           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    1003             :     return 0;
    1004           0 :   return fastEmitInst_r(AArch64::FRECPEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1005             : }
    1006             : 
    1007             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1008           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1009             :     return 0;
    1010           0 :   return fastEmitInst_r(AArch64::FRECPEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1011             : }
    1012             : 
    1013             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1014           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    1015             :     return 0;
    1016           0 :   return fastEmitInst_r(AArch64::FRECPEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1017             : }
    1018             : 
    1019             : unsigned fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1020           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1021             :     return 0;
    1022           0 :   return fastEmitInst_r(AArch64::FRECPEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1023             : }
    1024             : 
    1025           0 : unsigned fastEmit_AArch64ISD_FRECPE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1026           0 :   switch (VT.SimpleTy) {
    1027           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRECPE_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1028           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRECPE_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1029           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRECPE_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    1030           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRECPE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1031           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FRECPE_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
    1032           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRECPE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1033             :   default: return 0;
    1034             :   }
    1035             : }
    1036             : 
    1037             : // FastEmit functions for AArch64ISD::FRSQRTE.
    1038             : 
    1039             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1040           0 :   if (RetVT.SimpleTy != MVT::f32)
    1041             :     return 0;
    1042           0 :   return fastEmitInst_r(AArch64::FRSQRTEv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    1043             : }
    1044             : 
    1045             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1046           0 :   if (RetVT.SimpleTy != MVT::f64)
    1047             :     return 0;
    1048           0 :   return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1049             : }
    1050             : 
    1051             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1052           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    1053             :     return 0;
    1054           0 :   return fastEmitInst_r(AArch64::FRSQRTEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1055             : }
    1056             : 
    1057             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1058           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1059             :     return 0;
    1060           0 :   return fastEmitInst_r(AArch64::FRSQRTEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1061             : }
    1062             : 
    1063             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1064           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    1065             :     return 0;
    1066           0 :   return fastEmitInst_r(AArch64::FRSQRTEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1067             : }
    1068             : 
    1069             : unsigned fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1070           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1071             :     return 0;
    1072           0 :   return fastEmitInst_r(AArch64::FRSQRTEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1073             : }
    1074             : 
    1075           0 : unsigned fastEmit_AArch64ISD_FRSQRTE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1076           0 :   switch (VT.SimpleTy) {
    1077           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1078           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1079           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    1080           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTE_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1081           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
    1082           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTE_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1083             :   default: return 0;
    1084             :   }
    1085             : }
    1086             : 
    1087             : // FastEmit functions for AArch64ISD::NEG.
    1088             : 
    1089             : unsigned fastEmit_AArch64ISD_NEG_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1090           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1091             :     return 0;
    1092           0 :   return fastEmitInst_r(AArch64::NEGv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1093             : }
    1094             : 
    1095             : unsigned fastEmit_AArch64ISD_NEG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1096           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1097             :     return 0;
    1098           0 :   return fastEmitInst_r(AArch64::NEGv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1099             : }
    1100             : 
    1101             : unsigned fastEmit_AArch64ISD_NEG_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1102           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1103             :     return 0;
    1104           0 :   return fastEmitInst_r(AArch64::NEGv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1105             : }
    1106             : 
    1107             : unsigned fastEmit_AArch64ISD_NEG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1108           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1109             :     return 0;
    1110           0 :   return fastEmitInst_r(AArch64::NEGv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1111             : }
    1112             : 
    1113             : unsigned fastEmit_AArch64ISD_NEG_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1114           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1115             :     return 0;
    1116           0 :   return fastEmitInst_r(AArch64::NEGv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1117             : }
    1118             : 
    1119             : unsigned fastEmit_AArch64ISD_NEG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1120           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1121             :     return 0;
    1122           0 :   return fastEmitInst_r(AArch64::NEGv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1123             : }
    1124             : 
    1125             : unsigned fastEmit_AArch64ISD_NEG_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1126           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    1127             :     return 0;
    1128           0 :   return fastEmitInst_r(AArch64::NEGv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1129             : }
    1130             : 
    1131             : unsigned fastEmit_AArch64ISD_NEG_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1132           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1133             :     return 0;
    1134           0 :   return fastEmitInst_r(AArch64::NEGv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1135             : }
    1136             : 
    1137           0 : unsigned fastEmit_AArch64ISD_NEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1138           0 :   switch (VT.SimpleTy) {
    1139           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_NEG_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1140           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_NEG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1141           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_NEG_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1142           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_NEG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1143           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_NEG_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1144           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_NEG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1145           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_NEG_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    1146           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_NEG_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1147             :   default: return 0;
    1148             :   }
    1149             : }
    1150             : 
    1151             : // FastEmit functions for AArch64ISD::NOT.
    1152             : 
    1153             : unsigned fastEmit_AArch64ISD_NOT_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1154           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1155             :     return 0;
    1156           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1157             : }
    1158             : 
    1159             : unsigned fastEmit_AArch64ISD_NOT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1160           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1161             :     return 0;
    1162           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1163             : }
    1164             : 
    1165             : unsigned fastEmit_AArch64ISD_NOT_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1166           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1167             :     return 0;
    1168           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1169             : }
    1170             : 
    1171             : unsigned fastEmit_AArch64ISD_NOT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1172           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1173             :     return 0;
    1174           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1175             : }
    1176             : 
    1177             : unsigned fastEmit_AArch64ISD_NOT_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1178           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1179             :     return 0;
    1180           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1181             : }
    1182             : 
    1183             : unsigned fastEmit_AArch64ISD_NOT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1184           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1185             :     return 0;
    1186           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1187             : }
    1188             : 
    1189             : unsigned fastEmit_AArch64ISD_NOT_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1190           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    1191             :     return 0;
    1192           0 :   return fastEmitInst_r(AArch64::NOTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1193             : }
    1194             : 
    1195             : unsigned fastEmit_AArch64ISD_NOT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1196           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1197             :     return 0;
    1198           0 :   return fastEmitInst_r(AArch64::NOTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1199             : }
    1200             : 
    1201           0 : unsigned fastEmit_AArch64ISD_NOT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1202           0 :   switch (VT.SimpleTy) {
    1203           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_NOT_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1204           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_NOT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1205           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_NOT_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1206           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_NOT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1207           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_NOT_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1208           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_NOT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1209           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_NOT_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    1210           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_NOT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1211             :   default: return 0;
    1212             :   }
    1213             : }
    1214             : 
    1215             : // FastEmit functions for AArch64ISD::REV16.
    1216             : 
    1217             : unsigned fastEmit_AArch64ISD_REV16_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1218           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1219             :     return 0;
    1220           0 :   if ((Subtarget->hasNEON())) {
    1221           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1222             :   }
    1223             :   return 0;
    1224             : }
    1225             : 
    1226             : unsigned fastEmit_AArch64ISD_REV16_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1227           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1228             :     return 0;
    1229           0 :   if ((Subtarget->hasNEON())) {
    1230           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1231             :   }
    1232             :   return 0;
    1233             : }
    1234             : 
    1235           0 : unsigned fastEmit_AArch64ISD_REV16_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1236           0 :   switch (VT.SimpleTy) {
    1237           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_REV16_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1238           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_REV16_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1239             :   default: return 0;
    1240             :   }
    1241             : }
    1242             : 
    1243             : // FastEmit functions for AArch64ISD::REV32.
    1244             : 
    1245             : unsigned fastEmit_AArch64ISD_REV32_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1246           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1247             :     return 0;
    1248           0 :   if ((Subtarget->hasNEON())) {
    1249           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1250             :   }
    1251             :   return 0;
    1252             : }
    1253             : 
    1254             : unsigned fastEmit_AArch64ISD_REV32_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1255           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1256             :     return 0;
    1257           0 :   if ((Subtarget->hasNEON())) {
    1258           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1259             :   }
    1260             :   return 0;
    1261             : }
    1262             : 
    1263             : unsigned fastEmit_AArch64ISD_REV32_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1264           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1265             :     return 0;
    1266           0 :   if ((Subtarget->hasNEON())) {
    1267           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1268             :   }
    1269             :   return 0;
    1270             : }
    1271             : 
    1272             : unsigned fastEmit_AArch64ISD_REV32_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1273           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1274             :     return 0;
    1275           0 :   if ((Subtarget->hasNEON())) {
    1276           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1277             :   }
    1278             :   return 0;
    1279             : }
    1280             : 
    1281             : unsigned fastEmit_AArch64ISD_REV32_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1282           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    1283             :     return 0;
    1284           0 :   return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1285             : }
    1286             : 
    1287             : unsigned fastEmit_AArch64ISD_REV32_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1288           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    1289             :     return 0;
    1290           0 :   return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1291             : }
    1292             : 
    1293           0 : unsigned fastEmit_AArch64ISD_REV32_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1294           0 :   switch (VT.SimpleTy) {
    1295           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_REV32_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1296           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_REV32_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1297           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_REV32_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1298           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_REV32_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1299           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_REV32_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    1300           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_REV32_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    1301             :   default: return 0;
    1302             :   }
    1303             : }
    1304             : 
    1305             : // FastEmit functions for AArch64ISD::REV64.
    1306             : 
    1307             : unsigned fastEmit_AArch64ISD_REV64_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1308           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1309             :     return 0;
    1310           0 :   if ((Subtarget->hasNEON())) {
    1311           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1312             :   }
    1313             :   return 0;
    1314             : }
    1315             : 
    1316             : unsigned fastEmit_AArch64ISD_REV64_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1317           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1318             :     return 0;
    1319           0 :   if ((Subtarget->hasNEON())) {
    1320           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1321             :   }
    1322             :   return 0;
    1323             : }
    1324             : 
    1325             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1326           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1327             :     return 0;
    1328           0 :   if ((Subtarget->hasNEON())) {
    1329           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1330             :   }
    1331             :   return 0;
    1332             : }
    1333             : 
    1334             : unsigned fastEmit_AArch64ISD_REV64_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1335           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1336             :     return 0;
    1337           0 :   if ((Subtarget->hasNEON())) {
    1338           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1339             :   }
    1340             :   return 0;
    1341             : }
    1342             : 
    1343             : unsigned fastEmit_AArch64ISD_REV64_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1344           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1345             :     return 0;
    1346           0 :   if ((Subtarget->hasNEON())) {
    1347           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1348             :   }
    1349             :   return 0;
    1350             : }
    1351             : 
    1352             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1353           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1354             :     return 0;
    1355           0 :   if ((Subtarget->hasNEON())) {
    1356           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1357             :   }
    1358             :   return 0;
    1359             : }
    1360             : 
    1361             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1362           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    1363             :     return 0;
    1364           0 :   return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1365             : }
    1366             : 
    1367             : unsigned fastEmit_AArch64ISD_REV64_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1368           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    1369             :     return 0;
    1370           0 :   return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1371             : }
    1372             : 
    1373             : unsigned fastEmit_AArch64ISD_REV64_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1374           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    1375             :     return 0;
    1376           0 :   return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1377             : }
    1378             : 
    1379             : unsigned fastEmit_AArch64ISD_REV64_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1380           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1381             :     return 0;
    1382           0 :   return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1383             : }
    1384             : 
    1385           0 : unsigned fastEmit_AArch64ISD_REV64_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1386           0 :   switch (VT.SimpleTy) {
    1387           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_REV64_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1388           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_REV64_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1389           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_REV64_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1390           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_REV64_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1391           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_REV64_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1392           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_REV64_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1393           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_REV64_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    1394           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_REV64_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    1395           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_REV64_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    1396           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_REV64_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1397             :   default: return 0;
    1398             :   }
    1399             : }
    1400             : 
    1401             : // FastEmit functions for AArch64ISD::SITOF.
    1402             : 
    1403           0 : unsigned fastEmit_AArch64ISD_SITOF_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1404           0 :   if (RetVT.SimpleTy != MVT::f16)
    1405             :     return 0;
    1406           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    1407           0 :     return fastEmitInst_r(AArch64::SCVTFv1i16, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    1408             :   }
    1409             :   return 0;
    1410             : }
    1411             : 
    1412             : unsigned fastEmit_AArch64ISD_SITOF_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1413           0 :   if (RetVT.SimpleTy != MVT::f32)
    1414             :     return 0;
    1415           0 :   if ((Subtarget->hasNEON())) {
    1416           0 :     return fastEmitInst_r(AArch64::SCVTFv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    1417             :   }
    1418             :   return 0;
    1419             : }
    1420             : 
    1421             : unsigned fastEmit_AArch64ISD_SITOF_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1422           0 :   if (RetVT.SimpleTy != MVT::f64)
    1423             :     return 0;
    1424           0 :   if ((Subtarget->hasNEON())) {
    1425           0 :     return fastEmitInst_r(AArch64::SCVTFv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1426             :   }
    1427             :   return 0;
    1428             : }
    1429             : 
    1430           0 : unsigned fastEmit_AArch64ISD_SITOF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1431           0 :   switch (VT.SimpleTy) {
    1432           0 :   case MVT::f16: return fastEmit_AArch64ISD_SITOF_MVT_f16_r(RetVT, Op0, Op0IsKill);
    1433           0 :   case MVT::f32: return fastEmit_AArch64ISD_SITOF_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1434           0 :   case MVT::f64: return fastEmit_AArch64ISD_SITOF_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1435             :   default: return 0;
    1436             :   }
    1437             : }
    1438             : 
    1439             : // FastEmit functions for AArch64ISD::UITOF.
    1440             : 
    1441           0 : unsigned fastEmit_AArch64ISD_UITOF_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1442           0 :   if (RetVT.SimpleTy != MVT::f16)
    1443             :     return 0;
    1444           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    1445           0 :     return fastEmitInst_r(AArch64::UCVTFv1i16, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    1446             :   }
    1447             :   return 0;
    1448             : }
    1449             : 
    1450             : unsigned fastEmit_AArch64ISD_UITOF_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1451           0 :   if (RetVT.SimpleTy != MVT::f32)
    1452             :     return 0;
    1453           0 :   if ((Subtarget->hasNEON())) {
    1454           0 :     return fastEmitInst_r(AArch64::UCVTFv1i32, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    1455             :   }
    1456             :   return 0;
    1457             : }
    1458             : 
    1459             : unsigned fastEmit_AArch64ISD_UITOF_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1460           0 :   if (RetVT.SimpleTy != MVT::f64)
    1461             :     return 0;
    1462           0 :   if ((Subtarget->hasNEON())) {
    1463           0 :     return fastEmitInst_r(AArch64::UCVTFv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1464             :   }
    1465             :   return 0;
    1466             : }
    1467             : 
    1468           0 : unsigned fastEmit_AArch64ISD_UITOF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1469           0 :   switch (VT.SimpleTy) {
    1470           0 :   case MVT::f16: return fastEmit_AArch64ISD_UITOF_MVT_f16_r(RetVT, Op0, Op0IsKill);
    1471           0 :   case MVT::f32: return fastEmit_AArch64ISD_UITOF_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1472           0 :   case MVT::f64: return fastEmit_AArch64ISD_UITOF_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1473             :   default: return 0;
    1474             :   }
    1475             : }
    1476             : 
    1477             : // FastEmit functions for ISD::ABS.
    1478             : 
    1479             : unsigned fastEmit_ISD_ABS_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1480           0 :   if (RetVT.SimpleTy != MVT::i64)
    1481             :     return 0;
    1482           0 :   if ((Subtarget->hasNEON())) {
    1483           0 :     return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1484             :   }
    1485             :   return 0;
    1486             : }
    1487             : 
    1488             : unsigned fastEmit_ISD_ABS_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1489           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    1490             :     return 0;
    1491           0 :   if ((Subtarget->hasNEON())) {
    1492           0 :     return fastEmitInst_r(AArch64::ABSv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1493             :   }
    1494             :   return 0;
    1495             : }
    1496             : 
    1497             : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1498           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    1499             :     return 0;
    1500           0 :   if ((Subtarget->hasNEON())) {
    1501           0 :     return fastEmitInst_r(AArch64::ABSv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1502             :   }
    1503             :   return 0;
    1504             : }
    1505             : 
    1506             : unsigned fastEmit_ISD_ABS_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1507           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    1508             :     return 0;
    1509           0 :   if ((Subtarget->hasNEON())) {
    1510           0 :     return fastEmitInst_r(AArch64::ABSv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1511             :   }
    1512             :   return 0;
    1513             : }
    1514             : 
    1515             : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1516           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    1517             :     return 0;
    1518           0 :   if ((Subtarget->hasNEON())) {
    1519           0 :     return fastEmitInst_r(AArch64::ABSv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1520             :   }
    1521             :   return 0;
    1522             : }
    1523             : 
    1524             : unsigned fastEmit_ISD_ABS_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1525           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    1526             :     return 0;
    1527           0 :   if ((Subtarget->hasNEON())) {
    1528           0 :     return fastEmitInst_r(AArch64::ABSv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1529             :   }
    1530             :   return 0;
    1531             : }
    1532             : 
    1533             : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1534           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    1535             :     return 0;
    1536           0 :   if ((Subtarget->hasNEON())) {
    1537           0 :     return fastEmitInst_r(AArch64::ABSv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1538             :   }
    1539             :   return 0;
    1540             : }
    1541             : 
    1542             : unsigned fastEmit_ISD_ABS_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1543           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    1544             :     return 0;
    1545           0 :   if ((Subtarget->hasNEON())) {
    1546           0 :     return fastEmitInst_r(AArch64::ABSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1547             :   }
    1548             :   return 0;
    1549             : }
    1550             : 
    1551             : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1552           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1553             :     return 0;
    1554           0 :   if ((Subtarget->hasNEON())) {
    1555           0 :     return fastEmitInst_r(AArch64::ABSv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1556             :   }
    1557             :   return 0;
    1558             : }
    1559             : 
    1560           0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1561           0 :   switch (VT.SimpleTy) {
    1562           0 :   case MVT::i64: return fastEmit_ISD_ABS_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1563           0 :   case MVT::v8i8: return fastEmit_ISD_ABS_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    1564           0 :   case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1565           0 :   case MVT::v4i16: return fastEmit_ISD_ABS_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    1566           0 :   case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1567           0 :   case MVT::v2i32: return fastEmit_ISD_ABS_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    1568           0 :   case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1569           0 :   case MVT::v1i64: return fastEmit_ISD_ABS_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    1570           0 :   case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1571             :   default: return 0;
    1572             :   }
    1573             : }
    1574             : 
    1575             : // FastEmit functions for ISD::BITCAST.
    1576             : 
    1577             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1578           0 :   if ((!Subtarget->isLittleEndian())) {
    1579           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1580             :   }
    1581             :   return 0;
    1582             : }
    1583             : 
    1584             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1585           0 :   if ((!Subtarget->isLittleEndian())) {
    1586           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1587             :   }
    1588             :   return 0;
    1589             : }
    1590             : 
    1591             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1592           0 :   if ((!Subtarget->isLittleEndian())) {
    1593           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1594             :   }
    1595             :   return 0;
    1596             : }
    1597             : 
    1598             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1599           0 :   if ((!Subtarget->isLittleEndian())) {
    1600           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1601             :   }
    1602             :   return 0;
    1603             : }
    1604             : 
    1605             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1606           0 :   if ((!Subtarget->isLittleEndian())) {
    1607           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1608             :   }
    1609             :   return 0;
    1610             : }
    1611             : 
    1612           4 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1613           4 : switch (RetVT.SimpleTy) {
    1614           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v8i8_r(Op0, Op0IsKill);
    1615           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4i16_r(Op0, Op0IsKill);
    1616           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2i32_r(Op0, Op0IsKill);
    1617           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v4f16_r(Op0, Op0IsKill);
    1618           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_f64_MVT_v2f32_r(Op0, Op0IsKill);
    1619             :   default: return 0;
    1620             : }
    1621             : }
    1622             : 
    1623             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1624           2 :   if ((!Subtarget->isLittleEndian())) {
    1625           2 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1626             :   }
    1627             :   return 0;
    1628             : }
    1629             : 
    1630             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1631           0 :   if ((!Subtarget->isLittleEndian())) {
    1632           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1633             :   }
    1634             :   return 0;
    1635             : }
    1636             : 
    1637             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1638           0 :   if ((!Subtarget->isLittleEndian())) {
    1639           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1640             :   }
    1641             :   return 0;
    1642             : }
    1643             : 
    1644             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    1645           1 :   if ((!Subtarget->isLittleEndian())) {
    1646           1 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1647             :   }
    1648             :   return 0;
    1649             : }
    1650             : 
    1651             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1652           0 :   if ((!Subtarget->isLittleEndian())) {
    1653           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1654             :   }
    1655             :   return 0;
    1656             : }
    1657             : 
    1658             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1659           0 :   if ((!Subtarget->isLittleEndian())) {
    1660           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1661             :   }
    1662             :   return 0;
    1663             : }
    1664             : 
    1665             : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    1666           0 :   if ((!Subtarget->isLittleEndian())) {
    1667           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1668             :   }
    1669             :   return 0;
    1670             : }
    1671             : 
    1672           5 : unsigned fastEmit_ISD_BITCAST_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1673           5 : switch (RetVT.SimpleTy) {
    1674           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_f64_r(Op0, Op0IsKill);
    1675           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4i16_r(Op0, Op0IsKill);
    1676           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2i32_r(Op0, Op0IsKill);
    1677           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1i64_r(Op0, Op0IsKill);
    1678           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v4f16_r(Op0, Op0IsKill);
    1679           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v2f32_r(Op0, Op0IsKill);
    1680           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v8i8_MVT_v1f64_r(Op0, Op0IsKill);
    1681             :   default: return 0;
    1682             : }
    1683             : }
    1684             : 
    1685             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1686           0 :   if ((!Subtarget->isLittleEndian())) {
    1687           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1688             :   }
    1689             :   return 0;
    1690             : }
    1691             : 
    1692             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1693           0 :   if ((!Subtarget->isLittleEndian())) {
    1694           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1695             :   }
    1696             :   return 0;
    1697             : }
    1698             : 
    1699             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1700           0 :   if ((!Subtarget->isLittleEndian())) {
    1701           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1702             :   }
    1703             :   return 0;
    1704             : }
    1705             : 
    1706             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    1707           0 :   if ((!Subtarget->isLittleEndian())) {
    1708           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1709             :   }
    1710             :   return 0;
    1711             : }
    1712             : 
    1713             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1714           0 :   if ((!Subtarget->isLittleEndian())) {
    1715           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1716             :   }
    1717             :   return 0;
    1718             : }
    1719             : 
    1720             : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    1721           0 :   if ((!Subtarget->isLittleEndian())) {
    1722           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1723             :   }
    1724             :   return 0;
    1725             : }
    1726             : 
    1727           0 : unsigned fastEmit_ISD_BITCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1728           0 : switch (RetVT.SimpleTy) {
    1729           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1730           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1731           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1732           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v8f16_r(Op0, Op0IsKill);
    1733           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v4f32_r(Op0, Op0IsKill);
    1734           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v16i8_MVT_v2f64_r(Op0, Op0IsKill);
    1735             :   default: return 0;
    1736             : }
    1737             : }
    1738             : 
    1739             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1740           2 :   if ((!Subtarget->isLittleEndian())) {
    1741           2 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1742             :   }
    1743             :   return 0;
    1744             : }
    1745             : 
    1746             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1747           0 :   if ((!Subtarget->isLittleEndian())) {
    1748           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1749             :   }
    1750             :   return 0;
    1751             : }
    1752             : 
    1753             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1754           0 :   if ((!Subtarget->isLittleEndian())) {
    1755           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1756             :   }
    1757             :   return 0;
    1758             : }
    1759             : 
    1760             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    1761           1 :   if ((!Subtarget->isLittleEndian())) {
    1762           1 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1763             :   }
    1764             :   return 0;
    1765             : }
    1766             : 
    1767             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1768           0 :   if ((!Subtarget->isLittleEndian())) {
    1769           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1770             :   }
    1771             :   return 0;
    1772             : }
    1773             : 
    1774             : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    1775           0 :   if ((!Subtarget->isLittleEndian())) {
    1776           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1777             :   }
    1778             :   return 0;
    1779             : }
    1780             : 
    1781           5 : unsigned fastEmit_ISD_BITCAST_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1782           5 : switch (RetVT.SimpleTy) {
    1783           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_f64_r(Op0, Op0IsKill);
    1784           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v8i8_r(Op0, Op0IsKill);
    1785           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2i32_r(Op0, Op0IsKill);
    1786           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1i64_r(Op0, Op0IsKill);
    1787           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v2f32_r(Op0, Op0IsKill);
    1788           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4i16_MVT_v1f64_r(Op0, Op0IsKill);
    1789             :   default: return 0;
    1790             : }
    1791             : }
    1792             : 
    1793             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1794           0 :   if ((!Subtarget->isLittleEndian())) {
    1795           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1796             :   }
    1797             :   return 0;
    1798             : }
    1799             : 
    1800             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1801           0 :   if ((!Subtarget->isLittleEndian())) {
    1802           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1803             :   }
    1804             :   return 0;
    1805             : }
    1806             : 
    1807             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1808           0 :   if ((!Subtarget->isLittleEndian())) {
    1809           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1810             :   }
    1811             :   return 0;
    1812             : }
    1813             : 
    1814             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1815           0 :   if ((!Subtarget->isLittleEndian())) {
    1816           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1817             :   }
    1818             :   return 0;
    1819             : }
    1820             : 
    1821             : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    1822           0 :   if ((!Subtarget->isLittleEndian())) {
    1823           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1824             :   }
    1825             :   return 0;
    1826             : }
    1827             : 
    1828           0 : unsigned fastEmit_ISD_BITCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1829           0 : switch (RetVT.SimpleTy) {
    1830           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v16i8_r(Op0, Op0IsKill);
    1831           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    1832           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    1833           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
    1834           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8i16_MVT_v2f64_r(Op0, Op0IsKill);
    1835             :   default: return 0;
    1836             : }
    1837             : }
    1838             : 
    1839             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1840           2 :   if ((!Subtarget->isLittleEndian())) {
    1841           2 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1842             :   }
    1843             :   return 0;
    1844             : }
    1845             : 
    1846             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1847           0 :   if ((!Subtarget->isLittleEndian())) {
    1848           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1849             :   }
    1850             :   return 0;
    1851             : }
    1852             : 
    1853             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1854           0 :   if ((!Subtarget->isLittleEndian())) {
    1855           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1856             :   }
    1857             :   return 0;
    1858             : }
    1859             : 
    1860             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    1861           1 :   if ((!Subtarget->isLittleEndian())) {
    1862           1 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1863             :   }
    1864             :   return 0;
    1865             : }
    1866             : 
    1867             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1868           0 :   if ((!Subtarget->isLittleEndian())) {
    1869           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1870             :   }
    1871             :   return 0;
    1872             : }
    1873             : 
    1874             : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    1875           0 :   if ((!Subtarget->isLittleEndian())) {
    1876           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1877             :   }
    1878             :   return 0;
    1879             : }
    1880             : 
    1881           5 : unsigned fastEmit_ISD_BITCAST_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1882           5 : switch (RetVT.SimpleTy) {
    1883           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_f64_r(Op0, Op0IsKill);
    1884           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v8i8_r(Op0, Op0IsKill);
    1885           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4i16_r(Op0, Op0IsKill);
    1886           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1i64_r(Op0, Op0IsKill);
    1887           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v4f16_r(Op0, Op0IsKill);
    1888           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2i32_MVT_v1f64_r(Op0, Op0IsKill);
    1889             :   default: return 0;
    1890             : }
    1891             : }
    1892             : 
    1893             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1894           0 :   if ((!Subtarget->isLittleEndian())) {
    1895           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1896             :   }
    1897             :   return 0;
    1898             : }
    1899             : 
    1900             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1901           0 :   if ((!Subtarget->isLittleEndian())) {
    1902           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1903             :   }
    1904             :   return 0;
    1905             : }
    1906             : 
    1907             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1908           0 :   if ((!Subtarget->isLittleEndian())) {
    1909           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1910             :   }
    1911             :   return 0;
    1912             : }
    1913             : 
    1914             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    1915           0 :   if ((!Subtarget->isLittleEndian())) {
    1916           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1917             :   }
    1918             :   return 0;
    1919             : }
    1920             : 
    1921             : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    1922           0 :   if ((!Subtarget->isLittleEndian())) {
    1923           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1924             :   }
    1925             :   return 0;
    1926             : }
    1927             : 
    1928           0 : unsigned fastEmit_ISD_BITCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1929           0 : switch (RetVT.SimpleTy) {
    1930           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    1931           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    1932           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2i64_r(Op0, Op0IsKill);
    1933           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v8f16_r(Op0, Op0IsKill);
    1934           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4i32_MVT_v2f64_r(Op0, Op0IsKill);
    1935             :   default: return 0;
    1936             : }
    1937             : }
    1938             : 
    1939             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    1940           0 :   if ((!Subtarget->isLittleEndian())) {
    1941           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1942             :   }
    1943             :   return 0;
    1944             : }
    1945             : 
    1946             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    1947           0 :   if ((!Subtarget->isLittleEndian())) {
    1948           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1949             :   }
    1950             :   return 0;
    1951             : }
    1952             : 
    1953             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    1954           0 :   if ((!Subtarget->isLittleEndian())) {
    1955           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1956             :   }
    1957             :   return 0;
    1958             : }
    1959             : 
    1960             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    1961           0 :   if ((!Subtarget->isLittleEndian())) {
    1962           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1963             :   }
    1964             :   return 0;
    1965             : }
    1966             : 
    1967             : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    1968           0 :   if ((!Subtarget->isLittleEndian())) {
    1969           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    1970             :   }
    1971             :   return 0;
    1972             : }
    1973             : 
    1974           4 : unsigned fastEmit_ISD_BITCAST_MVT_v1i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1975           4 : switch (RetVT.SimpleTy) {
    1976           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v8i8_r(Op0, Op0IsKill);
    1977           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4i16_r(Op0, Op0IsKill);
    1978           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2i32_r(Op0, Op0IsKill);
    1979           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v4f16_r(Op0, Op0IsKill);
    1980           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1i64_MVT_v2f32_r(Op0, Op0IsKill);
    1981             :   default: return 0;
    1982             : }
    1983             : }
    1984             : 
    1985             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1986           0 :   if ((!Subtarget->isLittleEndian())) {
    1987           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1988             :   }
    1989             :   return 0;
    1990             : }
    1991             : 
    1992             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1993           0 :   if ((!Subtarget->isLittleEndian())) {
    1994           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    1995             :   }
    1996             :   return 0;
    1997             : }
    1998             : 
    1999             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2000           0 :   if ((!Subtarget->isLittleEndian())) {
    2001           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2002             :   }
    2003             :   return 0;
    2004             : }
    2005             : 
    2006             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    2007           0 :   if ((!Subtarget->isLittleEndian())) {
    2008           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2009             :   }
    2010             :   return 0;
    2011             : }
    2012             : 
    2013             : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2014           0 :   if ((!Subtarget->isLittleEndian())) {
    2015           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2016             :   }
    2017             :   return 0;
    2018             : }
    2019             : 
    2020           0 : unsigned fastEmit_ISD_BITCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2021           0 : switch (RetVT.SimpleTy) {
    2022           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    2023           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    2024           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    2025           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v8f16_r(Op0, Op0IsKill);
    2026           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2i64_MVT_v4f32_r(Op0, Op0IsKill);
    2027             :   default: return 0;
    2028             : }
    2029             : }
    2030             : 
    2031             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    2032           1 :   if ((!Subtarget->isLittleEndian())) {
    2033           1 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2034             :   }
    2035             :   return 0;
    2036             : }
    2037             : 
    2038             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    2039           0 :   if ((!Subtarget->isLittleEndian())) {
    2040           0 :     return fastEmitInst_r(AArch64::REV16v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2041             :   }
    2042             :   return 0;
    2043             : }
    2044             : 
    2045             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    2046           0 :   if ((!Subtarget->isLittleEndian())) {
    2047           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2048             :   }
    2049             :   return 0;
    2050             : }
    2051             : 
    2052             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    2053           0 :   if ((!Subtarget->isLittleEndian())) {
    2054           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2055             :   }
    2056             :   return 0;
    2057             : }
    2058             : 
    2059             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    2060           0 :   if ((!Subtarget->isLittleEndian())) {
    2061           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2062             :   }
    2063             :   return 0;
    2064             : }
    2065             : 
    2066             : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    2067           0 :   if ((!Subtarget->isLittleEndian())) {
    2068           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2069             :   }
    2070             :   return 0;
    2071             : }
    2072             : 
    2073           2 : unsigned fastEmit_ISD_BITCAST_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2074           2 : switch (RetVT.SimpleTy) {
    2075           1 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_f64_r(Op0, Op0IsKill);
    2076           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v8i8_r(Op0, Op0IsKill);
    2077           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2i32_r(Op0, Op0IsKill);
    2078           0 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1i64_r(Op0, Op0IsKill);
    2079           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v2f32_r(Op0, Op0IsKill);
    2080           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v4f16_MVT_v1f64_r(Op0, Op0IsKill);
    2081             :   default: return 0;
    2082             : }
    2083             : }
    2084             : 
    2085             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2086           0 :   if ((!Subtarget->isLittleEndian())) {
    2087           0 :     return fastEmitInst_r(AArch64::REV16v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2088             :   }
    2089             :   return 0;
    2090             : }
    2091             : 
    2092             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2093           0 :   if ((!Subtarget->isLittleEndian())) {
    2094           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2095             :   }
    2096             :   return 0;
    2097             : }
    2098             : 
    2099             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2100           0 :   if ((!Subtarget->isLittleEndian())) {
    2101           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2102             :   }
    2103             :   return 0;
    2104             : }
    2105             : 
    2106             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2107           0 :   if ((!Subtarget->isLittleEndian())) {
    2108           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2109             :   }
    2110             :   return 0;
    2111             : }
    2112             : 
    2113             : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    2114           0 :   if ((!Subtarget->isLittleEndian())) {
    2115           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2116             :   }
    2117             :   return 0;
    2118             : }
    2119             : 
    2120           0 : unsigned fastEmit_ISD_BITCAST_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2121           0 : switch (RetVT.SimpleTy) {
    2122           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v16i8_r(Op0, Op0IsKill);
    2123           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4i32_r(Op0, Op0IsKill);
    2124           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2i64_r(Op0, Op0IsKill);
    2125           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v4f32_r(Op0, Op0IsKill);
    2126           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v8f16_MVT_v2f64_r(Op0, Op0IsKill);
    2127             :   default: return 0;
    2128             : }
    2129             : }
    2130             : 
    2131             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    2132           2 :   if ((!Subtarget->isLittleEndian())) {
    2133           2 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2134             :   }
    2135             :   return 0;
    2136             : }
    2137             : 
    2138             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    2139           0 :   if ((!Subtarget->isLittleEndian())) {
    2140           0 :     return fastEmitInst_r(AArch64::REV32v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2141             :   }
    2142             :   return 0;
    2143             : }
    2144             : 
    2145             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    2146           0 :   if ((!Subtarget->isLittleEndian())) {
    2147           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2148             :   }
    2149             :   return 0;
    2150             : }
    2151             : 
    2152             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(unsigned Op0, bool Op0IsKill) {
    2153           1 :   if ((!Subtarget->isLittleEndian())) {
    2154           1 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2155             :   }
    2156             :   return 0;
    2157             : }
    2158             : 
    2159             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    2160           0 :   if ((!Subtarget->isLittleEndian())) {
    2161           0 :     return fastEmitInst_r(AArch64::REV32v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2162             :   }
    2163             :   return 0;
    2164             : }
    2165             : 
    2166             : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(unsigned Op0, bool Op0IsKill) {
    2167           0 :   if ((!Subtarget->isLittleEndian())) {
    2168           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2169             :   }
    2170             :   return 0;
    2171             : }
    2172             : 
    2173           5 : unsigned fastEmit_ISD_BITCAST_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2174           5 : switch (RetVT.SimpleTy) {
    2175           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_f64_r(Op0, Op0IsKill);
    2176           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v8i8_r(Op0, Op0IsKill);
    2177           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4i16_r(Op0, Op0IsKill);
    2178           1 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1i64_r(Op0, Op0IsKill);
    2179           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v4f16_r(Op0, Op0IsKill);
    2180           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v2f32_MVT_v1f64_r(Op0, Op0IsKill);
    2181             :   default: return 0;
    2182             : }
    2183             : }
    2184             : 
    2185             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2186           0 :   if ((!Subtarget->isLittleEndian())) {
    2187           0 :     return fastEmitInst_r(AArch64::REV32v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2188             :   }
    2189             :   return 0;
    2190             : }
    2191             : 
    2192             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2193           0 :   if ((!Subtarget->isLittleEndian())) {
    2194           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2195             :   }
    2196             :   return 0;
    2197             : }
    2198             : 
    2199             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2200           0 :   if ((!Subtarget->isLittleEndian())) {
    2201           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2202             :   }
    2203             :   return 0;
    2204             : }
    2205             : 
    2206             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    2207           0 :   if ((!Subtarget->isLittleEndian())) {
    2208           0 :     return fastEmitInst_r(AArch64::REV32v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2209             :   }
    2210             :   return 0;
    2211             : }
    2212             : 
    2213             : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    2214           0 :   if ((!Subtarget->isLittleEndian())) {
    2215           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2216             :   }
    2217             :   return 0;
    2218             : }
    2219             : 
    2220           0 : unsigned fastEmit_ISD_BITCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2221           0 : switch (RetVT.SimpleTy) {
    2222           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v16i8_r(Op0, Op0IsKill);
    2223           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8i16_r(Op0, Op0IsKill);
    2224           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2225           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v8f16_r(Op0, Op0IsKill);
    2226           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v4f32_MVT_v2f64_r(Op0, Op0IsKill);
    2227             :   default: return 0;
    2228             : }
    2229             : }
    2230             : 
    2231             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(unsigned Op0, bool Op0IsKill) {
    2232           0 :   if ((!Subtarget->isLittleEndian())) {
    2233           0 :     return fastEmitInst_r(AArch64::REV64v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2234             :   }
    2235             :   return 0;
    2236             : }
    2237             : 
    2238             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(unsigned Op0, bool Op0IsKill) {
    2239           0 :   if ((!Subtarget->isLittleEndian())) {
    2240           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2241             :   }
    2242             :   return 0;
    2243             : }
    2244             : 
    2245             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(unsigned Op0, bool Op0IsKill) {
    2246           0 :   if ((!Subtarget->isLittleEndian())) {
    2247           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2248             :   }
    2249             :   return 0;
    2250             : }
    2251             : 
    2252             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(unsigned Op0, bool Op0IsKill) {
    2253           0 :   if ((!Subtarget->isLittleEndian())) {
    2254           0 :     return fastEmitInst_r(AArch64::REV64v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2255             :   }
    2256             :   return 0;
    2257             : }
    2258             : 
    2259             : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(unsigned Op0, bool Op0IsKill) {
    2260           0 :   if ((!Subtarget->isLittleEndian())) {
    2261           0 :     return fastEmitInst_r(AArch64::REV64v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2262             :   }
    2263             :   return 0;
    2264             : }
    2265             : 
    2266           0 : unsigned fastEmit_ISD_BITCAST_MVT_v1f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2267           0 : switch (RetVT.SimpleTy) {
    2268           0 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v8i8_r(Op0, Op0IsKill);
    2269           0 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4i16_r(Op0, Op0IsKill);
    2270           0 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2i32_r(Op0, Op0IsKill);
    2271           0 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v4f16_r(Op0, Op0IsKill);
    2272           0 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v1f64_MVT_v2f32_r(Op0, Op0IsKill);
    2273             :   default: return 0;
    2274             : }
    2275             : }
    2276             : 
    2277             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2278           0 :   if ((!Subtarget->isLittleEndian())) {
    2279           0 :     return fastEmitInst_r(AArch64::REV64v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2280             :   }
    2281             :   return 0;
    2282             : }
    2283             : 
    2284             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2285           0 :   if ((!Subtarget->isLittleEndian())) {
    2286           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2287             :   }
    2288             :   return 0;
    2289             : }
    2290             : 
    2291             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2292           0 :   if ((!Subtarget->isLittleEndian())) {
    2293           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2294             :   }
    2295             :   return 0;
    2296             : }
    2297             : 
    2298             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(unsigned Op0, bool Op0IsKill) {
    2299           0 :   if ((!Subtarget->isLittleEndian())) {
    2300           0 :     return fastEmitInst_r(AArch64::REV64v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2301             :   }
    2302             :   return 0;
    2303             : }
    2304             : 
    2305             : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2306           0 :   if ((!Subtarget->isLittleEndian())) {
    2307           0 :     return fastEmitInst_r(AArch64::REV64v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2308             :   }
    2309             :   return 0;
    2310             : }
    2311             : 
    2312           0 : unsigned fastEmit_ISD_BITCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2313           0 : switch (RetVT.SimpleTy) {
    2314           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v16i8_r(Op0, Op0IsKill);
    2315           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8i16_r(Op0, Op0IsKill);
    2316           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2317           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v8f16_r(Op0, Op0IsKill);
    2318           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v2f64_MVT_v4f32_r(Op0, Op0IsKill);
    2319             :   default: return 0;
    2320             : }
    2321             : }
    2322             : 
    2323          36 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2324          36 :   switch (VT.SimpleTy) {
    2325           4 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2326           5 :   case MVT::v8i8: return fastEmit_ISD_BITCAST_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    2327           0 :   case MVT::v16i8: return fastEmit_ISD_BITCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2328           5 :   case MVT::v4i16: return fastEmit_ISD_BITCAST_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    2329           0 :   case MVT::v8i16: return fastEmit_ISD_BITCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2330           5 :   case MVT::v2i32: return fastEmit_ISD_BITCAST_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    2331           0 :   case MVT::v4i32: return fastEmit_ISD_BITCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2332           4 :   case MVT::v1i64: return fastEmit_ISD_BITCAST_MVT_v1i64_r(RetVT, Op0, Op0IsKill);
    2333           0 :   case MVT::v2i64: return fastEmit_ISD_BITCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2334           2 :   case MVT::v4f16: return fastEmit_ISD_BITCAST_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2335           0 :   case MVT::v8f16: return fastEmit_ISD_BITCAST_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2336           5 :   case MVT::v2f32: return fastEmit_ISD_BITCAST_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2337           0 :   case MVT::v4f32: return fastEmit_ISD_BITCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2338           0 :   case MVT::v1f64: return fastEmit_ISD_BITCAST_MVT_v1f64_r(RetVT, Op0, Op0IsKill);
    2339           0 :   case MVT::v2f64: return fastEmit_ISD_BITCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2340             :   default: return 0;
    2341             :   }
    2342             : }
    2343             : 
    2344             : // FastEmit functions for ISD::BITREVERSE.
    2345             : 
    2346             : unsigned fastEmit_ISD_BITREVERSE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2347           0 :   if (RetVT.SimpleTy != MVT::i32)
    2348             :     return 0;
    2349           0 :   return fastEmitInst_r(AArch64::RBITWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    2350             : }
    2351             : 
    2352             : unsigned fastEmit_ISD_BITREVERSE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2353           0 :   if (RetVT.SimpleTy != MVT::i64)
    2354             :     return 0;
    2355           0 :   return fastEmitInst_r(AArch64::RBITXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2356             : }
    2357             : 
    2358           0 : unsigned fastEmit_ISD_BITREVERSE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2359           0 :   switch (VT.SimpleTy) {
    2360           0 :   case MVT::i32: return fastEmit_ISD_BITREVERSE_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2361           0 :   case MVT::i64: return fastEmit_ISD_BITREVERSE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2362             :   default: return 0;
    2363             :   }
    2364             : }
    2365             : 
    2366             : // FastEmit functions for ISD::BRIND.
    2367             : 
    2368             : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2369           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2370             :     return 0;
    2371           0 :   return fastEmitInst_r(AArch64::BR, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2372             : }
    2373             : 
    2374             : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2375           0 :   switch (VT.SimpleTy) {
    2376           0 :   case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2377             :   default: return 0;
    2378             :   }
    2379             : }
    2380             : 
    2381             : // FastEmit functions for ISD::BSWAP.
    2382             : 
    2383             : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2384           0 :   if (RetVT.SimpleTy != MVT::i32)
    2385             :     return 0;
    2386           0 :   return fastEmitInst_r(AArch64::REVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    2387             : }
    2388             : 
    2389             : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2390           0 :   if (RetVT.SimpleTy != MVT::i64)
    2391             :     return 0;
    2392           0 :   return fastEmitInst_r(AArch64::REVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2393             : }
    2394             : 
    2395           0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2396           0 :   switch (VT.SimpleTy) {
    2397           0 :   case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2398           0 :   case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2399             :   default: return 0;
    2400             :   }
    2401             : }
    2402             : 
    2403             : // FastEmit functions for ISD::CTLZ.
    2404             : 
    2405             : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2406           0 :   if (RetVT.SimpleTy != MVT::i32)
    2407             :     return 0;
    2408           0 :   return fastEmitInst_r(AArch64::CLZWr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    2409             : }
    2410             : 
    2411             : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2412           0 :   if (RetVT.SimpleTy != MVT::i64)
    2413             :     return 0;
    2414           0 :   return fastEmitInst_r(AArch64::CLZXr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    2415             : }
    2416             : 
    2417             : unsigned fastEmit_ISD_CTLZ_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2418           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    2419             :     return 0;
    2420           0 :   if ((Subtarget->hasNEON())) {
    2421           0 :     return fastEmitInst_r(AArch64::CLZv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2422             :   }
    2423             :   return 0;
    2424             : }
    2425             : 
    2426             : unsigned fastEmit_ISD_CTLZ_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2427           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2428             :     return 0;
    2429           0 :   if ((Subtarget->hasNEON())) {
    2430           0 :     return fastEmitInst_r(AArch64::CLZv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2431             :   }
    2432             :   return 0;
    2433             : }
    2434             : 
    2435             : unsigned fastEmit_ISD_CTLZ_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2436           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    2437             :     return 0;
    2438           0 :   if ((Subtarget->hasNEON())) {
    2439           0 :     return fastEmitInst_r(AArch64::CLZv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2440             :   }
    2441             :   return 0;
    2442             : }
    2443             : 
    2444             : unsigned fastEmit_ISD_CTLZ_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2445           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    2446             :     return 0;
    2447           0 :   if ((Subtarget->hasNEON())) {
    2448           0 :     return fastEmitInst_r(AArch64::CLZv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2449             :   }
    2450             :   return 0;
    2451             : }
    2452             : 
    2453             : unsigned fastEmit_ISD_CTLZ_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2454           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    2455             :     return 0;
    2456           0 :   if ((Subtarget->hasNEON())) {
    2457           0 :     return fastEmitInst_r(AArch64::CLZv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2458             :   }
    2459             :   return 0;
    2460             : }
    2461             : 
    2462             : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2463           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2464             :     return 0;
    2465           0 :   if ((Subtarget->hasNEON())) {
    2466           0 :     return fastEmitInst_r(AArch64::CLZv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2467             :   }
    2468             :   return 0;
    2469             : }
    2470             : 
    2471           0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2472           0 :   switch (VT.SimpleTy) {
    2473           0 :   case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2474           0 :   case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2475           0 :   case MVT::v8i8: return fastEmit_ISD_CTLZ_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    2476           0 :   case MVT::v16i8: return fastEmit_ISD_CTLZ_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2477           0 :   case MVT::v4i16: return fastEmit_ISD_CTLZ_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    2478           0 :   case MVT::v8i16: return fastEmit_ISD_CTLZ_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2479           0 :   case MVT::v2i32: return fastEmit_ISD_CTLZ_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    2480           0 :   case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2481             :   default: return 0;
    2482             :   }
    2483             : }
    2484             : 
    2485             : // FastEmit functions for ISD::CTPOP.
    2486             : 
    2487             : unsigned fastEmit_ISD_CTPOP_MVT_v8i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2488           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    2489             :     return 0;
    2490           0 :   if ((Subtarget->hasNEON())) {
    2491           0 :     return fastEmitInst_r(AArch64::CNTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2492             :   }
    2493             :   return 0;
    2494             : }
    2495             : 
    2496             : unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2497           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2498             :     return 0;
    2499           0 :   if ((Subtarget->hasNEON())) {
    2500           0 :     return fastEmitInst_r(AArch64::CNTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2501             :   }
    2502             :   return 0;
    2503             : }
    2504             : 
    2505           0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2506           0 :   switch (VT.SimpleTy) {
    2507           0 :   case MVT::v8i8: return fastEmit_ISD_CTPOP_MVT_v8i8_r(RetVT, Op0, Op0IsKill);
    2508           0 :   case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2509             :   default: return 0;
    2510             :   }
    2511             : }
    2512             : 
    2513             : // FastEmit functions for ISD::FABS.
    2514             : 
    2515             : unsigned fastEmit_ISD_FABS_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2516           0 :   if (RetVT.SimpleTy != MVT::f16)
    2517             :     return 0;
    2518           0 :   if ((Subtarget->hasFullFP16())) {
    2519           0 :     return fastEmitInst_r(AArch64::FABSHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2520             :   }
    2521             :   return 0;
    2522             : }
    2523             : 
    2524             : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2525           0 :   if (RetVT.SimpleTy != MVT::f32)
    2526             :     return 0;
    2527           0 :   if ((Subtarget->hasFPARMv8())) {
    2528           0 :     return fastEmitInst_r(AArch64::FABSSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2529             :   }
    2530             :   return 0;
    2531             : }
    2532             : 
    2533             : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2534           0 :   if (RetVT.SimpleTy != MVT::f64)
    2535             :     return 0;
    2536           0 :   if ((Subtarget->hasFPARMv8())) {
    2537           0 :     return fastEmitInst_r(AArch64::FABSDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2538             :   }
    2539             :   return 0;
    2540             : }
    2541             : 
    2542           0 : unsigned fastEmit_ISD_FABS_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2543           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2544             :     return 0;
    2545           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2546           0 :     return fastEmitInst_r(AArch64::FABSv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2547             :   }
    2548             :   return 0;
    2549             : }
    2550             : 
    2551           0 : unsigned fastEmit_ISD_FABS_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2552           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2553             :     return 0;
    2554           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2555           0 :     return fastEmitInst_r(AArch64::FABSv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2556             :   }
    2557             :   return 0;
    2558             : }
    2559             : 
    2560             : unsigned fastEmit_ISD_FABS_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2561           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2562             :     return 0;
    2563           0 :   if ((Subtarget->hasNEON())) {
    2564           0 :     return fastEmitInst_r(AArch64::FABSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2565             :   }
    2566             :   return 0;
    2567             : }
    2568             : 
    2569             : unsigned fastEmit_ISD_FABS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2570           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2571             :     return 0;
    2572           0 :   if ((Subtarget->hasNEON())) {
    2573           0 :     return fastEmitInst_r(AArch64::FABSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2574             :   }
    2575             :   return 0;
    2576             : }
    2577             : 
    2578             : unsigned fastEmit_ISD_FABS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2579           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2580             :     return 0;
    2581           0 :   if ((Subtarget->hasNEON())) {
    2582           0 :     return fastEmitInst_r(AArch64::FABSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2583             :   }
    2584             :   return 0;
    2585             : }
    2586             : 
    2587           0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2588           0 :   switch (VT.SimpleTy) {
    2589           0 :   case MVT::f16: return fastEmit_ISD_FABS_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2590           0 :   case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2591           0 :   case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2592           0 :   case MVT::v4f16: return fastEmit_ISD_FABS_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2593           0 :   case MVT::v8f16: return fastEmit_ISD_FABS_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2594           0 :   case MVT::v2f32: return fastEmit_ISD_FABS_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2595           0 :   case MVT::v4f32: return fastEmit_ISD_FABS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2596           0 :   case MVT::v2f64: return fastEmit_ISD_FABS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2597             :   default: return 0;
    2598             :   }
    2599             : }
    2600             : 
    2601             : // FastEmit functions for ISD::FCEIL.
    2602             : 
    2603             : unsigned fastEmit_ISD_FCEIL_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2604           0 :   if (RetVT.SimpleTy != MVT::f16)
    2605             :     return 0;
    2606           0 :   if ((Subtarget->hasFullFP16())) {
    2607           0 :     return fastEmitInst_r(AArch64::FRINTPHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2608             :   }
    2609             :   return 0;
    2610             : }
    2611             : 
    2612             : unsigned fastEmit_ISD_FCEIL_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2613           0 :   if (RetVT.SimpleTy != MVT::f32)
    2614             :     return 0;
    2615           0 :   if ((Subtarget->hasFPARMv8())) {
    2616           0 :     return fastEmitInst_r(AArch64::FRINTPSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2617             :   }
    2618             :   return 0;
    2619             : }
    2620             : 
    2621             : unsigned fastEmit_ISD_FCEIL_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2622           0 :   if (RetVT.SimpleTy != MVT::f64)
    2623             :     return 0;
    2624           0 :   if ((Subtarget->hasFPARMv8())) {
    2625           0 :     return fastEmitInst_r(AArch64::FRINTPDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2626             :   }
    2627             :   return 0;
    2628             : }
    2629             : 
    2630           0 : unsigned fastEmit_ISD_FCEIL_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2631           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2632             :     return 0;
    2633           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2634           0 :     return fastEmitInst_r(AArch64::FRINTPv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2635             :   }
    2636             :   return 0;
    2637             : }
    2638             : 
    2639           0 : unsigned fastEmit_ISD_FCEIL_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2640           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2641             :     return 0;
    2642           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2643           0 :     return fastEmitInst_r(AArch64::FRINTPv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2644             :   }
    2645             :   return 0;
    2646             : }
    2647             : 
    2648             : unsigned fastEmit_ISD_FCEIL_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2649           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2650             :     return 0;
    2651           0 :   if ((Subtarget->hasNEON())) {
    2652           0 :     return fastEmitInst_r(AArch64::FRINTPv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2653             :   }
    2654             :   return 0;
    2655             : }
    2656             : 
    2657             : unsigned fastEmit_ISD_FCEIL_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2658           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2659             :     return 0;
    2660           0 :   if ((Subtarget->hasNEON())) {
    2661           0 :     return fastEmitInst_r(AArch64::FRINTPv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2662             :   }
    2663             :   return 0;
    2664             : }
    2665             : 
    2666             : unsigned fastEmit_ISD_FCEIL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2667           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2668             :     return 0;
    2669           0 :   if ((Subtarget->hasNEON())) {
    2670           0 :     return fastEmitInst_r(AArch64::FRINTPv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2671             :   }
    2672             :   return 0;
    2673             : }
    2674             : 
    2675           0 : unsigned fastEmit_ISD_FCEIL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2676           0 :   switch (VT.SimpleTy) {
    2677           0 :   case MVT::f16: return fastEmit_ISD_FCEIL_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2678           0 :   case MVT::f32: return fastEmit_ISD_FCEIL_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2679           0 :   case MVT::f64: return fastEmit_ISD_FCEIL_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2680           0 :   case MVT::v4f16: return fastEmit_ISD_FCEIL_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2681           0 :   case MVT::v8f16: return fastEmit_ISD_FCEIL_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2682           0 :   case MVT::v2f32: return fastEmit_ISD_FCEIL_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2683           0 :   case MVT::v4f32: return fastEmit_ISD_FCEIL_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2684           0 :   case MVT::v2f64: return fastEmit_ISD_FCEIL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2685             :   default: return 0;
    2686             :   }
    2687             : }
    2688             : 
    2689             : // FastEmit functions for ISD::FFLOOR.
    2690             : 
    2691             : unsigned fastEmit_ISD_FFLOOR_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2692           0 :   if (RetVT.SimpleTy != MVT::f16)
    2693             :     return 0;
    2694           0 :   if ((Subtarget->hasFullFP16())) {
    2695           0 :     return fastEmitInst_r(AArch64::FRINTMHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2696             :   }
    2697             :   return 0;
    2698             : }
    2699             : 
    2700             : unsigned fastEmit_ISD_FFLOOR_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2701           0 :   if (RetVT.SimpleTy != MVT::f32)
    2702             :     return 0;
    2703           0 :   if ((Subtarget->hasFPARMv8())) {
    2704           0 :     return fastEmitInst_r(AArch64::FRINTMSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2705             :   }
    2706             :   return 0;
    2707             : }
    2708             : 
    2709             : unsigned fastEmit_ISD_FFLOOR_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2710           0 :   if (RetVT.SimpleTy != MVT::f64)
    2711             :     return 0;
    2712           0 :   if ((Subtarget->hasFPARMv8())) {
    2713           0 :     return fastEmitInst_r(AArch64::FRINTMDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2714             :   }
    2715             :   return 0;
    2716             : }
    2717             : 
    2718           0 : unsigned fastEmit_ISD_FFLOOR_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2719           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2720             :     return 0;
    2721           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2722           0 :     return fastEmitInst_r(AArch64::FRINTMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2723             :   }
    2724             :   return 0;
    2725             : }
    2726             : 
    2727           0 : unsigned fastEmit_ISD_FFLOOR_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2728           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2729             :     return 0;
    2730           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2731           0 :     return fastEmitInst_r(AArch64::FRINTMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2732             :   }
    2733             :   return 0;
    2734             : }
    2735             : 
    2736             : unsigned fastEmit_ISD_FFLOOR_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2737           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2738             :     return 0;
    2739           0 :   if ((Subtarget->hasNEON())) {
    2740           0 :     return fastEmitInst_r(AArch64::FRINTMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2741             :   }
    2742             :   return 0;
    2743             : }
    2744             : 
    2745             : unsigned fastEmit_ISD_FFLOOR_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2746           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2747             :     return 0;
    2748           0 :   if ((Subtarget->hasNEON())) {
    2749           0 :     return fastEmitInst_r(AArch64::FRINTMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2750             :   }
    2751             :   return 0;
    2752             : }
    2753             : 
    2754             : unsigned fastEmit_ISD_FFLOOR_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2755           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2756             :     return 0;
    2757           0 :   if ((Subtarget->hasNEON())) {
    2758           0 :     return fastEmitInst_r(AArch64::FRINTMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2759             :   }
    2760             :   return 0;
    2761             : }
    2762             : 
    2763           0 : unsigned fastEmit_ISD_FFLOOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2764           0 :   switch (VT.SimpleTy) {
    2765           0 :   case MVT::f16: return fastEmit_ISD_FFLOOR_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2766           0 :   case MVT::f32: return fastEmit_ISD_FFLOOR_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2767           0 :   case MVT::f64: return fastEmit_ISD_FFLOOR_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2768           0 :   case MVT::v4f16: return fastEmit_ISD_FFLOOR_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2769           0 :   case MVT::v8f16: return fastEmit_ISD_FFLOOR_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2770           0 :   case MVT::v2f32: return fastEmit_ISD_FFLOOR_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2771           0 :   case MVT::v4f32: return fastEmit_ISD_FFLOOR_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2772           0 :   case MVT::v2f64: return fastEmit_ISD_FFLOOR_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2773             :   default: return 0;
    2774             :   }
    2775             : }
    2776             : 
    2777             : // FastEmit functions for ISD::FNEARBYINT.
    2778             : 
    2779             : unsigned fastEmit_ISD_FNEARBYINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2780           0 :   if (RetVT.SimpleTy != MVT::f16)
    2781             :     return 0;
    2782           0 :   if ((Subtarget->hasFullFP16())) {
    2783           0 :     return fastEmitInst_r(AArch64::FRINTIHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2784             :   }
    2785             :   return 0;
    2786             : }
    2787             : 
    2788             : unsigned fastEmit_ISD_FNEARBYINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2789           0 :   if (RetVT.SimpleTy != MVT::f32)
    2790             :     return 0;
    2791           0 :   if ((Subtarget->hasFPARMv8())) {
    2792           0 :     return fastEmitInst_r(AArch64::FRINTISr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2793             :   }
    2794             :   return 0;
    2795             : }
    2796             : 
    2797             : unsigned fastEmit_ISD_FNEARBYINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2798           0 :   if (RetVT.SimpleTy != MVT::f64)
    2799             :     return 0;
    2800           0 :   if ((Subtarget->hasFPARMv8())) {
    2801           0 :     return fastEmitInst_r(AArch64::FRINTIDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2802             :   }
    2803             :   return 0;
    2804             : }
    2805             : 
    2806           0 : unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2807           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2808             :     return 0;
    2809           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2810           0 :     return fastEmitInst_r(AArch64::FRINTIv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2811             :   }
    2812             :   return 0;
    2813             : }
    2814             : 
    2815           0 : unsigned fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2816           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2817             :     return 0;
    2818           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2819           0 :     return fastEmitInst_r(AArch64::FRINTIv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2820             :   }
    2821             :   return 0;
    2822             : }
    2823             : 
    2824             : unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2825           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2826             :     return 0;
    2827           0 :   if ((Subtarget->hasNEON())) {
    2828           0 :     return fastEmitInst_r(AArch64::FRINTIv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2829             :   }
    2830             :   return 0;
    2831             : }
    2832             : 
    2833             : unsigned fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2834           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2835             :     return 0;
    2836           0 :   if ((Subtarget->hasNEON())) {
    2837           0 :     return fastEmitInst_r(AArch64::FRINTIv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2838             :   }
    2839             :   return 0;
    2840             : }
    2841             : 
    2842             : unsigned fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2843           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2844             :     return 0;
    2845           0 :   if ((Subtarget->hasNEON())) {
    2846           0 :     return fastEmitInst_r(AArch64::FRINTIv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2847             :   }
    2848             :   return 0;
    2849             : }
    2850             : 
    2851           0 : unsigned fastEmit_ISD_FNEARBYINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2852           0 :   switch (VT.SimpleTy) {
    2853           0 :   case MVT::f16: return fastEmit_ISD_FNEARBYINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2854           0 :   case MVT::f32: return fastEmit_ISD_FNEARBYINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2855           0 :   case MVT::f64: return fastEmit_ISD_FNEARBYINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2856           0 :   case MVT::v4f16: return fastEmit_ISD_FNEARBYINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2857           0 :   case MVT::v8f16: return fastEmit_ISD_FNEARBYINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2858           0 :   case MVT::v2f32: return fastEmit_ISD_FNEARBYINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2859           0 :   case MVT::v4f32: return fastEmit_ISD_FNEARBYINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2860           0 :   case MVT::v2f64: return fastEmit_ISD_FNEARBYINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2861             :   default: return 0;
    2862             :   }
    2863             : }
    2864             : 
    2865             : // FastEmit functions for ISD::FNEG.
    2866             : 
    2867             : unsigned fastEmit_ISD_FNEG_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2868           0 :   if (RetVT.SimpleTy != MVT::f16)
    2869             :     return 0;
    2870           0 :   if ((Subtarget->hasFullFP16())) {
    2871           0 :     return fastEmitInst_r(AArch64::FNEGHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    2872             :   }
    2873             :   return 0;
    2874             : }
    2875             : 
    2876             : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2877           0 :   if (RetVT.SimpleTy != MVT::f32)
    2878             :     return 0;
    2879           0 :   if ((Subtarget->hasFPARMv8())) {
    2880           0 :     return fastEmitInst_r(AArch64::FNEGSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2881             :   }
    2882             :   return 0;
    2883             : }
    2884             : 
    2885             : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2886           0 :   if (RetVT.SimpleTy != MVT::f64)
    2887             :     return 0;
    2888           0 :   if ((Subtarget->hasFPARMv8())) {
    2889           0 :     return fastEmitInst_r(AArch64::FNEGDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2890             :   }
    2891             :   return 0;
    2892             : }
    2893             : 
    2894           0 : unsigned fastEmit_ISD_FNEG_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2895           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    2896             :     return 0;
    2897           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2898           0 :     return fastEmitInst_r(AArch64::FNEGv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2899             :   }
    2900             :   return 0;
    2901             : }
    2902             : 
    2903           0 : unsigned fastEmit_ISD_FNEG_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2904           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    2905             :     return 0;
    2906           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    2907           0 :     return fastEmitInst_r(AArch64::FNEGv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2908             :   }
    2909             :   return 0;
    2910             : }
    2911             : 
    2912             : unsigned fastEmit_ISD_FNEG_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2913           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    2914             :     return 0;
    2915           0 :   if ((Subtarget->hasNEON())) {
    2916           0 :     return fastEmitInst_r(AArch64::FNEGv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2917             :   }
    2918             :   return 0;
    2919             : }
    2920             : 
    2921             : unsigned fastEmit_ISD_FNEG_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2922           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2923             :     return 0;
    2924           0 :   if ((Subtarget->hasNEON())) {
    2925           0 :     return fastEmitInst_r(AArch64::FNEGv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2926             :   }
    2927             :   return 0;
    2928             : }
    2929             : 
    2930             : unsigned fastEmit_ISD_FNEG_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2931           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2932             :     return 0;
    2933           0 :   if ((Subtarget->hasNEON())) {
    2934           0 :     return fastEmitInst_r(AArch64::FNEGv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2935             :   }
    2936             :   return 0;
    2937             : }
    2938             : 
    2939           0 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2940           0 :   switch (VT.SimpleTy) {
    2941           0 :   case MVT::f16: return fastEmit_ISD_FNEG_MVT_f16_r(RetVT, Op0, Op0IsKill);
    2942           0 :   case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
    2943           0 :   case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
    2944           0 :   case MVT::v4f16: return fastEmit_ISD_FNEG_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    2945           0 :   case MVT::v8f16: return fastEmit_ISD_FNEG_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    2946           0 :   case MVT::v2f32: return fastEmit_ISD_FNEG_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    2947           0 :   case MVT::v4f32: return fastEmit_ISD_FNEG_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2948           0 :   case MVT::v2f64: return fastEmit_ISD_FNEG_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2949             :   default: return 0;
    2950             :   }
    2951             : }
    2952             : 
    2953             : // FastEmit functions for ISD::FP_EXTEND.
    2954             : 
    2955             : unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    2956           0 :   if ((Subtarget->hasFPARMv8())) {
    2957           0 :     return fastEmitInst_r(AArch64::FCVTSHr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    2958             :   }
    2959             :   return 0;
    2960             : }
    2961             : 
    2962             : unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    2963           0 :   if ((Subtarget->hasFPARMv8())) {
    2964           0 :     return fastEmitInst_r(AArch64::FCVTDHr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2965             :   }
    2966             :   return 0;
    2967             : }
    2968             : 
    2969           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2970           0 : switch (RetVT.SimpleTy) {
    2971           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f32_r(Op0, Op0IsKill);
    2972           0 :   case MVT::f64: return fastEmit_ISD_FP_EXTEND_MVT_f16_MVT_f64_r(Op0, Op0IsKill);
    2973             :   default: return 0;
    2974             : }
    2975             : }
    2976             : 
    2977             : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2978           0 :   if (RetVT.SimpleTy != MVT::f64)
    2979             :     return 0;
    2980           0 :   if ((Subtarget->hasFPARMv8())) {
    2981           0 :     return fastEmitInst_r(AArch64::FCVTDSr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    2982             :   }
    2983             :   return 0;
    2984             : }
    2985             : 
    2986             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2987           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2988             :     return 0;
    2989           0 :   return fastEmitInst_r(AArch64::FCVTLv4i16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2990             : }
    2991             : 
    2992             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2993           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2994             :     return 0;
    2995           0 :   return fastEmitInst_r(AArch64::FCVTLv2i32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    2996             : }
    2997             : 
    2998           0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2999           0 :   switch (VT.SimpleTy) {
    3000           0 :   case MVT::f16: return fastEmit_ISD_FP_EXTEND_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3001           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3002           0 :   case MVT::v4f16: return fastEmit_ISD_FP_EXTEND_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3003           0 :   case MVT::v2f32: return fastEmit_ISD_FP_EXTEND_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3004             :   default: return 0;
    3005             :   }
    3006             : }
    3007             : 
    3008             : // FastEmit functions for ISD::FP_ROUND.
    3009             : 
    3010             : unsigned fastEmit_ISD_FP_ROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3011           0 :   if (RetVT.SimpleTy != MVT::f16)
    3012             :     return 0;
    3013           0 :   if ((Subtarget->hasFPARMv8())) {
    3014           0 :     return fastEmitInst_r(AArch64::FCVTHSr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3015             :   }
    3016             :   return 0;
    3017             : }
    3018             : 
    3019             : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3020           0 :   if ((Subtarget->hasFPARMv8())) {
    3021           0 :     return fastEmitInst_r(AArch64::FCVTHDr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3022             :   }
    3023             :   return 0;
    3024             : }
    3025             : 
    3026             : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3027           0 :   if ((Subtarget->hasFPARMv8())) {
    3028           0 :     return fastEmitInst_r(AArch64::FCVTSDr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3029             :   }
    3030             :   return 0;
    3031             : }
    3032             : 
    3033           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3034           0 : switch (RetVT.SimpleTy) {
    3035           0 :   case MVT::f16: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f16_r(Op0, Op0IsKill);
    3036           0 :   case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f64_MVT_f32_r(Op0, Op0IsKill);
    3037             :   default: return 0;
    3038             : }
    3039             : }
    3040             : 
    3041             : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3042           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3043             :     return 0;
    3044           0 :   return fastEmitInst_r(AArch64::FCVTNv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3045             : }
    3046             : 
    3047             : unsigned fastEmit_ISD_FP_ROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3048           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3049             :     return 0;
    3050           0 :   return fastEmitInst_r(AArch64::FCVTNv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3051             : }
    3052             : 
    3053           0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3054           0 :   switch (VT.SimpleTy) {
    3055           0 :   case MVT::f32: return fastEmit_ISD_FP_ROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3056           0 :   case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3057           0 :   case MVT::v4f32: return fastEmit_ISD_FP_ROUND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3058           0 :   case MVT::v2f64: return fastEmit_ISD_FP_ROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3059             :   default: return 0;
    3060             :   }
    3061             : }
    3062             : 
    3063             : // FastEmit functions for ISD::FP_TO_SINT.
    3064             : 
    3065             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3066           1 :   if ((Subtarget->hasFullFP16())) {
    3067           0 :     return fastEmitInst_r(AArch64::FCVTZSUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3068             :   }
    3069             :   return 0;
    3070             : }
    3071             : 
    3072             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3073           0 :   if ((Subtarget->hasFullFP16())) {
    3074           0 :     return fastEmitInst_r(AArch64::FCVTZSUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3075             :   }
    3076             :   return 0;
    3077             : }
    3078             : 
    3079           1 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3080           1 : switch (RetVT.SimpleTy) {
    3081           1 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
    3082           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
    3083             :   default: return 0;
    3084             : }
    3085             : }
    3086             : 
    3087             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3088           1 :   if ((Subtarget->hasFPARMv8())) {
    3089           1 :     return fastEmitInst_r(AArch64::FCVTZSUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3090             :   }
    3091             :   return 0;
    3092             : }
    3093             : 
    3094             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3095           0 :   if ((Subtarget->hasFPARMv8())) {
    3096           0 :     return fastEmitInst_r(AArch64::FCVTZSUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3097             :   }
    3098             :   return 0;
    3099             : }
    3100             : 
    3101           1 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3102           1 : switch (RetVT.SimpleTy) {
    3103           1 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    3104           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    3105             :   default: return 0;
    3106             : }
    3107             : }
    3108             : 
    3109             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3110           1 :   if ((Subtarget->hasFPARMv8())) {
    3111           1 :     return fastEmitInst_r(AArch64::FCVTZSUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3112             :   }
    3113             :   return 0;
    3114             : }
    3115             : 
    3116             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3117           0 :   if ((Subtarget->hasFPARMv8())) {
    3118           0 :     return fastEmitInst_r(AArch64::FCVTZSUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3119             :   }
    3120             :   return 0;
    3121             : }
    3122             : 
    3123           1 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3124           1 : switch (RetVT.SimpleTy) {
    3125           1 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    3126           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    3127             :   default: return 0;
    3128             : }
    3129             : }
    3130             : 
    3131           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3132           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    3133             :     return 0;
    3134           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3135           0 :     return fastEmitInst_r(AArch64::FCVTZSv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3136             :   }
    3137             :   return 0;
    3138             : }
    3139             : 
    3140           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3141           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3142             :     return 0;
    3143           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3144           0 :     return fastEmitInst_r(AArch64::FCVTZSv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3145             :   }
    3146             :   return 0;
    3147             : }
    3148             : 
    3149             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3150           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    3151             :     return 0;
    3152           0 :   if ((Subtarget->hasNEON())) {
    3153           0 :     return fastEmitInst_r(AArch64::FCVTZSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3154             :   }
    3155             :   return 0;
    3156             : }
    3157             : 
    3158             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3159           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3160             :     return 0;
    3161           0 :   if ((Subtarget->hasNEON())) {
    3162           0 :     return fastEmitInst_r(AArch64::FCVTZSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3163             :   }
    3164             :   return 0;
    3165             : }
    3166             : 
    3167             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3168           1 :   if (RetVT.SimpleTy != MVT::v2i64)
    3169             :     return 0;
    3170           1 :   if ((Subtarget->hasNEON())) {
    3171           1 :     return fastEmitInst_r(AArch64::FCVTZSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3172             :   }
    3173             :   return 0;
    3174             : }
    3175             : 
    3176           4 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3177           4 :   switch (VT.SimpleTy) {
    3178           1 :   case MVT::f16: return fastEmit_ISD_FP_TO_SINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3179           1 :   case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3180           1 :   case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3181           0 :   case MVT::v4f16: return fastEmit_ISD_FP_TO_SINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3182           0 :   case MVT::v8f16: return fastEmit_ISD_FP_TO_SINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3183           0 :   case MVT::v2f32: return fastEmit_ISD_FP_TO_SINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3184           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3185           1 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3186             :   default: return 0;
    3187             :   }
    3188             : }
    3189             : 
    3190             : // FastEmit functions for ISD::FP_TO_UINT.
    3191             : 
    3192             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3193           0 :   if ((Subtarget->hasFullFP16())) {
    3194           0 :     return fastEmitInst_r(AArch64::FCVTZUUWHr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3195             :   }
    3196             :   return 0;
    3197             : }
    3198             : 
    3199             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3200           0 :   if ((Subtarget->hasFullFP16())) {
    3201           0 :     return fastEmitInst_r(AArch64::FCVTZUUXHr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3202             :   }
    3203             :   return 0;
    3204             : }
    3205             : 
    3206           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3207           0 : switch (RetVT.SimpleTy) {
    3208           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i32_r(Op0, Op0IsKill);
    3209           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f16_MVT_i64_r(Op0, Op0IsKill);
    3210             :   default: return 0;
    3211             : }
    3212             : }
    3213             : 
    3214             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3215           0 :   if ((Subtarget->hasFPARMv8())) {
    3216           0 :     return fastEmitInst_r(AArch64::FCVTZUUWSr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3217             :   }
    3218             :   return 0;
    3219             : }
    3220             : 
    3221             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3222           0 :   if ((Subtarget->hasFPARMv8())) {
    3223           0 :     return fastEmitInst_r(AArch64::FCVTZUUXSr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3224             :   }
    3225             :   return 0;
    3226             : }
    3227             : 
    3228           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3229           0 : switch (RetVT.SimpleTy) {
    3230           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    3231           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    3232             :   default: return 0;
    3233             : }
    3234             : }
    3235             : 
    3236             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3237           0 :   if ((Subtarget->hasFPARMv8())) {
    3238           0 :     return fastEmitInst_r(AArch64::FCVTZUUWDr, &AArch64::GPR32RegClass, Op0, Op0IsKill);
    3239             :   }
    3240             :   return 0;
    3241             : }
    3242             : 
    3243             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3244           0 :   if ((Subtarget->hasFPARMv8())) {
    3245           0 :     return fastEmitInst_r(AArch64::FCVTZUUXDr, &AArch64::GPR64RegClass, Op0, Op0IsKill);
    3246             :   }
    3247             :   return 0;
    3248             : }
    3249             : 
    3250           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3251           0 : switch (RetVT.SimpleTy) {
    3252           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    3253           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    3254             :   default: return 0;
    3255             : }
    3256             : }
    3257             : 
    3258           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3259           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    3260             :     return 0;
    3261           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3262           0 :     return fastEmitInst_r(AArch64::FCVTZUv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3263             :   }
    3264             :   return 0;
    3265             : }
    3266             : 
    3267           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3268           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3269             :     return 0;
    3270           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3271           0 :     return fastEmitInst_r(AArch64::FCVTZUv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3272             :   }
    3273             :   return 0;
    3274             : }
    3275             : 
    3276             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3277           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    3278             :     return 0;
    3279           0 :   if ((Subtarget->hasNEON())) {
    3280           0 :     return fastEmitInst_r(AArch64::FCVTZUv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3281             :   }
    3282             :   return 0;
    3283             : }
    3284             : 
    3285             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3286           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3287             :     return 0;
    3288           0 :   if ((Subtarget->hasNEON())) {
    3289           0 :     return fastEmitInst_r(AArch64::FCVTZUv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3290             :   }
    3291             :   return 0;
    3292             : }
    3293             : 
    3294             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3295           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3296             :     return 0;
    3297           0 :   if ((Subtarget->hasNEON())) {
    3298           0 :     return fastEmitInst_r(AArch64::FCVTZUv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3299             :   }
    3300             :   return 0;
    3301             : }
    3302             : 
    3303           0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3304           0 :   switch (VT.SimpleTy) {
    3305           0 :   case MVT::f16: return fastEmit_ISD_FP_TO_UINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3306           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3307           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3308           0 :   case MVT::v4f16: return fastEmit_ISD_FP_TO_UINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3309           0 :   case MVT::v8f16: return fastEmit_ISD_FP_TO_UINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3310           0 :   case MVT::v2f32: return fastEmit_ISD_FP_TO_UINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3311           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3312           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3313             :   default: return 0;
    3314             :   }
    3315             : }
    3316             : 
    3317             : // FastEmit functions for ISD::FRINT.
    3318             : 
    3319             : unsigned fastEmit_ISD_FRINT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3320           0 :   if (RetVT.SimpleTy != MVT::f16)
    3321             :     return 0;
    3322           0 :   if ((Subtarget->hasFullFP16())) {
    3323           0 :     return fastEmitInst_r(AArch64::FRINTXHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3324             :   }
    3325             :   return 0;
    3326             : }
    3327             : 
    3328             : unsigned fastEmit_ISD_FRINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3329           0 :   if (RetVT.SimpleTy != MVT::f32)
    3330             :     return 0;
    3331           0 :   if ((Subtarget->hasFPARMv8())) {
    3332           0 :     return fastEmitInst_r(AArch64::FRINTXSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3333             :   }
    3334             :   return 0;
    3335             : }
    3336             : 
    3337             : unsigned fastEmit_ISD_FRINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3338           0 :   if (RetVT.SimpleTy != MVT::f64)
    3339             :     return 0;
    3340           0 :   if ((Subtarget->hasFPARMv8())) {
    3341           0 :     return fastEmitInst_r(AArch64::FRINTXDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3342             :   }
    3343             :   return 0;
    3344             : }
    3345             : 
    3346           0 : unsigned fastEmit_ISD_FRINT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3347           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3348             :     return 0;
    3349           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3350           0 :     return fastEmitInst_r(AArch64::FRINTXv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3351             :   }
    3352             :   return 0;
    3353             : }
    3354             : 
    3355           0 : unsigned fastEmit_ISD_FRINT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3356           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3357             :     return 0;
    3358           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3359           0 :     return fastEmitInst_r(AArch64::FRINTXv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3360             :   }
    3361             :   return 0;
    3362             : }
    3363             : 
    3364             : unsigned fastEmit_ISD_FRINT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3365           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3366             :     return 0;
    3367           0 :   if ((Subtarget->hasNEON())) {
    3368           0 :     return fastEmitInst_r(AArch64::FRINTXv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3369             :   }
    3370             :   return 0;
    3371             : }
    3372             : 
    3373             : unsigned fastEmit_ISD_FRINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3374           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3375             :     return 0;
    3376           0 :   if ((Subtarget->hasNEON())) {
    3377           0 :     return fastEmitInst_r(AArch64::FRINTXv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3378             :   }
    3379             :   return 0;
    3380             : }
    3381             : 
    3382             : unsigned fastEmit_ISD_FRINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3383           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3384             :     return 0;
    3385           0 :   if ((Subtarget->hasNEON())) {
    3386           0 :     return fastEmitInst_r(AArch64::FRINTXv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3387             :   }
    3388             :   return 0;
    3389             : }
    3390             : 
    3391           0 : unsigned fastEmit_ISD_FRINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3392           0 :   switch (VT.SimpleTy) {
    3393           0 :   case MVT::f16: return fastEmit_ISD_FRINT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3394           0 :   case MVT::f32: return fastEmit_ISD_FRINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3395           0 :   case MVT::f64: return fastEmit_ISD_FRINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3396           0 :   case MVT::v4f16: return fastEmit_ISD_FRINT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3397           0 :   case MVT::v8f16: return fastEmit_ISD_FRINT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3398           0 :   case MVT::v2f32: return fastEmit_ISD_FRINT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3399           0 :   case MVT::v4f32: return fastEmit_ISD_FRINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3400           0 :   case MVT::v2f64: return fastEmit_ISD_FRINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3401             :   default: return 0;
    3402             :   }
    3403             : }
    3404             : 
    3405             : // FastEmit functions for ISD::FROUND.
    3406             : 
    3407             : unsigned fastEmit_ISD_FROUND_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3408           0 :   if (RetVT.SimpleTy != MVT::f16)
    3409             :     return 0;
    3410           0 :   if ((Subtarget->hasFullFP16())) {
    3411           0 :     return fastEmitInst_r(AArch64::FRINTAHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3412             :   }
    3413             :   return 0;
    3414             : }
    3415             : 
    3416             : unsigned fastEmit_ISD_FROUND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3417           0 :   if (RetVT.SimpleTy != MVT::f32)
    3418             :     return 0;
    3419           0 :   if ((Subtarget->hasFPARMv8())) {
    3420           0 :     return fastEmitInst_r(AArch64::FRINTASr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3421             :   }
    3422             :   return 0;
    3423             : }
    3424             : 
    3425             : unsigned fastEmit_ISD_FROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3426           0 :   if (RetVT.SimpleTy != MVT::f64)
    3427             :     return 0;
    3428           0 :   if ((Subtarget->hasFPARMv8())) {
    3429           0 :     return fastEmitInst_r(AArch64::FRINTADr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3430             :   }
    3431             :   return 0;
    3432             : }
    3433             : 
    3434           0 : unsigned fastEmit_ISD_FROUND_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3435           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3436             :     return 0;
    3437           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3438           0 :     return fastEmitInst_r(AArch64::FRINTAv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3439             :   }
    3440             :   return 0;
    3441             : }
    3442             : 
    3443           0 : unsigned fastEmit_ISD_FROUND_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3444           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3445             :     return 0;
    3446           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3447           0 :     return fastEmitInst_r(AArch64::FRINTAv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3448             :   }
    3449             :   return 0;
    3450             : }
    3451             : 
    3452             : unsigned fastEmit_ISD_FROUND_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3453           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3454             :     return 0;
    3455           0 :   if ((Subtarget->hasNEON())) {
    3456           0 :     return fastEmitInst_r(AArch64::FRINTAv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3457             :   }
    3458             :   return 0;
    3459             : }
    3460             : 
    3461             : unsigned fastEmit_ISD_FROUND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3462           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3463             :     return 0;
    3464           0 :   if ((Subtarget->hasNEON())) {
    3465           0 :     return fastEmitInst_r(AArch64::FRINTAv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3466             :   }
    3467             :   return 0;
    3468             : }
    3469             : 
    3470             : unsigned fastEmit_ISD_FROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3471           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3472             :     return 0;
    3473           0 :   if ((Subtarget->hasNEON())) {
    3474           0 :     return fastEmitInst_r(AArch64::FRINTAv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3475             :   }
    3476             :   return 0;
    3477             : }
    3478             : 
    3479           0 : unsigned fastEmit_ISD_FROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3480           0 :   switch (VT.SimpleTy) {
    3481           0 :   case MVT::f16: return fastEmit_ISD_FROUND_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3482           0 :   case MVT::f32: return fastEmit_ISD_FROUND_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3483           0 :   case MVT::f64: return fastEmit_ISD_FROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3484           0 :   case MVT::v4f16: return fastEmit_ISD_FROUND_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3485           0 :   case MVT::v8f16: return fastEmit_ISD_FROUND_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3486           0 :   case MVT::v2f32: return fastEmit_ISD_FROUND_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3487           0 :   case MVT::v4f32: return fastEmit_ISD_FROUND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3488           0 :   case MVT::v2f64: return fastEmit_ISD_FROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3489             :   default: return 0;
    3490             :   }
    3491             : }
    3492             : 
    3493             : // FastEmit functions for ISD::FSQRT.
    3494             : 
    3495             : unsigned fastEmit_ISD_FSQRT_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3496           0 :   if (RetVT.SimpleTy != MVT::f16)
    3497             :     return 0;
    3498           0 :   if ((Subtarget->hasFullFP16())) {
    3499           0 :     return fastEmitInst_r(AArch64::FSQRTHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3500             :   }
    3501             :   return 0;
    3502             : }
    3503             : 
    3504             : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3505           1 :   if (RetVT.SimpleTy != MVT::f32)
    3506             :     return 0;
    3507           1 :   if ((Subtarget->hasFPARMv8())) {
    3508           1 :     return fastEmitInst_r(AArch64::FSQRTSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3509             :   }
    3510             :   return 0;
    3511             : }
    3512             : 
    3513             : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3514           1 :   if (RetVT.SimpleTy != MVT::f64)
    3515             :     return 0;
    3516           1 :   if ((Subtarget->hasFPARMv8())) {
    3517           1 :     return fastEmitInst_r(AArch64::FSQRTDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3518             :   }
    3519             :   return 0;
    3520             : }
    3521             : 
    3522           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3523           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3524             :     return 0;
    3525           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3526           0 :     return fastEmitInst_r(AArch64::FSQRTv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3527             :   }
    3528             :   return 0;
    3529             : }
    3530             : 
    3531           0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3532           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3533             :     return 0;
    3534           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3535           0 :     return fastEmitInst_r(AArch64::FSQRTv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3536             :   }
    3537             :   return 0;
    3538             : }
    3539             : 
    3540             : unsigned fastEmit_ISD_FSQRT_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3541           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3542             :     return 0;
    3543           0 :   if ((Subtarget->hasNEON())) {
    3544           0 :     return fastEmitInst_r(AArch64::FSQRTv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3545             :   }
    3546             :   return 0;
    3547             : }
    3548             : 
    3549             : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3550           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3551             :     return 0;
    3552           0 :   if ((Subtarget->hasNEON())) {
    3553           0 :     return fastEmitInst_r(AArch64::FSQRTv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3554             :   }
    3555             :   return 0;
    3556             : }
    3557             : 
    3558             : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3559           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3560             :     return 0;
    3561           0 :   if ((Subtarget->hasNEON())) {
    3562           0 :     return fastEmitInst_r(AArch64::FSQRTv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3563             :   }
    3564             :   return 0;
    3565             : }
    3566             : 
    3567           2 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3568           2 :   switch (VT.SimpleTy) {
    3569           0 :   case MVT::f16: return fastEmit_ISD_FSQRT_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3570           1 :   case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3571           1 :   case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3572           0 :   case MVT::v4f16: return fastEmit_ISD_FSQRT_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3573           0 :   case MVT::v8f16: return fastEmit_ISD_FSQRT_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3574           0 :   case MVT::v2f32: return fastEmit_ISD_FSQRT_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3575           0 :   case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3576           0 :   case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3577             :   default: return 0;
    3578             :   }
    3579             : }
    3580             : 
    3581             : // FastEmit functions for ISD::FTRUNC.
    3582             : 
    3583             : unsigned fastEmit_ISD_FTRUNC_MVT_f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3584           0 :   if (RetVT.SimpleTy != MVT::f16)
    3585             :     return 0;
    3586           0 :   if ((Subtarget->hasFullFP16())) {
    3587           0 :     return fastEmitInst_r(AArch64::FRINTZHr, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3588             :   }
    3589             :   return 0;
    3590             : }
    3591             : 
    3592             : unsigned fastEmit_ISD_FTRUNC_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3593           0 :   if (RetVT.SimpleTy != MVT::f32)
    3594             :     return 0;
    3595           0 :   if ((Subtarget->hasFPARMv8())) {
    3596           0 :     return fastEmitInst_r(AArch64::FRINTZSr, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3597             :   }
    3598             :   return 0;
    3599             : }
    3600             : 
    3601             : unsigned fastEmit_ISD_FTRUNC_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3602           0 :   if (RetVT.SimpleTy != MVT::f64)
    3603             :     return 0;
    3604           0 :   if ((Subtarget->hasFPARMv8())) {
    3605           0 :     return fastEmitInst_r(AArch64::FRINTZDr, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3606             :   }
    3607             :   return 0;
    3608             : }
    3609             : 
    3610           0 : unsigned fastEmit_ISD_FTRUNC_MVT_v4f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3611           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3612             :     return 0;
    3613           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3614           0 :     return fastEmitInst_r(AArch64::FRINTZv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3615             :   }
    3616             :   return 0;
    3617             : }
    3618             : 
    3619           0 : unsigned fastEmit_ISD_FTRUNC_MVT_v8f16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3620           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3621             :     return 0;
    3622           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3623           0 :     return fastEmitInst_r(AArch64::FRINTZv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3624             :   }
    3625             :   return 0;
    3626             : }
    3627             : 
    3628             : unsigned fastEmit_ISD_FTRUNC_MVT_v2f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3629           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3630             :     return 0;
    3631           0 :   if ((Subtarget->hasNEON())) {
    3632           0 :     return fastEmitInst_r(AArch64::FRINTZv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3633             :   }
    3634             :   return 0;
    3635             : }
    3636             : 
    3637             : unsigned fastEmit_ISD_FTRUNC_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3638           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3639             :     return 0;
    3640           0 :   if ((Subtarget->hasNEON())) {
    3641           0 :     return fastEmitInst_r(AArch64::FRINTZv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3642             :   }
    3643             :   return 0;
    3644             : }
    3645             : 
    3646             : unsigned fastEmit_ISD_FTRUNC_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3647           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3648             :     return 0;
    3649           0 :   if ((Subtarget->hasNEON())) {
    3650           0 :     return fastEmitInst_r(AArch64::FRINTZv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3651             :   }
    3652             :   return 0;
    3653             : }
    3654             : 
    3655           0 : unsigned fastEmit_ISD_FTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3656           0 :   switch (VT.SimpleTy) {
    3657           0 :   case MVT::f16: return fastEmit_ISD_FTRUNC_MVT_f16_r(RetVT, Op0, Op0IsKill);
    3658           0 :   case MVT::f32: return fastEmit_ISD_FTRUNC_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3659           0 :   case MVT::f64: return fastEmit_ISD_FTRUNC_MVT_f64_r(RetVT, Op0, Op0IsKill);
    3660           0 :   case MVT::v4f16: return fastEmit_ISD_FTRUNC_MVT_v4f16_r(RetVT, Op0, Op0IsKill);
    3661           0 :   case MVT::v8f16: return fastEmit_ISD_FTRUNC_MVT_v8f16_r(RetVT, Op0, Op0IsKill);
    3662           0 :   case MVT::v2f32: return fastEmit_ISD_FTRUNC_MVT_v2f32_r(RetVT, Op0, Op0IsKill);
    3663           0 :   case MVT::v4f32: return fastEmit_ISD_FTRUNC_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3664           0 :   case MVT::v2f64: return fastEmit_ISD_FTRUNC_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3665             :   default: return 0;
    3666             :   }
    3667             : }
    3668             : 
    3669             : // FastEmit functions for ISD::SINT_TO_FP.
    3670             : 
    3671             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3672           1 :   if ((Subtarget->hasFullFP16())) {
    3673           0 :     return fastEmitInst_r(AArch64::SCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3674             :   }
    3675             :   return 0;
    3676             : }
    3677             : 
    3678             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3679           3 :   if ((Subtarget->hasFPARMv8())) {
    3680           3 :     return fastEmitInst_r(AArch64::SCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3681             :   }
    3682             :   return 0;
    3683             : }
    3684             : 
    3685             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3686           3 :   if ((Subtarget->hasFPARMv8())) {
    3687           3 :     return fastEmitInst_r(AArch64::SCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3688             :   }
    3689             :   return 0;
    3690             : }
    3691             : 
    3692           7 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3693           7 : switch (RetVT.SimpleTy) {
    3694           1 :   case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
    3695           3 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    3696           3 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    3697             :   default: return 0;
    3698             : }
    3699             : }
    3700             : 
    3701             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3702           1 :   if ((Subtarget->hasFullFP16())) {
    3703           0 :     return fastEmitInst_r(AArch64::SCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3704             :   }
    3705             :   return 0;
    3706             : }
    3707             : 
    3708             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3709           1 :   if ((Subtarget->hasFPARMv8())) {
    3710           1 :     return fastEmitInst_r(AArch64::SCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3711             :   }
    3712             :   return 0;
    3713             : }
    3714             : 
    3715             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3716           1 :   if ((Subtarget->hasFPARMv8())) {
    3717           1 :     return fastEmitInst_r(AArch64::SCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3718             :   }
    3719             :   return 0;
    3720             : }
    3721             : 
    3722           3 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3723           3 : switch (RetVT.SimpleTy) {
    3724           1 :   case MVT::f16: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
    3725           1 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    3726           1 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    3727             :   default: return 0;
    3728             : }
    3729             : }
    3730             : 
    3731           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3732           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3733             :     return 0;
    3734           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3735           0 :     return fastEmitInst_r(AArch64::SCVTFv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3736             :   }
    3737             :   return 0;
    3738             : }
    3739             : 
    3740           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3741           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3742             :     return 0;
    3743           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3744           0 :     return fastEmitInst_r(AArch64::SCVTFv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3745             :   }
    3746             :   return 0;
    3747             : }
    3748             : 
    3749             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3750           1 :   if (RetVT.SimpleTy != MVT::v2f32)
    3751             :     return 0;
    3752           0 :   if ((Subtarget->hasNEON())) {
    3753           0 :     return fastEmitInst_r(AArch64::SCVTFv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3754             :   }
    3755             :   return 0;
    3756             : }
    3757             : 
    3758             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3759           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3760             :     return 0;
    3761           0 :   if ((Subtarget->hasNEON())) {
    3762           0 :     return fastEmitInst_r(AArch64::SCVTFv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3763             :   }
    3764             :   return 0;
    3765             : }
    3766             : 
    3767             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3768           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3769             :     return 0;
    3770           0 :   if ((Subtarget->hasNEON())) {
    3771           0 :     return fastEmitInst_r(AArch64::SCVTFv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3772             :   }
    3773             :   return 0;
    3774             : }
    3775             : 
    3776          11 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3777          11 :   switch (VT.SimpleTy) {
    3778           7 :   case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3779           3 :   case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3780           0 :   case MVT::v4i16: return fastEmit_ISD_SINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    3781           0 :   case MVT::v8i16: return fastEmit_ISD_SINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3782           1 :   case MVT::v2i32: return fastEmit_ISD_SINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    3783           0 :   case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3784           0 :   case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3785             :   default: return 0;
    3786             :   }
    3787             : }
    3788             : 
    3789             : // FastEmit functions for ISD::TRUNCATE.
    3790             : 
    3791             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3792          12 :   if (RetVT.SimpleTy != MVT::i32)
    3793             :     return 0;
    3794          12 :   return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, AArch64::sub_32);
    3795             : }
    3796             : 
    3797             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3798           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    3799             :     return 0;
    3800           0 :   if ((Subtarget->hasNEON())) {
    3801           0 :     return fastEmitInst_r(AArch64::XTNv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3802             :   }
    3803             :   return 0;
    3804             : }
    3805             : 
    3806             : unsigned fastEmit_ISD_TRUNCATE_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3807           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    3808             :     return 0;
    3809           0 :   if ((Subtarget->hasNEON())) {
    3810           0 :     return fastEmitInst_r(AArch64::XTNv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3811             :   }
    3812             :   return 0;
    3813             : }
    3814             : 
    3815             : unsigned fastEmit_ISD_TRUNCATE_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3816           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    3817             :     return 0;
    3818           0 :   if ((Subtarget->hasNEON())) {
    3819           0 :     return fastEmitInst_r(AArch64::XTNv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3820             :   }
    3821             :   return 0;
    3822             : }
    3823             : 
    3824          12 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3825          12 :   switch (VT.SimpleTy) {
    3826          12 :   case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3827           0 :   case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3828           0 :   case MVT::v4i32: return fastEmit_ISD_TRUNCATE_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3829           0 :   case MVT::v2i64: return fastEmit_ISD_TRUNCATE_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3830             :   default: return 0;
    3831             :   }
    3832             : }
    3833             : 
    3834             : // FastEmit functions for ISD::UINT_TO_FP.
    3835             : 
    3836             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3837           0 :   if ((Subtarget->hasFullFP16())) {
    3838           0 :     return fastEmitInst_r(AArch64::UCVTFUWHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3839             :   }
    3840             :   return 0;
    3841             : }
    3842             : 
    3843             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3844           0 :   if ((Subtarget->hasFPARMv8())) {
    3845           0 :     return fastEmitInst_r(AArch64::UCVTFUWSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3846             :   }
    3847             :   return 0;
    3848             : }
    3849             : 
    3850             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3851           0 :   if ((Subtarget->hasFPARMv8())) {
    3852           0 :     return fastEmitInst_r(AArch64::UCVTFUWDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3853             :   }
    3854             :   return 0;
    3855             : }
    3856             : 
    3857           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3858           0 : switch (RetVT.SimpleTy) {
    3859           0 :   case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f16_r(Op0, Op0IsKill);
    3860           0 :   case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    3861           0 :   case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    3862             :   default: return 0;
    3863             : }
    3864             : }
    3865             : 
    3866             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(unsigned Op0, bool Op0IsKill) {
    3867           0 :   if ((Subtarget->hasFullFP16())) {
    3868           0 :     return fastEmitInst_r(AArch64::UCVTFUXHri, &AArch64::FPR16RegClass, Op0, Op0IsKill);
    3869             :   }
    3870             :   return 0;
    3871             : }
    3872             : 
    3873             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    3874           0 :   if ((Subtarget->hasFPARMv8())) {
    3875           0 :     return fastEmitInst_r(AArch64::UCVTFUXSri, &AArch64::FPR32RegClass, Op0, Op0IsKill);
    3876             :   }
    3877             :   return 0;
    3878             : }
    3879             : 
    3880             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    3881           0 :   if ((Subtarget->hasFPARMv8())) {
    3882           0 :     return fastEmitInst_r(AArch64::UCVTFUXDri, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3883             :   }
    3884             :   return 0;
    3885             : }
    3886             : 
    3887           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3888           0 : switch (RetVT.SimpleTy) {
    3889           0 :   case MVT::f16: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f16_r(Op0, Op0IsKill);
    3890           0 :   case MVT::f32: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    3891           0 :   case MVT::f64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    3892             :   default: return 0;
    3893             : }
    3894             : }
    3895             : 
    3896           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3897           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    3898             :     return 0;
    3899           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3900           0 :     return fastEmitInst_r(AArch64::UCVTFv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3901             :   }
    3902             :   return 0;
    3903             : }
    3904             : 
    3905           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3906           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    3907             :     return 0;
    3908           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    3909           0 :     return fastEmitInst_r(AArch64::UCVTFv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3910             :   }
    3911             :   return 0;
    3912             : }
    3913             : 
    3914             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3915           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    3916             :     return 0;
    3917           0 :   if ((Subtarget->hasNEON())) {
    3918           0 :     return fastEmitInst_r(AArch64::UCVTFv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill);
    3919             :   }
    3920             :   return 0;
    3921             : }
    3922             : 
    3923             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3924           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3925             :     return 0;
    3926           0 :   if ((Subtarget->hasNEON())) {
    3927           0 :     return fastEmitInst_r(AArch64::UCVTFv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3928             :   }
    3929             :   return 0;
    3930             : }
    3931             : 
    3932             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3933           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3934             :     return 0;
    3935           0 :   if ((Subtarget->hasNEON())) {
    3936           0 :     return fastEmitInst_r(AArch64::UCVTFv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill);
    3937             :   }
    3938             :   return 0;
    3939             : }
    3940             : 
    3941           0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3942           0 :   switch (VT.SimpleTy) {
    3943           0 :   case MVT::i32: return fastEmit_ISD_UINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3944           0 :   case MVT::i64: return fastEmit_ISD_UINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3945           0 :   case MVT::v4i16: return fastEmit_ISD_UINT_TO_FP_MVT_v4i16_r(RetVT, Op0, Op0IsKill);
    3946           0 :   case MVT::v8i16: return fastEmit_ISD_UINT_TO_FP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3947           0 :   case MVT::v2i32: return fastEmit_ISD_UINT_TO_FP_MVT_v2i32_r(RetVT, Op0, Op0IsKill);
    3948           0 :   case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3949           0 :   case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3950             :   default: return 0;
    3951             :   }
    3952             : }
    3953             : 
    3954             : // Top-level FastEmit function.
    3955             : 
    3956          65 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
    3957          65 :   switch (Opcode) {
    3958           0 :   case AArch64ISD::CALL: return fastEmit_AArch64ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
    3959           0 :   case AArch64ISD::CMEQz: return fastEmit_AArch64ISD_CMEQz_r(VT, RetVT, Op0, Op0IsKill);
    3960           0 :   case AArch64ISD::CMGEz: return fastEmit_AArch64ISD_CMGEz_r(VT, RetVT, Op0, Op0IsKill);
    3961           0 :   case AArch64ISD::CMGTz: return fastEmit_AArch64ISD_CMGTz_r(VT, RetVT, Op0, Op0IsKill);
    3962           0 :   case AArch64ISD::CMLEz: return fastEmit_AArch64ISD_CMLEz_r(VT, RetVT, Op0, Op0IsKill);
    3963           0 :   case AArch64ISD::CMLTz: return fastEmit_AArch64ISD_CMLTz_r(VT, RetVT, Op0, Op0IsKill);
    3964           0 :   case AArch64ISD::DUP: return fastEmit_AArch64ISD_DUP_r(VT, RetVT, Op0, Op0IsKill);
    3965           0 :   case AArch64ISD::FCMEQz: return fastEmit_AArch64ISD_FCMEQz_r(VT, RetVT, Op0, Op0IsKill);
    3966           0 :   case AArch64ISD::FCMGEz: return fastEmit_AArch64ISD_FCMGEz_r(VT, RetVT, Op0, Op0IsKill);
    3967           0 :   case AArch64ISD::FCMGTz: return fastEmit_AArch64ISD_FCMGTz_r(VT, RetVT, Op0, Op0IsKill);
    3968           0 :   case AArch64ISD::FCMLEz: return fastEmit_AArch64ISD_FCMLEz_r(VT, RetVT, Op0, Op0IsKill);
    3969           0 :   case AArch64ISD::FCMLTz: return fastEmit_AArch64ISD_FCMLTz_r(VT, RetVT, Op0, Op0IsKill);
    3970           0 :   case AArch64ISD::FRECPE: return fastEmit_AArch64ISD_FRECPE_r(VT, RetVT, Op0, Op0IsKill);
    3971           0 :   case AArch64ISD::FRSQRTE: return fastEmit_AArch64ISD_FRSQRTE_r(VT, RetVT, Op0, Op0IsKill);
    3972           0 :   case AArch64ISD::NEG: return fastEmit_AArch64ISD_NEG_r(VT, RetVT, Op0, Op0IsKill);
    3973           0 :   case AArch64ISD::NOT: return fastEmit_AArch64ISD_NOT_r(VT, RetVT, Op0, Op0IsKill);
    3974           0 :   case AArch64ISD::REV16: return fastEmit_AArch64ISD_REV16_r(VT, RetVT, Op0, Op0IsKill);
    3975           0 :   case AArch64ISD::REV32: return fastEmit_AArch64ISD_REV32_r(VT, RetVT, Op0, Op0IsKill);
    3976           0 :   case AArch64ISD::REV64: return fastEmit_AArch64ISD_REV64_r(VT, RetVT, Op0, Op0IsKill);
    3977           0 :   case AArch64ISD::SITOF: return fastEmit_AArch64ISD_SITOF_r(VT, RetVT, Op0, Op0IsKill);
    3978           0 :   case AArch64ISD::UITOF: return fastEmit_AArch64ISD_UITOF_r(VT, RetVT, Op0, Op0IsKill);
    3979           0 :   case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
    3980          36 :   case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
    3981           0 :   case ISD::BITREVERSE: return fastEmit_ISD_BITREVERSE_r(VT, RetVT, Op0, Op0IsKill);
    3982           0 :   case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    3983           0 :   case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
    3984           0 :   case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
    3985           0 :   case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
    3986           0 :   case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
    3987           0 :   case ISD::FCEIL: return fastEmit_ISD_FCEIL_r(VT, RetVT, Op0, Op0IsKill);
    3988           0 :   case ISD::FFLOOR: return fastEmit_ISD_FFLOOR_r(VT, RetVT, Op0, Op0IsKill);
    3989           0 :   case ISD::FNEARBYINT: return fastEmit_ISD_FNEARBYINT_r(VT, RetVT, Op0, Op0IsKill);
    3990           0 :   case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
    3991           0 :   case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    3992           0 :   case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
    3993           4 :   case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
    3994           0 :   case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
    3995           0 :   case ISD::FRINT: return fastEmit_ISD_FRINT_r(VT, RetVT, Op0, Op0IsKill);
    3996           0 :   case ISD::FROUND: return fastEmit_ISD_FROUND_r(VT, RetVT, Op0, Op0IsKill);
    3997           2 :   case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
    3998           0 :   case ISD::FTRUNC: return fastEmit_ISD_FTRUNC_r(VT, RetVT, Op0, Op0IsKill);
    3999          11 :   case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    4000          12 :   case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
    4001           0 :   case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    4002             :   default: return 0;
    4003             :   }
    4004             : }
    4005             : 
    4006             : // FastEmit functions for AArch64ISD::CMEQ.
    4007             : 
    4008             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4009           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4010             :     return 0;
    4011           0 :   if ((Subtarget->hasNEON())) {
    4012           0 :     return fastEmitInst_rr(AArch64::CMEQv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4013             :   }
    4014             :   return 0;
    4015             : }
    4016             : 
    4017             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4018           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4019             :     return 0;
    4020           0 :   if ((Subtarget->hasNEON())) {
    4021           0 :     return fastEmitInst_rr(AArch64::CMEQv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4022             :   }
    4023             :   return 0;
    4024             : }
    4025             : 
    4026             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4027           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4028             :     return 0;
    4029           0 :   if ((Subtarget->hasNEON())) {
    4030           0 :     return fastEmitInst_rr(AArch64::CMEQv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4031             :   }
    4032             :   return 0;
    4033             : }
    4034             : 
    4035             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4036           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4037             :     return 0;
    4038           0 :   if ((Subtarget->hasNEON())) {
    4039           0 :     return fastEmitInst_rr(AArch64::CMEQv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4040             :   }
    4041             :   return 0;
    4042             : }
    4043             : 
    4044             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4045           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4046             :     return 0;
    4047           0 :   if ((Subtarget->hasNEON())) {
    4048           0 :     return fastEmitInst_rr(AArch64::CMEQv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4049             :   }
    4050             :   return 0;
    4051             : }
    4052             : 
    4053             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4054           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4055             :     return 0;
    4056           0 :   if ((Subtarget->hasNEON())) {
    4057           0 :     return fastEmitInst_rr(AArch64::CMEQv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4058             :   }
    4059             :   return 0;
    4060             : }
    4061             : 
    4062             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4063           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4064             :     return 0;
    4065           0 :   if ((Subtarget->hasNEON())) {
    4066           0 :     return fastEmitInst_rr(AArch64::CMEQv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4067             :   }
    4068             :   return 0;
    4069             : }
    4070             : 
    4071             : unsigned fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4072           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4073             :     return 0;
    4074           0 :   if ((Subtarget->hasNEON())) {
    4075           0 :     return fastEmitInst_rr(AArch64::CMEQv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4076             :   }
    4077             :   return 0;
    4078             : }
    4079             : 
    4080           0 : unsigned fastEmit_AArch64ISD_CMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4081           0 :   switch (VT.SimpleTy) {
    4082           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMEQ_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4083           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMEQ_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4084           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMEQ_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4085           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMEQ_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4086           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMEQ_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4087           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMEQ_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4088           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMEQ_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4089           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMEQ_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4090             :   default: return 0;
    4091             :   }
    4092             : }
    4093             : 
    4094             : // FastEmit functions for AArch64ISD::CMGE.
    4095             : 
    4096             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4097           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4098             :     return 0;
    4099           0 :   if ((Subtarget->hasNEON())) {
    4100           0 :     return fastEmitInst_rr(AArch64::CMGEv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4101             :   }
    4102             :   return 0;
    4103             : }
    4104             : 
    4105             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4106           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4107             :     return 0;
    4108           0 :   if ((Subtarget->hasNEON())) {
    4109           0 :     return fastEmitInst_rr(AArch64::CMGEv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4110             :   }
    4111             :   return 0;
    4112             : }
    4113             : 
    4114             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4115           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4116             :     return 0;
    4117           0 :   if ((Subtarget->hasNEON())) {
    4118           0 :     return fastEmitInst_rr(AArch64::CMGEv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4119             :   }
    4120             :   return 0;
    4121             : }
    4122             : 
    4123             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4124           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4125             :     return 0;
    4126           0 :   if ((Subtarget->hasNEON())) {
    4127           0 :     return fastEmitInst_rr(AArch64::CMGEv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4128             :   }
    4129             :   return 0;
    4130             : }
    4131             : 
    4132             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4133           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4134             :     return 0;
    4135           0 :   if ((Subtarget->hasNEON())) {
    4136           0 :     return fastEmitInst_rr(AArch64::CMGEv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4137             :   }
    4138             :   return 0;
    4139             : }
    4140             : 
    4141             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4142           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4143             :     return 0;
    4144           0 :   if ((Subtarget->hasNEON())) {
    4145           0 :     return fastEmitInst_rr(AArch64::CMGEv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4146             :   }
    4147             :   return 0;
    4148             : }
    4149             : 
    4150             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4151           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4152             :     return 0;
    4153           0 :   if ((Subtarget->hasNEON())) {
    4154           0 :     return fastEmitInst_rr(AArch64::CMGEv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4155             :   }
    4156             :   return 0;
    4157             : }
    4158             : 
    4159             : unsigned fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4160           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4161             :     return 0;
    4162           0 :   if ((Subtarget->hasNEON())) {
    4163           0 :     return fastEmitInst_rr(AArch64::CMGEv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4164             :   }
    4165             :   return 0;
    4166             : }
    4167             : 
    4168           0 : unsigned fastEmit_AArch64ISD_CMGE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4169           0 :   switch (VT.SimpleTy) {
    4170           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGE_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4171           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGE_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4172           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGE_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4173           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGE_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4174           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGE_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4175           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGE_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4176           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGE_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4177           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGE_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4178             :   default: return 0;
    4179             :   }
    4180             : }
    4181             : 
    4182             : // FastEmit functions for AArch64ISD::CMGT.
    4183             : 
    4184             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4185           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4186             :     return 0;
    4187           0 :   if ((Subtarget->hasNEON())) {
    4188           0 :     return fastEmitInst_rr(AArch64::CMGTv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4189             :   }
    4190             :   return 0;
    4191             : }
    4192             : 
    4193             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4194           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4195             :     return 0;
    4196           0 :   if ((Subtarget->hasNEON())) {
    4197           0 :     return fastEmitInst_rr(AArch64::CMGTv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4198             :   }
    4199             :   return 0;
    4200             : }
    4201             : 
    4202             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4203           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4204             :     return 0;
    4205           0 :   if ((Subtarget->hasNEON())) {
    4206           0 :     return fastEmitInst_rr(AArch64::CMGTv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4207             :   }
    4208             :   return 0;
    4209             : }
    4210             : 
    4211             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4212           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4213             :     return 0;
    4214           0 :   if ((Subtarget->hasNEON())) {
    4215           0 :     return fastEmitInst_rr(AArch64::CMGTv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4216             :   }
    4217             :   return 0;
    4218             : }
    4219             : 
    4220             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4221           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4222             :     return 0;
    4223           0 :   if ((Subtarget->hasNEON())) {
    4224           0 :     return fastEmitInst_rr(AArch64::CMGTv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4225             :   }
    4226             :   return 0;
    4227             : }
    4228             : 
    4229             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4230           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4231             :     return 0;
    4232           0 :   if ((Subtarget->hasNEON())) {
    4233           0 :     return fastEmitInst_rr(AArch64::CMGTv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4234             :   }
    4235             :   return 0;
    4236             : }
    4237             : 
    4238             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4239           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4240             :     return 0;
    4241           0 :   if ((Subtarget->hasNEON())) {
    4242           0 :     return fastEmitInst_rr(AArch64::CMGTv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4243             :   }
    4244             :   return 0;
    4245             : }
    4246             : 
    4247             : unsigned fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4248           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4249             :     return 0;
    4250           0 :   if ((Subtarget->hasNEON())) {
    4251           0 :     return fastEmitInst_rr(AArch64::CMGTv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4252             :   }
    4253             :   return 0;
    4254             : }
    4255             : 
    4256           0 : unsigned fastEmit_AArch64ISD_CMGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4257           0 :   switch (VT.SimpleTy) {
    4258           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMGT_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4259           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMGT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4260           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMGT_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4261           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMGT_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4262           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMGT_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4263           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMGT_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4264           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMGT_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4265           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMGT_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4266             :   default: return 0;
    4267             :   }
    4268             : }
    4269             : 
    4270             : // FastEmit functions for AArch64ISD::CMHI.
    4271             : 
    4272             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4273           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4274             :     return 0;
    4275           0 :   if ((Subtarget->hasNEON())) {
    4276           0 :     return fastEmitInst_rr(AArch64::CMHIv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4277             :   }
    4278             :   return 0;
    4279             : }
    4280             : 
    4281             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4282           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4283             :     return 0;
    4284           0 :   if ((Subtarget->hasNEON())) {
    4285           0 :     return fastEmitInst_rr(AArch64::CMHIv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4286             :   }
    4287             :   return 0;
    4288             : }
    4289             : 
    4290             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4291           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4292             :     return 0;
    4293           0 :   if ((Subtarget->hasNEON())) {
    4294           0 :     return fastEmitInst_rr(AArch64::CMHIv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4295             :   }
    4296             :   return 0;
    4297             : }
    4298             : 
    4299             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4300           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4301             :     return 0;
    4302           0 :   if ((Subtarget->hasNEON())) {
    4303           0 :     return fastEmitInst_rr(AArch64::CMHIv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4304             :   }
    4305             :   return 0;
    4306             : }
    4307             : 
    4308             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4309           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4310             :     return 0;
    4311           0 :   if ((Subtarget->hasNEON())) {
    4312           0 :     return fastEmitInst_rr(AArch64::CMHIv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4313             :   }
    4314             :   return 0;
    4315             : }
    4316             : 
    4317             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4318           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4319             :     return 0;
    4320           0 :   if ((Subtarget->hasNEON())) {
    4321           0 :     return fastEmitInst_rr(AArch64::CMHIv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4322             :   }
    4323             :   return 0;
    4324             : }
    4325             : 
    4326             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4327           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4328             :     return 0;
    4329           0 :   if ((Subtarget->hasNEON())) {
    4330           0 :     return fastEmitInst_rr(AArch64::CMHIv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4331             :   }
    4332             :   return 0;
    4333             : }
    4334             : 
    4335             : unsigned fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4336           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4337             :     return 0;
    4338           0 :   if ((Subtarget->hasNEON())) {
    4339           0 :     return fastEmitInst_rr(AArch64::CMHIv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4340             :   }
    4341             :   return 0;
    4342             : }
    4343             : 
    4344           0 : unsigned fastEmit_AArch64ISD_CMHI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4345           0 :   switch (VT.SimpleTy) {
    4346           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMHI_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4347           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMHI_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4348           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMHI_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4349           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMHI_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4350           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMHI_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4351           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMHI_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4352           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMHI_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4353           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMHI_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4354             :   default: return 0;
    4355             :   }
    4356             : }
    4357             : 
    4358             : // FastEmit functions for AArch64ISD::CMHS.
    4359             : 
    4360             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4361           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4362             :     return 0;
    4363           0 :   if ((Subtarget->hasNEON())) {
    4364           0 :     return fastEmitInst_rr(AArch64::CMHSv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4365             :   }
    4366             :   return 0;
    4367             : }
    4368             : 
    4369             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4370           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4371             :     return 0;
    4372           0 :   if ((Subtarget->hasNEON())) {
    4373           0 :     return fastEmitInst_rr(AArch64::CMHSv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4374             :   }
    4375             :   return 0;
    4376             : }
    4377             : 
    4378             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4379           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4380             :     return 0;
    4381           0 :   if ((Subtarget->hasNEON())) {
    4382           0 :     return fastEmitInst_rr(AArch64::CMHSv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4383             :   }
    4384             :   return 0;
    4385             : }
    4386             : 
    4387             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4388           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4389             :     return 0;
    4390           0 :   if ((Subtarget->hasNEON())) {
    4391           0 :     return fastEmitInst_rr(AArch64::CMHSv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4392             :   }
    4393             :   return 0;
    4394             : }
    4395             : 
    4396             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4397           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4398             :     return 0;
    4399           0 :   if ((Subtarget->hasNEON())) {
    4400           0 :     return fastEmitInst_rr(AArch64::CMHSv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4401             :   }
    4402             :   return 0;
    4403             : }
    4404             : 
    4405             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4406           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4407             :     return 0;
    4408           0 :   if ((Subtarget->hasNEON())) {
    4409           0 :     return fastEmitInst_rr(AArch64::CMHSv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4410             :   }
    4411             :   return 0;
    4412             : }
    4413             : 
    4414             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4415           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4416             :     return 0;
    4417           0 :   if ((Subtarget->hasNEON())) {
    4418           0 :     return fastEmitInst_rr(AArch64::CMHSv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4419             :   }
    4420             :   return 0;
    4421             : }
    4422             : 
    4423             : unsigned fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4424           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4425             :     return 0;
    4426           0 :   if ((Subtarget->hasNEON())) {
    4427           0 :     return fastEmitInst_rr(AArch64::CMHSv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4428             :   }
    4429             :   return 0;
    4430             : }
    4431             : 
    4432           0 : unsigned fastEmit_AArch64ISD_CMHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4433           0 :   switch (VT.SimpleTy) {
    4434           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_CMHS_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4435           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_CMHS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4436           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_CMHS_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4437           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_CMHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4438           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_CMHS_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4439           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_CMHS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4440           0 :   case MVT::v1i64: return fastEmit_AArch64ISD_CMHS_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4441           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_CMHS_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4442             :   default: return 0;
    4443             :   }
    4444             : }
    4445             : 
    4446             : // FastEmit functions for AArch64ISD::FCMEQ.
    4447             : 
    4448             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4449           0 :   if (RetVT.SimpleTy != MVT::i32)
    4450             :     return 0;
    4451           0 :   if ((Subtarget->hasNEON())) {
    4452           0 :     return fastEmitInst_rr(AArch64::FCMEQ32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4453             :   }
    4454             :   return 0;
    4455             : }
    4456             : 
    4457             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4458           0 :   if (RetVT.SimpleTy != MVT::i64)
    4459             :     return 0;
    4460           0 :   if ((Subtarget->hasNEON())) {
    4461           0 :     return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4462             :   }
    4463             :   return 0;
    4464             : }
    4465             : 
    4466           0 : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4467           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4468             :     return 0;
    4469           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4470           0 :     return fastEmitInst_rr(AArch64::FCMEQv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4471             :   }
    4472             :   return 0;
    4473             : }
    4474             : 
    4475           0 : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4476           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4477             :     return 0;
    4478           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4479           0 :     return fastEmitInst_rr(AArch64::FCMEQv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4480             :   }
    4481             :   return 0;
    4482             : }
    4483             : 
    4484             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4485           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4486             :     return 0;
    4487           0 :   if ((Subtarget->hasNEON())) {
    4488           0 :     return fastEmitInst_rr(AArch64::FCMEQv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4489             :   }
    4490             :   return 0;
    4491             : }
    4492             : 
    4493             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4494           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4495             :     return 0;
    4496           0 :   if ((Subtarget->hasNEON())) {
    4497           0 :     return fastEmitInst_rr(AArch64::FCMEQv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4498             :   }
    4499             :   return 0;
    4500             : }
    4501             : 
    4502             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4503           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4504             :     return 0;
    4505           0 :   if ((Subtarget->hasNEON())) {
    4506           0 :     return fastEmitInst_rr(AArch64::FCMEQ64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4507             :   }
    4508             :   return 0;
    4509             : }
    4510             : 
    4511             : unsigned fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4512           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4513             :     return 0;
    4514           0 :   if ((Subtarget->hasNEON())) {
    4515           0 :     return fastEmitInst_rr(AArch64::FCMEQv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4516             :   }
    4517             :   return 0;
    4518             : }
    4519             : 
    4520           0 : unsigned fastEmit_AArch64ISD_FCMEQ_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4521           0 :   switch (VT.SimpleTy) {
    4522           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMEQ_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4523           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMEQ_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4524           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4525           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMEQ_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4526           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4527           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMEQ_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4528           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4529           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMEQ_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4530             :   default: return 0;
    4531             :   }
    4532             : }
    4533             : 
    4534             : // FastEmit functions for AArch64ISD::FCMGE.
    4535             : 
    4536             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4537           0 :   if (RetVT.SimpleTy != MVT::i32)
    4538             :     return 0;
    4539           0 :   if ((Subtarget->hasNEON())) {
    4540           0 :     return fastEmitInst_rr(AArch64::FCMGE32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4541             :   }
    4542             :   return 0;
    4543             : }
    4544             : 
    4545             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4546           0 :   if (RetVT.SimpleTy != MVT::i64)
    4547             :     return 0;
    4548           0 :   if ((Subtarget->hasNEON())) {
    4549           0 :     return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4550             :   }
    4551             :   return 0;
    4552             : }
    4553             : 
    4554           0 : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4555           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4556             :     return 0;
    4557           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4558           0 :     return fastEmitInst_rr(AArch64::FCMGEv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4559             :   }
    4560             :   return 0;
    4561             : }
    4562             : 
    4563           0 : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4564           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4565             :     return 0;
    4566           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4567           0 :     return fastEmitInst_rr(AArch64::FCMGEv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4568             :   }
    4569             :   return 0;
    4570             : }
    4571             : 
    4572             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4573           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4574             :     return 0;
    4575           0 :   if ((Subtarget->hasNEON())) {
    4576           0 :     return fastEmitInst_rr(AArch64::FCMGEv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4577             :   }
    4578             :   return 0;
    4579             : }
    4580             : 
    4581             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4582           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4583             :     return 0;
    4584           0 :   if ((Subtarget->hasNEON())) {
    4585           0 :     return fastEmitInst_rr(AArch64::FCMGEv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4586             :   }
    4587             :   return 0;
    4588             : }
    4589             : 
    4590             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4591           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4592             :     return 0;
    4593           0 :   if ((Subtarget->hasNEON())) {
    4594           0 :     return fastEmitInst_rr(AArch64::FCMGE64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4595             :   }
    4596             :   return 0;
    4597             : }
    4598             : 
    4599             : unsigned fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4600           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4601             :     return 0;
    4602           0 :   if ((Subtarget->hasNEON())) {
    4603           0 :     return fastEmitInst_rr(AArch64::FCMGEv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4604             :   }
    4605             :   return 0;
    4606             : }
    4607             : 
    4608           0 : unsigned fastEmit_AArch64ISD_FCMGE_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4609           0 :   switch (VT.SimpleTy) {
    4610           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMGE_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4611           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMGE_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4612           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGE_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4613           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGE_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4614           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGE_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4615           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGE_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4616           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGE_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4617           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGE_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4618             :   default: return 0;
    4619             :   }
    4620             : }
    4621             : 
    4622             : // FastEmit functions for AArch64ISD::FCMGT.
    4623             : 
    4624             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4625           0 :   if (RetVT.SimpleTy != MVT::i32)
    4626             :     return 0;
    4627           0 :   if ((Subtarget->hasNEON())) {
    4628           0 :     return fastEmitInst_rr(AArch64::FCMGT32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4629             :   }
    4630             :   return 0;
    4631             : }
    4632             : 
    4633             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4634           0 :   if (RetVT.SimpleTy != MVT::i64)
    4635             :     return 0;
    4636           0 :   if ((Subtarget->hasNEON())) {
    4637           0 :     return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4638             :   }
    4639             :   return 0;
    4640             : }
    4641             : 
    4642           0 : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4643           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4644             :     return 0;
    4645           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4646           0 :     return fastEmitInst_rr(AArch64::FCMGTv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4647             :   }
    4648             :   return 0;
    4649             : }
    4650             : 
    4651           0 : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4652           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4653             :     return 0;
    4654           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    4655           0 :     return fastEmitInst_rr(AArch64::FCMGTv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4656             :   }
    4657             :   return 0;
    4658             : }
    4659             : 
    4660             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4661           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4662             :     return 0;
    4663           0 :   if ((Subtarget->hasNEON())) {
    4664           0 :     return fastEmitInst_rr(AArch64::FCMGTv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4665             :   }
    4666             :   return 0;
    4667             : }
    4668             : 
    4669             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4670           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4671             :     return 0;
    4672           0 :   if ((Subtarget->hasNEON())) {
    4673           0 :     return fastEmitInst_rr(AArch64::FCMGTv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4674             :   }
    4675             :   return 0;
    4676             : }
    4677             : 
    4678             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4679           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    4680             :     return 0;
    4681           0 :   if ((Subtarget->hasNEON())) {
    4682           0 :     return fastEmitInst_rr(AArch64::FCMGT64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4683             :   }
    4684             :   return 0;
    4685             : }
    4686             : 
    4687             : unsigned fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4688           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4689             :     return 0;
    4690           0 :   if ((Subtarget->hasNEON())) {
    4691           0 :     return fastEmitInst_rr(AArch64::FCMGTv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4692             :   }
    4693             :   return 0;
    4694             : }
    4695             : 
    4696           0 : unsigned fastEmit_AArch64ISD_FCMGT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4697           0 :   switch (VT.SimpleTy) {
    4698           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMGT_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4699           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMGT_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4700           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FCMGT_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4701           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FCMGT_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4702           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FCMGT_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4703           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FCMGT_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4704           0 :   case MVT::v1f64: return fastEmit_AArch64ISD_FCMGT_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4705           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FCMGT_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4706             :   default: return 0;
    4707             :   }
    4708             : }
    4709             : 
    4710             : // FastEmit functions for AArch64ISD::FCMP.
    4711             : 
    4712             : unsigned fastEmit_AArch64ISD_FCMP_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4713           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4714             :     return 0;
    4715           0 :   if ((Subtarget->hasFullFP16())) {
    4716           0 :     return fastEmitInst_rr(AArch64::FCMPHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4717             :   }
    4718             :   return 0;
    4719             : }
    4720             : 
    4721             : unsigned fastEmit_AArch64ISD_FCMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4722           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4723             :     return 0;
    4724           0 :   if ((Subtarget->hasFPARMv8())) {
    4725           0 :     return fastEmitInst_rr(AArch64::FCMPSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4726             :   }
    4727             :   return 0;
    4728             : }
    4729             : 
    4730             : unsigned fastEmit_AArch64ISD_FCMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4731           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4732             :     return 0;
    4733           0 :   if ((Subtarget->hasFPARMv8())) {
    4734           0 :     return fastEmitInst_rr(AArch64::FCMPDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4735             :   }
    4736             :   return 0;
    4737             : }
    4738             : 
    4739           0 : unsigned fastEmit_AArch64ISD_FCMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4740           0 :   switch (VT.SimpleTy) {
    4741           0 :   case MVT::f16: return fastEmit_AArch64ISD_FCMP_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4742           0 :   case MVT::f32: return fastEmit_AArch64ISD_FCMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4743           0 :   case MVT::f64: return fastEmit_AArch64ISD_FCMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4744             :   default: return 0;
    4745             :   }
    4746             : }
    4747             : 
    4748             : // FastEmit functions for AArch64ISD::FRECPS.
    4749             : 
    4750             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4751           0 :   if (RetVT.SimpleTy != MVT::f32)
    4752             :     return 0;
    4753           0 :   return fastEmitInst_rr(AArch64::FRECPS32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4754             : }
    4755             : 
    4756             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4757           0 :   if (RetVT.SimpleTy != MVT::f64)
    4758             :     return 0;
    4759           0 :   return fastEmitInst_rr(AArch64::FRECPS64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4760             : }
    4761             : 
    4762             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4763           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    4764             :     return 0;
    4765           0 :   return fastEmitInst_rr(AArch64::FRECPSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4766             : }
    4767             : 
    4768             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4769           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4770             :     return 0;
    4771           0 :   return fastEmitInst_rr(AArch64::FRECPSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4772             : }
    4773             : 
    4774             : unsigned fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4775           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4776             :     return 0;
    4777           0 :   return fastEmitInst_rr(AArch64::FRECPSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4778             : }
    4779             : 
    4780           0 : unsigned fastEmit_AArch64ISD_FRECPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4781           0 :   switch (VT.SimpleTy) {
    4782           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRECPS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4783           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRECPS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4784           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRECPS_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4785           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRECPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4786           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRECPS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4787             :   default: return 0;
    4788             :   }
    4789             : }
    4790             : 
    4791             : // FastEmit functions for AArch64ISD::FRSQRTS.
    4792             : 
    4793             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4794           0 :   if (RetVT.SimpleTy != MVT::f32)
    4795             :     return 0;
    4796           0 :   return fastEmitInst_rr(AArch64::FRSQRTS32, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4797             : }
    4798             : 
    4799             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4800           0 :   if (RetVT.SimpleTy != MVT::f64)
    4801             :     return 0;
    4802           0 :   return fastEmitInst_rr(AArch64::FRSQRTS64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4803             : }
    4804             : 
    4805             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4806           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    4807             :     return 0;
    4808           0 :   return fastEmitInst_rr(AArch64::FRSQRTSv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4809             : }
    4810             : 
    4811             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4812           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4813             :     return 0;
    4814           0 :   return fastEmitInst_rr(AArch64::FRSQRTSv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4815             : }
    4816             : 
    4817             : unsigned fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4818           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4819             :     return 0;
    4820           0 :   return fastEmitInst_rr(AArch64::FRSQRTSv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4821             : }
    4822             : 
    4823           0 : unsigned fastEmit_AArch64ISD_FRSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4824           0 :   switch (VT.SimpleTy) {
    4825           0 :   case MVT::f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4826           0 :   case MVT::f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4827           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4828           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4829           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4830             :   default: return 0;
    4831             :   }
    4832             : }
    4833             : 
    4834             : // FastEmit functions for AArch64ISD::SMULL.
    4835             : 
    4836             : unsigned fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4837           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4838             :     return 0;
    4839           0 :   return fastEmitInst_rr(AArch64::SMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4840             : }
    4841             : 
    4842             : unsigned fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4843           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4844             :     return 0;
    4845           0 :   return fastEmitInst_rr(AArch64::SMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4846             : }
    4847             : 
    4848             : unsigned fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4849           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4850             :     return 0;
    4851           0 :   return fastEmitInst_rr(AArch64::SMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4852             : }
    4853             : 
    4854           0 : unsigned fastEmit_AArch64ISD_SMULL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4855           0 :   switch (VT.SimpleTy) {
    4856           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_SMULL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4857           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_SMULL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4858           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_SMULL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4859             :   default: return 0;
    4860             :   }
    4861             : }
    4862             : 
    4863             : // FastEmit functions for AArch64ISD::TRN1.
    4864             : 
    4865             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4866           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4867             :     return 0;
    4868           0 :   if ((Subtarget->hasNEON())) {
    4869           0 :     return fastEmitInst_rr(AArch64::TRN1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4870             :   }
    4871             :   return 0;
    4872             : }
    4873             : 
    4874             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4875           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4876             :     return 0;
    4877           0 :   if ((Subtarget->hasNEON())) {
    4878           0 :     return fastEmitInst_rr(AArch64::TRN1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4879             :   }
    4880             :   return 0;
    4881             : }
    4882             : 
    4883             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4884           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    4885             :     return 0;
    4886           0 :   if ((Subtarget->hasNEON())) {
    4887           0 :     return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4888             :   }
    4889             :   return 0;
    4890             : }
    4891             : 
    4892             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4893           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4894             :     return 0;
    4895           0 :   if ((Subtarget->hasNEON())) {
    4896           0 :     return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4897             :   }
    4898             :   return 0;
    4899             : }
    4900             : 
    4901             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4902           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    4903             :     return 0;
    4904           0 :   if ((Subtarget->hasNEON())) {
    4905           0 :     return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4906             :   }
    4907             :   return 0;
    4908             : }
    4909             : 
    4910             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4911           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4912             :     return 0;
    4913           0 :   if ((Subtarget->hasNEON())) {
    4914           0 :     return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4915             :   }
    4916             :   return 0;
    4917             : }
    4918             : 
    4919             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4920           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4921             :     return 0;
    4922           0 :   if ((Subtarget->hasNEON())) {
    4923           0 :     return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4924             :   }
    4925             :   return 0;
    4926             : }
    4927             : 
    4928             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4929           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    4930             :     return 0;
    4931           0 :   if ((Subtarget->hasNEON())) {
    4932           0 :     return fastEmitInst_rr(AArch64::TRN1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4933             :   }
    4934             :   return 0;
    4935             : }
    4936             : 
    4937             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4938           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    4939             :     return 0;
    4940           0 :   if ((Subtarget->hasNEON())) {
    4941           0 :     return fastEmitInst_rr(AArch64::TRN1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4942             :   }
    4943             :   return 0;
    4944             : }
    4945             : 
    4946             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4947           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    4948             :     return 0;
    4949           0 :   if ((Subtarget->hasNEON())) {
    4950           0 :     return fastEmitInst_rr(AArch64::TRN1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4951             :   }
    4952             :   return 0;
    4953             : }
    4954             : 
    4955             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4956           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4957             :     return 0;
    4958           0 :   if ((Subtarget->hasNEON())) {
    4959           0 :     return fastEmitInst_rr(AArch64::TRN1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4960             :   }
    4961             :   return 0;
    4962             : }
    4963             : 
    4964             : unsigned fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4965           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4966             :     return 0;
    4967           0 :   if ((Subtarget->hasNEON())) {
    4968           0 :     return fastEmitInst_rr(AArch64::TRN1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4969             :   }
    4970             :   return 0;
    4971             : }
    4972             : 
    4973           0 : unsigned fastEmit_AArch64ISD_TRN1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4974           0 :   switch (VT.SimpleTy) {
    4975           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_TRN1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4976           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_TRN1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4977           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_TRN1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4978           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_TRN1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4979           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_TRN1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4980           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_TRN1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4981           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_TRN1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4982           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_TRN1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4983           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_TRN1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4984           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_TRN1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4985           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_TRN1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4986           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_TRN1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    4987             :   default: return 0;
    4988             :   }
    4989             : }
    4990             : 
    4991             : // FastEmit functions for AArch64ISD::TRN2.
    4992             : 
    4993             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    4994           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    4995             :     return 0;
    4996           0 :   if ((Subtarget->hasNEON())) {
    4997           0 :     return fastEmitInst_rr(AArch64::TRN2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    4998             :   }
    4999             :   return 0;
    5000             : }
    5001             : 
    5002             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5003           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5004             :     return 0;
    5005           0 :   if ((Subtarget->hasNEON())) {
    5006           0 :     return fastEmitInst_rr(AArch64::TRN2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5007             :   }
    5008             :   return 0;
    5009             : }
    5010             : 
    5011             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5012           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5013             :     return 0;
    5014           0 :   if ((Subtarget->hasNEON())) {
    5015           0 :     return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5016             :   }
    5017             :   return 0;
    5018             : }
    5019             : 
    5020             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5021           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5022             :     return 0;
    5023           0 :   if ((Subtarget->hasNEON())) {
    5024           0 :     return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5025             :   }
    5026             :   return 0;
    5027             : }
    5028             : 
    5029             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5030           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5031             :     return 0;
    5032           0 :   if ((Subtarget->hasNEON())) {
    5033           0 :     return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5034             :   }
    5035             :   return 0;
    5036             : }
    5037             : 
    5038             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5039           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5040             :     return 0;
    5041           0 :   if ((Subtarget->hasNEON())) {
    5042           0 :     return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5043             :   }
    5044             :   return 0;
    5045             : }
    5046             : 
    5047             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5048           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5049             :     return 0;
    5050           0 :   if ((Subtarget->hasNEON())) {
    5051           0 :     return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5052             :   }
    5053             :   return 0;
    5054             : }
    5055             : 
    5056             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5057           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5058             :     return 0;
    5059           0 :   if ((Subtarget->hasNEON())) {
    5060           0 :     return fastEmitInst_rr(AArch64::TRN2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5061             :   }
    5062             :   return 0;
    5063             : }
    5064             : 
    5065             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5066           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5067             :     return 0;
    5068           0 :   if ((Subtarget->hasNEON())) {
    5069           0 :     return fastEmitInst_rr(AArch64::TRN2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5070             :   }
    5071             :   return 0;
    5072             : }
    5073             : 
    5074             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5075           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5076             :     return 0;
    5077           0 :   if ((Subtarget->hasNEON())) {
    5078           0 :     return fastEmitInst_rr(AArch64::TRN2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5079             :   }
    5080             :   return 0;
    5081             : }
    5082             : 
    5083             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5084           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5085             :     return 0;
    5086           0 :   if ((Subtarget->hasNEON())) {
    5087           0 :     return fastEmitInst_rr(AArch64::TRN2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5088             :   }
    5089             :   return 0;
    5090             : }
    5091             : 
    5092             : unsigned fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5093           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5094             :     return 0;
    5095           0 :   if ((Subtarget->hasNEON())) {
    5096           0 :     return fastEmitInst_rr(AArch64::TRN2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5097             :   }
    5098             :   return 0;
    5099             : }
    5100             : 
    5101           0 : unsigned fastEmit_AArch64ISD_TRN2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5102           0 :   switch (VT.SimpleTy) {
    5103           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_TRN2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5104           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_TRN2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5105           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_TRN2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5106           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_TRN2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5107           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_TRN2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5108           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_TRN2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5109           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_TRN2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5110           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_TRN2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5111           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_TRN2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5112           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_TRN2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5113           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_TRN2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5114           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_TRN2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5115             :   default: return 0;
    5116             :   }
    5117             : }
    5118             : 
    5119             : // FastEmit functions for AArch64ISD::UMULL.
    5120             : 
    5121             : unsigned fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5122           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5123             :     return 0;
    5124           0 :   return fastEmitInst_rr(AArch64::UMULLv8i8_v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5125             : }
    5126             : 
    5127             : unsigned fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5128           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5129             :     return 0;
    5130           0 :   return fastEmitInst_rr(AArch64::UMULLv4i16_v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5131             : }
    5132             : 
    5133             : unsigned fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5134           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5135             :     return 0;
    5136           0 :   return fastEmitInst_rr(AArch64::UMULLv2i32_v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5137             : }
    5138             : 
    5139           0 : unsigned fastEmit_AArch64ISD_UMULL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5140           0 :   switch (VT.SimpleTy) {
    5141           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_UMULL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5142           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_UMULL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5143           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_UMULL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5144             :   default: return 0;
    5145             :   }
    5146             : }
    5147             : 
    5148             : // FastEmit functions for AArch64ISD::UZP1.
    5149             : 
    5150             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5151           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5152             :     return 0;
    5153           0 :   if ((Subtarget->hasNEON())) {
    5154           0 :     return fastEmitInst_rr(AArch64::UZP1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5155             :   }
    5156             :   return 0;
    5157             : }
    5158             : 
    5159             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5160           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5161             :     return 0;
    5162           0 :   if ((Subtarget->hasNEON())) {
    5163           0 :     return fastEmitInst_rr(AArch64::UZP1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5164             :   }
    5165             :   return 0;
    5166             : }
    5167             : 
    5168             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5169           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5170             :     return 0;
    5171           0 :   if ((Subtarget->hasNEON())) {
    5172           0 :     return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5173             :   }
    5174             :   return 0;
    5175             : }
    5176             : 
    5177             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5178           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5179             :     return 0;
    5180           0 :   if ((Subtarget->hasNEON())) {
    5181           0 :     return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5182             :   }
    5183             :   return 0;
    5184             : }
    5185             : 
    5186             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5187           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5188             :     return 0;
    5189           0 :   if ((Subtarget->hasNEON())) {
    5190           0 :     return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5191             :   }
    5192             :   return 0;
    5193             : }
    5194             : 
    5195             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5196           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5197             :     return 0;
    5198           0 :   if ((Subtarget->hasNEON())) {
    5199           0 :     return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5200             :   }
    5201             :   return 0;
    5202             : }
    5203             : 
    5204             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5205           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5206             :     return 0;
    5207           0 :   if ((Subtarget->hasNEON())) {
    5208           0 :     return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5209             :   }
    5210             :   return 0;
    5211             : }
    5212             : 
    5213             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5214           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5215             :     return 0;
    5216           0 :   if ((Subtarget->hasNEON())) {
    5217           0 :     return fastEmitInst_rr(AArch64::UZP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5218             :   }
    5219             :   return 0;
    5220             : }
    5221             : 
    5222             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5223           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5224             :     return 0;
    5225           0 :   if ((Subtarget->hasNEON())) {
    5226           0 :     return fastEmitInst_rr(AArch64::UZP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5227             :   }
    5228             :   return 0;
    5229             : }
    5230             : 
    5231             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5232           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5233             :     return 0;
    5234           0 :   if ((Subtarget->hasNEON())) {
    5235           0 :     return fastEmitInst_rr(AArch64::UZP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5236             :   }
    5237             :   return 0;
    5238             : }
    5239             : 
    5240             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5241           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5242             :     return 0;
    5243           0 :   if ((Subtarget->hasNEON())) {
    5244           0 :     return fastEmitInst_rr(AArch64::UZP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5245             :   }
    5246             :   return 0;
    5247             : }
    5248             : 
    5249             : unsigned fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5250           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5251             :     return 0;
    5252           0 :   if ((Subtarget->hasNEON())) {
    5253           0 :     return fastEmitInst_rr(AArch64::UZP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5254             :   }
    5255             :   return 0;
    5256             : }
    5257             : 
    5258           0 : unsigned fastEmit_AArch64ISD_UZP1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5259           0 :   switch (VT.SimpleTy) {
    5260           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_UZP1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5261           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_UZP1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5262           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_UZP1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5263           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_UZP1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5264           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_UZP1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5265           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_UZP1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5266           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_UZP1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5267           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_UZP1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5268           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_UZP1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5269           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_UZP1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5270           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_UZP1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5271           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_UZP1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5272             :   default: return 0;
    5273             :   }
    5274             : }
    5275             : 
    5276             : // FastEmit functions for AArch64ISD::UZP2.
    5277             : 
    5278             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5279           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5280             :     return 0;
    5281           0 :   if ((Subtarget->hasNEON())) {
    5282           0 :     return fastEmitInst_rr(AArch64::UZP2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5283             :   }
    5284             :   return 0;
    5285             : }
    5286             : 
    5287             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5288           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5289             :     return 0;
    5290           0 :   if ((Subtarget->hasNEON())) {
    5291           0 :     return fastEmitInst_rr(AArch64::UZP2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5292             :   }
    5293             :   return 0;
    5294             : }
    5295             : 
    5296             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5297           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5298             :     return 0;
    5299           0 :   if ((Subtarget->hasNEON())) {
    5300           0 :     return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5301             :   }
    5302             :   return 0;
    5303             : }
    5304             : 
    5305             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5306           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5307             :     return 0;
    5308           0 :   if ((Subtarget->hasNEON())) {
    5309           0 :     return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5310             :   }
    5311             :   return 0;
    5312             : }
    5313             : 
    5314             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5315           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5316             :     return 0;
    5317           0 :   if ((Subtarget->hasNEON())) {
    5318           0 :     return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5319             :   }
    5320             :   return 0;
    5321             : }
    5322             : 
    5323             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5324           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5325             :     return 0;
    5326           0 :   if ((Subtarget->hasNEON())) {
    5327           0 :     return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5328             :   }
    5329             :   return 0;
    5330             : }
    5331             : 
    5332             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5333           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5334             :     return 0;
    5335           0 :   if ((Subtarget->hasNEON())) {
    5336           0 :     return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5337             :   }
    5338             :   return 0;
    5339             : }
    5340             : 
    5341             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5342           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5343             :     return 0;
    5344           0 :   if ((Subtarget->hasNEON())) {
    5345           0 :     return fastEmitInst_rr(AArch64::UZP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5346             :   }
    5347             :   return 0;
    5348             : }
    5349             : 
    5350             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5351           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5352             :     return 0;
    5353           0 :   if ((Subtarget->hasNEON())) {
    5354           0 :     return fastEmitInst_rr(AArch64::UZP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5355             :   }
    5356             :   return 0;
    5357             : }
    5358             : 
    5359             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5360           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5361             :     return 0;
    5362           0 :   if ((Subtarget->hasNEON())) {
    5363           0 :     return fastEmitInst_rr(AArch64::UZP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5364             :   }
    5365             :   return 0;
    5366             : }
    5367             : 
    5368             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5369           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5370             :     return 0;
    5371           0 :   if ((Subtarget->hasNEON())) {
    5372           0 :     return fastEmitInst_rr(AArch64::UZP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5373             :   }
    5374             :   return 0;
    5375             : }
    5376             : 
    5377             : unsigned fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5378           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5379             :     return 0;
    5380           0 :   if ((Subtarget->hasNEON())) {
    5381           0 :     return fastEmitInst_rr(AArch64::UZP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5382             :   }
    5383             :   return 0;
    5384             : }
    5385             : 
    5386           0 : unsigned fastEmit_AArch64ISD_UZP2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5387           0 :   switch (VT.SimpleTy) {
    5388           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_UZP2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5389           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_UZP2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5390           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_UZP2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5391           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_UZP2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5392           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_UZP2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5393           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_UZP2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5394           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_UZP2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5395           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_UZP2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5396           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_UZP2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5397           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_UZP2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5398           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_UZP2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5399           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_UZP2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5400             :   default: return 0;
    5401             :   }
    5402             : }
    5403             : 
    5404             : // FastEmit functions for AArch64ISD::ZIP1.
    5405             : 
    5406             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5407           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5408             :     return 0;
    5409           0 :   if ((Subtarget->hasNEON())) {
    5410           0 :     return fastEmitInst_rr(AArch64::ZIP1v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5411             :   }
    5412             :   return 0;
    5413             : }
    5414             : 
    5415             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5416           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5417             :     return 0;
    5418           0 :   if ((Subtarget->hasNEON())) {
    5419           0 :     return fastEmitInst_rr(AArch64::ZIP1v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5420             :   }
    5421             :   return 0;
    5422             : }
    5423             : 
    5424             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5425           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5426             :     return 0;
    5427           0 :   if ((Subtarget->hasNEON())) {
    5428           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5429             :   }
    5430             :   return 0;
    5431             : }
    5432             : 
    5433             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5434           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5435             :     return 0;
    5436           0 :   if ((Subtarget->hasNEON())) {
    5437           0 :     return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5438             :   }
    5439             :   return 0;
    5440             : }
    5441             : 
    5442             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5443           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5444             :     return 0;
    5445           0 :   if ((Subtarget->hasNEON())) {
    5446           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5447             :   }
    5448             :   return 0;
    5449             : }
    5450             : 
    5451             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5452           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5453             :     return 0;
    5454           0 :   if ((Subtarget->hasNEON())) {
    5455           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5456             :   }
    5457             :   return 0;
    5458             : }
    5459             : 
    5460             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5461           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5462             :     return 0;
    5463           0 :   if ((Subtarget->hasNEON())) {
    5464           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5465             :   }
    5466             :   return 0;
    5467             : }
    5468             : 
    5469             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5470           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5471             :     return 0;
    5472           0 :   if ((Subtarget->hasNEON())) {
    5473           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5474             :   }
    5475             :   return 0;
    5476             : }
    5477             : 
    5478             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5479           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5480             :     return 0;
    5481           0 :   if ((Subtarget->hasNEON())) {
    5482           0 :     return fastEmitInst_rr(AArch64::ZIP1v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5483             :   }
    5484             :   return 0;
    5485             : }
    5486             : 
    5487             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5488           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5489             :     return 0;
    5490           0 :   if ((Subtarget->hasNEON())) {
    5491           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5492             :   }
    5493             :   return 0;
    5494             : }
    5495             : 
    5496             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5497           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5498             :     return 0;
    5499           0 :   if ((Subtarget->hasNEON())) {
    5500           0 :     return fastEmitInst_rr(AArch64::ZIP1v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5501             :   }
    5502             :   return 0;
    5503             : }
    5504             : 
    5505             : unsigned fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5506           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5507             :     return 0;
    5508           0 :   if ((Subtarget->hasNEON())) {
    5509           0 :     return fastEmitInst_rr(AArch64::ZIP1v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5510             :   }
    5511             :   return 0;
    5512             : }
    5513             : 
    5514           0 : unsigned fastEmit_AArch64ISD_ZIP1_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5515           0 :   switch (VT.SimpleTy) {
    5516           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_ZIP1_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5517           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_ZIP1_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5518           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_ZIP1_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5519           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_ZIP1_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5520           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_ZIP1_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5521           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_ZIP1_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5522           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_ZIP1_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5523           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_ZIP1_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5524           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_ZIP1_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5525           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_ZIP1_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5526           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_ZIP1_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5527           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_ZIP1_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5528             :   default: return 0;
    5529             :   }
    5530             : }
    5531             : 
    5532             : // FastEmit functions for AArch64ISD::ZIP2.
    5533             : 
    5534             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5535           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5536             :     return 0;
    5537           0 :   if ((Subtarget->hasNEON())) {
    5538           0 :     return fastEmitInst_rr(AArch64::ZIP2v8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5539             :   }
    5540             :   return 0;
    5541             : }
    5542             : 
    5543             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5544           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5545             :     return 0;
    5546           0 :   if ((Subtarget->hasNEON())) {
    5547           0 :     return fastEmitInst_rr(AArch64::ZIP2v16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5548             :   }
    5549             :   return 0;
    5550             : }
    5551             : 
    5552             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5553           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5554             :     return 0;
    5555           0 :   if ((Subtarget->hasNEON())) {
    5556           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5557             :   }
    5558             :   return 0;
    5559             : }
    5560             : 
    5561             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5562           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5563             :     return 0;
    5564           0 :   if ((Subtarget->hasNEON())) {
    5565           0 :     return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5566             :   }
    5567             :   return 0;
    5568             : }
    5569             : 
    5570             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5571           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5572             :     return 0;
    5573           0 :   if ((Subtarget->hasNEON())) {
    5574           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5575             :   }
    5576             :   return 0;
    5577             : }
    5578             : 
    5579             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5580           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5581             :     return 0;
    5582           0 :   if ((Subtarget->hasNEON())) {
    5583           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5584             :   }
    5585             :   return 0;
    5586             : }
    5587             : 
    5588             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5589           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5590             :     return 0;
    5591           0 :   if ((Subtarget->hasNEON())) {
    5592           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5593             :   }
    5594             :   return 0;
    5595             : }
    5596             : 
    5597             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5598           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5599             :     return 0;
    5600           0 :   if ((Subtarget->hasNEON())) {
    5601           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5602             :   }
    5603             :   return 0;
    5604             : }
    5605             : 
    5606             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5607           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5608             :     return 0;
    5609           0 :   if ((Subtarget->hasNEON())) {
    5610           0 :     return fastEmitInst_rr(AArch64::ZIP2v8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5611             :   }
    5612             :   return 0;
    5613             : }
    5614             : 
    5615             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5616           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    5617             :     return 0;
    5618           0 :   if ((Subtarget->hasNEON())) {
    5619           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5620             :   }
    5621             :   return 0;
    5622             : }
    5623             : 
    5624             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5625           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    5626             :     return 0;
    5627           0 :   if ((Subtarget->hasNEON())) {
    5628           0 :     return fastEmitInst_rr(AArch64::ZIP2v4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5629             :   }
    5630             :   return 0;
    5631             : }
    5632             : 
    5633             : unsigned fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5634           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5635             :     return 0;
    5636           0 :   if ((Subtarget->hasNEON())) {
    5637           0 :     return fastEmitInst_rr(AArch64::ZIP2v2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5638             :   }
    5639             :   return 0;
    5640             : }
    5641             : 
    5642           0 : unsigned fastEmit_AArch64ISD_ZIP2_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5643           0 :   switch (VT.SimpleTy) {
    5644           0 :   case MVT::v8i8: return fastEmit_AArch64ISD_ZIP2_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5645           0 :   case MVT::v16i8: return fastEmit_AArch64ISD_ZIP2_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5646           0 :   case MVT::v4i16: return fastEmit_AArch64ISD_ZIP2_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5647           0 :   case MVT::v8i16: return fastEmit_AArch64ISD_ZIP2_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5648           0 :   case MVT::v2i32: return fastEmit_AArch64ISD_ZIP2_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5649           0 :   case MVT::v4i32: return fastEmit_AArch64ISD_ZIP2_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5650           0 :   case MVT::v2i64: return fastEmit_AArch64ISD_ZIP2_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5651           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_ZIP2_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5652           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_ZIP2_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5653           0 :   case MVT::v2f32: return fastEmit_AArch64ISD_ZIP2_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5654           0 :   case MVT::v4f32: return fastEmit_AArch64ISD_ZIP2_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5655           0 :   case MVT::v2f64: return fastEmit_AArch64ISD_ZIP2_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5656             :   default: return 0;
    5657             :   }
    5658             : }
    5659             : 
    5660             : // FastEmit functions for ISD::ADD.
    5661             : 
    5662             : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5663           0 :   if (RetVT.SimpleTy != MVT::i32)
    5664             :     return 0;
    5665           0 :   return fastEmitInst_rr(AArch64::ADDWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5666             : }
    5667             : 
    5668             : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5669           3 :   if (RetVT.SimpleTy != MVT::i64)
    5670             :     return 0;
    5671           3 :   return fastEmitInst_rr(AArch64::ADDXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5672             : }
    5673             : 
    5674             : unsigned fastEmit_ISD_ADD_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5675          11 :   if (RetVT.SimpleTy != MVT::v8i8)
    5676             :     return 0;
    5677          11 :   if ((Subtarget->hasNEON())) {
    5678          11 :     return fastEmitInst_rr(AArch64::ADDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5679             :   }
    5680             :   return 0;
    5681             : }
    5682             : 
    5683             : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5684           7 :   if (RetVT.SimpleTy != MVT::v16i8)
    5685             :     return 0;
    5686           7 :   if ((Subtarget->hasNEON())) {
    5687           7 :     return fastEmitInst_rr(AArch64::ADDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5688             :   }
    5689             :   return 0;
    5690             : }
    5691             : 
    5692             : unsigned fastEmit_ISD_ADD_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5693          10 :   if (RetVT.SimpleTy != MVT::v4i16)
    5694             :     return 0;
    5695          10 :   if ((Subtarget->hasNEON())) {
    5696          10 :     return fastEmitInst_rr(AArch64::ADDv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5697             :   }
    5698             :   return 0;
    5699             : }
    5700             : 
    5701             : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5702           7 :   if (RetVT.SimpleTy != MVT::v8i16)
    5703             :     return 0;
    5704           7 :   if ((Subtarget->hasNEON())) {
    5705           7 :     return fastEmitInst_rr(AArch64::ADDv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5706             :   }
    5707             :   return 0;
    5708             : }
    5709             : 
    5710             : unsigned fastEmit_ISD_ADD_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5711          10 :   if (RetVT.SimpleTy != MVT::v2i32)
    5712             :     return 0;
    5713          10 :   if ((Subtarget->hasNEON())) {
    5714          10 :     return fastEmitInst_rr(AArch64::ADDv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5715             :   }
    5716             :   return 0;
    5717             : }
    5718             : 
    5719             : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5720           7 :   if (RetVT.SimpleTy != MVT::v4i32)
    5721             :     return 0;
    5722           7 :   if ((Subtarget->hasNEON())) {
    5723           7 :     return fastEmitInst_rr(AArch64::ADDv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5724             :   }
    5725             :   return 0;
    5726             : }
    5727             : 
    5728             : unsigned fastEmit_ISD_ADD_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5729          12 :   if (RetVT.SimpleTy != MVT::v1i64)
    5730             :     return 0;
    5731          12 :   if ((Subtarget->hasNEON())) {
    5732          12 :     return fastEmitInst_rr(AArch64::ADDv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5733             :   }
    5734             :   return 0;
    5735             : }
    5736             : 
    5737             : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5738           7 :   if (RetVT.SimpleTy != MVT::v2i64)
    5739             :     return 0;
    5740           7 :   if ((Subtarget->hasNEON())) {
    5741           7 :     return fastEmitInst_rr(AArch64::ADDv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5742             :   }
    5743             :   return 0;
    5744             : }
    5745             : 
    5746          74 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5747          74 :   switch (VT.SimpleTy) {
    5748           0 :   case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5749           3 :   case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5750          11 :   case MVT::v8i8: return fastEmit_ISD_ADD_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5751           7 :   case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5752          10 :   case MVT::v4i16: return fastEmit_ISD_ADD_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5753           7 :   case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5754          10 :   case MVT::v2i32: return fastEmit_ISD_ADD_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5755           7 :   case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5756          12 :   case MVT::v1i64: return fastEmit_ISD_ADD_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5757           7 :   case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5758             :   default: return 0;
    5759             :   }
    5760             : }
    5761             : 
    5762             : // FastEmit functions for ISD::AND.
    5763             : 
    5764             : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5765          30 :   if (RetVT.SimpleTy != MVT::i32)
    5766             :     return 0;
    5767          30 :   return fastEmitInst_rr(AArch64::ANDWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5768             : }
    5769             : 
    5770             : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5771           4 :   if (RetVT.SimpleTy != MVT::i64)
    5772             :     return 0;
    5773           4 :   return fastEmitInst_rr(AArch64::ANDXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5774             : }
    5775             : 
    5776             : unsigned fastEmit_ISD_AND_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5777           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    5778             :     return 0;
    5779           0 :   if ((Subtarget->hasNEON())) {
    5780           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5781             :   }
    5782             :   return 0;
    5783             : }
    5784             : 
    5785             : unsigned fastEmit_ISD_AND_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5786           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5787             :     return 0;
    5788           0 :   if ((Subtarget->hasNEON())) {
    5789           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5790             :   }
    5791             :   return 0;
    5792             : }
    5793             : 
    5794             : unsigned fastEmit_ISD_AND_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5795           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    5796             :     return 0;
    5797           0 :   if ((Subtarget->hasNEON())) {
    5798           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5799             :   }
    5800             :   return 0;
    5801             : }
    5802             : 
    5803             : unsigned fastEmit_ISD_AND_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5804           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    5805             :     return 0;
    5806           0 :   if ((Subtarget->hasNEON())) {
    5807           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5808             :   }
    5809             :   return 0;
    5810             : }
    5811             : 
    5812             : unsigned fastEmit_ISD_AND_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5813           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    5814             :     return 0;
    5815           0 :   if ((Subtarget->hasNEON())) {
    5816           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5817             :   }
    5818             :   return 0;
    5819             : }
    5820             : 
    5821             : unsigned fastEmit_ISD_AND_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5822           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    5823             :     return 0;
    5824           0 :   if ((Subtarget->hasNEON())) {
    5825           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5826             :   }
    5827             :   return 0;
    5828             : }
    5829             : 
    5830             : unsigned fastEmit_ISD_AND_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5831           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    5832             :     return 0;
    5833           0 :   if ((Subtarget->hasNEON())) {
    5834           0 :     return fastEmitInst_rr(AArch64::ANDv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5835             :   }
    5836             :   return 0;
    5837             : }
    5838             : 
    5839             : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5840           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5841             :     return 0;
    5842           0 :   if ((Subtarget->hasNEON())) {
    5843           0 :     return fastEmitInst_rr(AArch64::ANDv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5844             :   }
    5845             :   return 0;
    5846             : }
    5847             : 
    5848          34 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5849          34 :   switch (VT.SimpleTy) {
    5850          30 :   case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5851           4 :   case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5852           0 :   case MVT::v8i8: return fastEmit_ISD_AND_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5853           0 :   case MVT::v16i8: return fastEmit_ISD_AND_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5854           0 :   case MVT::v4i16: return fastEmit_ISD_AND_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5855           0 :   case MVT::v8i16: return fastEmit_ISD_AND_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5856           0 :   case MVT::v2i32: return fastEmit_ISD_AND_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5857           0 :   case MVT::v4i32: return fastEmit_ISD_AND_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5858           0 :   case MVT::v1i64: return fastEmit_ISD_AND_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5859           0 :   case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5860             :   default: return 0;
    5861             :   }
    5862             : }
    5863             : 
    5864             : // FastEmit functions for ISD::FADD.
    5865             : 
    5866             : unsigned fastEmit_ISD_FADD_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5867           0 :   if (RetVT.SimpleTy != MVT::f16)
    5868             :     return 0;
    5869           0 :   if ((Subtarget->hasFullFP16())) {
    5870           0 :     return fastEmitInst_rr(AArch64::FADDHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5871             :   }
    5872             :   return 0;
    5873             : }
    5874             : 
    5875             : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5876          28 :   if (RetVT.SimpleTy != MVT::f32)
    5877             :     return 0;
    5878          28 :   if ((Subtarget->hasFPARMv8())) {
    5879          28 :     return fastEmitInst_rr(AArch64::FADDSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5880             :   }
    5881             :   return 0;
    5882             : }
    5883             : 
    5884             : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5885          57 :   if (RetVT.SimpleTy != MVT::f64)
    5886             :     return 0;
    5887          57 :   if ((Subtarget->hasFPARMv8())) {
    5888          57 :     return fastEmitInst_rr(AArch64::FADDDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5889             :   }
    5890             :   return 0;
    5891             : }
    5892             : 
    5893           1 : unsigned fastEmit_ISD_FADD_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5894           1 :   if (RetVT.SimpleTy != MVT::v4f16)
    5895             :     return 0;
    5896           1 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    5897           0 :     return fastEmitInst_rr(AArch64::FADDv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5898             :   }
    5899             :   return 0;
    5900             : }
    5901             : 
    5902           0 : unsigned fastEmit_ISD_FADD_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5903           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5904             :     return 0;
    5905           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    5906           0 :     return fastEmitInst_rr(AArch64::FADDv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5907             :   }
    5908             :   return 0;
    5909             : }
    5910             : 
    5911             : unsigned fastEmit_ISD_FADD_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5912          10 :   if (RetVT.SimpleTy != MVT::v2f32)
    5913             :     return 0;
    5914          10 :   if ((Subtarget->hasNEON())) {
    5915          10 :     return fastEmitInst_rr(AArch64::FADDv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5916             :   }
    5917             :   return 0;
    5918             : }
    5919             : 
    5920             : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5921          10 :   if (RetVT.SimpleTy != MVT::v4f32)
    5922             :     return 0;
    5923          10 :   if ((Subtarget->hasNEON())) {
    5924          10 :     return fastEmitInst_rr(AArch64::FADDv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5925             :   }
    5926             :   return 0;
    5927             : }
    5928             : 
    5929             : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5930           7 :   if (RetVT.SimpleTy != MVT::v2f64)
    5931             :     return 0;
    5932           7 :   if ((Subtarget->hasNEON())) {
    5933           7 :     return fastEmitInst_rr(AArch64::FADDv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5934             :   }
    5935             :   return 0;
    5936             : }
    5937             : 
    5938         119 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5939         119 :   switch (VT.SimpleTy) {
    5940           0 :   case MVT::f16: return fastEmit_ISD_FADD_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5941          28 :   case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5942          57 :   case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5943           1 :   case MVT::v4f16: return fastEmit_ISD_FADD_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5944           0 :   case MVT::v8f16: return fastEmit_ISD_FADD_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5945          10 :   case MVT::v2f32: return fastEmit_ISD_FADD_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5946          10 :   case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5947           7 :   case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5948             :   default: return 0;
    5949             :   }
    5950             : }
    5951             : 
    5952             : // FastEmit functions for ISD::FDIV.
    5953             : 
    5954             : unsigned fastEmit_ISD_FDIV_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5955           0 :   if (RetVT.SimpleTy != MVT::f16)
    5956             :     return 0;
    5957           0 :   if ((Subtarget->hasFullFP16())) {
    5958           0 :     return fastEmitInst_rr(AArch64::FDIVHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5959             :   }
    5960             :   return 0;
    5961             : }
    5962             : 
    5963             : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5964           0 :   if (RetVT.SimpleTy != MVT::f32)
    5965             :     return 0;
    5966           0 :   if ((Subtarget->hasFPARMv8())) {
    5967           0 :     return fastEmitInst_rr(AArch64::FDIVSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5968             :   }
    5969             :   return 0;
    5970             : }
    5971             : 
    5972             : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5973           0 :   if (RetVT.SimpleTy != MVT::f64)
    5974             :     return 0;
    5975           0 :   if ((Subtarget->hasFPARMv8())) {
    5976           0 :     return fastEmitInst_rr(AArch64::FDIVDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5977             :   }
    5978             :   return 0;
    5979             : }
    5980             : 
    5981           0 : unsigned fastEmit_ISD_FDIV_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5982           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    5983             :     return 0;
    5984           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    5985           0 :     return fastEmitInst_rr(AArch64::FDIVv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5986             :   }
    5987             :   return 0;
    5988             : }
    5989             : 
    5990           0 : unsigned fastEmit_ISD_FDIV_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5991           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    5992             :     return 0;
    5993           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    5994           0 :     return fastEmitInst_rr(AArch64::FDIVv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5995             :   }
    5996             :   return 0;
    5997             : }
    5998             : 
    5999             : unsigned fastEmit_ISD_FDIV_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6000           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6001             :     return 0;
    6002           0 :   if ((Subtarget->hasNEON())) {
    6003           0 :     return fastEmitInst_rr(AArch64::FDIVv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6004             :   }
    6005             :   return 0;
    6006             : }
    6007             : 
    6008             : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6009           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6010             :     return 0;
    6011           0 :   if ((Subtarget->hasNEON())) {
    6012           0 :     return fastEmitInst_rr(AArch64::FDIVv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6013             :   }
    6014             :   return 0;
    6015             : }
    6016             : 
    6017             : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6018           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6019             :     return 0;
    6020           0 :   if ((Subtarget->hasNEON())) {
    6021           0 :     return fastEmitInst_rr(AArch64::FDIVv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6022             :   }
    6023             :   return 0;
    6024             : }
    6025             : 
    6026           0 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6027           0 :   switch (VT.SimpleTy) {
    6028           0 :   case MVT::f16: return fastEmit_ISD_FDIV_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6029           0 :   case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6030           0 :   case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6031           0 :   case MVT::v4f16: return fastEmit_ISD_FDIV_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6032           0 :   case MVT::v8f16: return fastEmit_ISD_FDIV_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6033           0 :   case MVT::v2f32: return fastEmit_ISD_FDIV_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6034           0 :   case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6035           0 :   case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6036             :   default: return 0;
    6037             :   }
    6038             : }
    6039             : 
    6040             : // FastEmit functions for ISD::FMAXNAN.
    6041             : 
    6042             : unsigned fastEmit_ISD_FMAXNAN_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6043           0 :   if (RetVT.SimpleTy != MVT::f16)
    6044             :     return 0;
    6045           0 :   if ((Subtarget->hasFullFP16())) {
    6046           0 :     return fastEmitInst_rr(AArch64::FMAXHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6047             :   }
    6048             :   return 0;
    6049             : }
    6050             : 
    6051             : unsigned fastEmit_ISD_FMAXNAN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6052           0 :   if (RetVT.SimpleTy != MVT::f32)
    6053             :     return 0;
    6054           0 :   if ((Subtarget->hasFPARMv8())) {
    6055           0 :     return fastEmitInst_rr(AArch64::FMAXSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6056             :   }
    6057             :   return 0;
    6058             : }
    6059             : 
    6060             : unsigned fastEmit_ISD_FMAXNAN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6061           0 :   if (RetVT.SimpleTy != MVT::f64)
    6062             :     return 0;
    6063           0 :   if ((Subtarget->hasFPARMv8())) {
    6064           0 :     return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6065             :   }
    6066             :   return 0;
    6067             : }
    6068             : 
    6069           0 : unsigned fastEmit_ISD_FMAXNAN_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6070           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6071             :     return 0;
    6072           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6073           0 :     return fastEmitInst_rr(AArch64::FMAXv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6074             :   }
    6075             :   return 0;
    6076             : }
    6077             : 
    6078           0 : unsigned fastEmit_ISD_FMAXNAN_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6079           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6080             :     return 0;
    6081           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6082           0 :     return fastEmitInst_rr(AArch64::FMAXv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6083             :   }
    6084             :   return 0;
    6085             : }
    6086             : 
    6087             : unsigned fastEmit_ISD_FMAXNAN_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6088           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6089             :     return 0;
    6090           0 :   if ((Subtarget->hasNEON())) {
    6091           0 :     return fastEmitInst_rr(AArch64::FMAXv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6092             :   }
    6093             :   return 0;
    6094             : }
    6095             : 
    6096             : unsigned fastEmit_ISD_FMAXNAN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6097           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6098             :     return 0;
    6099           0 :   if ((Subtarget->hasNEON())) {
    6100           0 :     return fastEmitInst_rr(AArch64::FMAXv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6101             :   }
    6102             :   return 0;
    6103             : }
    6104             : 
    6105             : unsigned fastEmit_ISD_FMAXNAN_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6106           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6107             :     return 0;
    6108           0 :   return fastEmitInst_rr(AArch64::FMAXDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6109             : }
    6110             : 
    6111             : unsigned fastEmit_ISD_FMAXNAN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6112           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6113             :     return 0;
    6114           0 :   if ((Subtarget->hasNEON())) {
    6115           0 :     return fastEmitInst_rr(AArch64::FMAXv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6116             :   }
    6117             :   return 0;
    6118             : }
    6119             : 
    6120           0 : unsigned fastEmit_ISD_FMAXNAN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6121           0 :   switch (VT.SimpleTy) {
    6122           0 :   case MVT::f16: return fastEmit_ISD_FMAXNAN_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6123           0 :   case MVT::f32: return fastEmit_ISD_FMAXNAN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6124           0 :   case MVT::f64: return fastEmit_ISD_FMAXNAN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6125           0 :   case MVT::v4f16: return fastEmit_ISD_FMAXNAN_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6126           0 :   case MVT::v8f16: return fastEmit_ISD_FMAXNAN_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6127           0 :   case MVT::v2f32: return fastEmit_ISD_FMAXNAN_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6128           0 :   case MVT::v4f32: return fastEmit_ISD_FMAXNAN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6129           0 :   case MVT::v1f64: return fastEmit_ISD_FMAXNAN_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6130           0 :   case MVT::v2f64: return fastEmit_ISD_FMAXNAN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6131             :   default: return 0;
    6132             :   }
    6133             : }
    6134             : 
    6135             : // FastEmit functions for ISD::FMAXNUM.
    6136             : 
    6137             : unsigned fastEmit_ISD_FMAXNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6138           0 :   if (RetVT.SimpleTy != MVT::f16)
    6139             :     return 0;
    6140           0 :   if ((Subtarget->hasFullFP16())) {
    6141           0 :     return fastEmitInst_rr(AArch64::FMAXNMHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6142             :   }
    6143             :   return 0;
    6144             : }
    6145             : 
    6146             : unsigned fastEmit_ISD_FMAXNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6147           0 :   if (RetVT.SimpleTy != MVT::f32)
    6148             :     return 0;
    6149           0 :   if ((Subtarget->hasFPARMv8())) {
    6150           0 :     return fastEmitInst_rr(AArch64::FMAXNMSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6151             :   }
    6152             :   return 0;
    6153             : }
    6154             : 
    6155             : unsigned fastEmit_ISD_FMAXNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6156           0 :   if (RetVT.SimpleTy != MVT::f64)
    6157             :     return 0;
    6158           0 :   if ((Subtarget->hasFPARMv8())) {
    6159           0 :     return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6160             :   }
    6161             :   return 0;
    6162             : }
    6163             : 
    6164           0 : unsigned fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6165           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6166             :     return 0;
    6167           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6168           0 :     return fastEmitInst_rr(AArch64::FMAXNMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6169             :   }
    6170             :   return 0;
    6171             : }
    6172             : 
    6173           0 : unsigned fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6174           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6175             :     return 0;
    6176           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6177           0 :     return fastEmitInst_rr(AArch64::FMAXNMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6178             :   }
    6179             :   return 0;
    6180             : }
    6181             : 
    6182             : unsigned fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6183           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6184             :     return 0;
    6185           0 :   if ((Subtarget->hasNEON())) {
    6186           0 :     return fastEmitInst_rr(AArch64::FMAXNMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6187             :   }
    6188             :   return 0;
    6189             : }
    6190             : 
    6191             : unsigned fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6192           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6193             :     return 0;
    6194           0 :   if ((Subtarget->hasNEON())) {
    6195           0 :     return fastEmitInst_rr(AArch64::FMAXNMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6196             :   }
    6197             :   return 0;
    6198             : }
    6199             : 
    6200             : unsigned fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6201           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6202             :     return 0;
    6203           0 :   return fastEmitInst_rr(AArch64::FMAXNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6204             : }
    6205             : 
    6206             : unsigned fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6207           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6208             :     return 0;
    6209           0 :   if ((Subtarget->hasNEON())) {
    6210           0 :     return fastEmitInst_rr(AArch64::FMAXNMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6211             :   }
    6212             :   return 0;
    6213             : }
    6214             : 
    6215           0 : unsigned fastEmit_ISD_FMAXNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6216           0 :   switch (VT.SimpleTy) {
    6217           0 :   case MVT::f16: return fastEmit_ISD_FMAXNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6218           0 :   case MVT::f32: return fastEmit_ISD_FMAXNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6219           0 :   case MVT::f64: return fastEmit_ISD_FMAXNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6220           0 :   case MVT::v4f16: return fastEmit_ISD_FMAXNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6221           0 :   case MVT::v8f16: return fastEmit_ISD_FMAXNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6222           0 :   case MVT::v2f32: return fastEmit_ISD_FMAXNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6223           0 :   case MVT::v4f32: return fastEmit_ISD_FMAXNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6224           0 :   case MVT::v1f64: return fastEmit_ISD_FMAXNUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6225           0 :   case MVT::v2f64: return fastEmit_ISD_FMAXNUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6226             :   default: return 0;
    6227             :   }
    6228             : }
    6229             : 
    6230             : // FastEmit functions for ISD::FMINNAN.
    6231             : 
    6232             : unsigned fastEmit_ISD_FMINNAN_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6233           0 :   if (RetVT.SimpleTy != MVT::f16)
    6234             :     return 0;
    6235           0 :   if ((Subtarget->hasFullFP16())) {
    6236           0 :     return fastEmitInst_rr(AArch64::FMINHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6237             :   }
    6238             :   return 0;
    6239             : }
    6240             : 
    6241             : unsigned fastEmit_ISD_FMINNAN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6242           0 :   if (RetVT.SimpleTy != MVT::f32)
    6243             :     return 0;
    6244           0 :   if ((Subtarget->hasFPARMv8())) {
    6245           0 :     return fastEmitInst_rr(AArch64::FMINSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6246             :   }
    6247             :   return 0;
    6248             : }
    6249             : 
    6250             : unsigned fastEmit_ISD_FMINNAN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6251           0 :   if (RetVT.SimpleTy != MVT::f64)
    6252             :     return 0;
    6253           0 :   if ((Subtarget->hasFPARMv8())) {
    6254           0 :     return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6255             :   }
    6256             :   return 0;
    6257             : }
    6258             : 
    6259           0 : unsigned fastEmit_ISD_FMINNAN_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6260           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6261             :     return 0;
    6262           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6263           0 :     return fastEmitInst_rr(AArch64::FMINv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6264             :   }
    6265             :   return 0;
    6266             : }
    6267             : 
    6268           0 : unsigned fastEmit_ISD_FMINNAN_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6269           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6270             :     return 0;
    6271           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6272           0 :     return fastEmitInst_rr(AArch64::FMINv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6273             :   }
    6274             :   return 0;
    6275             : }
    6276             : 
    6277             : unsigned fastEmit_ISD_FMINNAN_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6278           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6279             :     return 0;
    6280           0 :   if ((Subtarget->hasNEON())) {
    6281           0 :     return fastEmitInst_rr(AArch64::FMINv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6282             :   }
    6283             :   return 0;
    6284             : }
    6285             : 
    6286             : unsigned fastEmit_ISD_FMINNAN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6287           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6288             :     return 0;
    6289           0 :   if ((Subtarget->hasNEON())) {
    6290           0 :     return fastEmitInst_rr(AArch64::FMINv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6291             :   }
    6292             :   return 0;
    6293             : }
    6294             : 
    6295             : unsigned fastEmit_ISD_FMINNAN_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6296           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6297             :     return 0;
    6298           0 :   return fastEmitInst_rr(AArch64::FMINDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6299             : }
    6300             : 
    6301             : unsigned fastEmit_ISD_FMINNAN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6302           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6303             :     return 0;
    6304           0 :   if ((Subtarget->hasNEON())) {
    6305           0 :     return fastEmitInst_rr(AArch64::FMINv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6306             :   }
    6307             :   return 0;
    6308             : }
    6309             : 
    6310           0 : unsigned fastEmit_ISD_FMINNAN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6311           0 :   switch (VT.SimpleTy) {
    6312           0 :   case MVT::f16: return fastEmit_ISD_FMINNAN_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6313           0 :   case MVT::f32: return fastEmit_ISD_FMINNAN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6314           0 :   case MVT::f64: return fastEmit_ISD_FMINNAN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6315           0 :   case MVT::v4f16: return fastEmit_ISD_FMINNAN_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6316           0 :   case MVT::v8f16: return fastEmit_ISD_FMINNAN_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6317           0 :   case MVT::v2f32: return fastEmit_ISD_FMINNAN_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6318           0 :   case MVT::v4f32: return fastEmit_ISD_FMINNAN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6319           0 :   case MVT::v1f64: return fastEmit_ISD_FMINNAN_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6320           0 :   case MVT::v2f64: return fastEmit_ISD_FMINNAN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6321             :   default: return 0;
    6322             :   }
    6323             : }
    6324             : 
    6325             : // FastEmit functions for ISD::FMINNUM.
    6326             : 
    6327             : unsigned fastEmit_ISD_FMINNUM_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6328           0 :   if (RetVT.SimpleTy != MVT::f16)
    6329             :     return 0;
    6330           0 :   if ((Subtarget->hasFullFP16())) {
    6331           0 :     return fastEmitInst_rr(AArch64::FMINNMHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6332             :   }
    6333             :   return 0;
    6334             : }
    6335             : 
    6336             : unsigned fastEmit_ISD_FMINNUM_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6337           0 :   if (RetVT.SimpleTy != MVT::f32)
    6338             :     return 0;
    6339           0 :   if ((Subtarget->hasFPARMv8())) {
    6340           0 :     return fastEmitInst_rr(AArch64::FMINNMSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6341             :   }
    6342             :   return 0;
    6343             : }
    6344             : 
    6345             : unsigned fastEmit_ISD_FMINNUM_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6346           0 :   if (RetVT.SimpleTy != MVT::f64)
    6347             :     return 0;
    6348           0 :   if ((Subtarget->hasFPARMv8())) {
    6349           0 :     return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6350             :   }
    6351             :   return 0;
    6352             : }
    6353             : 
    6354           0 : unsigned fastEmit_ISD_FMINNUM_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6355           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6356             :     return 0;
    6357           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6358           0 :     return fastEmitInst_rr(AArch64::FMINNMv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6359             :   }
    6360             :   return 0;
    6361             : }
    6362             : 
    6363           0 : unsigned fastEmit_ISD_FMINNUM_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6364           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6365             :     return 0;
    6366           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6367           0 :     return fastEmitInst_rr(AArch64::FMINNMv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6368             :   }
    6369             :   return 0;
    6370             : }
    6371             : 
    6372             : unsigned fastEmit_ISD_FMINNUM_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6373           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6374             :     return 0;
    6375           0 :   if ((Subtarget->hasNEON())) {
    6376           0 :     return fastEmitInst_rr(AArch64::FMINNMv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6377             :   }
    6378             :   return 0;
    6379             : }
    6380             : 
    6381             : unsigned fastEmit_ISD_FMINNUM_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6382           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6383             :     return 0;
    6384           0 :   if ((Subtarget->hasNEON())) {
    6385           0 :     return fastEmitInst_rr(AArch64::FMINNMv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6386             :   }
    6387             :   return 0;
    6388             : }
    6389             : 
    6390             : unsigned fastEmit_ISD_FMINNUM_MVT_v1f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6391           0 :   if (RetVT.SimpleTy != MVT::v1f64)
    6392             :     return 0;
    6393           0 :   return fastEmitInst_rr(AArch64::FMINNMDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6394             : }
    6395             : 
    6396             : unsigned fastEmit_ISD_FMINNUM_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6397           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6398             :     return 0;
    6399           0 :   if ((Subtarget->hasNEON())) {
    6400           0 :     return fastEmitInst_rr(AArch64::FMINNMv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6401             :   }
    6402             :   return 0;
    6403             : }
    6404             : 
    6405           0 : unsigned fastEmit_ISD_FMINNUM_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6406           0 :   switch (VT.SimpleTy) {
    6407           0 :   case MVT::f16: return fastEmit_ISD_FMINNUM_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6408           0 :   case MVT::f32: return fastEmit_ISD_FMINNUM_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6409           0 :   case MVT::f64: return fastEmit_ISD_FMINNUM_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6410           0 :   case MVT::v4f16: return fastEmit_ISD_FMINNUM_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6411           0 :   case MVT::v8f16: return fastEmit_ISD_FMINNUM_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6412           0 :   case MVT::v2f32: return fastEmit_ISD_FMINNUM_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6413           0 :   case MVT::v4f32: return fastEmit_ISD_FMINNUM_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6414           0 :   case MVT::v1f64: return fastEmit_ISD_FMINNUM_MVT_v1f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6415           0 :   case MVT::v2f64: return fastEmit_ISD_FMINNUM_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6416             :   default: return 0;
    6417             :   }
    6418             : }
    6419             : 
    6420             : // FastEmit functions for ISD::FMUL.
    6421             : 
    6422             : unsigned fastEmit_ISD_FMUL_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6423           0 :   if (RetVT.SimpleTy != MVT::f16)
    6424             :     return 0;
    6425           0 :   if ((Subtarget->hasFullFP16())) {
    6426           0 :     return fastEmitInst_rr(AArch64::FMULHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6427             :   }
    6428             :   return 0;
    6429             : }
    6430             : 
    6431             : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6432           0 :   if (RetVT.SimpleTy != MVT::f32)
    6433             :     return 0;
    6434           0 :   if ((Subtarget->hasFPARMv8())) {
    6435           0 :     return fastEmitInst_rr(AArch64::FMULSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6436             :   }
    6437             :   return 0;
    6438             : }
    6439             : 
    6440             : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6441           0 :   if (RetVT.SimpleTy != MVT::f64)
    6442             :     return 0;
    6443           0 :   if ((Subtarget->hasFPARMv8())) {
    6444           0 :     return fastEmitInst_rr(AArch64::FMULDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6445             :   }
    6446             :   return 0;
    6447             : }
    6448             : 
    6449           0 : unsigned fastEmit_ISD_FMUL_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6450           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6451             :     return 0;
    6452           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6453           0 :     return fastEmitInst_rr(AArch64::FMULv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6454             :   }
    6455             :   return 0;
    6456             : }
    6457             : 
    6458           0 : unsigned fastEmit_ISD_FMUL_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6459           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6460             :     return 0;
    6461           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6462           0 :     return fastEmitInst_rr(AArch64::FMULv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6463             :   }
    6464             :   return 0;
    6465             : }
    6466             : 
    6467             : unsigned fastEmit_ISD_FMUL_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6468           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6469             :     return 0;
    6470           0 :   if ((Subtarget->hasNEON())) {
    6471           0 :     return fastEmitInst_rr(AArch64::FMULv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6472             :   }
    6473             :   return 0;
    6474             : }
    6475             : 
    6476             : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6477           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6478             :     return 0;
    6479           0 :   if ((Subtarget->hasNEON())) {
    6480           0 :     return fastEmitInst_rr(AArch64::FMULv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6481             :   }
    6482             :   return 0;
    6483             : }
    6484             : 
    6485             : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6486           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6487             :     return 0;
    6488           0 :   if ((Subtarget->hasNEON())) {
    6489           0 :     return fastEmitInst_rr(AArch64::FMULv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6490             :   }
    6491             :   return 0;
    6492             : }
    6493             : 
    6494           0 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6495           0 :   switch (VT.SimpleTy) {
    6496           0 :   case MVT::f16: return fastEmit_ISD_FMUL_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6497           0 :   case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6498           0 :   case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6499           0 :   case MVT::v4f16: return fastEmit_ISD_FMUL_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6500           0 :   case MVT::v8f16: return fastEmit_ISD_FMUL_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6501           0 :   case MVT::v2f32: return fastEmit_ISD_FMUL_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6502           0 :   case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6503           0 :   case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6504             :   default: return 0;
    6505             :   }
    6506             : }
    6507             : 
    6508             : // FastEmit functions for ISD::FSUB.
    6509             : 
    6510             : unsigned fastEmit_ISD_FSUB_MVT_f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6511           0 :   if (RetVT.SimpleTy != MVT::f16)
    6512             :     return 0;
    6513           0 :   if ((Subtarget->hasFullFP16())) {
    6514           0 :     return fastEmitInst_rr(AArch64::FSUBHrr, &AArch64::FPR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6515             :   }
    6516             :   return 0;
    6517             : }
    6518             : 
    6519             : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6520           0 :   if (RetVT.SimpleTy != MVT::f32)
    6521             :     return 0;
    6522           0 :   if ((Subtarget->hasFPARMv8())) {
    6523           0 :     return fastEmitInst_rr(AArch64::FSUBSrr, &AArch64::FPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6524             :   }
    6525             :   return 0;
    6526             : }
    6527             : 
    6528             : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6529           0 :   if (RetVT.SimpleTy != MVT::f64)
    6530             :     return 0;
    6531           0 :   if ((Subtarget->hasFPARMv8())) {
    6532           0 :     return fastEmitInst_rr(AArch64::FSUBDrr, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6533             :   }
    6534             :   return 0;
    6535             : }
    6536             : 
    6537           0 : unsigned fastEmit_ISD_FSUB_MVT_v4f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6538           0 :   if (RetVT.SimpleTy != MVT::v4f16)
    6539             :     return 0;
    6540           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6541           0 :     return fastEmitInst_rr(AArch64::FSUBv4f16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6542             :   }
    6543             :   return 0;
    6544             : }
    6545             : 
    6546           0 : unsigned fastEmit_ISD_FSUB_MVT_v8f16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6547           0 :   if (RetVT.SimpleTy != MVT::v8f16)
    6548             :     return 0;
    6549           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    6550           0 :     return fastEmitInst_rr(AArch64::FSUBv8f16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6551             :   }
    6552             :   return 0;
    6553             : }
    6554             : 
    6555             : unsigned fastEmit_ISD_FSUB_MVT_v2f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6556           0 :   if (RetVT.SimpleTy != MVT::v2f32)
    6557             :     return 0;
    6558           0 :   if ((Subtarget->hasNEON())) {
    6559           0 :     return fastEmitInst_rr(AArch64::FSUBv2f32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6560             :   }
    6561             :   return 0;
    6562             : }
    6563             : 
    6564             : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6565           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    6566             :     return 0;
    6567           0 :   if ((Subtarget->hasNEON())) {
    6568           0 :     return fastEmitInst_rr(AArch64::FSUBv4f32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6569             :   }
    6570             :   return 0;
    6571             : }
    6572             : 
    6573             : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6574           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    6575             :     return 0;
    6576           0 :   if ((Subtarget->hasNEON())) {
    6577           0 :     return fastEmitInst_rr(AArch64::FSUBv2f64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6578             :   }
    6579             :   return 0;
    6580             : }
    6581             : 
    6582           0 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6583           0 :   switch (VT.SimpleTy) {
    6584           0 :   case MVT::f16: return fastEmit_ISD_FSUB_MVT_f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6585           0 :   case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6586           0 :   case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6587           0 :   case MVT::v4f16: return fastEmit_ISD_FSUB_MVT_v4f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6588           0 :   case MVT::v8f16: return fastEmit_ISD_FSUB_MVT_v8f16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6589           0 :   case MVT::v2f32: return fastEmit_ISD_FSUB_MVT_v2f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6590           0 :   case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6591           0 :   case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6592             :   default: return 0;
    6593             :   }
    6594             : }
    6595             : 
    6596             : // FastEmit functions for ISD::MUL.
    6597             : 
    6598             : unsigned fastEmit_ISD_MUL_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6599           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6600             :     return 0;
    6601           0 :   if ((Subtarget->hasNEON())) {
    6602           0 :     return fastEmitInst_rr(AArch64::MULv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6603             :   }
    6604             :   return 0;
    6605             : }
    6606             : 
    6607             : unsigned fastEmit_ISD_MUL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6608           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6609             :     return 0;
    6610           0 :   if ((Subtarget->hasNEON())) {
    6611           0 :     return fastEmitInst_rr(AArch64::MULv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6612             :   }
    6613             :   return 0;
    6614             : }
    6615             : 
    6616             : unsigned fastEmit_ISD_MUL_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6617           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6618             :     return 0;
    6619           0 :   if ((Subtarget->hasNEON())) {
    6620           0 :     return fastEmitInst_rr(AArch64::MULv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6621             :   }
    6622             :   return 0;
    6623             : }
    6624             : 
    6625             : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6626           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6627             :     return 0;
    6628           0 :   if ((Subtarget->hasNEON())) {
    6629           0 :     return fastEmitInst_rr(AArch64::MULv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6630             :   }
    6631             :   return 0;
    6632             : }
    6633             : 
    6634             : unsigned fastEmit_ISD_MUL_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6635           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6636             :     return 0;
    6637           0 :   if ((Subtarget->hasNEON())) {
    6638           0 :     return fastEmitInst_rr(AArch64::MULv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6639             :   }
    6640             :   return 0;
    6641             : }
    6642             : 
    6643             : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6644           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6645             :     return 0;
    6646           0 :   if ((Subtarget->hasNEON())) {
    6647           0 :     return fastEmitInst_rr(AArch64::MULv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6648             :   }
    6649             :   return 0;
    6650             : }
    6651             : 
    6652           0 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6653           0 :   switch (VT.SimpleTy) {
    6654           0 :   case MVT::v8i8: return fastEmit_ISD_MUL_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6655           0 :   case MVT::v16i8: return fastEmit_ISD_MUL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6656           0 :   case MVT::v4i16: return fastEmit_ISD_MUL_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6657           0 :   case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6658           0 :   case MVT::v2i32: return fastEmit_ISD_MUL_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6659           0 :   case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6660             :   default: return 0;
    6661             :   }
    6662             : }
    6663             : 
    6664             : // FastEmit functions for ISD::MULHS.
    6665             : 
    6666             : unsigned fastEmit_ISD_MULHS_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6667           3 :   if (RetVT.SimpleTy != MVT::i64)
    6668             :     return 0;
    6669           3 :   return fastEmitInst_rr(AArch64::SMULHrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6670             : }
    6671             : 
    6672             : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6673           3 :   switch (VT.SimpleTy) {
    6674           3 :   case MVT::i64: return fastEmit_ISD_MULHS_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6675             :   default: return 0;
    6676             :   }
    6677             : }
    6678             : 
    6679             : // FastEmit functions for ISD::MULHU.
    6680             : 
    6681             : unsigned fastEmit_ISD_MULHU_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6682           4 :   if (RetVT.SimpleTy != MVT::i64)
    6683             :     return 0;
    6684           4 :   return fastEmitInst_rr(AArch64::UMULHrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6685             : }
    6686             : 
    6687             : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6688           4 :   switch (VT.SimpleTy) {
    6689           4 :   case MVT::i64: return fastEmit_ISD_MULHU_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6690             :   default: return 0;
    6691             :   }
    6692             : }
    6693             : 
    6694             : // FastEmit functions for ISD::OR.
    6695             : 
    6696             : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6697           8 :   if (RetVT.SimpleTy != MVT::i32)
    6698             :     return 0;
    6699           8 :   return fastEmitInst_rr(AArch64::ORRWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6700             : }
    6701             : 
    6702             : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6703           2 :   if (RetVT.SimpleTy != MVT::i64)
    6704             :     return 0;
    6705           2 :   return fastEmitInst_rr(AArch64::ORRXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6706             : }
    6707             : 
    6708             : unsigned fastEmit_ISD_OR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6709           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6710             :     return 0;
    6711           0 :   if ((Subtarget->hasNEON())) {
    6712           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6713             :   }
    6714             :   return 0;
    6715             : }
    6716             : 
    6717             : unsigned fastEmit_ISD_OR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6718           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6719             :     return 0;
    6720           0 :   if ((Subtarget->hasNEON())) {
    6721           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6722             :   }
    6723             :   return 0;
    6724             : }
    6725             : 
    6726             : unsigned fastEmit_ISD_OR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6727           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6728             :     return 0;
    6729           0 :   if ((Subtarget->hasNEON())) {
    6730           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6731             :   }
    6732             :   return 0;
    6733             : }
    6734             : 
    6735             : unsigned fastEmit_ISD_OR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6736           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6737             :     return 0;
    6738           0 :   if ((Subtarget->hasNEON())) {
    6739           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6740             :   }
    6741             :   return 0;
    6742             : }
    6743             : 
    6744             : unsigned fastEmit_ISD_OR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6745           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6746             :     return 0;
    6747           0 :   if ((Subtarget->hasNEON())) {
    6748           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6749             :   }
    6750             :   return 0;
    6751             : }
    6752             : 
    6753             : unsigned fastEmit_ISD_OR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6754           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6755             :     return 0;
    6756           0 :   if ((Subtarget->hasNEON())) {
    6757           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6758             :   }
    6759             :   return 0;
    6760             : }
    6761             : 
    6762             : unsigned fastEmit_ISD_OR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6763           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    6764             :     return 0;
    6765           0 :   if ((Subtarget->hasNEON())) {
    6766           0 :     return fastEmitInst_rr(AArch64::ORRv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6767             :   }
    6768             :   return 0;
    6769             : }
    6770             : 
    6771             : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6772           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6773             :     return 0;
    6774           0 :   if ((Subtarget->hasNEON())) {
    6775           0 :     return fastEmitInst_rr(AArch64::ORRv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6776             :   }
    6777             :   return 0;
    6778             : }
    6779             : 
    6780          10 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6781          10 :   switch (VT.SimpleTy) {
    6782           8 :   case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6783           2 :   case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6784           0 :   case MVT::v8i8: return fastEmit_ISD_OR_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6785           0 :   case MVT::v16i8: return fastEmit_ISD_OR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6786           0 :   case MVT::v4i16: return fastEmit_ISD_OR_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6787           0 :   case MVT::v8i16: return fastEmit_ISD_OR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6788           0 :   case MVT::v2i32: return fastEmit_ISD_OR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6789           0 :   case MVT::v4i32: return fastEmit_ISD_OR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6790           0 :   case MVT::v1i64: return fastEmit_ISD_OR_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6791           0 :   case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6792             :   default: return 0;
    6793             :   }
    6794             : }
    6795             : 
    6796             : // FastEmit functions for ISD::ROTR.
    6797             : 
    6798             : unsigned fastEmit_ISD_ROTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6799           0 :   if (RetVT.SimpleTy != MVT::i64)
    6800             :     return 0;
    6801           0 :   return fastEmitInst_rr(AArch64::RORVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6802             : }
    6803             : 
    6804             : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6805           0 :   switch (VT.SimpleTy) {
    6806           0 :   case MVT::i64: return fastEmit_ISD_ROTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6807             :   default: return 0;
    6808             :   }
    6809             : }
    6810             : 
    6811             : // FastEmit functions for ISD::SDIV.
    6812             : 
    6813             : unsigned fastEmit_ISD_SDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6814           0 :   if (RetVT.SimpleTy != MVT::i32)
    6815             :     return 0;
    6816           0 :   return fastEmitInst_rr(AArch64::SDIVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6817             : }
    6818             : 
    6819             : unsigned fastEmit_ISD_SDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6820           0 :   if (RetVT.SimpleTy != MVT::i64)
    6821             :     return 0;
    6822           0 :   return fastEmitInst_rr(AArch64::SDIVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6823             : }
    6824             : 
    6825           0 : unsigned fastEmit_ISD_SDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6826           0 :   switch (VT.SimpleTy) {
    6827           0 :   case MVT::i32: return fastEmit_ISD_SDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6828           0 :   case MVT::i64: return fastEmit_ISD_SDIV_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6829             :   default: return 0;
    6830             :   }
    6831             : }
    6832             : 
    6833             : // FastEmit functions for ISD::SHL.
    6834             : 
    6835             : unsigned fastEmit_ISD_SHL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6836           0 :   if (RetVT.SimpleTy != MVT::i64)
    6837             :     return 0;
    6838           0 :   return fastEmitInst_rr(AArch64::LSLVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6839             : }
    6840             : 
    6841             : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6842           0 :   switch (VT.SimpleTy) {
    6843           0 :   case MVT::i64: return fastEmit_ISD_SHL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6844             :   default: return 0;
    6845             :   }
    6846             : }
    6847             : 
    6848             : // FastEmit functions for ISD::SMAX.
    6849             : 
    6850             : unsigned fastEmit_ISD_SMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6851           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6852             :     return 0;
    6853           0 :   if ((Subtarget->hasNEON())) {
    6854           0 :     return fastEmitInst_rr(AArch64::SMAXv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6855             :   }
    6856             :   return 0;
    6857             : }
    6858             : 
    6859             : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6860           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6861             :     return 0;
    6862           0 :   if ((Subtarget->hasNEON())) {
    6863           0 :     return fastEmitInst_rr(AArch64::SMAXv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6864             :   }
    6865             :   return 0;
    6866             : }
    6867             : 
    6868             : unsigned fastEmit_ISD_SMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6869           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6870             :     return 0;
    6871           0 :   if ((Subtarget->hasNEON())) {
    6872           0 :     return fastEmitInst_rr(AArch64::SMAXv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6873             :   }
    6874             :   return 0;
    6875             : }
    6876             : 
    6877             : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6878           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6879             :     return 0;
    6880           0 :   if ((Subtarget->hasNEON())) {
    6881           0 :     return fastEmitInst_rr(AArch64::SMAXv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6882             :   }
    6883             :   return 0;
    6884             : }
    6885             : 
    6886             : unsigned fastEmit_ISD_SMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6887           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6888             :     return 0;
    6889           0 :   if ((Subtarget->hasNEON())) {
    6890           0 :     return fastEmitInst_rr(AArch64::SMAXv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6891             :   }
    6892             :   return 0;
    6893             : }
    6894             : 
    6895             : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6896           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6897             :     return 0;
    6898           0 :   if ((Subtarget->hasNEON())) {
    6899           0 :     return fastEmitInst_rr(AArch64::SMAXv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6900             :   }
    6901             :   return 0;
    6902             : }
    6903             : 
    6904           0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6905           0 :   switch (VT.SimpleTy) {
    6906           0 :   case MVT::v8i8: return fastEmit_ISD_SMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6907           0 :   case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6908           0 :   case MVT::v4i16: return fastEmit_ISD_SMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6909           0 :   case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6910           0 :   case MVT::v2i32: return fastEmit_ISD_SMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6911           0 :   case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6912             :   default: return 0;
    6913             :   }
    6914             : }
    6915             : 
    6916             : // FastEmit functions for ISD::SMIN.
    6917             : 
    6918             : unsigned fastEmit_ISD_SMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6919           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    6920             :     return 0;
    6921           0 :   if ((Subtarget->hasNEON())) {
    6922           0 :     return fastEmitInst_rr(AArch64::SMINv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6923             :   }
    6924             :   return 0;
    6925             : }
    6926             : 
    6927             : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6928           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6929             :     return 0;
    6930           0 :   if ((Subtarget->hasNEON())) {
    6931           0 :     return fastEmitInst_rr(AArch64::SMINv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6932             :   }
    6933             :   return 0;
    6934             : }
    6935             : 
    6936             : unsigned fastEmit_ISD_SMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6937           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    6938             :     return 0;
    6939           0 :   if ((Subtarget->hasNEON())) {
    6940           0 :     return fastEmitInst_rr(AArch64::SMINv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6941             :   }
    6942             :   return 0;
    6943             : }
    6944             : 
    6945             : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6946           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6947             :     return 0;
    6948           0 :   if ((Subtarget->hasNEON())) {
    6949           0 :     return fastEmitInst_rr(AArch64::SMINv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6950             :   }
    6951             :   return 0;
    6952             : }
    6953             : 
    6954             : unsigned fastEmit_ISD_SMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6955           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    6956             :     return 0;
    6957           0 :   if ((Subtarget->hasNEON())) {
    6958           0 :     return fastEmitInst_rr(AArch64::SMINv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6959             :   }
    6960             :   return 0;
    6961             : }
    6962             : 
    6963             : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6964           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6965             :     return 0;
    6966           0 :   if ((Subtarget->hasNEON())) {
    6967           0 :     return fastEmitInst_rr(AArch64::SMINv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6968             :   }
    6969             :   return 0;
    6970             : }
    6971             : 
    6972           0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6973           0 :   switch (VT.SimpleTy) {
    6974           0 :   case MVT::v8i8: return fastEmit_ISD_SMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6975           0 :   case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6976           0 :   case MVT::v4i16: return fastEmit_ISD_SMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6977           0 :   case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6978           0 :   case MVT::v2i32: return fastEmit_ISD_SMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6979           0 :   case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6980             :   default: return 0;
    6981             :   }
    6982             : }
    6983             : 
    6984             : // FastEmit functions for ISD::SRA.
    6985             : 
    6986             : unsigned fastEmit_ISD_SRA_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6987           0 :   if (RetVT.SimpleTy != MVT::i64)
    6988             :     return 0;
    6989           0 :   return fastEmitInst_rr(AArch64::ASRVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6990             : }
    6991             : 
    6992             : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6993           0 :   switch (VT.SimpleTy) {
    6994           0 :   case MVT::i64: return fastEmit_ISD_SRA_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6995             :   default: return 0;
    6996             :   }
    6997             : }
    6998             : 
    6999             : // FastEmit functions for ISD::SRL.
    7000             : 
    7001             : unsigned fastEmit_ISD_SRL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7002           3 :   if (RetVT.SimpleTy != MVT::i64)
    7003             :     return 0;
    7004           3 :   return fastEmitInst_rr(AArch64::LSRVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7005             : }
    7006             : 
    7007             : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7008           3 :   switch (VT.SimpleTy) {
    7009           3 :   case MVT::i64: return fastEmit_ISD_SRL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7010             :   default: return 0;
    7011             :   }
    7012             : }
    7013             : 
    7014             : // FastEmit functions for ISD::SUB.
    7015             : 
    7016             : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7017           0 :   if (RetVT.SimpleTy != MVT::i32)
    7018             :     return 0;
    7019           0 :   return fastEmitInst_rr(AArch64::SUBSWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7020             : }
    7021             : 
    7022             : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7023           0 :   if (RetVT.SimpleTy != MVT::i64)
    7024             :     return 0;
    7025           0 :   return fastEmitInst_rr(AArch64::SUBSXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7026             : }
    7027             : 
    7028             : unsigned fastEmit_ISD_SUB_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7029           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7030             :     return 0;
    7031           0 :   if ((Subtarget->hasNEON())) {
    7032           0 :     return fastEmitInst_rr(AArch64::SUBv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7033             :   }
    7034             :   return 0;
    7035             : }
    7036             : 
    7037             : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7038           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7039             :     return 0;
    7040           0 :   if ((Subtarget->hasNEON())) {
    7041           0 :     return fastEmitInst_rr(AArch64::SUBv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7042             :   }
    7043             :   return 0;
    7044             : }
    7045             : 
    7046             : unsigned fastEmit_ISD_SUB_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7047           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7048             :     return 0;
    7049           0 :   if ((Subtarget->hasNEON())) {
    7050           0 :     return fastEmitInst_rr(AArch64::SUBv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7051             :   }
    7052             :   return 0;
    7053             : }
    7054             : 
    7055             : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7056           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7057             :     return 0;
    7058           0 :   if ((Subtarget->hasNEON())) {
    7059           0 :     return fastEmitInst_rr(AArch64::SUBv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7060             :   }
    7061             :   return 0;
    7062             : }
    7063             : 
    7064             : unsigned fastEmit_ISD_SUB_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7065           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7066             :     return 0;
    7067           0 :   if ((Subtarget->hasNEON())) {
    7068           0 :     return fastEmitInst_rr(AArch64::SUBv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7069             :   }
    7070             :   return 0;
    7071             : }
    7072             : 
    7073             : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7074           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7075             :     return 0;
    7076           0 :   if ((Subtarget->hasNEON())) {
    7077           0 :     return fastEmitInst_rr(AArch64::SUBv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7078             :   }
    7079             :   return 0;
    7080             : }
    7081             : 
    7082             : unsigned fastEmit_ISD_SUB_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7083           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    7084             :     return 0;
    7085           0 :   if ((Subtarget->hasNEON())) {
    7086           0 :     return fastEmitInst_rr(AArch64::SUBv1i64, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7087             :   }
    7088             :   return 0;
    7089             : }
    7090             : 
    7091             : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7092           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7093             :     return 0;
    7094           0 :   if ((Subtarget->hasNEON())) {
    7095           0 :     return fastEmitInst_rr(AArch64::SUBv2i64, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7096             :   }
    7097             :   return 0;
    7098             : }
    7099             : 
    7100           0 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7101           0 :   switch (VT.SimpleTy) {
    7102           0 :   case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7103           0 :   case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7104           0 :   case MVT::v8i8: return fastEmit_ISD_SUB_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7105           0 :   case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7106           0 :   case MVT::v4i16: return fastEmit_ISD_SUB_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7107           0 :   case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7108           0 :   case MVT::v2i32: return fastEmit_ISD_SUB_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7109           0 :   case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7110           0 :   case MVT::v1i64: return fastEmit_ISD_SUB_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7111           0 :   case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7112             :   default: return 0;
    7113             :   }
    7114             : }
    7115             : 
    7116             : // FastEmit functions for ISD::UDIV.
    7117             : 
    7118             : unsigned fastEmit_ISD_UDIV_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7119           0 :   if (RetVT.SimpleTy != MVT::i32)
    7120             :     return 0;
    7121           0 :   return fastEmitInst_rr(AArch64::UDIVWr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7122             : }
    7123             : 
    7124             : unsigned fastEmit_ISD_UDIV_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7125           0 :   if (RetVT.SimpleTy != MVT::i64)
    7126             :     return 0;
    7127           0 :   return fastEmitInst_rr(AArch64::UDIVXr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7128             : }
    7129             : 
    7130           0 : unsigned fastEmit_ISD_UDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7131           0 :   switch (VT.SimpleTy) {
    7132           0 :   case MVT::i32: return fastEmit_ISD_UDIV_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7133           0 :   case MVT::i64: return fastEmit_ISD_UDIV_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7134             :   default: return 0;
    7135             :   }
    7136             : }
    7137             : 
    7138             : // FastEmit functions for ISD::UMAX.
    7139             : 
    7140             : unsigned fastEmit_ISD_UMAX_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7141           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7142             :     return 0;
    7143           0 :   if ((Subtarget->hasNEON())) {
    7144           0 :     return fastEmitInst_rr(AArch64::UMAXv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7145             :   }
    7146             :   return 0;
    7147             : }
    7148             : 
    7149             : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7150           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7151             :     return 0;
    7152           0 :   if ((Subtarget->hasNEON())) {
    7153           0 :     return fastEmitInst_rr(AArch64::UMAXv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7154             :   }
    7155             :   return 0;
    7156             : }
    7157             : 
    7158             : unsigned fastEmit_ISD_UMAX_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7159           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7160             :     return 0;
    7161           0 :   if ((Subtarget->hasNEON())) {
    7162           0 :     return fastEmitInst_rr(AArch64::UMAXv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7163             :   }
    7164             :   return 0;
    7165             : }
    7166             : 
    7167             : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7168           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7169             :     return 0;
    7170           0 :   if ((Subtarget->hasNEON())) {
    7171           0 :     return fastEmitInst_rr(AArch64::UMAXv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7172             :   }
    7173             :   return 0;
    7174             : }
    7175             : 
    7176             : unsigned fastEmit_ISD_UMAX_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7177           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7178             :     return 0;
    7179           0 :   if ((Subtarget->hasNEON())) {
    7180           0 :     return fastEmitInst_rr(AArch64::UMAXv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7181             :   }
    7182             :   return 0;
    7183             : }
    7184             : 
    7185             : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7186           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7187             :     return 0;
    7188           0 :   if ((Subtarget->hasNEON())) {
    7189           0 :     return fastEmitInst_rr(AArch64::UMAXv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7190             :   }
    7191             :   return 0;
    7192             : }
    7193             : 
    7194           0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7195           0 :   switch (VT.SimpleTy) {
    7196           0 :   case MVT::v8i8: return fastEmit_ISD_UMAX_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7197           0 :   case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7198           0 :   case MVT::v4i16: return fastEmit_ISD_UMAX_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7199           0 :   case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7200           0 :   case MVT::v2i32: return fastEmit_ISD_UMAX_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7201           0 :   case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7202             :   default: return 0;
    7203             :   }
    7204             : }
    7205             : 
    7206             : // FastEmit functions for ISD::UMIN.
    7207             : 
    7208             : unsigned fastEmit_ISD_UMIN_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7209           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7210             :     return 0;
    7211           0 :   if ((Subtarget->hasNEON())) {
    7212           0 :     return fastEmitInst_rr(AArch64::UMINv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7213             :   }
    7214             :   return 0;
    7215             : }
    7216             : 
    7217             : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7218           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7219             :     return 0;
    7220           0 :   if ((Subtarget->hasNEON())) {
    7221           0 :     return fastEmitInst_rr(AArch64::UMINv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7222             :   }
    7223             :   return 0;
    7224             : }
    7225             : 
    7226             : unsigned fastEmit_ISD_UMIN_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7227           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7228             :     return 0;
    7229           0 :   if ((Subtarget->hasNEON())) {
    7230           0 :     return fastEmitInst_rr(AArch64::UMINv4i16, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7231             :   }
    7232             :   return 0;
    7233             : }
    7234             : 
    7235             : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7236           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7237             :     return 0;
    7238           0 :   if ((Subtarget->hasNEON())) {
    7239           0 :     return fastEmitInst_rr(AArch64::UMINv8i16, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7240             :   }
    7241             :   return 0;
    7242             : }
    7243             : 
    7244             : unsigned fastEmit_ISD_UMIN_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7245           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7246             :     return 0;
    7247           0 :   if ((Subtarget->hasNEON())) {
    7248           0 :     return fastEmitInst_rr(AArch64::UMINv2i32, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7249             :   }
    7250             :   return 0;
    7251             : }
    7252             : 
    7253             : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7254           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7255             :     return 0;
    7256           0 :   if ((Subtarget->hasNEON())) {
    7257           0 :     return fastEmitInst_rr(AArch64::UMINv4i32, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7258             :   }
    7259             :   return 0;
    7260             : }
    7261             : 
    7262           0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7263           0 :   switch (VT.SimpleTy) {
    7264           0 :   case MVT::v8i8: return fastEmit_ISD_UMIN_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7265           0 :   case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7266           0 :   case MVT::v4i16: return fastEmit_ISD_UMIN_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7267           0 :   case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7268           0 :   case MVT::v2i32: return fastEmit_ISD_UMIN_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7269           0 :   case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7270             :   default: return 0;
    7271             :   }
    7272             : }
    7273             : 
    7274             : // FastEmit functions for ISD::XOR.
    7275             : 
    7276             : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7277           7 :   if (RetVT.SimpleTy != MVT::i32)
    7278             :     return 0;
    7279           7 :   return fastEmitInst_rr(AArch64::EORWrr, &AArch64::GPR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7280             : }
    7281             : 
    7282             : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7283           2 :   if (RetVT.SimpleTy != MVT::i64)
    7284             :     return 0;
    7285           2 :   return fastEmitInst_rr(AArch64::EORXrr, &AArch64::GPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7286             : }
    7287             : 
    7288             : unsigned fastEmit_ISD_XOR_MVT_v8i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7289           0 :   if (RetVT.SimpleTy != MVT::v8i8)
    7290             :     return 0;
    7291           0 :   if ((Subtarget->hasNEON())) {
    7292           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7293             :   }
    7294             :   return 0;
    7295             : }
    7296             : 
    7297             : unsigned fastEmit_ISD_XOR_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7298           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7299             :     return 0;
    7300           0 :   if ((Subtarget->hasNEON())) {
    7301           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7302             :   }
    7303             :   return 0;
    7304             : }
    7305             : 
    7306             : unsigned fastEmit_ISD_XOR_MVT_v4i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7307           0 :   if (RetVT.SimpleTy != MVT::v4i16)
    7308             :     return 0;
    7309           0 :   if ((Subtarget->hasNEON())) {
    7310           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7311             :   }
    7312             :   return 0;
    7313             : }
    7314             : 
    7315             : unsigned fastEmit_ISD_XOR_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7316           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7317             :     return 0;
    7318           0 :   if ((Subtarget->hasNEON())) {
    7319           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7320             :   }
    7321             :   return 0;
    7322             : }
    7323             : 
    7324             : unsigned fastEmit_ISD_XOR_MVT_v2i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7325           0 :   if (RetVT.SimpleTy != MVT::v2i32)
    7326             :     return 0;
    7327           0 :   if ((Subtarget->hasNEON())) {
    7328           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7329             :   }
    7330             :   return 0;
    7331             : }
    7332             : 
    7333             : unsigned fastEmit_ISD_XOR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7334           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7335             :     return 0;
    7336           0 :   if ((Subtarget->hasNEON())) {
    7337           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7338             :   }
    7339             :   return 0;
    7340             : }
    7341             : 
    7342             : unsigned fastEmit_ISD_XOR_MVT_v1i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7343           0 :   if (RetVT.SimpleTy != MVT::v1i64)
    7344             :     return 0;
    7345           0 :   if ((Subtarget->hasNEON())) {
    7346           0 :     return fastEmitInst_rr(AArch64::EORv8i8, &AArch64::FPR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7347             :   }
    7348             :   return 0;
    7349             : }
    7350             : 
    7351             : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7352           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7353             :     return 0;
    7354           0 :   if ((Subtarget->hasNEON())) {
    7355           0 :     return fastEmitInst_rr(AArch64::EORv16i8, &AArch64::FPR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7356             :   }
    7357             :   return 0;
    7358             : }
    7359             : 
    7360           9 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7361           9 :   switch (VT.SimpleTy) {
    7362           7 :   case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7363           2 :   case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7364           0 :   case MVT::v8i8: return fastEmit_ISD_XOR_MVT_v8i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7365           0 :   case MVT::v16i8: return fastEmit_ISD_XOR_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7366           0 :   case MVT::v4i16: return fastEmit_ISD_XOR_MVT_v4i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7367           0 :   case MVT::v8i16: return fastEmit_ISD_XOR_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7368           0 :   case MVT::v2i32: return fastEmit_ISD_XOR_MVT_v2i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7369           0 :   case MVT::v4i32: return fastEmit_ISD_XOR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7370           0 :   case MVT::v1i64: return fastEmit_ISD_XOR_MVT_v1i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7371           0 :   case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7372             :   default: return 0;
    7373             :   }
    7374             : }
    7375             : 
    7376             : // Top-level FastEmit function.
    7377             : 
    7378         264 : unsigned fastEmit_rr(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) override {
    7379         264 :   switch (Opcode) {
    7380           0 :   case AArch64ISD::CMEQ: return fastEmit_AArch64ISD_CMEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7381           0 :   case AArch64ISD::CMGE: return fastEmit_AArch64ISD_CMGE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7382           0 :   case AArch64ISD::CMGT: return fastEmit_AArch64ISD_CMGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7383           0 :   case AArch64ISD::CMHI: return fastEmit_AArch64ISD_CMHI_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7384           0 :   case AArch64ISD::CMHS: return fastEmit_AArch64ISD_CMHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7385           0 :   case AArch64ISD::FCMEQ: return fastEmit_AArch64ISD_FCMEQ_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7386           0 :   case AArch64ISD::FCMGE: return fastEmit_AArch64ISD_FCMGE_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7387           0 :   case AArch64ISD::FCMGT: return fastEmit_AArch64ISD_FCMGT_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7388           0 :   case AArch64ISD::FCMP: return fastEmit_AArch64ISD_FCMP_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7389           0 :   case AArch64ISD::FRECPS: return fastEmit_AArch64ISD_FRECPS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7390           0 :   case AArch64ISD::FRSQRTS: return fastEmit_AArch64ISD_FRSQRTS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7391           0 :   case AArch64ISD::SMULL: return fastEmit_AArch64ISD_SMULL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7392           0 :   case AArch64ISD::TRN1: return fastEmit_AArch64ISD_TRN1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7393           0 :   case AArch64ISD::TRN2: return fastEmit_AArch64ISD_TRN2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7394           0 :   case AArch64ISD::UMULL: return fastEmit_AArch64ISD_UMULL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7395           0 :   case AArch64ISD::UZP1: return fastEmit_AArch64ISD_UZP1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7396           0 :   case AArch64ISD::UZP2: return fastEmit_AArch64ISD_UZP2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7397           0 :   case AArch64ISD::ZIP1: return fastEmit_AArch64ISD_ZIP1_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7398           0 :   case AArch64ISD::ZIP2: return fastEmit_AArch64ISD_ZIP2_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7399          74 :   case ISD::ADD: return fastEmit_ISD_ADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7400          34 :   case ISD::AND: return fastEmit_ISD_AND_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7401         119 :   case ISD::FADD: return fastEmit_ISD_FADD_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7402           0 :   case ISD::FDIV: return fastEmit_ISD_FDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7403           0 :   case ISD::FMAXNAN: return fastEmit_ISD_FMAXNAN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7404           0 :   case ISD::FMAXNUM: return fastEmit_ISD_FMAXNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7405           0 :   case ISD::FMINNAN: return fastEmit_ISD_FMINNAN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7406           0 :   case ISD::FMINNUM: return fastEmit_ISD_FMINNUM_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7407           0 :   case ISD::FMUL: return fastEmit_ISD_FMUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7408           0 :   case ISD::FSUB: return fastEmit_ISD_FSUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7409           0 :   case ISD::MUL: return fastEmit_ISD_MUL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7410           3 :   case ISD::MULHS: return fastEmit_ISD_MULHS_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7411           4 :   case ISD::MULHU: return fastEmit_ISD_MULHU_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7412          10 :   case ISD::OR: return fastEmit_ISD_OR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7413           0 :   case ISD::ROTR: return fastEmit_ISD_ROTR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7414           0 :   case ISD::SDIV: return fastEmit_ISD_SDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7415           0 :   case ISD::SHL: return fastEmit_ISD_SHL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7416           0 :   case ISD::SMAX: return fastEmit_ISD_SMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7417           0 :   case ISD::SMIN: return fastEmit_ISD_SMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7418           0 :   case ISD::SRA: return fastEmit_ISD_SRA_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7419           3 :   case ISD::SRL: return fastEmit_ISD_SRL_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7420           0 :   case ISD::SUB: return fastEmit_ISD_SUB_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7421           0 :   case ISD::UDIV: return fastEmit_ISD_UDIV_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7422           0 :   case ISD::UMAX: return fastEmit_ISD_UMAX_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7423           0 :   case ISD::UMIN: return fastEmit_ISD_UMIN_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7424           9 :   case ISD::XOR: return fastEmit_ISD_XOR_rr(VT, RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7425             :   default: return 0;
    7426             :   }
    7427             : }
    7428             : 
    7429             : // FastEmit functions for AArch64ISD::DUPLANE64.
    7430             : 
    7431             : unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7432             :   if (RetVT.SimpleTy != MVT::v2i64)
    7433             :     return 0;
    7434             :   if ((Subtarget->hasNEON())) {
    7435             :     return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7436             :   }
    7437             :   return 0;
    7438             : }
    7439             : 
    7440             : unsigned fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7441             :   if (RetVT.SimpleTy != MVT::v2f64)
    7442             :     return 0;
    7443             :   return fastEmitInst_ri(AArch64::DUPv2i64lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7444             : }
    7445             : 
    7446             : unsigned fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7447             :   switch (VT.SimpleTy) {
    7448             :   case MVT::v2i64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7449             :   case MVT::v2f64: return fastEmit_AArch64ISD_DUPLANE64_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7450             :   default: return 0;
    7451             :   }
    7452             : }
    7453             : 
    7454             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7455             : 
    7456             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7457             :   if (RetVT.SimpleTy != MVT::i64)
    7458             :     return 0;
    7459             :   if ((Subtarget->hasNEON())) {
    7460             :     return fastEmitInst_ri(AArch64::UMOVvi64, &AArch64::GPR64RegClass, Op0, Op0IsKill, imm1);
    7461             :   }
    7462             :   return 0;
    7463             : }
    7464             : 
    7465             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7466             :   if (RetVT.SimpleTy != MVT::f64)
    7467             :     return 0;
    7468             :   return fastEmitInst_ri(AArch64::CPYi64, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7469             : }
    7470             : 
    7471             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7472             :   switch (VT.SimpleTy) {
    7473             :   case MVT::v2i64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2i64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7474             :   case MVT::v2f64: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v2f64_ri_Predicate_VectorIndexD(RetVT, Op0, Op0IsKill, imm1);
    7475             :   default: return 0;
    7476             :   }
    7477             : }
    7478             : 
    7479             : // Top-level FastEmit function.
    7480             : 
    7481             : unsigned fastEmit_ri_Predicate_VectorIndexD(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7482             :   switch (Opcode) {
    7483             :   case AArch64ISD::DUPLANE64: return fastEmit_AArch64ISD_DUPLANE64_ri_Predicate_VectorIndexD(VT, RetVT, Op0, Op0IsKill, imm1);
    7484             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexD(VT, RetVT, Op0, Op0IsKill, imm1);
    7485             :   default: return 0;
    7486             :   }
    7487             : }
    7488             : 
    7489             : // FastEmit functions for AArch64ISD::DUPLANE32.
    7490             : 
    7491             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7492             :   if ((Subtarget->hasNEON())) {
    7493             :     return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7494             :   }
    7495             :   return 0;
    7496             : }
    7497             : 
    7498             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7499             :   if ((Subtarget->hasNEON())) {
    7500             :     return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7501             :   }
    7502             :   return 0;
    7503             : }
    7504             : 
    7505             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7506             : switch (RetVT.SimpleTy) {
    7507             :   case MVT::v2i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v2i32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7508             :   case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_MVT_v4i32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7509             :   default: return 0;
    7510             : }
    7511             : }
    7512             : 
    7513             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7514             :   return fastEmitInst_ri(AArch64::DUPv2i32lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7515             : }
    7516             : 
    7517             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7518             :   return fastEmitInst_ri(AArch64::DUPv4i32lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7519             : }
    7520             : 
    7521             : unsigned fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7522             : switch (RetVT.SimpleTy) {
    7523             :   case MVT::v2f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v2f32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7524             :   case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_MVT_v4f32_ri_Predicate_VectorIndexS(Op0, Op0IsKill, imm1);
    7525             :   default: return 0;
    7526             : }
    7527             : }
    7528             : 
    7529             : unsigned fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7530             :   switch (VT.SimpleTy) {
    7531             :   case MVT::v4i32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7532             :   case MVT::v4f32: return fastEmit_AArch64ISD_DUPLANE32_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7533             :   default: return 0;
    7534             :   }
    7535             : }
    7536             : 
    7537             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7538             : 
    7539             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7540             :   if (RetVT.SimpleTy != MVT::i32)
    7541             :     return 0;
    7542             :   if ((Subtarget->hasNEON())) {
    7543             :     return fastEmitInst_ri(AArch64::UMOVvi32, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
    7544             :   }
    7545             :   return 0;
    7546             : }
    7547             : 
    7548             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7549             :   if (RetVT.SimpleTy != MVT::f32)
    7550             :     return 0;
    7551             :   return fastEmitInst_ri(AArch64::CPYi32, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7552             : }
    7553             : 
    7554             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7555             :   switch (VT.SimpleTy) {
    7556             :   case MVT::v4i32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4i32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7557             :   case MVT::v4f32: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v4f32_ri_Predicate_VectorIndexS(RetVT, Op0, Op0IsKill, imm1);
    7558             :   default: return 0;
    7559             :   }
    7560             : }
    7561             : 
    7562             : // Top-level FastEmit function.
    7563             : 
    7564             : unsigned fastEmit_ri_Predicate_VectorIndexS(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7565             :   switch (Opcode) {
    7566             :   case AArch64ISD::DUPLANE32: return fastEmit_AArch64ISD_DUPLANE32_ri_Predicate_VectorIndexS(VT, RetVT, Op0, Op0IsKill, imm1);
    7567             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexS(VT, RetVT, Op0, Op0IsKill, imm1);
    7568             :   default: return 0;
    7569             :   }
    7570             : }
    7571             : 
    7572             : // FastEmit functions for AArch64ISD::DUPLANE16.
    7573             : 
    7574             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7575             :   if ((Subtarget->hasNEON())) {
    7576             :     return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7577             :   }
    7578             :   return 0;
    7579             : }
    7580             : 
    7581             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7582             :   if ((Subtarget->hasNEON())) {
    7583             :     return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7584             :   }
    7585             :   return 0;
    7586             : }
    7587             : 
    7588             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7589             : switch (RetVT.SimpleTy) {
    7590             :   case MVT::v4i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v4i16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7591             :   case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_MVT_v8i16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7592             :   default: return 0;
    7593             : }
    7594             : }
    7595             : 
    7596             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7597             :   return fastEmitInst_ri(AArch64::DUPv4i16lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7598             : }
    7599             : 
    7600             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7601             :   return fastEmitInst_ri(AArch64::DUPv8i16lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7602             : }
    7603             : 
    7604             : unsigned fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7605             : switch (RetVT.SimpleTy) {
    7606             :   case MVT::v4f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v4f16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7607             :   case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_MVT_v8f16_ri_Predicate_VectorIndexH(Op0, Op0IsKill, imm1);
    7608             :   default: return 0;
    7609             : }
    7610             : }
    7611             : 
    7612             : unsigned fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7613             :   switch (VT.SimpleTy) {
    7614             :   case MVT::v8i16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7615             :   case MVT::v8f16: return fastEmit_AArch64ISD_DUPLANE16_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7616             :   default: return 0;
    7617             :   }
    7618             : }
    7619             : 
    7620             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7621             : 
    7622             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7623             :   if (RetVT.SimpleTy != MVT::i32)
    7624             :     return 0;
    7625             :   if ((Subtarget->hasNEON())) {
    7626             :     return fastEmitInst_ri(AArch64::UMOVvi16, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
    7627             :   }
    7628             :   return 0;
    7629             : }
    7630             : 
    7631             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7632             :   if (RetVT.SimpleTy != MVT::f16)
    7633             :     return 0;
    7634             :   return fastEmitInst_ri(AArch64::CPYi16, &AArch64::FPR16RegClass, Op0, Op0IsKill, imm1);
    7635             : }
    7636             : 
    7637             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7638             :   switch (VT.SimpleTy) {
    7639             :   case MVT::v8i16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8i16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7640             :   case MVT::v8f16: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v8f16_ri_Predicate_VectorIndexH(RetVT, Op0, Op0IsKill, imm1);
    7641             :   default: return 0;
    7642             :   }
    7643             : }
    7644             : 
    7645             : // Top-level FastEmit function.
    7646             : 
    7647             : unsigned fastEmit_ri_Predicate_VectorIndexH(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7648             :   switch (Opcode) {
    7649             :   case AArch64ISD::DUPLANE16: return fastEmit_AArch64ISD_DUPLANE16_ri_Predicate_VectorIndexH(VT, RetVT, Op0, Op0IsKill, imm1);
    7650             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexH(VT, RetVT, Op0, Op0IsKill, imm1);
    7651             :   default: return 0;
    7652             :   }
    7653             : }
    7654             : 
    7655             : // FastEmit functions for AArch64ISD::DUPLANE8.
    7656             : 
    7657             : unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7658             :   if ((Subtarget->hasNEON())) {
    7659             :     return fastEmitInst_ri(AArch64::DUPv8i8lane, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7660             :   }
    7661             :   return 0;
    7662             : }
    7663             : 
    7664             : unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7665             :   if ((Subtarget->hasNEON())) {
    7666             :     return fastEmitInst_ri(AArch64::DUPv16i8lane, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7667             :   }
    7668             :   return 0;
    7669             : }
    7670             : 
    7671             : unsigned fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7672             : switch (RetVT.SimpleTy) {
    7673             :   case MVT::v8i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v8i8_ri_Predicate_VectorIndexB(Op0, Op0IsKill, imm1);
    7674             :   case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_MVT_v16i8_ri_Predicate_VectorIndexB(Op0, Op0IsKill, imm1);
    7675             :   default: return 0;
    7676             : }
    7677             : }
    7678             : 
    7679             : unsigned fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7680             :   switch (VT.SimpleTy) {
    7681             :   case MVT::v16i8: return fastEmit_AArch64ISD_DUPLANE8_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, Op0IsKill, imm1);
    7682             :   default: return 0;
    7683             :   }
    7684             : }
    7685             : 
    7686             : // FastEmit functions for ISD::EXTRACT_VECTOR_ELT.
    7687             : 
    7688             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7689             :   if (RetVT.SimpleTy != MVT::i32)
    7690             :     return 0;
    7691             :   if ((Subtarget->hasNEON())) {
    7692             :     return fastEmitInst_ri(AArch64::UMOVvi8, &AArch64::GPR32RegClass, Op0, Op0IsKill, imm1);
    7693             :   }
    7694             :   return 0;
    7695             : }
    7696             : 
    7697             : unsigned fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7698             :   switch (VT.SimpleTy) {
    7699             :   case MVT::v16i8: return fastEmit_ISD_EXTRACT_VECTOR_ELT_MVT_v16i8_ri_Predicate_VectorIndexB(RetVT, Op0, Op0IsKill, imm1);
    7700             :   default: return 0;
    7701             :   }
    7702             : }
    7703             : 
    7704             : // Top-level FastEmit function.
    7705             : 
    7706             : unsigned fastEmit_ri_Predicate_VectorIndexB(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7707             :   switch (Opcode) {
    7708             :   case AArch64ISD::DUPLANE8: return fastEmit_AArch64ISD_DUPLANE8_ri_Predicate_VectorIndexB(VT, RetVT, Op0, Op0IsKill, imm1);
    7709             :   case ISD::EXTRACT_VECTOR_ELT: return fastEmit_ISD_EXTRACT_VECTOR_ELT_ri_Predicate_VectorIndexB(VT, RetVT, Op0, Op0IsKill, imm1);
    7710             :   default: return 0;
    7711             :   }
    7712             : }
    7713             : 
    7714             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    7715             : 
    7716             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7717             :   if (RetVT.SimpleTy != MVT::i64)
    7718             :     return 0;
    7719             :   if ((Subtarget->hasNEON())) {
    7720             :     return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7721             :   }
    7722             :   return 0;
    7723             : }
    7724             : 
    7725             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7726             :   if (RetVT.SimpleTy != MVT::v1i64)
    7727             :     return 0;
    7728             :   if ((Subtarget->hasNEON())) {
    7729             :     return fastEmitInst_ri(AArch64::SQSHLUd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7730             :   }
    7731             :   return 0;
    7732             : }
    7733             : 
    7734             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7735             :   if (RetVT.SimpleTy != MVT::v2i64)
    7736             :     return 0;
    7737             :   if ((Subtarget->hasNEON())) {
    7738             :     return fastEmitInst_ri(AArch64::SQSHLUv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7739             :   }
    7740             :   return 0;
    7741             : }
    7742             : 
    7743             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7744             :   switch (VT.SimpleTy) {
    7745             :   case MVT::i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7746             :   case MVT::v1i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7747             :   case MVT::v2i64: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7748             :   default: return 0;
    7749             :   }
    7750             : }
    7751             : 
    7752             : // FastEmit functions for AArch64ISD::SQSHL_I.
    7753             : 
    7754             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7755             :   if (RetVT.SimpleTy != MVT::i64)
    7756             :     return 0;
    7757             :   if ((Subtarget->hasNEON())) {
    7758             :     return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7759             :   }
    7760             :   return 0;
    7761             : }
    7762             : 
    7763             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7764             :   if (RetVT.SimpleTy != MVT::v1i64)
    7765             :     return 0;
    7766             :   if ((Subtarget->hasNEON())) {
    7767             :     return fastEmitInst_ri(AArch64::SQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7768             :   }
    7769             :   return 0;
    7770             : }
    7771             : 
    7772             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7773             :   if (RetVT.SimpleTy != MVT::v2i64)
    7774             :     return 0;
    7775             :   if ((Subtarget->hasNEON())) {
    7776             :     return fastEmitInst_ri(AArch64::SQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7777             :   }
    7778             :   return 0;
    7779             : }
    7780             : 
    7781             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7782             :   switch (VT.SimpleTy) {
    7783             :   case MVT::i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7784             :   case MVT::v1i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7785             :   case MVT::v2i64: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7786             :   default: return 0;
    7787             :   }
    7788             : }
    7789             : 
    7790             : // FastEmit functions for AArch64ISD::UQSHL_I.
    7791             : 
    7792             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7793             :   if (RetVT.SimpleTy != MVT::i64)
    7794             :     return 0;
    7795             :   if ((Subtarget->hasNEON())) {
    7796             :     return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7797             :   }
    7798             :   return 0;
    7799             : }
    7800             : 
    7801             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7802             :   if (RetVT.SimpleTy != MVT::v1i64)
    7803             :     return 0;
    7804             :   if ((Subtarget->hasNEON())) {
    7805             :     return fastEmitInst_ri(AArch64::UQSHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7806             :   }
    7807             :   return 0;
    7808             : }
    7809             : 
    7810             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7811             :   if (RetVT.SimpleTy != MVT::v2i64)
    7812             :     return 0;
    7813             :   if ((Subtarget->hasNEON())) {
    7814             :     return fastEmitInst_ri(AArch64::UQSHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7815             :   }
    7816             :   return 0;
    7817             : }
    7818             : 
    7819             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7820             :   switch (VT.SimpleTy) {
    7821             :   case MVT::i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7822             :   case MVT::v1i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7823             :   case MVT::v2i64: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7824             :   default: return 0;
    7825             :   }
    7826             : }
    7827             : 
    7828             : // FastEmit functions for AArch64ISD::VSHL.
    7829             : 
    7830             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7831             :   if (RetVT.SimpleTy != MVT::v1i64)
    7832             :     return 0;
    7833             :   if ((Subtarget->hasNEON())) {
    7834             :     return fastEmitInst_ri(AArch64::SHLd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7835             :   }
    7836             :   return 0;
    7837             : }
    7838             : 
    7839             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7840             :   if (RetVT.SimpleTy != MVT::v2i64)
    7841             :     return 0;
    7842             :   if ((Subtarget->hasNEON())) {
    7843             :     return fastEmitInst_ri(AArch64::SHLv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7844             :   }
    7845             :   return 0;
    7846             : }
    7847             : 
    7848             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7849             :   switch (VT.SimpleTy) {
    7850             :   case MVT::v1i64: return fastEmit_AArch64ISD_VSHL_MVT_v1i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7851             :   case MVT::v2i64: return fastEmit_AArch64ISD_VSHL_MVT_v2i64_ri_Predicate_vecshiftL64(RetVT, Op0, Op0IsKill, imm1);
    7852             :   default: return 0;
    7853             :   }
    7854             : }
    7855             : 
    7856             : // Top-level FastEmit function.
    7857             : 
    7858             : unsigned fastEmit_ri_Predicate_vecshiftL64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7859             :   switch (Opcode) {
    7860             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7861             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7862             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7863             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL64(VT, RetVT, Op0, Op0IsKill, imm1);
    7864             :   default: return 0;
    7865             :   }
    7866             : }
    7867             : 
    7868             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    7869             : 
    7870             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7871             :   if (RetVT.SimpleTy != MVT::i32)
    7872             :     return 0;
    7873             :   if ((Subtarget->hasNEON())) {
    7874             :     return fastEmitInst_ri(AArch64::SQSHLUs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7875             :   }
    7876             :   return 0;
    7877             : }
    7878             : 
    7879             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7880             :   if (RetVT.SimpleTy != MVT::v2i32)
    7881             :     return 0;
    7882             :   if ((Subtarget->hasNEON())) {
    7883             :     return fastEmitInst_ri(AArch64::SQSHLUv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7884             :   }
    7885             :   return 0;
    7886             : }
    7887             : 
    7888             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7889             :   if (RetVT.SimpleTy != MVT::v4i32)
    7890             :     return 0;
    7891             :   if ((Subtarget->hasNEON())) {
    7892             :     return fastEmitInst_ri(AArch64::SQSHLUv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7893             :   }
    7894             :   return 0;
    7895             : }
    7896             : 
    7897             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7898             :   switch (VT.SimpleTy) {
    7899             :   case MVT::i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7900             :   case MVT::v2i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7901             :   case MVT::v4i32: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7902             :   default: return 0;
    7903             :   }
    7904             : }
    7905             : 
    7906             : // FastEmit functions for AArch64ISD::SQSHL_I.
    7907             : 
    7908             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7909             :   if (RetVT.SimpleTy != MVT::i32)
    7910             :     return 0;
    7911             :   if ((Subtarget->hasNEON())) {
    7912             :     return fastEmitInst_ri(AArch64::SQSHLs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7913             :   }
    7914             :   return 0;
    7915             : }
    7916             : 
    7917             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7918             :   if (RetVT.SimpleTy != MVT::v2i32)
    7919             :     return 0;
    7920             :   if ((Subtarget->hasNEON())) {
    7921             :     return fastEmitInst_ri(AArch64::SQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7922             :   }
    7923             :   return 0;
    7924             : }
    7925             : 
    7926             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7927             :   if (RetVT.SimpleTy != MVT::v4i32)
    7928             :     return 0;
    7929             :   if ((Subtarget->hasNEON())) {
    7930             :     return fastEmitInst_ri(AArch64::SQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7931             :   }
    7932             :   return 0;
    7933             : }
    7934             : 
    7935             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7936             :   switch (VT.SimpleTy) {
    7937             :   case MVT::i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7938             :   case MVT::v2i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7939             :   case MVT::v4i32: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7940             :   default: return 0;
    7941             :   }
    7942             : }
    7943             : 
    7944             : // FastEmit functions for AArch64ISD::UQSHL_I.
    7945             : 
    7946             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7947             :   if (RetVT.SimpleTy != MVT::i32)
    7948             :     return 0;
    7949             :   if ((Subtarget->hasNEON())) {
    7950             :     return fastEmitInst_ri(AArch64::UQSHLs, &AArch64::FPR32RegClass, Op0, Op0IsKill, imm1);
    7951             :   }
    7952             :   return 0;
    7953             : }
    7954             : 
    7955             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7956             :   if (RetVT.SimpleTy != MVT::v2i32)
    7957             :     return 0;
    7958             :   if ((Subtarget->hasNEON())) {
    7959             :     return fastEmitInst_ri(AArch64::UQSHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7960             :   }
    7961             :   return 0;
    7962             : }
    7963             : 
    7964             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7965             :   if (RetVT.SimpleTy != MVT::v4i32)
    7966             :     return 0;
    7967             :   if ((Subtarget->hasNEON())) {
    7968             :     return fastEmitInst_ri(AArch64::UQSHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7969             :   }
    7970             :   return 0;
    7971             : }
    7972             : 
    7973             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7974             :   switch (VT.SimpleTy) {
    7975             :   case MVT::i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7976             :   case MVT::v2i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7977             :   case MVT::v4i32: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    7978             :   default: return 0;
    7979             :   }
    7980             : }
    7981             : 
    7982             : // FastEmit functions for AArch64ISD::VSHL.
    7983             : 
    7984             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7985             :   if (RetVT.SimpleTy != MVT::v2i32)
    7986             :     return 0;
    7987             :   if ((Subtarget->hasNEON())) {
    7988             :     return fastEmitInst_ri(AArch64::SHLv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    7989             :   }
    7990             :   return 0;
    7991             : }
    7992             : 
    7993             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    7994             :   if (RetVT.SimpleTy != MVT::v4i32)
    7995             :     return 0;
    7996             :   if ((Subtarget->hasNEON())) {
    7997             :     return fastEmitInst_ri(AArch64::SHLv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    7998             :   }
    7999             :   return 0;
    8000             : }
    8001             : 
    8002             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8003             :   switch (VT.SimpleTy) {
    8004             :   case MVT::v2i32: return fastEmit_AArch64ISD_VSHL_MVT_v2i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    8005             :   case MVT::v4i32: return fastEmit_AArch64ISD_VSHL_MVT_v4i32_ri_Predicate_vecshiftL32(RetVT, Op0, Op0IsKill, imm1);
    8006             :   default: return 0;
    8007             :   }
    8008             : }
    8009             : 
    8010             : // Top-level FastEmit function.
    8011             : 
    8012             : unsigned fastEmit_ri_Predicate_vecshiftL32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8013             :   switch (Opcode) {
    8014             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8015             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8016             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8017             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL32(VT, RetVT, Op0, Op0IsKill, imm1);
    8018             :   default: return 0;
    8019             :   }
    8020             : }
    8021             : 
    8022             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8023             : 
    8024             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8025             :   if (RetVT.SimpleTy != MVT::i64)
    8026             :     return 0;
    8027             :   if ((Subtarget->hasNEON())) {
    8028             :     return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8029             :   }
    8030             :   return 0;
    8031             : }
    8032             : 
    8033             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8034             :   if (RetVT.SimpleTy != MVT::v1i64)
    8035             :     return 0;
    8036             :   if ((Subtarget->hasNEON())) {
    8037             :     return fastEmitInst_ri(AArch64::SRSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8038             :   }
    8039             :   return 0;
    8040             : }
    8041             : 
    8042             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8043             :   if (RetVT.SimpleTy != MVT::v2i64)
    8044             :     return 0;
    8045             :   if ((Subtarget->hasNEON())) {
    8046             :     return fastEmitInst_ri(AArch64::SRSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8047             :   }
    8048             :   return 0;
    8049             : }
    8050             : 
    8051             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8052             :   switch (VT.SimpleTy) {
    8053             :   case MVT::i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8054             :   case MVT::v1i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8055             :   case MVT::v2i64: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8056             :   default: return 0;
    8057             :   }
    8058             : }
    8059             : 
    8060             : // FastEmit functions for AArch64ISD::URSHR_I.
    8061             : 
    8062             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8063             :   if (RetVT.SimpleTy != MVT::i64)
    8064             :     return 0;
    8065             :   if ((Subtarget->hasNEON())) {
    8066             :     return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8067             :   }
    8068             :   return 0;
    8069             : }
    8070             : 
    8071             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8072             :   if (RetVT.SimpleTy != MVT::v1i64)
    8073             :     return 0;
    8074             :   if ((Subtarget->hasNEON())) {
    8075             :     return fastEmitInst_ri(AArch64::URSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8076             :   }
    8077             :   return 0;
    8078             : }
    8079             : 
    8080             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8081             :   if (RetVT.SimpleTy != MVT::v2i64)
    8082             :     return 0;
    8083             :   if ((Subtarget->hasNEON())) {
    8084             :     return fastEmitInst_ri(AArch64::URSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8085             :   }
    8086             :   return 0;
    8087             : }
    8088             : 
    8089             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8090             :   switch (VT.SimpleTy) {
    8091             :   case MVT::i64: return fastEmit_AArch64ISD_URSHR_I_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8092             :   case MVT::v1i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8093             :   case MVT::v2i64: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8094             :   default: return 0;
    8095             :   }
    8096             : }
    8097             : 
    8098             : // FastEmit functions for AArch64ISD::VASHR.
    8099             : 
    8100             : unsigned fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8101             :   if (RetVT.SimpleTy != MVT::i64)
    8102             :     return 0;
    8103             :   if ((Subtarget->hasNEON())) {
    8104             :     return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8105             :   }
    8106             :   return 0;
    8107             : }
    8108             : 
    8109             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8110             :   if (RetVT.SimpleTy != MVT::v1i64)
    8111             :     return 0;
    8112             :   if ((Subtarget->hasNEON())) {
    8113             :     return fastEmitInst_ri(AArch64::SSHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8114             :   }
    8115             :   return 0;
    8116             : }
    8117             : 
    8118             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8119             :   if (RetVT.SimpleTy != MVT::v2i64)
    8120             :     return 0;
    8121             :   if ((Subtarget->hasNEON())) {
    8122             :     return fastEmitInst_ri(AArch64::SSHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8123             :   }
    8124             :   return 0;
    8125             : }
    8126             : 
    8127             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8128             :   switch (VT.SimpleTy) {
    8129             :   case MVT::i64: return fastEmit_AArch64ISD_VASHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8130             :   case MVT::v1i64: return fastEmit_AArch64ISD_VASHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8131             :   case MVT::v2i64: return fastEmit_AArch64ISD_VASHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8132             :   default: return 0;
    8133             :   }
    8134             : }
    8135             : 
    8136             : // FastEmit functions for AArch64ISD::VLSHR.
    8137             : 
    8138             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8139             :   if (RetVT.SimpleTy != MVT::i64)
    8140             :     return 0;
    8141             :   if ((Subtarget->hasNEON())) {
    8142             :     return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8143             :   }
    8144             :   return 0;
    8145             : }
    8146             : 
    8147             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8148             :   if (RetVT.SimpleTy != MVT::v1i64)
    8149             :     return 0;
    8150             :   if ((Subtarget->hasNEON())) {
    8151             :     return fastEmitInst_ri(AArch64::USHRd, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8152             :   }
    8153             :   return 0;
    8154             : }
    8155             : 
    8156             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8157             :   if (RetVT.SimpleTy != MVT::v2i64)
    8158             :     return 0;
    8159             :   if ((Subtarget->hasNEON())) {
    8160             :     return fastEmitInst_ri(AArch64::USHRv2i64_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8161             :   }
    8162             :   return 0;
    8163             : }
    8164             : 
    8165             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8166             :   switch (VT.SimpleTy) {
    8167             :   case MVT::i64: return fastEmit_AArch64ISD_VLSHR_MVT_i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8168             :   case MVT::v1i64: return fastEmit_AArch64ISD_VLSHR_MVT_v1i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8169             :   case MVT::v2i64: return fastEmit_AArch64ISD_VLSHR_MVT_v2i64_ri_Predicate_vecshiftR64(RetVT, Op0, Op0IsKill, imm1);
    8170             :   default: return 0;
    8171             :   }
    8172             : }
    8173             : 
    8174             : // Top-level FastEmit function.
    8175             : 
    8176             : unsigned fastEmit_ri_Predicate_vecshiftR64(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8177             :   switch (Opcode) {
    8178             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8179             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8180             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8181             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR64(VT, RetVT, Op0, Op0IsKill, imm1);
    8182             :   default: return 0;
    8183             :   }
    8184             : }
    8185             : 
    8186             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    8187             : 
    8188             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8189             :   if (RetVT.SimpleTy != MVT::v8i8)
    8190             :     return 0;
    8191             :   if ((Subtarget->hasNEON())) {
    8192             :     return fastEmitInst_ri(AArch64::SQSHLUv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8193             :   }
    8194             :   return 0;
    8195             : }
    8196             : 
    8197             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8198             :   if (RetVT.SimpleTy != MVT::v16i8)
    8199             :     return 0;
    8200             :   if ((Subtarget->hasNEON())) {
    8201             :     return fastEmitInst_ri(AArch64::SQSHLUv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8202             :   }
    8203             :   return 0;
    8204             : }
    8205             : 
    8206             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8207             :   switch (VT.SimpleTy) {
    8208             :   case MVT::v8i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8209             :   case MVT::v16i8: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8210             :   default: return 0;
    8211             :   }
    8212             : }
    8213             : 
    8214             : // FastEmit functions for AArch64ISD::SQSHL_I.
    8215             : 
    8216             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8217             :   if (RetVT.SimpleTy != MVT::v8i8)
    8218             :     return 0;
    8219             :   if ((Subtarget->hasNEON())) {
    8220             :     return fastEmitInst_ri(AArch64::SQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8221             :   }
    8222             :   return 0;
    8223             : }
    8224             : 
    8225             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8226             :   if (RetVT.SimpleTy != MVT::v16i8)
    8227             :     return 0;
    8228             :   if ((Subtarget->hasNEON())) {
    8229             :     return fastEmitInst_ri(AArch64::SQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8230             :   }
    8231             :   return 0;
    8232             : }
    8233             : 
    8234             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8235             :   switch (VT.SimpleTy) {
    8236             :   case MVT::v8i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8237             :   case MVT::v16i8: return fastEmit_AArch64ISD_SQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8238             :   default: return 0;
    8239             :   }
    8240             : }
    8241             : 
    8242             : // FastEmit functions for AArch64ISD::UQSHL_I.
    8243             : 
    8244             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8245             :   if (RetVT.SimpleTy != MVT::v8i8)
    8246             :     return 0;
    8247             :   if ((Subtarget->hasNEON())) {
    8248             :     return fastEmitInst_ri(AArch64::UQSHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8249             :   }
    8250             :   return 0;
    8251             : }
    8252             : 
    8253             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8254             :   if (RetVT.SimpleTy != MVT::v16i8)
    8255             :     return 0;
    8256             :   if ((Subtarget->hasNEON())) {
    8257             :     return fastEmitInst_ri(AArch64::UQSHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8258             :   }
    8259             :   return 0;
    8260             : }
    8261             : 
    8262             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8263             :   switch (VT.SimpleTy) {
    8264             :   case MVT::v8i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8265             :   case MVT::v16i8: return fastEmit_AArch64ISD_UQSHL_I_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8266             :   default: return 0;
    8267             :   }
    8268             : }
    8269             : 
    8270             : // FastEmit functions for AArch64ISD::VSHL.
    8271             : 
    8272             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8273             :   if (RetVT.SimpleTy != MVT::v8i8)
    8274             :     return 0;
    8275             :   if ((Subtarget->hasNEON())) {
    8276             :     return fastEmitInst_ri(AArch64::SHLv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8277             :   }
    8278             :   return 0;
    8279             : }
    8280             : 
    8281             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8282             :   if (RetVT.SimpleTy != MVT::v16i8)
    8283             :     return 0;
    8284             :   if ((Subtarget->hasNEON())) {
    8285             :     return fastEmitInst_ri(AArch64::SHLv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8286             :   }
    8287             :   return 0;
    8288             : }
    8289             : 
    8290             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8291             :   switch (VT.SimpleTy) {
    8292             :   case MVT::v8i8: return fastEmit_AArch64ISD_VSHL_MVT_v8i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8293             :   case MVT::v16i8: return fastEmit_AArch64ISD_VSHL_MVT_v16i8_ri_Predicate_vecshiftL8(RetVT, Op0, Op0IsKill, imm1);
    8294             :   default: return 0;
    8295             :   }
    8296             : }
    8297             : 
    8298             : // Top-level FastEmit function.
    8299             : 
    8300             : unsigned fastEmit_ri_Predicate_vecshiftL8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8301             :   switch (Opcode) {
    8302             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8303             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8304             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8305             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL8(VT, RetVT, Op0, Op0IsKill, imm1);
    8306             :   default: return 0;
    8307             :   }
    8308             : }
    8309             : 
    8310             : // FastEmit functions for AArch64ISD::SQSHLU_I.
    8311             : 
    8312             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8313             :   if (RetVT.SimpleTy != MVT::v4i16)
    8314             :     return 0;
    8315             :   if ((Subtarget->hasNEON())) {
    8316             :     return fastEmitInst_ri(AArch64::SQSHLUv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8317             :   }
    8318             :   return 0;
    8319             : }
    8320             : 
    8321             : unsigned fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8322             :   if (RetVT.SimpleTy != MVT::v8i16)
    8323             :     return 0;
    8324             :   if ((Subtarget->hasNEON())) {
    8325             :     return fastEmitInst_ri(AArch64::SQSHLUv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8326             :   }
    8327             :   return 0;
    8328             : }
    8329             : 
    8330             : unsigned fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8331             :   switch (VT.SimpleTy) {
    8332             :   case MVT::v4i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8333             :   case MVT::v8i16: return fastEmit_AArch64ISD_SQSHLU_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8334             :   default: return 0;
    8335             :   }
    8336             : }
    8337             : 
    8338             : // FastEmit functions for AArch64ISD::SQSHL_I.
    8339             : 
    8340             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8341             :   if (RetVT.SimpleTy != MVT::v4i16)
    8342             :     return 0;
    8343             :   if ((Subtarget->hasNEON())) {
    8344             :     return fastEmitInst_ri(AArch64::SQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8345             :   }
    8346             :   return 0;
    8347             : }
    8348             : 
    8349             : unsigned fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8350             :   if (RetVT.SimpleTy != MVT::v8i16)
    8351             :     return 0;
    8352             :   if ((Subtarget->hasNEON())) {
    8353             :     return fastEmitInst_ri(AArch64::SQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8354             :   }
    8355             :   return 0;
    8356             : }
    8357             : 
    8358             : unsigned fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8359             :   switch (VT.SimpleTy) {
    8360             :   case MVT::v4i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8361             :   case MVT::v8i16: return fastEmit_AArch64ISD_SQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8362             :   default: return 0;
    8363             :   }
    8364             : }
    8365             : 
    8366             : // FastEmit functions for AArch64ISD::UQSHL_I.
    8367             : 
    8368             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8369             :   if (RetVT.SimpleTy != MVT::v4i16)
    8370             :     return 0;
    8371             :   if ((Subtarget->hasNEON())) {
    8372             :     return fastEmitInst_ri(AArch64::UQSHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8373             :   }
    8374             :   return 0;
    8375             : }
    8376             : 
    8377             : unsigned fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8378             :   if (RetVT.SimpleTy != MVT::v8i16)
    8379             :     return 0;
    8380             :   if ((Subtarget->hasNEON())) {
    8381             :     return fastEmitInst_ri(AArch64::UQSHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8382             :   }
    8383             :   return 0;
    8384             : }
    8385             : 
    8386             : unsigned fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8387             :   switch (VT.SimpleTy) {
    8388             :   case MVT::v4i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8389             :   case MVT::v8i16: return fastEmit_AArch64ISD_UQSHL_I_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8390             :   default: return 0;
    8391             :   }
    8392             : }
    8393             : 
    8394             : // FastEmit functions for AArch64ISD::VSHL.
    8395             : 
    8396             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8397             :   if (RetVT.SimpleTy != MVT::v4i16)
    8398             :     return 0;
    8399             :   if ((Subtarget->hasNEON())) {
    8400             :     return fastEmitInst_ri(AArch64::SHLv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8401             :   }
    8402             :   return 0;
    8403             : }
    8404             : 
    8405             : unsigned fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8406             :   if (RetVT.SimpleTy != MVT::v8i16)
    8407             :     return 0;
    8408             :   if ((Subtarget->hasNEON())) {
    8409             :     return fastEmitInst_ri(AArch64::SHLv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8410             :   }
    8411             :   return 0;
    8412             : }
    8413             : 
    8414             : unsigned fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8415             :   switch (VT.SimpleTy) {
    8416             :   case MVT::v4i16: return fastEmit_AArch64ISD_VSHL_MVT_v4i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8417             :   case MVT::v8i16: return fastEmit_AArch64ISD_VSHL_MVT_v8i16_ri_Predicate_vecshiftL16(RetVT, Op0, Op0IsKill, imm1);
    8418             :   default: return 0;
    8419             :   }
    8420             : }
    8421             : 
    8422             : // Top-level FastEmit function.
    8423             : 
    8424             : unsigned fastEmit_ri_Predicate_vecshiftL16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8425             :   switch (Opcode) {
    8426             :   case AArch64ISD::SQSHLU_I: return fastEmit_AArch64ISD_SQSHLU_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8427             :   case AArch64ISD::SQSHL_I: return fastEmit_AArch64ISD_SQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8428             :   case AArch64ISD::UQSHL_I: return fastEmit_AArch64ISD_UQSHL_I_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8429             :   case AArch64ISD::VSHL: return fastEmit_AArch64ISD_VSHL_ri_Predicate_vecshiftL16(VT, RetVT, Op0, Op0IsKill, imm1);
    8430             :   default: return 0;
    8431             :   }
    8432             : }
    8433             : 
    8434             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8435             : 
    8436             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8437             :   if (RetVT.SimpleTy != MVT::v8i8)
    8438             :     return 0;
    8439             :   if ((Subtarget->hasNEON())) {
    8440             :     return fastEmitInst_ri(AArch64::SRSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8441             :   }
    8442             :   return 0;
    8443             : }
    8444             : 
    8445             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8446             :   if (RetVT.SimpleTy != MVT::v16i8)
    8447             :     return 0;
    8448             :   if ((Subtarget->hasNEON())) {
    8449             :     return fastEmitInst_ri(AArch64::SRSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8450             :   }
    8451             :   return 0;
    8452             : }
    8453             : 
    8454             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8455             :   switch (VT.SimpleTy) {
    8456             :   case MVT::v8i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8457             :   case MVT::v16i8: return fastEmit_AArch64ISD_SRSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8458             :   default: return 0;
    8459             :   }
    8460             : }
    8461             : 
    8462             : // FastEmit functions for AArch64ISD::URSHR_I.
    8463             : 
    8464             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8465             :   if (RetVT.SimpleTy != MVT::v8i8)
    8466             :     return 0;
    8467             :   if ((Subtarget->hasNEON())) {
    8468             :     return fastEmitInst_ri(AArch64::URSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8469             :   }
    8470             :   return 0;
    8471             : }
    8472             : 
    8473             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8474             :   if (RetVT.SimpleTy != MVT::v16i8)
    8475             :     return 0;
    8476             :   if ((Subtarget->hasNEON())) {
    8477             :     return fastEmitInst_ri(AArch64::URSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8478             :   }
    8479             :   return 0;
    8480             : }
    8481             : 
    8482             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8483             :   switch (VT.SimpleTy) {
    8484             :   case MVT::v8i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8485             :   case MVT::v16i8: return fastEmit_AArch64ISD_URSHR_I_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8486             :   default: return 0;
    8487             :   }
    8488             : }
    8489             : 
    8490             : // FastEmit functions for AArch64ISD::VASHR.
    8491             : 
    8492             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8493             :   if (RetVT.SimpleTy != MVT::v8i8)
    8494             :     return 0;
    8495             :   if ((Subtarget->hasNEON())) {
    8496             :     return fastEmitInst_ri(AArch64::SSHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8497             :   }
    8498             :   return 0;
    8499             : }
    8500             : 
    8501             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8502             :   if (RetVT.SimpleTy != MVT::v16i8)
    8503             :     return 0;
    8504             :   if ((Subtarget->hasNEON())) {
    8505             :     return fastEmitInst_ri(AArch64::SSHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8506             :   }
    8507             :   return 0;
    8508             : }
    8509             : 
    8510             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8511             :   switch (VT.SimpleTy) {
    8512             :   case MVT::v8i8: return fastEmit_AArch64ISD_VASHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8513             :   case MVT::v16i8: return fastEmit_AArch64ISD_VASHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8514             :   default: return 0;
    8515             :   }
    8516             : }
    8517             : 
    8518             : // FastEmit functions for AArch64ISD::VLSHR.
    8519             : 
    8520             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8521             :   if (RetVT.SimpleTy != MVT::v8i8)
    8522             :     return 0;
    8523             :   if ((Subtarget->hasNEON())) {
    8524             :     return fastEmitInst_ri(AArch64::USHRv8i8_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8525             :   }
    8526             :   return 0;
    8527             : }
    8528             : 
    8529             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8530             :   if (RetVT.SimpleTy != MVT::v16i8)
    8531             :     return 0;
    8532             :   if ((Subtarget->hasNEON())) {
    8533             :     return fastEmitInst_ri(AArch64::USHRv16i8_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8534             :   }
    8535             :   return 0;
    8536             : }
    8537             : 
    8538             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8539             :   switch (VT.SimpleTy) {
    8540             :   case MVT::v8i8: return fastEmit_AArch64ISD_VLSHR_MVT_v8i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8541             :   case MVT::v16i8: return fastEmit_AArch64ISD_VLSHR_MVT_v16i8_ri_Predicate_vecshiftR8(RetVT, Op0, Op0IsKill, imm1);
    8542             :   default: return 0;
    8543             :   }
    8544             : }
    8545             : 
    8546             : // Top-level FastEmit function.
    8547             : 
    8548             : unsigned fastEmit_ri_Predicate_vecshiftR8(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8549             :   switch (Opcode) {
    8550             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8551             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8552             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8553             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR8(VT, RetVT, Op0, Op0IsKill, imm1);
    8554             :   default: return 0;
    8555             :   }
    8556             : }
    8557             : 
    8558             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8559             : 
    8560             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8561             :   if (RetVT.SimpleTy != MVT::v4i16)
    8562             :     return 0;
    8563             :   if ((Subtarget->hasNEON())) {
    8564             :     return fastEmitInst_ri(AArch64::SRSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8565             :   }
    8566             :   return 0;
    8567             : }
    8568             : 
    8569             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8570             :   if (RetVT.SimpleTy != MVT::v8i16)
    8571             :     return 0;
    8572             :   if ((Subtarget->hasNEON())) {
    8573             :     return fastEmitInst_ri(AArch64::SRSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8574             :   }
    8575             :   return 0;
    8576             : }
    8577             : 
    8578             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8579             :   switch (VT.SimpleTy) {
    8580             :   case MVT::v4i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8581             :   case MVT::v8i16: return fastEmit_AArch64ISD_SRSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8582             :   default: return 0;
    8583             :   }
    8584             : }
    8585             : 
    8586             : // FastEmit functions for AArch64ISD::URSHR_I.
    8587             : 
    8588             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8589             :   if (RetVT.SimpleTy != MVT::v4i16)
    8590             :     return 0;
    8591             :   if ((Subtarget->hasNEON())) {
    8592             :     return fastEmitInst_ri(AArch64::URSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8593             :   }
    8594             :   return 0;
    8595             : }
    8596             : 
    8597             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8598             :   if (RetVT.SimpleTy != MVT::v8i16)
    8599             :     return 0;
    8600             :   if ((Subtarget->hasNEON())) {
    8601             :     return fastEmitInst_ri(AArch64::URSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8602             :   }
    8603             :   return 0;
    8604             : }
    8605             : 
    8606             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8607             :   switch (VT.SimpleTy) {
    8608             :   case MVT::v4i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8609             :   case MVT::v8i16: return fastEmit_AArch64ISD_URSHR_I_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8610             :   default: return 0;
    8611             :   }
    8612             : }
    8613             : 
    8614             : // FastEmit functions for AArch64ISD::VASHR.
    8615             : 
    8616             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8617             :   if (RetVT.SimpleTy != MVT::v4i16)
    8618             :     return 0;
    8619             :   if ((Subtarget->hasNEON())) {
    8620             :     return fastEmitInst_ri(AArch64::SSHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8621             :   }
    8622             :   return 0;
    8623             : }
    8624             : 
    8625             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8626             :   if (RetVT.SimpleTy != MVT::v8i16)
    8627             :     return 0;
    8628             :   if ((Subtarget->hasNEON())) {
    8629             :     return fastEmitInst_ri(AArch64::SSHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8630             :   }
    8631             :   return 0;
    8632             : }
    8633             : 
    8634             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8635             :   switch (VT.SimpleTy) {
    8636             :   case MVT::v4i16: return fastEmit_AArch64ISD_VASHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8637             :   case MVT::v8i16: return fastEmit_AArch64ISD_VASHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8638             :   default: return 0;
    8639             :   }
    8640             : }
    8641             : 
    8642             : // FastEmit functions for AArch64ISD::VLSHR.
    8643             : 
    8644             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8645             :   if (RetVT.SimpleTy != MVT::v4i16)
    8646             :     return 0;
    8647             :   if ((Subtarget->hasNEON())) {
    8648             :     return fastEmitInst_ri(AArch64::USHRv4i16_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8649             :   }
    8650             :   return 0;
    8651             : }
    8652             : 
    8653             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8654             :   if (RetVT.SimpleTy != MVT::v8i16)
    8655             :     return 0;
    8656             :   if ((Subtarget->hasNEON())) {
    8657             :     return fastEmitInst_ri(AArch64::USHRv8i16_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8658             :   }
    8659             :   return 0;
    8660             : }
    8661             : 
    8662             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8663             :   switch (VT.SimpleTy) {
    8664             :   case MVT::v4i16: return fastEmit_AArch64ISD_VLSHR_MVT_v4i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8665             :   case MVT::v8i16: return fastEmit_AArch64ISD_VLSHR_MVT_v8i16_ri_Predicate_vecshiftR16(RetVT, Op0, Op0IsKill, imm1);
    8666             :   default: return 0;
    8667             :   }
    8668             : }
    8669             : 
    8670             : // Top-level FastEmit function.
    8671             : 
    8672             : unsigned fastEmit_ri_Predicate_vecshiftR16(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8673             :   switch (Opcode) {
    8674             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8675             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8676             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8677             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR16(VT, RetVT, Op0, Op0IsKill, imm1);
    8678             :   default: return 0;
    8679             :   }
    8680             : }
    8681             : 
    8682             : // FastEmit functions for AArch64ISD::SRSHR_I.
    8683             : 
    8684             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8685             :   if (RetVT.SimpleTy != MVT::v2i32)
    8686             :     return 0;
    8687             :   if ((Subtarget->hasNEON())) {
    8688             :     return fastEmitInst_ri(AArch64::SRSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8689             :   }
    8690             :   return 0;
    8691             : }
    8692             : 
    8693             : unsigned fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8694             :   if (RetVT.SimpleTy != MVT::v4i32)
    8695             :     return 0;
    8696             :   if ((Subtarget->hasNEON())) {
    8697             :     return fastEmitInst_ri(AArch64::SRSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8698             :   }
    8699             :   return 0;
    8700             : }
    8701             : 
    8702             : unsigned fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8703             :   switch (VT.SimpleTy) {
    8704             :   case MVT::v2i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8705             :   case MVT::v4i32: return fastEmit_AArch64ISD_SRSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8706             :   default: return 0;
    8707             :   }
    8708             : }
    8709             : 
    8710             : // FastEmit functions for AArch64ISD::URSHR_I.
    8711             : 
    8712             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8713             :   if (RetVT.SimpleTy != MVT::v2i32)
    8714             :     return 0;
    8715             :   if ((Subtarget->hasNEON())) {
    8716             :     return fastEmitInst_ri(AArch64::URSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8717             :   }
    8718             :   return 0;
    8719             : }
    8720             : 
    8721             : unsigned fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8722             :   if (RetVT.SimpleTy != MVT::v4i32)
    8723             :     return 0;
    8724             :   if ((Subtarget->hasNEON())) {
    8725             :     return fastEmitInst_ri(AArch64::URSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8726             :   }
    8727             :   return 0;
    8728             : }
    8729             : 
    8730             : unsigned fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8731             :   switch (VT.SimpleTy) {
    8732             :   case MVT::v2i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8733             :   case MVT::v4i32: return fastEmit_AArch64ISD_URSHR_I_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8734             :   default: return 0;
    8735             :   }
    8736             : }
    8737             : 
    8738             : // FastEmit functions for AArch64ISD::VASHR.
    8739             : 
    8740             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8741             :   if (RetVT.SimpleTy != MVT::v2i32)
    8742             :     return 0;
    8743             :   if ((Subtarget->hasNEON())) {
    8744             :     return fastEmitInst_ri(AArch64::SSHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8745             :   }
    8746             :   return 0;
    8747             : }
    8748             : 
    8749             : unsigned fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8750             :   if (RetVT.SimpleTy != MVT::v4i32)
    8751             :     return 0;
    8752             :   if ((Subtarget->hasNEON())) {
    8753             :     return fastEmitInst_ri(AArch64::SSHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8754             :   }
    8755             :   return 0;
    8756             : }
    8757             : 
    8758             : unsigned fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8759             :   switch (VT.SimpleTy) {
    8760             :   case MVT::v2i32: return fastEmit_AArch64ISD_VASHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8761             :   case MVT::v4i32: return fastEmit_AArch64ISD_VASHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8762             :   default: return 0;
    8763             :   }
    8764             : }
    8765             : 
    8766             : // FastEmit functions for AArch64ISD::VLSHR.
    8767             : 
    8768             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8769             :   if (RetVT.SimpleTy != MVT::v2i32)
    8770             :     return 0;
    8771             :   if ((Subtarget->hasNEON())) {
    8772             :     return fastEmitInst_ri(AArch64::USHRv2i32_shift, &AArch64::FPR64RegClass, Op0, Op0IsKill, imm1);
    8773             :   }
    8774             :   return 0;
    8775             : }
    8776             : 
    8777             : unsigned fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8778             :   if (RetVT.SimpleTy != MVT::v4i32)
    8779             :     return 0;
    8780             :   if ((Subtarget->hasNEON())) {
    8781             :     return fastEmitInst_ri(AArch64::USHRv4i32_shift, &AArch64::FPR128RegClass, Op0, Op0IsKill, imm1);
    8782             :   }
    8783             :   return 0;
    8784             : }
    8785             : 
    8786             : unsigned fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8787             :   switch (VT.SimpleTy) {
    8788             :   case MVT::v2i32: return fastEmit_AArch64ISD_VLSHR_MVT_v2i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8789             :   case MVT::v4i32: return fastEmit_AArch64ISD_VLSHR_MVT_v4i32_ri_Predicate_vecshiftR32(RetVT, Op0, Op0IsKill, imm1);
    8790             :   default: return 0;
    8791             :   }
    8792             : }
    8793             : 
    8794             : // Top-level FastEmit function.
    8795             : 
    8796             : unsigned fastEmit_ri_Predicate_vecshiftR32(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill, uint64_t imm1) {
    8797             :   switch (Opcode) {
    8798             :   case AArch64ISD::SRSHR_I: return fastEmit_AArch64ISD_SRSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8799             :   case AArch64ISD::URSHR_I: return fastEmit_AArch64ISD_URSHR_I_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8800             :   case AArch64ISD::VASHR: return fastEmit_AArch64ISD_VASHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8801             :   case AArch64ISD::VLSHR: return fastEmit_AArch64ISD_VLSHR_ri_Predicate_vecshiftR32(VT, RetVT, Op0, Op0IsKill, imm1);
    8802             :   default: return 0;
    8803             :   }
    8804             : }
    8805             : 
    8806             : // FastEmit functions for ISD::Constant.
    8807             : 
    8808             : unsigned fastEmit_ISD_Constant_MVT_i32_i(MVT RetVT, uint64_t imm0) {
    8809         180 :   if (RetVT.SimpleTy != MVT::i32)
    8810             :     return 0;
    8811         180 :   return fastEmitInst_i(AArch64::MOVi32imm, &AArch64::GPR32RegClass, imm0);
    8812             : }
    8813             : 
    8814             : unsigned fastEmit_ISD_Constant_MVT_i64_i(MVT RetVT, uint64_t imm0) {
    8815          62 :   if (RetVT.SimpleTy != MVT::i64)
    8816             :     return 0;
    8817          62 :   return fastEmitInst_i(AArch64::MOVi64imm, &AArch64::GPR64RegClass, imm0);
    8818             : }
    8819             : 
    8820         326 : unsigned fastEmit_ISD_Constant_i(MVT VT, MVT RetVT, uint64_t imm0) {
    8821         326 :   switch (VT.SimpleTy) {
    8822         180 :   case MVT::i32: return fastEmit_ISD_Constant_MVT_i32_i(RetVT, imm0);
    8823          62 :   case MVT::i64: return fastEmit_ISD_Constant_MVT_i64_i(RetVT, imm0);
    8824             :   default: return 0;
    8825             :   }
    8826             : }
    8827             : 
    8828             : // Top-level FastEmit function.
    8829             : 
    8830         326 : unsigned fastEmit_i(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) override {
    8831         506 :   if (VT == MVT::i32 && Predicate_imm0_255(imm0))
    8832         171 :     if (unsigned Reg = fastEmit_i_Predicate_imm0_255(VT, RetVT, Opcode, imm0))
    8833             :       return Reg;
    8834             : 
    8835         326 :   switch (Opcode) {
    8836         326 :   case ISD::Constant: return fastEmit_ISD_Constant_i(VT, RetVT, imm0);
    8837             :   default: return 0;
    8838             :   }
    8839             : }
    8840             : 
    8841             : // FastEmit functions for AArch64ISD::FMOV.
    8842             : 
    8843           0 : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(uint64_t imm0) {
    8844           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    8845           0 :     return fastEmitInst_i(AArch64::FMOVv4f16_ns, &AArch64::FPR64RegClass, imm0);
    8846             :   }
    8847             :   return 0;
    8848             : }
    8849             : 
    8850           0 : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(uint64_t imm0) {
    8851           0 :   if ((Subtarget->hasFullFP16()) && (Subtarget->hasNEON())) {
    8852           0 :     return fastEmitInst_i(AArch64::FMOVv8f16_ns, &AArch64::FPR128RegClass, imm0);
    8853             :   }
    8854             :   return 0;
    8855             : }
    8856             : 
    8857             : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(uint64_t imm0) {
    8858           0 :   if ((Subtarget->hasNEON())) {
    8859           0 :     return fastEmitInst_i(AArch64::FMOVv2f32_ns, &AArch64::FPR64RegClass, imm0);
    8860             :   }
    8861             :   return 0;
    8862             : }
    8863             : 
    8864             : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(uint64_t imm0) {
    8865           0 :   if ((Subtarget->hasNEON())) {
    8866           0 :     return fastEmitInst_i(AArch64::FMOVv4f32_ns, &AArch64::FPR128RegClass, imm0);
    8867             :   }
    8868             :   return 0;
    8869             : }
    8870             : 
    8871             : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(uint64_t imm0) {
    8872           0 :   if ((Subtarget->hasNEON())) {
    8873           0 :     return fastEmitInst_i(AArch64::FMOVv2f64_ns, &AArch64::FPR128RegClass, imm0);
    8874             :   }
    8875             :   return 0;
    8876             : }
    8877             : 
    8878           0 : unsigned fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
    8879           0 : switch (RetVT.SimpleTy) {
    8880           0 :   case MVT::v4f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f16_i_Predicate_imm0_255(imm0);
    8881           0 :   case MVT::v8f16: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v8f16_i_Predicate_imm0_255(imm0);
    8882             :   case MVT::v2f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f32_i_Predicate_imm0_255(imm0);
    8883             :   case MVT::v4f32: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v4f32_i_Predicate_imm0_255(imm0);
    8884             :   case MVT::v2f64: return fastEmit_AArch64ISD_FMOV_MVT_i32_MVT_v2f64_i_Predicate_imm0_255(imm0);
    8885             :   default: return 0;
    8886             : }
    8887             : }
    8888             : 
    8889             : unsigned fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
    8890           0 :   switch (VT.SimpleTy) {
    8891           0 :   case MVT::i32: return fastEmit_AArch64ISD_FMOV_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
    8892             :   default: return 0;
    8893             :   }
    8894             : }
    8895             : 
    8896             : // FastEmit functions for AArch64ISD::MOVI.
    8897             : 
    8898             : unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(uint64_t imm0) {
    8899           0 :   if ((Subtarget->hasNEON())) {
    8900           0 :     return fastEmitInst_i(AArch64::MOVIv8b_ns, &AArch64::FPR64RegClass, imm0);
    8901             :   }
    8902             :   return 0;
    8903             : }
    8904             : 
    8905             : unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(uint64_t imm0) {
    8906           0 :   if ((Subtarget->hasNEON())) {
    8907           0 :     return fastEmitInst_i(AArch64::MOVIv16b_ns, &AArch64::FPR128RegClass, imm0);
    8908             :   }
    8909             :   return 0;
    8910             : }
    8911             : 
    8912           0 : unsigned fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
    8913           0 : switch (RetVT.SimpleTy) {
    8914             :   case MVT::v8i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v8i8_i_Predicate_imm0_255(imm0);
    8915             :   case MVT::v16i8: return fastEmit_AArch64ISD_MOVI_MVT_i32_MVT_v16i8_i_Predicate_imm0_255(imm0);
    8916             :   default: return 0;
    8917             : }
    8918             : }
    8919             : 
    8920             : unsigned fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
    8921           0 :   switch (VT.SimpleTy) {
    8922           0 :   case MVT::i32: return fastEmit_AArch64ISD_MOVI_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
    8923             :   default: return 0;
    8924             :   }
    8925             : }
    8926             : 
    8927             : // FastEmit functions for AArch64ISD::MOVIedit.
    8928             : 
    8929             : unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(uint64_t imm0) {
    8930           0 :   return fastEmitInst_i(AArch64::MOVID, &AArch64::FPR64RegClass, imm0);
    8931             : }
    8932             : 
    8933             : unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(uint64_t imm0) {
    8934           0 :   if ((Subtarget->hasNEON())) {
    8935           0 :     return fastEmitInst_i(AArch64::MOVIv2d_ns, &AArch64::FPR128RegClass, imm0);
    8936             :   }
    8937             :   return 0;
    8938             : }
    8939             : 
    8940           0 : unsigned fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(MVT RetVT, uint64_t imm0) {
    8941           0 : switch (RetVT.SimpleTy) {
    8942           0 :   case MVT::f64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_f64_i_Predicate_imm0_255(imm0);
    8943             :   case MVT::v2i64: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_MVT_v2i64_i_Predicate_imm0_255(imm0);
    8944             :   default: return 0;
    8945             : }
    8946             : }
    8947             : 
    8948             : unsigned fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(MVT VT, MVT RetVT, uint64_t imm0) {
    8949           0 :   switch (VT.SimpleTy) {
    8950           0 :   case MVT::i32: return fastEmit_AArch64ISD_MOVIedit_MVT_i32_i_Predicate_imm0_255(RetVT, imm0);
    8951             :   default: return 0;
    8952             :   }
    8953             : }
    8954             : 
    8955             : // Top-level FastEmit function.
    8956             : 
    8957         171 : unsigned fastEmit_i_Predicate_imm0_255(MVT VT, MVT RetVT, unsigned Opcode, uint64_t imm0) {
    8958         171 :   switch (Opcode) {
    8959           0 :   case AArch64ISD::FMOV: return fastEmit_AArch64ISD_FMOV_i_Predicate_imm0_255(VT, RetVT, imm0);
    8960           0 :   case AArch64ISD::MOVI: return fastEmit_AArch64ISD_MOVI_i_Predicate_imm0_255(VT, RetVT, imm0);
    8961           0 :   case AArch64ISD::MOVIedit: return fastEmit_AArch64ISD_MOVIedit_i_Predicate_imm0_255(VT, RetVT, imm0);
    8962             :   default: return 0;
    8963             :   }
    8964             : }
    8965             : 

Generated by: LCOV version 1.13