LCOV - code coverage report
Current view: top level - build-llvm/lib/Target/X86 - X86GenFastISel.inc (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 935 6872 13.6 %
Date: 2018-07-13 00:08:38 Functions: 127 869 14.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
       2             : |*                                                                            *|
       3             : |* "Fast" Instruction Selector for the X86 target                             *|
       4             : |*                                                                            *|
       5             : |* Automatically generated file, do not edit!                                 *|
       6             : |*                                                                            *|
       7             : \*===----------------------------------------------------------------------===*/
       8             : 
       9             : 
      10             : // FastEmit Immediate Predicate functions.
      11             : static bool Predicate_i16immSExt8(int64_t Imm) {
      12             :  return isInt<8>(Imm); 
      13             : }
      14             : static bool Predicate_i32immSExt8(int64_t Imm) {
      15             :  return isInt<8>(Imm); 
      16             : }
      17             : static bool Predicate_i64immSExt8(int64_t Imm) {
      18             :  return isInt<8>(Imm); 
      19             : }
      20             : static bool Predicate_i64immSExt32(int64_t Imm) {
      21             :  return isInt<32>(Imm); 
      22             : }
      23             : static bool Predicate_AndMask64(int64_t Imm) {
      24             : 
      25             :   return isMask_64(Imm) && !isUInt<32>(Imm);
      26             : 
      27             : }
      28             : static bool Predicate_BTRMask64(int64_t Imm) {
      29             : 
      30             :   return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
      31             : 
      32             : }
      33             : static bool Predicate_BTCBTSMask64(int64_t Imm) {
      34             : 
      35             :   return !isInt<32>(Imm) && isPowerOf2_64(Imm);
      36             : 
      37             : }
      38             : 
      39             : 
      40             : // FastEmit functions for ISD::ABS.
      41             : 
      42           0 : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      43           0 :   if (RetVT.SimpleTy != MVT::v16i8)
      44             :     return 0;
      45           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      46           0 :     return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      47             :   }
      48           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      49           0 :     return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      50             :   }
      51           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      52           0 :     return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      53             :   }
      54             :   return 0;
      55             : }
      56             : 
      57           0 : unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      58           0 :   if (RetVT.SimpleTy != MVT::v32i8)
      59             :     return 0;
      60           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      61           0 :     return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
      62             :   }
      63           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      64           0 :     return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
      65             :   }
      66             :   return 0;
      67             : }
      68             : 
      69             : unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      70           0 :   if (RetVT.SimpleTy != MVT::v64i8)
      71             :     return 0;
      72           0 :   if ((Subtarget->hasBWI())) {
      73           0 :     return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
      74             :   }
      75             :   return 0;
      76             : }
      77             : 
      78           0 : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      79           0 :   if (RetVT.SimpleTy != MVT::v8i16)
      80             :     return 0;
      81           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      82           0 :     return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      83             :   }
      84           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      85           0 :     return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      86             :   }
      87           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      88           0 :     return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      89             :   }
      90             :   return 0;
      91             : }
      92             : 
      93           0 : unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      94           0 :   if (RetVT.SimpleTy != MVT::v16i16)
      95             :     return 0;
      96           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      97           0 :     return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
      98             :   }
      99           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
     100           0 :     return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     101             :   }
     102             :   return 0;
     103             : }
     104             : 
     105             : unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     106           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     107             :     return 0;
     108           0 :   if ((Subtarget->hasBWI())) {
     109           0 :     return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     110             :   }
     111             :   return 0;
     112             : }
     113             : 
     114           0 : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     115           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     116             :     return 0;
     117           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     118           0 :     return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     119             :   }
     120           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
     121           0 :     return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     122             :   }
     123           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     124           0 :     return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     125             :   }
     126             :   return 0;
     127             : }
     128             : 
     129           0 : unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     130           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     131             :     return 0;
     132           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     133           0 :     return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     134             :   }
     135           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
     136           0 :     return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     137             :   }
     138             :   return 0;
     139             : }
     140             : 
     141             : unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     142           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     143             :     return 0;
     144           0 :   if ((Subtarget->hasAVX512())) {
     145           0 :     return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     146             :   }
     147             :   return 0;
     148             : }
     149             : 
     150           0 : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     151           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     152             :     return 0;
     153           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     154           0 :     return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     155             :   }
     156             :   return 0;
     157             : }
     158             : 
     159           0 : unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     160           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     161             :     return 0;
     162           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     163           0 :     return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     164             :   }
     165             :   return 0;
     166             : }
     167             : 
     168             : unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     169           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     170             :     return 0;
     171           0 :   if ((Subtarget->hasAVX512())) {
     172           0 :     return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     173             :   }
     174             :   return 0;
     175             : }
     176             : 
     177           0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     178           0 :   switch (VT.SimpleTy) {
     179           0 :   case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     180           0 :   case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     181           0 :   case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     182           0 :   case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     183           0 :   case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     184           0 :   case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     185           0 :   case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     186           0 :   case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     187           0 :   case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     188           0 :   case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     189           0 :   case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     190           0 :   case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     191             :   default: return 0;
     192             :   }
     193             : }
     194             : 
     195             : // FastEmit functions for ISD::ANY_EXTEND.
     196             : 
     197             : unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     198          11 :   if (RetVT.SimpleTy != MVT::i32)
     199             :     return 0;
     200          11 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
     201             : }
     202             : 
     203             : unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     204          14 :   switch (VT.SimpleTy) {
     205          11 :   case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
     206             :   default: return 0;
     207             :   }
     208             : }
     209             : 
     210             : // FastEmit functions for ISD::BITCAST.
     211             : 
     212          13 : unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     213          13 :   if (RetVT.SimpleTy != MVT::f32)
     214             :     return 0;
     215           2 :   if ((Subtarget->hasAVX512())) {
     216           0 :     return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
     217             :   }
     218           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     219           1 :     return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     220             :   }
     221           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     222           0 :     return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     223             :   }
     224             :   return 0;
     225             : }
     226             : 
     227           1 : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     228           2 :   if ((Subtarget->hasAVX512())) {
     229           0 :     return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
     230             :   }
     231           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     232           1 :     return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     233             :   }
     234           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     235           0 :     return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     236             :   }
     237             :   return 0;
     238             : }
     239             : 
     240             : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     241           0 :   if ((Subtarget->hasMMX())) {
     242           0 :     return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
     243             :   }
     244             :   return 0;
     245             : }
     246             : 
     247           4 : unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     248           4 : switch (RetVT.SimpleTy) {
     249           1 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
     250           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill);
     251             :   default: return 0;
     252             : }
     253             : }
     254             : 
     255           7 : unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     256           7 :   if (RetVT.SimpleTy != MVT::i32)
     257             :     return 0;
     258          14 :   if ((Subtarget->hasAVX512())) {
     259           2 :     return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     260             :   }
     261           5 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     262           3 :     return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     263             :   }
     264           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     265           2 :     return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     266             :   }
     267             :   return 0;
     268             : }
     269             : 
     270           2 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     271           4 :   if ((Subtarget->hasAVX512())) {
     272           0 :     return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     273             :   }
     274           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     275           1 :     return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     276             :   }
     277           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     278           1 :     return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     279             :   }
     280             :   return 0;
     281             : }
     282             : 
     283           0 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     284           0 :   if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
     285           0 :     return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
     286             :   }
     287             :   return 0;
     288             : }
     289             : 
     290           2 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     291           2 : switch (RetVT.SimpleTy) {
     292           2 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     293           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill);
     294             :   default: return 0;
     295             : }
     296             : }
     297             : 
     298             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     299           0 :   if ((Subtarget->hasMMX())) {
     300           0 :     return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     301             :   }
     302             :   return 0;
     303             : }
     304             : 
     305           0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     306           0 :   if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
     307           0 :     return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
     308             :   }
     309             :   return 0;
     310             : }
     311             : 
     312           0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     313           0 : switch (RetVT.SimpleTy) {
     314           0 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
     315           0 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
     316             :   default: return 0;
     317             : }
     318             : }
     319             : 
     320        4024 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     321        4024 :   switch (VT.SimpleTy) {
     322          13 :   case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
     323           4 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
     324           7 :   case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
     325           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
     326           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
     327             :   default: return 0;
     328             :   }
     329             : }
     330             : 
     331             : // FastEmit functions for ISD::BRIND.
     332             : 
     333             : unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     334           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     335             :     return 0;
     336           0 :   if ((!Subtarget->is64Bit())) {
     337           0 :     return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
     338             :   }
     339             :   return 0;
     340             : }
     341             : 
     342             : unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     343           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     344             :     return 0;
     345           0 :   if ((!Subtarget->is64Bit())) {
     346           0 :     return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     347             :   }
     348             :   return 0;
     349             : }
     350             : 
     351             : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     352           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     353             :     return 0;
     354           0 :   if ((Subtarget->is64Bit())) {
     355           0 :     return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     356             :   }
     357             :   return 0;
     358             : }
     359             : 
     360           0 : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     361           0 :   switch (VT.SimpleTy) {
     362           0 :   case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
     363           0 :   case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
     364           0 :   case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
     365             :   default: return 0;
     366             :   }
     367             : }
     368             : 
     369             : // FastEmit functions for ISD::BSWAP.
     370             : 
     371             : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     372           0 :   if (RetVT.SimpleTy != MVT::i32)
     373             :     return 0;
     374           0 :   return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     375             : }
     376             : 
     377             : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     378           0 :   if (RetVT.SimpleTy != MVT::i64)
     379             :     return 0;
     380           0 :   return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     381             : }
     382             : 
     383           0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     384           0 :   switch (VT.SimpleTy) {
     385           0 :   case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     386           0 :   case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     387             :   default: return 0;
     388             :   }
     389             : }
     390             : 
     391             : // FastEmit functions for ISD::CTLZ.
     392             : 
     393             : unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     394           0 :   if (RetVT.SimpleTy != MVT::i16)
     395             :     return 0;
     396           0 :   if ((Subtarget->hasLZCNT())) {
     397           0 :     return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     398             :   }
     399             :   return 0;
     400             : }
     401             : 
     402             : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     403           0 :   if (RetVT.SimpleTy != MVT::i32)
     404             :     return 0;
     405           0 :   if ((Subtarget->hasLZCNT())) {
     406           0 :     return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     407             :   }
     408             :   return 0;
     409             : }
     410             : 
     411             : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     412           0 :   if (RetVT.SimpleTy != MVT::i64)
     413             :     return 0;
     414           0 :   if ((Subtarget->hasLZCNT())) {
     415           0 :     return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     416             :   }
     417             :   return 0;
     418             : }
     419             : 
     420           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     421           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     422             :     return 0;
     423           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     424           0 :     return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     425             :   }
     426             :   return 0;
     427             : }
     428             : 
     429           0 : unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     430           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     431             :     return 0;
     432           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     433           0 :     return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     434             :   }
     435             :   return 0;
     436             : }
     437             : 
     438             : unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     439           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     440             :     return 0;
     441           0 :   if ((Subtarget->hasCDI())) {
     442           0 :     return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     443             :   }
     444             :   return 0;
     445             : }
     446             : 
     447           0 : unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     448           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     449             :     return 0;
     450           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     451           0 :     return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     452             :   }
     453             :   return 0;
     454             : }
     455             : 
     456           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     457           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     458             :     return 0;
     459           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     460           0 :     return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     461             :   }
     462             :   return 0;
     463             : }
     464             : 
     465             : unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     466           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     467             :     return 0;
     468           0 :   if ((Subtarget->hasCDI())) {
     469           0 :     return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     470             :   }
     471             :   return 0;
     472             : }
     473             : 
     474           0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     475           0 :   switch (VT.SimpleTy) {
     476           0 :   case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     477           0 :   case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     478           0 :   case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     479           0 :   case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     480           0 :   case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     481           0 :   case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     482           0 :   case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     483           0 :   case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     484           0 :   case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     485             :   default: return 0;
     486             :   }
     487             : }
     488             : 
     489             : // FastEmit functions for ISD::CTPOP.
     490             : 
     491             : unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     492           0 :   if (RetVT.SimpleTy != MVT::i16)
     493             :     return 0;
     494           0 :   if ((Subtarget->hasPOPCNT())) {
     495           0 :     return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     496             :   }
     497             :   return 0;
     498             : }
     499             : 
     500             : unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     501           0 :   if (RetVT.SimpleTy != MVT::i32)
     502             :     return 0;
     503           0 :   if ((Subtarget->hasPOPCNT())) {
     504           0 :     return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     505             :   }
     506             :   return 0;
     507             : }
     508             : 
     509             : unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     510           0 :   if (RetVT.SimpleTy != MVT::i64)
     511             :     return 0;
     512           0 :   if ((Subtarget->hasPOPCNT())) {
     513           0 :     return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     514             :   }
     515             :   return 0;
     516             : }
     517             : 
     518           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     519           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     520             :     return 0;
     521           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     522           0 :     return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     523             :   }
     524             :   return 0;
     525             : }
     526             : 
     527           0 : unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     528           0 :   if (RetVT.SimpleTy != MVT::v32i8)
     529             :     return 0;
     530           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     531           0 :     return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     532             :   }
     533             :   return 0;
     534             : }
     535             : 
     536             : unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     537           0 :   if (RetVT.SimpleTy != MVT::v64i8)
     538             :     return 0;
     539           0 :   if ((Subtarget->hasBITALG())) {
     540           0 :     return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     541             :   }
     542             :   return 0;
     543             : }
     544             : 
     545           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     546           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     547             :     return 0;
     548           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     549           0 :     return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     550             :   }
     551             :   return 0;
     552             : }
     553             : 
     554           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     555           0 :   if (RetVT.SimpleTy != MVT::v16i16)
     556             :     return 0;
     557           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     558           0 :     return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     559             :   }
     560             :   return 0;
     561             : }
     562             : 
     563             : unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     564           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     565             :     return 0;
     566           0 :   if ((Subtarget->hasBITALG())) {
     567           0 :     return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     568             :   }
     569             :   return 0;
     570             : }
     571             : 
     572           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     573           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     574             :     return 0;
     575           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     576           0 :     return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     577             :   }
     578             :   return 0;
     579             : }
     580             : 
     581           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     582           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     583             :     return 0;
     584           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     585           0 :     return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     586             :   }
     587             :   return 0;
     588             : }
     589             : 
     590             : unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     591           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     592             :     return 0;
     593           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     594           0 :     return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     595             :   }
     596             :   return 0;
     597             : }
     598             : 
     599           0 : unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     600           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     601             :     return 0;
     602           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     603           0 :     return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     604             :   }
     605             :   return 0;
     606             : }
     607             : 
     608           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     609           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     610             :     return 0;
     611           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     612           0 :     return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     613             :   }
     614             :   return 0;
     615             : }
     616             : 
     617             : unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     618           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     619             :     return 0;
     620           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     621           0 :     return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     622             :   }
     623             :   return 0;
     624             : }
     625             : 
     626           0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     627           0 :   switch (VT.SimpleTy) {
     628           0 :   case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
     629           0 :   case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     630           0 :   case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     631           0 :   case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     632           0 :   case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     633           0 :   case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     634           0 :   case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     635           0 :   case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     636           0 :   case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     637           0 :   case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     638           0 :   case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     639           0 :   case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     640           0 :   case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     641           0 :   case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     642           0 :   case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     643             :   default: return 0;
     644             :   }
     645             : }
     646             : 
     647             : // FastEmit functions for ISD::CTTZ.
     648             : 
     649             : unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     650           0 :   if (RetVT.SimpleTy != MVT::i16)
     651             :     return 0;
     652           0 :   if ((Subtarget->hasBMI())) {
     653           0 :     return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     654             :   }
     655             :   return 0;
     656             : }
     657             : 
     658             : unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     659           0 :   if (RetVT.SimpleTy != MVT::i32)
     660             :     return 0;
     661           0 :   if ((Subtarget->hasBMI())) {
     662           0 :     return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     663             :   }
     664             :   return 0;
     665             : }
     666             : 
     667             : unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     668           0 :   if (RetVT.SimpleTy != MVT::i64)
     669             :     return 0;
     670           0 :   if ((Subtarget->hasBMI())) {
     671           0 :     return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     672             :   }
     673             :   return 0;
     674             : }
     675             : 
     676           0 : unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     677           0 :   switch (VT.SimpleTy) {
     678           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     679           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     680           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     681             :   default: return 0;
     682             :   }
     683             : }
     684             : 
     685             : // FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
     686             : 
     687             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     688           0 :   if (RetVT.SimpleTy != MVT::i16)
     689             :     return 0;
     690           0 :   return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     691             : }
     692             : 
     693             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     694           0 :   if (RetVT.SimpleTy != MVT::i32)
     695             :     return 0;
     696           0 :   return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     697             : }
     698             : 
     699             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     700           0 :   if (RetVT.SimpleTy != MVT::i64)
     701             :     return 0;
     702           0 :   return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     703             : }
     704             : 
     705           0 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     706           0 :   switch (VT.SimpleTy) {
     707           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
     708           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
     709           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
     710             :   default: return 0;
     711             :   }
     712             : }
     713             : 
     714             : // FastEmit functions for ISD::FABS.
     715             : 
     716             : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     717           0 :   if (RetVT.SimpleTy != MVT::f32)
     718             :     return 0;
     719           0 :   if ((!Subtarget->hasSSE1())) {
     720           0 :     return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     721             :   }
     722             :   return 0;
     723             : }
     724             : 
     725             : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     726           0 :   if (RetVT.SimpleTy != MVT::f64)
     727             :     return 0;
     728           0 :   if ((!Subtarget->hasSSE2())) {
     729           0 :     return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     730             :   }
     731             :   return 0;
     732             : }
     733             : 
     734             : unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     735           0 :   if (RetVT.SimpleTy != MVT::f80)
     736             :     return 0;
     737           0 :   return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     738             : }
     739             : 
     740           0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     741           0 :   switch (VT.SimpleTy) {
     742           0 :   case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     743           0 :   case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     744           0 :   case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     745             :   default: return 0;
     746             :   }
     747             : }
     748             : 
     749             : // FastEmit functions for ISD::FCOS.
     750             : 
     751             : unsigned fastEmit_ISD_FCOS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     752           0 :   if (RetVT.SimpleTy != MVT::f32)
     753             :     return 0;
     754           0 :   if ((!Subtarget->hasSSE1())) {
     755           0 :     return fastEmitInst_r(X86::COS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     756             :   }
     757             :   return 0;
     758             : }
     759             : 
     760             : unsigned fastEmit_ISD_FCOS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     761           0 :   if (RetVT.SimpleTy != MVT::f64)
     762             :     return 0;
     763           0 :   if ((!Subtarget->hasSSE2())) {
     764           0 :     return fastEmitInst_r(X86::COS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     765             :   }
     766             :   return 0;
     767             : }
     768             : 
     769             : unsigned fastEmit_ISD_FCOS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     770           0 :   if (RetVT.SimpleTy != MVT::f80)
     771             :     return 0;
     772           0 :   return fastEmitInst_r(X86::COS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     773             : }
     774             : 
     775           0 : unsigned fastEmit_ISD_FCOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     776           0 :   switch (VT.SimpleTy) {
     777           0 :   case MVT::f32: return fastEmit_ISD_FCOS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     778           0 :   case MVT::f64: return fastEmit_ISD_FCOS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     779           0 :   case MVT::f80: return fastEmit_ISD_FCOS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     780             :   default: return 0;
     781             :   }
     782             : }
     783             : 
     784             : // FastEmit functions for ISD::FNEG.
     785             : 
     786             : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     787           1 :   if (RetVT.SimpleTy != MVT::f32)
     788             :     return 0;
     789           2 :   if ((!Subtarget->hasSSE1())) {
     790           0 :     return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     791             :   }
     792             :   return 0;
     793             : }
     794             : 
     795             : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     796           1 :   if (RetVT.SimpleTy != MVT::f64)
     797             :     return 0;
     798           2 :   if ((!Subtarget->hasSSE2())) {
     799           0 :     return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     800             :   }
     801             :   return 0;
     802             : }
     803             : 
     804             : unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     805           0 :   if (RetVT.SimpleTy != MVT::f80)
     806             :     return 0;
     807           0 :   return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     808             : }
     809             : 
     810          42 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     811          42 :   switch (VT.SimpleTy) {
     812           1 :   case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
     813           1 :   case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
     814           0 :   case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill);
     815             :   default: return 0;
     816             :   }
     817             : }
     818             : 
     819             : // FastEmit functions for ISD::FP_EXTEND.
     820             : 
     821           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     822           0 :   if (RetVT.SimpleTy != MVT::f64)
     823             :     return 0;
     824           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     825           0 :     return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     826             :   }
     827             :   return 0;
     828             : }
     829             : 
     830           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     831           0 :   if (RetVT.SimpleTy != MVT::v4f64)
     832             :     return 0;
     833           0 :   if ((Subtarget->hasVLX())) {
     834           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     835             :   }
     836           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     837           0 :     return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     838             :   }
     839             :   return 0;
     840             : }
     841             : 
     842             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     843           0 :   if (RetVT.SimpleTy != MVT::v8f64)
     844             :     return 0;
     845           0 :   if ((Subtarget->hasAVX512())) {
     846           0 :     return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     847             :   }
     848             :   return 0;
     849             : }
     850             : 
     851           0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     852           0 :   switch (VT.SimpleTy) {
     853           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
     854           0 :   case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     855           0 :   case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
     856             :   default: return 0;
     857             :   }
     858             : }
     859             : 
     860             : // FastEmit functions for ISD::FP_ROUND.
     861             : 
     862           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     863           0 :   if (RetVT.SimpleTy != MVT::f32)
     864             :     return 0;
     865           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     866           0 :     return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     867             :   }
     868             :   return 0;
     869             : }
     870             : 
     871           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     872           0 :   if (RetVT.SimpleTy != MVT::v4f32)
     873             :     return 0;
     874           0 :   if ((Subtarget->hasVLX())) {
     875           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     876             :   }
     877           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     878           0 :     return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
     879             :   }
     880             :   return 0;
     881             : }
     882             : 
     883             : unsigned fastEmit_ISD_FP_ROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     884           0 :   if (RetVT.SimpleTy != MVT::v8f32)
     885             :     return 0;
     886           0 :   if ((Subtarget->hasAVX512())) {
     887           0 :     return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
     888             :   }
     889             :   return 0;
     890             : }
     891             : 
     892           0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     893           0 :   switch (VT.SimpleTy) {
     894           0 :   case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
     895           0 :   case MVT::v4f64: return fastEmit_ISD_FP_ROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
     896           0 :   case MVT::v8f64: return fastEmit_ISD_FP_ROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
     897             :   default: return 0;
     898             :   }
     899             : }
     900             : 
     901             : // FastEmit functions for ISD::FP_TO_SINT.
     902             : 
     903           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     904           0 :   if ((Subtarget->hasAVX512())) {
     905           0 :     return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     906             :   }
     907           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     908           0 :     return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     909             :   }
     910           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     911           0 :     return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     912             :   }
     913             :   return 0;
     914             : }
     915             : 
     916           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     917           0 :   if ((Subtarget->hasAVX512())) {
     918           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     919             :   }
     920           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     921           0 :     return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     922             :   }
     923           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     924           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     925             :   }
     926             :   return 0;
     927             : }
     928             : 
     929           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     930           0 : switch (RetVT.SimpleTy) {
     931           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
     932           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
     933             :   default: return 0;
     934             : }
     935             : }
     936             : 
     937           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     938           0 :   if ((Subtarget->hasAVX512())) {
     939           0 :     return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     940             :   }
     941           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     942           0 :     return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     943             :   }
     944           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     945           0 :     return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     946             :   }
     947             :   return 0;
     948             : }
     949             : 
     950           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     951           0 :   if ((Subtarget->hasAVX512())) {
     952           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     953             :   }
     954           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     955           0 :     return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     956             :   }
     957           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     958           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     959             :   }
     960             :   return 0;
     961             : }
     962             : 
     963           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     964           0 : switch (RetVT.SimpleTy) {
     965           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
     966           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     967             :   default: return 0;
     968             : }
     969             : }
     970             : 
     971           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     972           0 :   if ((Subtarget->hasVLX())) {
     973           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     974             :   }
     975           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     976           0 :     return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     977             :   }
     978           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     979           0 :     return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     980             :   }
     981             :   return 0;
     982             : }
     983             : 
     984           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
     985           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
     986           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     987             :   }
     988             :   return 0;
     989             : }
     990             : 
     991           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     992           0 : switch (RetVT.SimpleTy) {
     993           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
     994           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
     995             :   default: return 0;
     996             : }
     997             : }
     998             : 
     999           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1000           0 :   if ((Subtarget->hasVLX())) {
    1001           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1002             :   }
    1003           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1004           0 :     return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1005             :   }
    1006             :   return 0;
    1007             : }
    1008             : 
    1009             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1010           0 :   if ((Subtarget->hasDQI())) {
    1011           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1012             :   }
    1013             :   return 0;
    1014             : }
    1015             : 
    1016           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1017           0 : switch (RetVT.SimpleTy) {
    1018           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1019           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1020             :   default: return 0;
    1021             : }
    1022             : }
    1023             : 
    1024             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1025           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1026             :     return 0;
    1027           0 :   if ((Subtarget->hasAVX512())) {
    1028           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1029             :   }
    1030             :   return 0;
    1031             : }
    1032             : 
    1033           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1034           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1035             :     return 0;
    1036           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1037           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1038             :   }
    1039             :   return 0;
    1040             : }
    1041             : 
    1042           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1043           0 :   if ((Subtarget->hasVLX())) {
    1044           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1045             :   }
    1046           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1047           0 :     return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1048             :   }
    1049             :   return 0;
    1050             : }
    1051             : 
    1052           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1053           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1054           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1055             :   }
    1056             :   return 0;
    1057             : }
    1058             : 
    1059           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1060           0 : switch (RetVT.SimpleTy) {
    1061           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1062           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1063             :   default: return 0;
    1064             : }
    1065             : }
    1066             : 
    1067             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1068           0 :   if ((Subtarget->hasAVX512())) {
    1069           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1070             :   }
    1071             :   return 0;
    1072             : }
    1073             : 
    1074             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1075           0 :   if ((Subtarget->hasDQI())) {
    1076           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1077             :   }
    1078             :   return 0;
    1079             : }
    1080             : 
    1081           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1082           0 : switch (RetVT.SimpleTy) {
    1083           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1084           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1085             :   default: return 0;
    1086             : }
    1087             : }
    1088             : 
    1089           0 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1090           0 :   switch (VT.SimpleTy) {
    1091           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1092           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1093           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1094           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1095           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1096           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1097           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1098           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1099             :   default: return 0;
    1100             :   }
    1101             : }
    1102             : 
    1103             : // FastEmit functions for ISD::FP_TO_UINT.
    1104             : 
    1105             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1106           0 :   if ((Subtarget->hasAVX512())) {
    1107           0 :     return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1108             :   }
    1109             :   return 0;
    1110             : }
    1111             : 
    1112             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1113           0 :   if ((Subtarget->hasAVX512())) {
    1114           0 :     return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1115             :   }
    1116             :   return 0;
    1117             : }
    1118             : 
    1119           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1120           0 : switch (RetVT.SimpleTy) {
    1121           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    1122           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    1123             :   default: return 0;
    1124             : }
    1125             : }
    1126             : 
    1127             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1128           0 :   if ((Subtarget->hasAVX512())) {
    1129           0 :     return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1130             :   }
    1131             :   return 0;
    1132             : }
    1133             : 
    1134             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1135           0 :   if ((Subtarget->hasAVX512())) {
    1136           0 :     return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1137             :   }
    1138             :   return 0;
    1139             : }
    1140             : 
    1141           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1142           0 : switch (RetVT.SimpleTy) {
    1143           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    1144           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    1145             :   default: return 0;
    1146             : }
    1147             : }
    1148             : 
    1149             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1150           0 :   if ((Subtarget->hasVLX())) {
    1151           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1152             :   }
    1153             :   return 0;
    1154             : }
    1155             : 
    1156           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1157           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1158           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1159             :   }
    1160             :   return 0;
    1161             : }
    1162             : 
    1163           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1164           0 : switch (RetVT.SimpleTy) {
    1165           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    1166           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    1167             :   default: return 0;
    1168             : }
    1169             : }
    1170             : 
    1171             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1172           0 :   if ((Subtarget->hasVLX())) {
    1173           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1174             :   }
    1175             :   return 0;
    1176             : }
    1177             : 
    1178             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1179           0 :   if ((Subtarget->hasDQI())) {
    1180           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1181             :   }
    1182             :   return 0;
    1183             : }
    1184             : 
    1185           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1186           0 : switch (RetVT.SimpleTy) {
    1187           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1188           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1189             :   default: return 0;
    1190             : }
    1191             : }
    1192             : 
    1193             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1194           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1195             :     return 0;
    1196           0 :   if ((Subtarget->hasAVX512())) {
    1197           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1198             :   }
    1199             :   return 0;
    1200             : }
    1201             : 
    1202           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1203           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1204             :     return 0;
    1205           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1206           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1207             :   }
    1208             :   return 0;
    1209             : }
    1210             : 
    1211             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1212           0 :   if ((Subtarget->hasVLX())) {
    1213           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1214             :   }
    1215             :   return 0;
    1216             : }
    1217             : 
    1218           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1219           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1220           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1221             :   }
    1222             :   return 0;
    1223             : }
    1224             : 
    1225           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1226           0 : switch (RetVT.SimpleTy) {
    1227           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1228           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1229             :   default: return 0;
    1230             : }
    1231             : }
    1232             : 
    1233             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1234           0 :   if ((Subtarget->hasAVX512())) {
    1235           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1236             :   }
    1237             :   return 0;
    1238             : }
    1239             : 
    1240             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1241           0 :   if ((Subtarget->hasDQI())) {
    1242           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1243             :   }
    1244             :   return 0;
    1245             : }
    1246             : 
    1247           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1248           0 : switch (RetVT.SimpleTy) {
    1249           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1250           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1251             :   default: return 0;
    1252             : }
    1253             : }
    1254             : 
    1255           0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1256           0 :   switch (VT.SimpleTy) {
    1257           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1258           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1259           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1260           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1261           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1262           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1263           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1264           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1265             :   default: return 0;
    1266             :   }
    1267             : }
    1268             : 
    1269             : // FastEmit functions for ISD::FSIN.
    1270             : 
    1271             : unsigned fastEmit_ISD_FSIN_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1272           0 :   if (RetVT.SimpleTy != MVT::f32)
    1273             :     return 0;
    1274           0 :   if ((!Subtarget->hasSSE1())) {
    1275           0 :     return fastEmitInst_r(X86::SIN_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1276             :   }
    1277             :   return 0;
    1278             : }
    1279             : 
    1280             : unsigned fastEmit_ISD_FSIN_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1281           0 :   if (RetVT.SimpleTy != MVT::f64)
    1282             :     return 0;
    1283           0 :   if ((!Subtarget->hasSSE2())) {
    1284           0 :     return fastEmitInst_r(X86::SIN_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1285             :   }
    1286             :   return 0;
    1287             : }
    1288             : 
    1289             : unsigned fastEmit_ISD_FSIN_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1290           0 :   if (RetVT.SimpleTy != MVT::f80)
    1291             :     return 0;
    1292           0 :   return fastEmitInst_r(X86::SIN_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1293             : }
    1294             : 
    1295           0 : unsigned fastEmit_ISD_FSIN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1296           0 :   switch (VT.SimpleTy) {
    1297           0 :   case MVT::f32: return fastEmit_ISD_FSIN_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1298           0 :   case MVT::f64: return fastEmit_ISD_FSIN_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1299           0 :   case MVT::f80: return fastEmit_ISD_FSIN_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1300             :   default: return 0;
    1301             :   }
    1302             : }
    1303             : 
    1304             : // FastEmit functions for ISD::FSQRT.
    1305             : 
    1306           0 : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1307           0 :   if (RetVT.SimpleTy != MVT::f32)
    1308             :     return 0;
    1309           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1310           0 :     return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    1311             :   }
    1312           0 :   if ((!Subtarget->hasSSE1())) {
    1313           0 :     return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1314             :   }
    1315             :   return 0;
    1316             : }
    1317             : 
    1318           0 : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1319           0 :   if (RetVT.SimpleTy != MVT::f64)
    1320             :     return 0;
    1321           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1322           0 :     return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
    1323             :   }
    1324           0 :   if ((!Subtarget->hasSSE2())) {
    1325           0 :     return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1326             :   }
    1327             :   return 0;
    1328             : }
    1329             : 
    1330             : unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1331           0 :   if (RetVT.SimpleTy != MVT::f80)
    1332             :     return 0;
    1333           0 :   return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1334             : }
    1335             : 
    1336           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1337           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1338             :     return 0;
    1339           0 :   if ((Subtarget->hasVLX())) {
    1340           0 :     return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1341             :   }
    1342           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1343           0 :     return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1344             :   }
    1345           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1346           0 :     return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1347             :   }
    1348             :   return 0;
    1349             : }
    1350             : 
    1351           0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1352           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    1353             :     return 0;
    1354           0 :   if ((Subtarget->hasVLX())) {
    1355           0 :     return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1356             :   }
    1357           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1358           0 :     return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1359             :   }
    1360             :   return 0;
    1361             : }
    1362             : 
    1363             : unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1364           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1365             :     return 0;
    1366           0 :   if ((Subtarget->hasAVX512())) {
    1367           0 :     return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1368             :   }
    1369             :   return 0;
    1370             : }
    1371             : 
    1372           0 : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1373           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1374             :     return 0;
    1375           0 :   if ((Subtarget->hasVLX())) {
    1376           0 :     return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1377             :   }
    1378           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1379           0 :     return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1380             :   }
    1381           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1382           0 :     return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1383             :   }
    1384             :   return 0;
    1385             : }
    1386             : 
    1387           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1388           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    1389             :     return 0;
    1390           0 :   if ((Subtarget->hasVLX())) {
    1391           0 :     return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1392             :   }
    1393           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1394           0 :     return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1395             :   }
    1396             :   return 0;
    1397             : }
    1398             : 
    1399             : unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1400           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    1401             :     return 0;
    1402           0 :   if ((Subtarget->hasAVX512())) {
    1403           0 :     return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1404             :   }
    1405             :   return 0;
    1406             : }
    1407             : 
    1408           0 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1409           0 :   switch (VT.SimpleTy) {
    1410           0 :   case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1411           0 :   case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1412           0 :   case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1413           0 :   case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1414           0 :   case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1415           0 :   case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1416           0 :   case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1417           0 :   case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1418           0 :   case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1419             :   default: return 0;
    1420             :   }
    1421             : }
    1422             : 
    1423             : // FastEmit functions for ISD::SCALAR_TO_VECTOR.
    1424             : 
    1425           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1426           2 :   if ((Subtarget->hasAVX512())) {
    1427           0 :     return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1428             :   }
    1429           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1430           0 :     return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1431             :   }
    1432           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1433           1 :     return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1434             :   }
    1435             :   return 0;
    1436             : }
    1437             : 
    1438             : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
    1439           0 :   if ((Subtarget->hasMMX())) {
    1440           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    1441             :   }
    1442             :   return 0;
    1443             : }
    1444             : 
    1445           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1446           1 : switch (RetVT.SimpleTy) {
    1447           1 :   case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    1448           0 :   case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill);
    1449             :   default: return 0;
    1450             : }
    1451             : }
    1452             : 
    1453           0 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1454           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1455             :     return 0;
    1456           0 :   if ((Subtarget->hasAVX512())) {
    1457           0 :     return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1458             :   }
    1459           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1460           0 :     return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1461             :   }
    1462           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1463           0 :     return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1464             :   }
    1465             :   return 0;
    1466             : }
    1467             : 
    1468           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1469           1 :   switch (VT.SimpleTy) {
    1470           1 :   case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1471           0 :   case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1472             :   default: return 0;
    1473             :   }
    1474             : }
    1475             : 
    1476             : // FastEmit functions for ISD::SIGN_EXTEND.
    1477             : 
    1478             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1479         294 :   return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    1480             : }
    1481             : 
    1482             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1483           5 :   return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
    1484             : }
    1485             : 
    1486         307 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1487         307 : switch (RetVT.SimpleTy) {
    1488         588 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill);
    1489          10 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill);
    1490             :   default: return 0;
    1491             : }
    1492             : }
    1493             : 
    1494             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1495          50 :   return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    1496             : }
    1497             : 
    1498             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1499           3 :   return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
    1500             : }
    1501             : 
    1502          53 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1503          53 : switch (RetVT.SimpleTy) {
    1504         100 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill);
    1505           6 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill);
    1506             :   default: return 0;
    1507             : }
    1508             : }
    1509             : 
    1510             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1511         728 :   if (RetVT.SimpleTy != MVT::i64)
    1512             :     return 0;
    1513         728 :   if ((Subtarget->is64Bit())) {
    1514         728 :     return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
    1515             :   }
    1516             :   return 0;
    1517             : }
    1518             : 
    1519          28 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1520          28 :   if (RetVT.SimpleTy != MVT::v2i64)
    1521             :     return 0;
    1522          28 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1523          28 :     return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1524             :   }
    1525             :   return 0;
    1526             : }
    1527             : 
    1528          30 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1529          30 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1530          30 :     return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1531             :   }
    1532             :   return 0;
    1533             : }
    1534             : 
    1535           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1536           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1537           0 :     return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1538             :   }
    1539             :   return 0;
    1540             : }
    1541             : 
    1542          30 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1543          30 : switch (RetVT.SimpleTy) {
    1544          30 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
    1545           0 :   case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill);
    1546             :   default: return 0;
    1547             : }
    1548             : }
    1549             : 
    1550           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1551           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1552           6 :     return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1553             :   }
    1554             :   return 0;
    1555             : }
    1556             : 
    1557           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1558           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1559           0 :     return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1560             :   }
    1561             :   return 0;
    1562             : }
    1563             : 
    1564             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1565           0 :   if ((Subtarget->hasDQI())) {
    1566           0 :     return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1567             :   }
    1568             :   return 0;
    1569             : }
    1570             : 
    1571           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1572           6 : switch (RetVT.SimpleTy) {
    1573           6 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
    1574           0 :   case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill);
    1575           0 :   case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    1576             :   default: return 0;
    1577             : }
    1578             : }
    1579             : 
    1580           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1581           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1582           6 :     return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1583             :   }
    1584             :   return 0;
    1585             : }
    1586             : 
    1587           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    1588           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1589           0 :     return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1590             :   }
    1591             :   return 0;
    1592             : }
    1593             : 
    1594             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    1595           0 :   if ((Subtarget->hasDQI())) {
    1596           0 :     return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1597             :   }
    1598             :   return 0;
    1599             : }
    1600             : 
    1601           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1602           6 : switch (RetVT.SimpleTy) {
    1603           6 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
    1604           0 :   case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill);
    1605           0 :   case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    1606             :   default: return 0;
    1607             : }
    1608             : }
    1609             : 
    1610           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    1611           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1612           0 :     return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1613             :   }
    1614             :   return 0;
    1615             : }
    1616             : 
    1617             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    1618           0 :   if ((Subtarget->hasBWI())) {
    1619           0 :     return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1620             :   }
    1621             :   return 0;
    1622             : }
    1623             : 
    1624           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1625           0 : switch (RetVT.SimpleTy) {
    1626           0 :   case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
    1627           0 :   case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
    1628             :   default: return 0;
    1629             : }
    1630             : }
    1631             : 
    1632             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1633           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    1634             :     return 0;
    1635           0 :   if ((Subtarget->hasBWI())) {
    1636           0 :     return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1637             :   }
    1638             :   return 0;
    1639             : }
    1640             : 
    1641        1164 : unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1642        1164 :   switch (VT.SimpleTy) {
    1643         307 :   case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    1644          53 :   case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    1645         728 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1646          28 :   case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill);
    1647          30 :   case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill);
    1648           6 :   case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    1649           6 :   case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    1650           0 :   case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill);
    1651           0 :   case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill);
    1652             :   default: return 0;
    1653             :   }
    1654             : }
    1655             : 
    1656             : // FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
    1657             : 
    1658           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1659           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1660           0 :     return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1661             :   }
    1662           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    1663           0 :     return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1664             :   }
    1665           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1666           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1667             :   }
    1668             :   return 0;
    1669             : }
    1670             : 
    1671           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1672           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1673           0 :     return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1674             :   }
    1675           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1676           0 :     return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1677             :   }
    1678           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1679           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1680             :   }
    1681             :   return 0;
    1682             : }
    1683             : 
    1684           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1685           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1686           0 :     return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1687             :   }
    1688           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1689           0 :     return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1690             :   }
    1691           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1692           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1693             :   }
    1694             :   return 0;
    1695             : }
    1696             : 
    1697           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1698           0 : switch (RetVT.SimpleTy) {
    1699           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1700           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1701           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1702             :   default: return 0;
    1703             : }
    1704             : }
    1705             : 
    1706           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1707           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1708           0 :     return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1709             :   }
    1710           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1711           0 :     return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1712             :   }
    1713           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1714           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1715             :   }
    1716             :   return 0;
    1717             : }
    1718             : 
    1719           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1720           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1721           0 :     return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1722             :   }
    1723           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1724           0 :     return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1725             :   }
    1726           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1727           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1728             :   }
    1729             :   return 0;
    1730             : }
    1731             : 
    1732           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1733           0 : switch (RetVT.SimpleTy) {
    1734           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    1735           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    1736             :   default: return 0;
    1737             : }
    1738             : }
    1739             : 
    1740           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1741           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1742             :     return 0;
    1743           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1744           0 :     return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1745             :   }
    1746           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1747           0 :     return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1748             :   }
    1749           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1750           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1751             :   }
    1752             :   return 0;
    1753             : }
    1754             : 
    1755           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1756           0 :   switch (VT.SimpleTy) {
    1757           0 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1758           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1759           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1760             :   default: return 0;
    1761             :   }
    1762             : }
    1763             : 
    1764             : // FastEmit functions for ISD::SINT_TO_FP.
    1765             : 
    1766          12 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1767          24 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1768           9 :     return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1769             :   }
    1770             :   return 0;
    1771             : }
    1772             : 
    1773          14 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1774          28 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1775           9 :     return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1776             :   }
    1777             :   return 0;
    1778             : }
    1779             : 
    1780          26 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1781          26 : switch (RetVT.SimpleTy) {
    1782          12 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    1783          14 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    1784             :   default: return 0;
    1785             : }
    1786             : }
    1787             : 
    1788           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1789          12 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1790           3 :     return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1791             :   }
    1792             :   return 0;
    1793             : }
    1794             : 
    1795           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1796          12 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1797           3 :     return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1798             :   }
    1799             :   return 0;
    1800             : }
    1801             : 
    1802          12 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1803          12 : switch (RetVT.SimpleTy) {
    1804           6 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    1805           6 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    1806             :   default: return 0;
    1807             : }
    1808             : }
    1809             : 
    1810           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1811           6 :   if ((Subtarget->hasVLX())) {
    1812           2 :     return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1813             :   }
    1814           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1815           2 :     return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1816             :   }
    1817           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1818           2 :     return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1819             :   }
    1820             :   return 0;
    1821             : }
    1822             : 
    1823           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1824           2 :   if ((Subtarget->hasVLX())) {
    1825           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1826             :   }
    1827           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1828           2 :     return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1829             :   }
    1830             :   return 0;
    1831             : }
    1832             : 
    1833           8 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1834           8 : switch (RetVT.SimpleTy) {
    1835           6 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    1836           2 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    1837             :   default: return 0;
    1838             : }
    1839             : }
    1840             : 
    1841           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1842           2 :   if ((Subtarget->hasVLX())) {
    1843           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1844             :   }
    1845           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1846           2 :     return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1847             :   }
    1848             :   return 0;
    1849             : }
    1850             : 
    1851             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1852           0 :   if ((Subtarget->hasAVX512())) {
    1853           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1854             :   }
    1855             :   return 0;
    1856             : }
    1857             : 
    1858           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1859           2 : switch (RetVT.SimpleTy) {
    1860           2 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    1861           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    1862             :   default: return 0;
    1863             : }
    1864             : }
    1865             : 
    1866             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1867           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1868             :     return 0;
    1869           0 :   if ((Subtarget->hasAVX512())) {
    1870           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1871             :   }
    1872             :   return 0;
    1873             : }
    1874             : 
    1875           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1876           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1877             :     return 0;
    1878           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1879           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1880             :   }
    1881             :   return 0;
    1882             : }
    1883             : 
    1884           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1885           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1886           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1887             :   }
    1888             :   return 0;
    1889             : }
    1890             : 
    1891           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1892           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1893           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1894             :   }
    1895             :   return 0;
    1896             : }
    1897             : 
    1898           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1899           0 : switch (RetVT.SimpleTy) {
    1900           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    1901           0 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    1902             :   default: return 0;
    1903             : }
    1904             : }
    1905             : 
    1906             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1907           0 :   if ((Subtarget->hasDQI())) {
    1908           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1909             :   }
    1910             :   return 0;
    1911             : }
    1912             : 
    1913             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1914           0 :   if ((Subtarget->hasDQI())) {
    1915           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1916             :   }
    1917             :   return 0;
    1918             : }
    1919             : 
    1920           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1921           0 : switch (RetVT.SimpleTy) {
    1922           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    1923           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    1924             :   default: return 0;
    1925             : }
    1926             : }
    1927             : 
    1928          48 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1929          48 :   switch (VT.SimpleTy) {
    1930          26 :   case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1931          12 :   case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1932           8 :   case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1933           2 :   case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    1934           0 :   case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    1935           0 :   case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1936           0 :   case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    1937           0 :   case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    1938             :   default: return 0;
    1939             :   }
    1940             : }
    1941             : 
    1942             : // FastEmit functions for ISD::TRUNCATE.
    1943             : 
    1944             : unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1945           2 :   if (RetVT.SimpleTy != MVT::i8)
    1946             :     return 0;
    1947           2 :   if ((Subtarget->is64Bit())) {
    1948           1 :     return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
    1949             :   }
    1950             :   return 0;
    1951             : }
    1952             : 
    1953             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1954          44 :   if ((Subtarget->is64Bit())) {
    1955          80 :     return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1956             :   }
    1957             :   return 0;
    1958             : }
    1959             : 
    1960             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1961          50 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1962             : }
    1963             : 
    1964          69 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1965          69 : switch (RetVT.SimpleTy) {
    1966          44 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill);
    1967          50 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill);
    1968             :   default: return 0;
    1969             : }
    1970             : }
    1971             : 
    1972             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1973          18 :   return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1974             : }
    1975             : 
    1976             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1977           4 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1978             : }
    1979             : 
    1980             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1981         200 :   return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
    1982             : }
    1983             : 
    1984         111 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1985         111 : switch (RetVT.SimpleTy) {
    1986          18 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill);
    1987           4 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill);
    1988         200 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill);
    1989             :   default: return 0;
    1990             : }
    1991             : }
    1992             : 
    1993           6 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1994           6 :   if (RetVT.SimpleTy != MVT::v16i8)
    1995             :     return 0;
    1996           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1997           4 :     return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1998             :   }
    1999             :   return 0;
    2000             : }
    2001             : 
    2002             : unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2003           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    2004             :     return 0;
    2005           2 :   if ((Subtarget->hasBWI())) {
    2006           2 :     return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2007             :   }
    2008             :   return 0;
    2009             : }
    2010             : 
    2011           6 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2012           6 :   if (RetVT.SimpleTy != MVT::v8i16)
    2013             :     return 0;
    2014          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2015           4 :     return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2016             :   }
    2017             :   return 0;
    2018             : }
    2019             : 
    2020             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2021           4 :   if ((Subtarget->hasAVX512())) {
    2022           2 :     return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2023             :   }
    2024             :   return 0;
    2025             : }
    2026             : 
    2027             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    2028           0 :   if ((Subtarget->hasAVX512())) {
    2029           0 :     return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2030             :   }
    2031             :   return 0;
    2032             : }
    2033             : 
    2034           2 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2035           2 : switch (RetVT.SimpleTy) {
    2036           2 :   case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    2037           0 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    2038             :   default: return 0;
    2039             : }
    2040             : }
    2041             : 
    2042           4 : unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2043           4 :   if (RetVT.SimpleTy != MVT::v4i32)
    2044             :     return 0;
    2045           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2046           2 :     return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2047             :   }
    2048             :   return 0;
    2049             : }
    2050             : 
    2051             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2052           4 :   if ((Subtarget->hasAVX512())) {
    2053           2 :     return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2054             :   }
    2055             :   return 0;
    2056             : }
    2057             : 
    2058             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2059           4 :   if ((Subtarget->hasAVX512())) {
    2060           2 :     return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2061             :   }
    2062             :   return 0;
    2063             : }
    2064             : 
    2065           8 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2066           8 : switch (RetVT.SimpleTy) {
    2067           2 :   case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    2068           2 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    2069             :   default: return 0;
    2070             : }
    2071             : }
    2072             : 
    2073         213 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2074         213 :   switch (VT.SimpleTy) {
    2075           2 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2076          69 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2077         111 :   case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2078           6 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2079           2 :   case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2080           6 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2081           2 :   case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2082           4 :   case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2083           8 :   case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2084             :   default: return 0;
    2085             :   }
    2086             : }
    2087             : 
    2088             : // FastEmit functions for ISD::UINT_TO_FP.
    2089             : 
    2090             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2091           0 :   if ((Subtarget->hasVLX())) {
    2092           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2093             :   }
    2094             :   return 0;
    2095             : }
    2096             : 
    2097             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2098           0 :   if ((Subtarget->hasVLX())) {
    2099           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2100             :   }
    2101             :   return 0;
    2102             : }
    2103             : 
    2104           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2105           0 : switch (RetVT.SimpleTy) {
    2106           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    2107           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    2108             :   default: return 0;
    2109             : }
    2110             : }
    2111             : 
    2112             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2113           0 :   if ((Subtarget->hasVLX())) {
    2114           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2115             :   }
    2116             :   return 0;
    2117             : }
    2118             : 
    2119             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2120           0 :   if ((Subtarget->hasAVX512())) {
    2121           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2122             :   }
    2123             :   return 0;
    2124             : }
    2125             : 
    2126           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2127           0 : switch (RetVT.SimpleTy) {
    2128           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    2129           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    2130             :   default: return 0;
    2131             : }
    2132             : }
    2133             : 
    2134             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2135           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2136             :     return 0;
    2137           0 :   if ((Subtarget->hasAVX512())) {
    2138           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2139             :   }
    2140             :   return 0;
    2141             : }
    2142             : 
    2143           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2144           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2145             :     return 0;
    2146           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2147           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2148             :   }
    2149             :   return 0;
    2150             : }
    2151             : 
    2152           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2153           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2154           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2155             :   }
    2156             :   return 0;
    2157             : }
    2158             : 
    2159           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2160           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2161           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2162             :   }
    2163             :   return 0;
    2164             : }
    2165             : 
    2166           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2167           0 : switch (RetVT.SimpleTy) {
    2168           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    2169           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    2170             :   default: return 0;
    2171             : }
    2172             : }
    2173             : 
    2174             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2175           0 :   if ((Subtarget->hasDQI())) {
    2176           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2177             :   }
    2178             :   return 0;
    2179             : }
    2180             : 
    2181             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2182           0 :   if ((Subtarget->hasDQI())) {
    2183           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2184             :   }
    2185             :   return 0;
    2186             : }
    2187             : 
    2188           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2189           0 : switch (RetVT.SimpleTy) {
    2190           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    2191           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    2192             :   default: return 0;
    2193             : }
    2194             : }
    2195             : 
    2196           0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2197           0 :   switch (VT.SimpleTy) {
    2198           0 :   case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2199           0 :   case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2200           0 :   case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2201           0 :   case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2202           0 :   case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2203           0 :   case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2204             :   default: return 0;
    2205             :   }
    2206             : }
    2207             : 
    2208             : // FastEmit functions for ISD::ZERO_EXTEND.
    2209             : 
    2210             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2211        1242 :   if (RetVT.SimpleTy != MVT::i32)
    2212             :     return 0;
    2213        1228 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    2214             : }
    2215             : 
    2216             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2217          75 :   if (RetVT.SimpleTy != MVT::i32)
    2218             :     return 0;
    2219          73 :   return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    2220             : }
    2221             : 
    2222        1363 : unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2223        1363 :   switch (VT.SimpleTy) {
    2224        1242 :   case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    2225          75 :   case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2226             :   default: return 0;
    2227             :   }
    2228             : }
    2229             : 
    2230             : // FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
    2231             : 
    2232           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2233           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2234           0 :     return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2235             :   }
    2236           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    2237           0 :     return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2238             :   }
    2239           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2240           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2241             :   }
    2242             :   return 0;
    2243             : }
    2244             : 
    2245           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2246           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2247           0 :     return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2248             :   }
    2249           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2250           0 :     return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2251             :   }
    2252           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2253           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2254             :   }
    2255             :   return 0;
    2256             : }
    2257             : 
    2258           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2259           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2260           0 :     return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2261             :   }
    2262           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2263           0 :     return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2264             :   }
    2265           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2266           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2267             :   }
    2268             :   return 0;
    2269             : }
    2270             : 
    2271           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2272           0 : switch (RetVT.SimpleTy) {
    2273           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    2274           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    2275           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    2276             :   default: return 0;
    2277             : }
    2278             : }
    2279             : 
    2280           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2281           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2282           0 :     return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2283             :   }
    2284           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2285           0 :     return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2286             :   }
    2287           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2288           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2289             :   }
    2290             :   return 0;
    2291             : }
    2292             : 
    2293           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2294           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2295           0 :     return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2296             :   }
    2297           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2298           0 :     return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2299             :   }
    2300           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2301           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2302             :   }
    2303             :   return 0;
    2304             : }
    2305             : 
    2306           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2307           0 : switch (RetVT.SimpleTy) {
    2308           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    2309           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    2310             :   default: return 0;
    2311             : }
    2312             : }
    2313             : 
    2314           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2315           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2316             :     return 0;
    2317           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2318           0 :     return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2319             :   }
    2320           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2321           0 :     return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2322             :   }
    2323           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2324           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2325             :   }
    2326             :   return 0;
    2327             : }
    2328             : 
    2329           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2330           0 :   switch (VT.SimpleTy) {
    2331           0 :   case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2332           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2333           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2334             :   default: return 0;
    2335             :   }
    2336             : }
    2337             : 
    2338             : // FastEmit functions for X86ISD::CALL.
    2339             : 
    2340             : unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2341           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2342             :     return 0;
    2343           0 :   if ((!Subtarget->is64Bit())) {
    2344           0 :     return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
    2345             :   }
    2346             :   return 0;
    2347             : }
    2348             : 
    2349           0 : unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2350           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2351             :     return 0;
    2352           0 :   if ((!Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
    2353           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL32, &X86::GR32RegClass, Op0, Op0IsKill);
    2354             :   }
    2355           0 :   if ((!Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
    2356           0 :     return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
    2357             :   }
    2358             :   return 0;
    2359             : }
    2360             : 
    2361           0 : unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2362           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2363             :     return 0;
    2364           0 :   if ((Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
    2365           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL64, &X86::GR64RegClass, Op0, Op0IsKill);
    2366             :   }
    2367           0 :   if ((Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
    2368           0 :     return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
    2369             :   }
    2370             :   return 0;
    2371             : }
    2372             : 
    2373           0 : unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2374           0 :   switch (VT.SimpleTy) {
    2375           0 :   case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2376           0 :   case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2377           0 :   case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2378             :   default: return 0;
    2379             :   }
    2380             : }
    2381             : 
    2382             : // FastEmit functions for X86ISD::COMPRESS.
    2383             : 
    2384           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2385           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2386             :     return 0;
    2387           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2388           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2389             :   }
    2390             :   return 0;
    2391             : }
    2392             : 
    2393           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2394           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    2395             :     return 0;
    2396           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2397           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2398             :   }
    2399             :   return 0;
    2400             : }
    2401             : 
    2402             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2403           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    2404             :     return 0;
    2405           0 :   if ((Subtarget->hasVBMI2())) {
    2406           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2407             :   }
    2408             :   return 0;
    2409             : }
    2410             : 
    2411           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2412           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    2413             :     return 0;
    2414           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2415           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2416             :   }
    2417             :   return 0;
    2418             : }
    2419             : 
    2420           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2421           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    2422             :     return 0;
    2423           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2424           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2425             :   }
    2426             :   return 0;
    2427             : }
    2428             : 
    2429             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2430           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    2431             :     return 0;
    2432           0 :   if ((Subtarget->hasVBMI2())) {
    2433           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2434             :   }
    2435             :   return 0;
    2436             : }
    2437             : 
    2438           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2439           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2440             :     return 0;
    2441           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2442           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2443             :   }
    2444             :   return 0;
    2445             : }
    2446             : 
    2447           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2448           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2449             :     return 0;
    2450           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2451           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2452             :   }
    2453             :   return 0;
    2454             : }
    2455             : 
    2456             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2457           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2458             :     return 0;
    2459           0 :   if ((Subtarget->hasAVX512())) {
    2460           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2461             :   }
    2462             :   return 0;
    2463             : }
    2464             : 
    2465           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2466           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2467             :     return 0;
    2468           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2469           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2470             :   }
    2471             :   return 0;
    2472             : }
    2473             : 
    2474           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2475           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2476             :     return 0;
    2477           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2478           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2479             :   }
    2480             :   return 0;
    2481             : }
    2482             : 
    2483             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2484           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2485             :     return 0;
    2486           0 :   if ((Subtarget->hasAVX512())) {
    2487           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2488             :   }
    2489             :   return 0;
    2490             : }
    2491             : 
    2492           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2493           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2494             :     return 0;
    2495           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2496           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2497             :   }
    2498             :   return 0;
    2499             : }
    2500             : 
    2501           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2502           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    2503             :     return 0;
    2504           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2505           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2506             :   }
    2507             :   return 0;
    2508             : }
    2509             : 
    2510             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2511           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2512             :     return 0;
    2513           0 :   if ((Subtarget->hasAVX512())) {
    2514           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2515             :   }
    2516             :   return 0;
    2517             : }
    2518             : 
    2519           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2520           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2521             :     return 0;
    2522           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2523           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2524             :   }
    2525             :   return 0;
    2526             : }
    2527             : 
    2528           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2529           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    2530             :     return 0;
    2531           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2532           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2533             :   }
    2534             :   return 0;
    2535             : }
    2536             : 
    2537             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2538           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    2539             :     return 0;
    2540           0 :   if ((Subtarget->hasAVX512())) {
    2541           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2542             :   }
    2543             :   return 0;
    2544             : }
    2545             : 
    2546           0 : unsigned fastEmit_X86ISD_COMPRESS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2547           0 :   switch (VT.SimpleTy) {
    2548           0 :   case MVT::v16i8: return fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2549           0 :   case MVT::v32i8: return fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    2550           0 :   case MVT::v64i8: return fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    2551           0 :   case MVT::v8i16: return fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2552           0 :   case MVT::v16i16: return fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2553           0 :   case MVT::v32i16: return fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2554           0 :   case MVT::v4i32: return fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2555           0 :   case MVT::v8i32: return fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2556           0 :   case MVT::v16i32: return fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2557           0 :   case MVT::v2i64: return fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2558           0 :   case MVT::v4i64: return fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2559           0 :   case MVT::v8i64: return fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2560           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2561           0 :   case MVT::v8f32: return fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2562           0 :   case MVT::v16f32: return fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2563           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2564           0 :   case MVT::v4f64: return fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2565           0 :   case MVT::v8f64: return fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2566             :   default: return 0;
    2567             :   }
    2568             : }
    2569             : 
    2570             : // FastEmit functions for X86ISD::CONFLICT.
    2571             : 
    2572           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2573           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2574             :     return 0;
    2575           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2576           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2577             :   }
    2578             :   return 0;
    2579             : }
    2580             : 
    2581           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2582           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2583             :     return 0;
    2584           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2585           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2586             :   }
    2587             :   return 0;
    2588             : }
    2589             : 
    2590             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2591           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2592             :     return 0;
    2593           0 :   if ((Subtarget->hasCDI())) {
    2594           0 :     return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2595             :   }
    2596             :   return 0;
    2597             : }
    2598             : 
    2599           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2600           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2601             :     return 0;
    2602           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2603           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2604             :   }
    2605             :   return 0;
    2606             : }
    2607             : 
    2608           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2609           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2610             :     return 0;
    2611           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2612           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2613             :   }
    2614             :   return 0;
    2615             : }
    2616             : 
    2617             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2618           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2619             :     return 0;
    2620           0 :   if ((Subtarget->hasCDI())) {
    2621           0 :     return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2622             :   }
    2623             :   return 0;
    2624             : }
    2625             : 
    2626           0 : unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2627           0 :   switch (VT.SimpleTy) {
    2628           0 :   case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2629           0 :   case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2630           0 :   case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2631           0 :   case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2632           0 :   case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2633           0 :   case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2634             :   default: return 0;
    2635             :   }
    2636             : }
    2637             : 
    2638             : // FastEmit functions for X86ISD::CVTP2SI.
    2639             : 
    2640           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2641           0 :   if ((Subtarget->hasVLX())) {
    2642           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2643             :   }
    2644           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2645           0 :     return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2646             :   }
    2647           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2648           0 :     return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2649             :   }
    2650             :   return 0;
    2651             : }
    2652             : 
    2653           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2654           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2655           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2656             :   }
    2657             :   return 0;
    2658             : }
    2659             : 
    2660           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2661           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2662           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2663             :   }
    2664             :   return 0;
    2665             : }
    2666             : 
    2667           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2668           0 : switch (RetVT.SimpleTy) {
    2669           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2670           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2671           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2672             :   default: return 0;
    2673             : }
    2674             : }
    2675             : 
    2676           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2677           0 :   if ((Subtarget->hasVLX())) {
    2678           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2679             :   }
    2680           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2681           0 :     return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    2682             :   }
    2683             :   return 0;
    2684             : }
    2685             : 
    2686             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2687           0 :   if ((Subtarget->hasDQI())) {
    2688           0 :     return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2689             :   }
    2690             :   return 0;
    2691             : }
    2692             : 
    2693           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2694           0 : switch (RetVT.SimpleTy) {
    2695           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2696           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2697             :   default: return 0;
    2698             : }
    2699             : }
    2700             : 
    2701             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2702           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2703             :     return 0;
    2704           0 :   if ((Subtarget->hasAVX512())) {
    2705           0 :     return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2706             :   }
    2707             :   return 0;
    2708             : }
    2709             : 
    2710           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2711           0 :   if ((Subtarget->hasVLX())) {
    2712           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2713             :   }
    2714           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2715           0 :     return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2716             :   }
    2717           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2718           0 :     return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2719             :   }
    2720             :   return 0;
    2721             : }
    2722             : 
    2723           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2724           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2725           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2726             :   }
    2727             :   return 0;
    2728             : }
    2729             : 
    2730           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2731           0 : switch (RetVT.SimpleTy) {
    2732           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2733           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2734             :   default: return 0;
    2735             : }
    2736             : }
    2737             : 
    2738           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2739           0 :   if ((Subtarget->hasVLX())) {
    2740           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2741             :   }
    2742           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2743           0 :     return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2744             :   }
    2745             :   return 0;
    2746             : }
    2747             : 
    2748           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2749           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2750           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2751             :   }
    2752             :   return 0;
    2753             : }
    2754             : 
    2755           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2756           0 : switch (RetVT.SimpleTy) {
    2757           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2758           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2759             :   default: return 0;
    2760             : }
    2761             : }
    2762             : 
    2763             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2764           0 :   if ((Subtarget->hasAVX512())) {
    2765           0 :     return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2766             :   }
    2767             :   return 0;
    2768             : }
    2769             : 
    2770             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2771           0 :   if ((Subtarget->hasDQI())) {
    2772           0 :     return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2773             :   }
    2774             :   return 0;
    2775             : }
    2776             : 
    2777           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2778           0 : switch (RetVT.SimpleTy) {
    2779           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2780           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2781             :   default: return 0;
    2782             : }
    2783             : }
    2784             : 
    2785           0 : unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2786           0 :   switch (VT.SimpleTy) {
    2787           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2788           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2789           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2790           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2791           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2792           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2793             :   default: return 0;
    2794             :   }
    2795             : }
    2796             : 
    2797             : // FastEmit functions for X86ISD::CVTP2UI.
    2798             : 
    2799             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2800           0 :   if ((Subtarget->hasVLX())) {
    2801           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2802             :   }
    2803             :   return 0;
    2804             : }
    2805             : 
    2806           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2807           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2808           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2809             :   }
    2810             :   return 0;
    2811             : }
    2812             : 
    2813           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2814           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2815           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2816             :   }
    2817             :   return 0;
    2818             : }
    2819             : 
    2820           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2821           0 : switch (RetVT.SimpleTy) {
    2822           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2823           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2824           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2825             :   default: return 0;
    2826             : }
    2827             : }
    2828             : 
    2829             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2830           0 :   if ((Subtarget->hasVLX())) {
    2831           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2832             :   }
    2833             :   return 0;
    2834             : }
    2835             : 
    2836             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2837           0 :   if ((Subtarget->hasDQI())) {
    2838           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2839             :   }
    2840             :   return 0;
    2841             : }
    2842             : 
    2843           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2844           0 : switch (RetVT.SimpleTy) {
    2845           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2846           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2847             :   default: return 0;
    2848             : }
    2849             : }
    2850             : 
    2851             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2852           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2853             :     return 0;
    2854           0 :   if ((Subtarget->hasAVX512())) {
    2855           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2856             :   }
    2857             :   return 0;
    2858             : }
    2859             : 
    2860             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2861           0 :   if ((Subtarget->hasVLX())) {
    2862           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2863             :   }
    2864             :   return 0;
    2865             : }
    2866             : 
    2867           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2868           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2869           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2870             :   }
    2871             :   return 0;
    2872             : }
    2873             : 
    2874           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2875           0 : switch (RetVT.SimpleTy) {
    2876           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2877           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2878             :   default: return 0;
    2879             : }
    2880             : }
    2881             : 
    2882             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2883           0 :   if ((Subtarget->hasVLX())) {
    2884           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2885             :   }
    2886             :   return 0;
    2887             : }
    2888             : 
    2889           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2890           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2891           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2892             :   }
    2893             :   return 0;
    2894             : }
    2895             : 
    2896           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2897           0 : switch (RetVT.SimpleTy) {
    2898           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2899           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2900             :   default: return 0;
    2901             : }
    2902             : }
    2903             : 
    2904             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2905           0 :   if ((Subtarget->hasAVX512())) {
    2906           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2907             :   }
    2908             :   return 0;
    2909             : }
    2910             : 
    2911             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2912           0 :   if ((Subtarget->hasDQI())) {
    2913           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2914             :   }
    2915             :   return 0;
    2916             : }
    2917             : 
    2918           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2919           0 : switch (RetVT.SimpleTy) {
    2920           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2921           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2922             :   default: return 0;
    2923             : }
    2924             : }
    2925             : 
    2926           0 : unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2927           0 :   switch (VT.SimpleTy) {
    2928           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2929           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2930           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2931           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2932           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2933           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2934             :   default: return 0;
    2935             :   }
    2936             : }
    2937             : 
    2938             : // FastEmit functions for X86ISD::CVTPH2PS.
    2939             : 
    2940           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2941           0 :   if ((Subtarget->hasVLX())) {
    2942           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2943             :   }
    2944           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2945           0 :     return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2946             :   }
    2947             :   return 0;
    2948             : }
    2949             : 
    2950           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2951           0 :   if ((Subtarget->hasVLX())) {
    2952           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2953             :   }
    2954           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2955           0 :     return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    2956             :   }
    2957             :   return 0;
    2958             : }
    2959             : 
    2960           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2961           0 : switch (RetVT.SimpleTy) {
    2962           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
    2963           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
    2964             :   default: return 0;
    2965             : }
    2966             : }
    2967             : 
    2968             : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2969           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2970             :     return 0;
    2971           0 :   if ((Subtarget->hasAVX512())) {
    2972           0 :     return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2973             :   }
    2974             :   return 0;
    2975             : }
    2976             : 
    2977           0 : unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2978           0 :   switch (VT.SimpleTy) {
    2979           0 :   case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2980           0 :   case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2981             :   default: return 0;
    2982             :   }
    2983             : }
    2984             : 
    2985             : // FastEmit functions for X86ISD::CVTSI2P.
    2986             : 
    2987           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2988           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2989             :     return 0;
    2990           0 :   if ((Subtarget->hasVLX())) {
    2991           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2992             :   }
    2993           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2994           0 :     return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2995             :   }
    2996           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2997           0 :     return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2998             :   }
    2999             :   return 0;
    3000             : }
    3001             : 
    3002           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3003           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3004             :     return 0;
    3005           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3006           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3007             :   }
    3008             :   return 0;
    3009             : }
    3010             : 
    3011           0 : unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3012           0 :   switch (VT.SimpleTy) {
    3013           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3014           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3015             :   default: return 0;
    3016             :   }
    3017             : }
    3018             : 
    3019             : // FastEmit functions for X86ISD::CVTTP2SI.
    3020             : 
    3021           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3022           0 :   if ((Subtarget->hasVLX())) {
    3023           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3024             :   }
    3025           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3026           0 :     return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3027             :   }
    3028           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3029           0 :     return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3030             :   }
    3031             :   return 0;
    3032             : }
    3033             : 
    3034           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3035           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3036           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3037             :   }
    3038             :   return 0;
    3039             : }
    3040             : 
    3041           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3042           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3043           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3044             :   }
    3045             :   return 0;
    3046             : }
    3047             : 
    3048           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3049           0 : switch (RetVT.SimpleTy) {
    3050           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    3051           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    3052           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    3053             :   default: return 0;
    3054             : }
    3055             : }
    3056             : 
    3057           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3058           0 :   if ((Subtarget->hasVLX())) {
    3059           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3060             :   }
    3061           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3062           0 :     return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3063             :   }
    3064             :   return 0;
    3065             : }
    3066             : 
    3067             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3068           0 :   if ((Subtarget->hasDQI())) {
    3069           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3070             :   }
    3071             :   return 0;
    3072             : }
    3073             : 
    3074           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3075           0 : switch (RetVT.SimpleTy) {
    3076           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    3077           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    3078             :   default: return 0;
    3079             : }
    3080             : }
    3081             : 
    3082             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3083           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3084             :     return 0;
    3085           0 :   if ((Subtarget->hasAVX512())) {
    3086           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3087             :   }
    3088             :   return 0;
    3089             : }
    3090             : 
    3091           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3092           0 :   if ((Subtarget->hasVLX())) {
    3093           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3094             :   }
    3095           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3096           0 :     return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3097             :   }
    3098           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3099           0 :     return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3100             :   }
    3101             :   return 0;
    3102             : }
    3103             : 
    3104           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3105           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3106           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3107             :   }
    3108             :   return 0;
    3109             : }
    3110             : 
    3111           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3112           0 : switch (RetVT.SimpleTy) {
    3113           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    3114           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    3115             :   default: return 0;
    3116             : }
    3117             : }
    3118             : 
    3119           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3120           0 :   if ((Subtarget->hasVLX())) {
    3121           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3122             :   }
    3123           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3124           0 :     return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3125             :   }
    3126             :   return 0;
    3127             : }
    3128             : 
    3129           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3130           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3131           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3132             :   }
    3133             :   return 0;
    3134             : }
    3135             : 
    3136           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3137           0 : switch (RetVT.SimpleTy) {
    3138           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    3139           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    3140             :   default: return 0;
    3141             : }
    3142             : }
    3143             : 
    3144             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3145           0 :   if ((Subtarget->hasAVX512())) {
    3146           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3147             :   }
    3148             :   return 0;
    3149             : }
    3150             : 
    3151             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3152           0 :   if ((Subtarget->hasDQI())) {
    3153           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3154             :   }
    3155             :   return 0;
    3156             : }
    3157             : 
    3158           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3159           0 : switch (RetVT.SimpleTy) {
    3160           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    3161           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    3162             :   default: return 0;
    3163             : }
    3164             : }
    3165             : 
    3166           0 : unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3167           0 :   switch (VT.SimpleTy) {
    3168           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3169           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3170           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3171           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3172           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3173           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3174             :   default: return 0;
    3175             :   }
    3176             : }
    3177             : 
    3178             : // FastEmit functions for X86ISD::CVTTP2UI.
    3179             : 
    3180             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3181           0 :   if ((Subtarget->hasVLX())) {
    3182           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3183             :   }
    3184             :   return 0;
    3185             : }
    3186             : 
    3187           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3188           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3189           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3190             :   }
    3191             :   return 0;
    3192             : }
    3193             : 
    3194           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3195           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3196           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3197             :   }
    3198             :   return 0;
    3199             : }
    3200             : 
    3201           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3202           0 : switch (RetVT.SimpleTy) {
    3203           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    3204           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    3205           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    3206             :   default: return 0;
    3207             : }
    3208             : }
    3209             : 
    3210             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3211           0 :   if ((Subtarget->hasVLX())) {
    3212           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3213             :   }
    3214             :   return 0;
    3215             : }
    3216             : 
    3217             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3218           0 :   if ((Subtarget->hasDQI())) {
    3219           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3220             :   }
    3221             :   return 0;
    3222             : }
    3223             : 
    3224           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3225           0 : switch (RetVT.SimpleTy) {
    3226           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    3227           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    3228             :   default: return 0;
    3229             : }
    3230             : }
    3231             : 
    3232             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3233           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3234             :     return 0;
    3235           0 :   if ((Subtarget->hasAVX512())) {
    3236           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3237             :   }
    3238             :   return 0;
    3239             : }
    3240             : 
    3241             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3242           0 :   if ((Subtarget->hasVLX())) {
    3243           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3244             :   }
    3245             :   return 0;
    3246             : }
    3247             : 
    3248           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3249           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3250           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3251             :   }
    3252             :   return 0;
    3253             : }
    3254             : 
    3255           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3256           0 : switch (RetVT.SimpleTy) {
    3257           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    3258           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    3259             :   default: return 0;
    3260             : }
    3261             : }
    3262             : 
    3263             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3264           0 :   if ((Subtarget->hasVLX())) {
    3265           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3266             :   }
    3267             :   return 0;
    3268             : }
    3269             : 
    3270           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3271           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3272           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3273             :   }
    3274             :   return 0;
    3275             : }
    3276             : 
    3277           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3278           0 : switch (RetVT.SimpleTy) {
    3279           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    3280           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    3281             :   default: return 0;
    3282             : }
    3283             : }
    3284             : 
    3285             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3286           0 :   if ((Subtarget->hasAVX512())) {
    3287           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3288             :   }
    3289             :   return 0;
    3290             : }
    3291             : 
    3292             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3293           0 :   if ((Subtarget->hasDQI())) {
    3294           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3295             :   }
    3296             :   return 0;
    3297             : }
    3298             : 
    3299           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3300           0 : switch (RetVT.SimpleTy) {
    3301           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    3302           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    3303             :   default: return 0;
    3304             : }
    3305             : }
    3306             : 
    3307           0 : unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3308           0 :   switch (VT.SimpleTy) {
    3309           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3310           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3311           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3312           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3313           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3314           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3315             :   default: return 0;
    3316             :   }
    3317             : }
    3318             : 
    3319             : // FastEmit functions for X86ISD::CVTUI2P.
    3320             : 
    3321             : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3322           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3323             :     return 0;
    3324           0 :   if ((Subtarget->hasVLX())) {
    3325           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3326             :   }
    3327             :   return 0;
    3328             : }
    3329             : 
    3330           0 : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3331           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3332             :     return 0;
    3333           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3334           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3335             :   }
    3336             :   return 0;
    3337             : }
    3338             : 
    3339           0 : unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3340           0 :   switch (VT.SimpleTy) {
    3341           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3342           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3343             :   default: return 0;
    3344             :   }
    3345             : }
    3346             : 
    3347             : // FastEmit functions for X86ISD::EH_RETURN.
    3348             : 
    3349             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3350           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3351             :     return 0;
    3352           0 :   return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
    3353             : }
    3354             : 
    3355             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3356           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3357             :     return 0;
    3358           0 :   return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
    3359             : }
    3360             : 
    3361           0 : unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3362           0 :   switch (VT.SimpleTy) {
    3363           0 :   case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3364           0 :   case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3365             :   default: return 0;
    3366             :   }
    3367             : }
    3368             : 
    3369             : // FastEmit functions for X86ISD::EXPAND.
    3370             : 
    3371           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3372           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    3373             :     return 0;
    3374           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3375           0 :     return fastEmitInst_r(X86::VPEXPANDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3376             :   }
    3377             :   return 0;
    3378             : }
    3379             : 
    3380           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3381           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    3382             :     return 0;
    3383           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3384           0 :     return fastEmitInst_r(X86::VPEXPANDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3385             :   }
    3386             :   return 0;
    3387             : }
    3388             : 
    3389             : unsigned fastEmit_X86ISD_EXPAND_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3390           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    3391             :     return 0;
    3392           0 :   if ((Subtarget->hasVBMI2())) {
    3393           0 :     return fastEmitInst_r(X86::VPEXPANDBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3394             :   }
    3395             :   return 0;
    3396             : }
    3397             : 
    3398           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3399           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3400             :     return 0;
    3401           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3402           0 :     return fastEmitInst_r(X86::VPEXPANDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3403             :   }
    3404             :   return 0;
    3405             : }
    3406             : 
    3407           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3408           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    3409             :     return 0;
    3410           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3411           0 :     return fastEmitInst_r(X86::VPEXPANDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3412             :   }
    3413             :   return 0;
    3414             : }
    3415             : 
    3416             : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3417           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    3418             :     return 0;
    3419           0 :   if ((Subtarget->hasVBMI2())) {
    3420           0 :     return fastEmitInst_r(X86::VPEXPANDWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3421             :   }
    3422             :   return 0;
    3423             : }
    3424             : 
    3425           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3426           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3427             :     return 0;
    3428           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3429           0 :     return fastEmitInst_r(X86::VPEXPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3430             :   }
    3431             :   return 0;
    3432             : }
    3433             : 
    3434           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3435           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3436             :     return 0;
    3437           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3438           0 :     return fastEmitInst_r(X86::VPEXPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3439             :   }
    3440             :   return 0;
    3441             : }
    3442             : 
    3443             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3444           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3445             :     return 0;
    3446           0 :   if ((Subtarget->hasAVX512())) {
    3447           0 :     return fastEmitInst_r(X86::VPEXPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3448             :   }
    3449             :   return 0;
    3450             : }
    3451             : 
    3452           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3453           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3454             :     return 0;
    3455           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3456           0 :     return fastEmitInst_r(X86::VPEXPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3457             :   }
    3458             :   return 0;
    3459             : }
    3460             : 
    3461           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3462           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    3463             :     return 0;
    3464           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3465           0 :     return fastEmitInst_r(X86::VPEXPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3466             :   }
    3467             :   return 0;
    3468             : }
    3469             : 
    3470             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3471           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    3472             :     return 0;
    3473           0 :   if ((Subtarget->hasAVX512())) {
    3474           0 :     return fastEmitInst_r(X86::VPEXPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3475             :   }
    3476             :   return 0;
    3477             : }
    3478             : 
    3479           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3480           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3481             :     return 0;
    3482           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3483           0 :     return fastEmitInst_r(X86::VEXPANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3484             :   }
    3485             :   return 0;
    3486             : }
    3487             : 
    3488           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3489           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3490             :     return 0;
    3491           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3492           0 :     return fastEmitInst_r(X86::VEXPANDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3493             :   }
    3494             :   return 0;
    3495             : }
    3496             : 
    3497             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3498           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3499             :     return 0;
    3500           0 :   if ((Subtarget->hasAVX512())) {
    3501           0 :     return fastEmitInst_r(X86::VEXPANDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3502             :   }
    3503             :   return 0;
    3504             : }
    3505             : 
    3506           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3507           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3508             :     return 0;
    3509           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3510           0 :     return fastEmitInst_r(X86::VEXPANDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3511             :   }
    3512             :   return 0;
    3513             : }
    3514             : 
    3515           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3516           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3517             :     return 0;
    3518           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3519           0 :     return fastEmitInst_r(X86::VEXPANDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3520             :   }
    3521             :   return 0;
    3522             : }
    3523             : 
    3524             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3525           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3526             :     return 0;
    3527           0 :   if ((Subtarget->hasAVX512())) {
    3528           0 :     return fastEmitInst_r(X86::VEXPANDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3529             :   }
    3530             :   return 0;
    3531             : }
    3532             : 
    3533           0 : unsigned fastEmit_X86ISD_EXPAND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3534           0 :   switch (VT.SimpleTy) {
    3535           0 :   case MVT::v16i8: return fastEmit_X86ISD_EXPAND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3536           0 :   case MVT::v32i8: return fastEmit_X86ISD_EXPAND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3537           0 :   case MVT::v64i8: return fastEmit_X86ISD_EXPAND_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    3538           0 :   case MVT::v8i16: return fastEmit_X86ISD_EXPAND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3539           0 :   case MVT::v16i16: return fastEmit_X86ISD_EXPAND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    3540           0 :   case MVT::v32i16: return fastEmit_X86ISD_EXPAND_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    3541           0 :   case MVT::v4i32: return fastEmit_X86ISD_EXPAND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3542           0 :   case MVT::v8i32: return fastEmit_X86ISD_EXPAND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3543           0 :   case MVT::v16i32: return fastEmit_X86ISD_EXPAND_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    3544           0 :   case MVT::v2i64: return fastEmit_X86ISD_EXPAND_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3545           0 :   case MVT::v4i64: return fastEmit_X86ISD_EXPAND_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    3546           0 :   case MVT::v8i64: return fastEmit_X86ISD_EXPAND_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    3547           0 :   case MVT::v4f32: return fastEmit_X86ISD_EXPAND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3548           0 :   case MVT::v8f32: return fastEmit_X86ISD_EXPAND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3549           0 :   case MVT::v16f32: return fastEmit_X86ISD_EXPAND_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3550           0 :   case MVT::v2f64: return fastEmit_X86ISD_EXPAND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3551           0 :   case MVT::v4f64: return fastEmit_X86ISD_EXPAND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3552           0 :   case MVT::v8f64: return fastEmit_X86ISD_EXPAND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3553             :   default: return 0;
    3554             :   }
    3555             : }
    3556             : 
    3557             : // FastEmit functions for X86ISD::FRCP.
    3558             : 
    3559           0 : unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3560           0 :   if (RetVT.SimpleTy != MVT::f32)
    3561             :     return 0;
    3562           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3563           0 :     return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3564             :   }
    3565             :   return 0;
    3566             : }
    3567             : 
    3568           0 : unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3569           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3570             :     return 0;
    3571           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3572           0 :     return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3573             :   }
    3574           0 :   if ((Subtarget->hasAVX())) {
    3575           0 :     return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3576             :   }
    3577             :   return 0;
    3578             : }
    3579             : 
    3580             : unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3581           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3582             :     return 0;
    3583           0 :   if ((Subtarget->hasAVX())) {
    3584           0 :     return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3585             :   }
    3586             :   return 0;
    3587             : }
    3588             : 
    3589           0 : unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3590           0 :   switch (VT.SimpleTy) {
    3591           0 :   case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3592           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3593           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3594             :   default: return 0;
    3595             :   }
    3596             : }
    3597             : 
    3598             : // FastEmit functions for X86ISD::FRSQRT.
    3599             : 
    3600           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3601           0 :   if (RetVT.SimpleTy != MVT::f32)
    3602             :     return 0;
    3603           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3604           0 :     return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3605             :   }
    3606             :   return 0;
    3607             : }
    3608             : 
    3609           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3610           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3611             :     return 0;
    3612           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3613           0 :     return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3614             :   }
    3615           0 :   if ((Subtarget->hasAVX())) {
    3616           0 :     return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3617             :   }
    3618             :   return 0;
    3619             : }
    3620             : 
    3621             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3622           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3623             :     return 0;
    3624           0 :   if ((Subtarget->hasAVX())) {
    3625           0 :     return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3626             :   }
    3627             :   return 0;
    3628             : }
    3629             : 
    3630           0 : unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3631           0 :   switch (VT.SimpleTy) {
    3632           0 :   case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3633           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3634           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3635             :   default: return 0;
    3636             :   }
    3637             : }
    3638             : 
    3639             : // FastEmit functions for X86ISD::MMX_MOVD2W.
    3640             : 
    3641             : unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3642           0 :   if (RetVT.SimpleTy != MVT::i32)
    3643             :     return 0;
    3644           0 :   if ((Subtarget->hasMMX())) {
    3645           0 :     return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
    3646             :   }
    3647             :   return 0;
    3648             : }
    3649             : 
    3650           0 : unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3651           0 :   switch (VT.SimpleTy) {
    3652           0 :   case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
    3653             :   default: return 0;
    3654             :   }
    3655             : }
    3656             : 
    3657             : // FastEmit functions for X86ISD::MMX_MOVW2D.
    3658             : 
    3659             : unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3660           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3661             :     return 0;
    3662           0 :   if ((Subtarget->hasMMX())) {
    3663           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    3664             :   }
    3665             :   return 0;
    3666             : }
    3667             : 
    3668           0 : unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3669           0 :   switch (VT.SimpleTy) {
    3670           0 :   case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3671             :   default: return 0;
    3672             :   }
    3673             : }
    3674             : 
    3675             : // FastEmit functions for X86ISD::MOVDDUP.
    3676             : 
    3677           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3678           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3679             :     return 0;
    3680           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3681           0 :     return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3682             :   }
    3683           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3684           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3685             :   }
    3686             :   return 0;
    3687             : }
    3688             : 
    3689           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3690           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3691             :     return 0;
    3692           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3693           0 :     return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3694             :   }
    3695           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3696           0 :     return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3697             :   }
    3698             :   return 0;
    3699             : }
    3700             : 
    3701             : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3702           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3703             :     return 0;
    3704           0 :   if ((Subtarget->hasAVX512())) {
    3705           0 :     return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3706             :   }
    3707             :   return 0;
    3708             : }
    3709             : 
    3710           0 : unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3711           0 :   switch (VT.SimpleTy) {
    3712           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3713           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3714           0 :   case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3715             :   default: return 0;
    3716             :   }
    3717             : }
    3718             : 
    3719             : // FastEmit functions for X86ISD::MOVDQ2Q.
    3720             : 
    3721             : unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3722           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3723             :     return 0;
    3724           0 :   return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
    3725             : }
    3726             : 
    3727             : unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3728           0 :   switch (VT.SimpleTy) {
    3729           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3730             :   default: return 0;
    3731             :   }
    3732             : }
    3733             : 
    3734             : // FastEmit functions for X86ISD::MOVMSK.
    3735             : 
    3736           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3737           0 :   if (RetVT.SimpleTy != MVT::i32)
    3738             :     return 0;
    3739           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3740           0 :     return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3741             :   }
    3742           0 :   if ((Subtarget->hasAVX())) {
    3743           0 :     return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3744             :   }
    3745             :   return 0;
    3746             : }
    3747             : 
    3748             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3749           0 :   if (RetVT.SimpleTy != MVT::i32)
    3750             :     return 0;
    3751           0 :   if ((Subtarget->hasAVX2())) {
    3752           0 :     return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3753             :   }
    3754             :   return 0;
    3755             : }
    3756             : 
    3757           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3758           0 :   if (RetVT.SimpleTy != MVT::i32)
    3759             :     return 0;
    3760           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3761           0 :     return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3762             :   }
    3763           0 :   if ((Subtarget->hasAVX())) {
    3764           0 :     return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3765             :   }
    3766             :   return 0;
    3767             : }
    3768             : 
    3769             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3770           0 :   if (RetVT.SimpleTy != MVT::i32)
    3771             :     return 0;
    3772           0 :   if ((Subtarget->hasAVX())) {
    3773           0 :     return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3774             :   }
    3775             :   return 0;
    3776             : }
    3777             : 
    3778           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3779           0 :   if (RetVT.SimpleTy != MVT::i32)
    3780             :     return 0;
    3781           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3782           0 :     return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3783             :   }
    3784           0 :   if ((Subtarget->hasAVX())) {
    3785           0 :     return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3786             :   }
    3787             :   return 0;
    3788             : }
    3789             : 
    3790             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3791           0 :   if (RetVT.SimpleTy != MVT::i32)
    3792             :     return 0;
    3793           0 :   if ((Subtarget->hasAVX())) {
    3794           0 :     return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3795             :   }
    3796             :   return 0;
    3797             : }
    3798             : 
    3799           0 : unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3800           0 :   switch (VT.SimpleTy) {
    3801           0 :   case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3802           0 :   case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3803           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3804           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3805           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3806           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3807             :   default: return 0;
    3808             :   }
    3809             : }
    3810             : 
    3811             : // FastEmit functions for X86ISD::MOVSHDUP.
    3812             : 
    3813           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3814           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3815             :     return 0;
    3816           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3817           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3818             :   }
    3819           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3820           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3821             :   }
    3822             :   return 0;
    3823             : }
    3824             : 
    3825           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3826           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3827             :     return 0;
    3828           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3829           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3830             :   }
    3831             :   return 0;
    3832             : }
    3833             : 
    3834           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3835           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3836             :     return 0;
    3837           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3838           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3839             :   }
    3840           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3841           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3842             :   }
    3843           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3844           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3845             :   }
    3846             :   return 0;
    3847             : }
    3848             : 
    3849           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3850           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3851             :     return 0;
    3852           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3853           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3854             :   }
    3855           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3856           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3857             :   }
    3858             :   return 0;
    3859             : }
    3860             : 
    3861             : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3862           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3863             :     return 0;
    3864           0 :   if ((Subtarget->hasAVX512())) {
    3865           0 :     return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3866             :   }
    3867             :   return 0;
    3868             : }
    3869             : 
    3870           0 : unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3871           0 :   switch (VT.SimpleTy) {
    3872           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3873           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3874           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3875           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3876           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3877             :   default: return 0;
    3878             :   }
    3879             : }
    3880             : 
    3881             : // FastEmit functions for X86ISD::MOVSLDUP.
    3882             : 
    3883           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3884           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3885             :     return 0;
    3886           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3887           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3888             :   }
    3889           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3890           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3891             :   }
    3892             :   return 0;
    3893             : }
    3894             : 
    3895           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3896           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3897             :     return 0;
    3898           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3899           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3900             :   }
    3901             :   return 0;
    3902             : }
    3903             : 
    3904           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3905           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3906             :     return 0;
    3907           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3908           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3909             :   }
    3910           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3911           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3912             :   }
    3913           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3914           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3915             :   }
    3916             :   return 0;
    3917             : }
    3918             : 
    3919           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3920           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3921             :     return 0;
    3922           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3923           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3924             :   }
    3925           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3926           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3927             :   }
    3928             :   return 0;
    3929             : }
    3930             : 
    3931             : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3932           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3933             :     return 0;
    3934           0 :   if ((Subtarget->hasAVX512())) {
    3935           0 :     return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3936             :   }
    3937             :   return 0;
    3938             : }
    3939             : 
    3940           0 : unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3941           0 :   switch (VT.SimpleTy) {
    3942           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3943           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3944           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3945           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3946           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3947             :   default: return 0;
    3948             :   }
    3949             : }
    3950             : 
    3951             : // FastEmit functions for X86ISD::NT_BRIND.
    3952             : 
    3953             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3954           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3955             :     return 0;
    3956           0 :   if ((!Subtarget->is64Bit())) {
    3957           0 :     return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
    3958             :   }
    3959             :   return 0;
    3960             : }
    3961             : 
    3962             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3963           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3964             :     return 0;
    3965           0 :   if ((!Subtarget->is64Bit())) {
    3966           0 :     return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
    3967             :   }
    3968             :   return 0;
    3969             : }
    3970             : 
    3971             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3972           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3973             :     return 0;
    3974           0 :   if ((Subtarget->is64Bit())) {
    3975           0 :     return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
    3976             :   }
    3977             :   return 0;
    3978             : }
    3979             : 
    3980           0 : unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3981           0 :   switch (VT.SimpleTy) {
    3982           0 :   case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    3983           0 :   case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3984           0 :   case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3985             :   default: return 0;
    3986             :   }
    3987             : }
    3988             : 
    3989             : // FastEmit functions for X86ISD::NT_CALL.
    3990             : 
    3991             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3992           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3993             :     return 0;
    3994           0 :   if ((!Subtarget->is64Bit())) {
    3995           0 :     return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
    3996             :   }
    3997             :   return 0;
    3998             : }
    3999             : 
    4000             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4001           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4002             :     return 0;
    4003           0 :   if ((!Subtarget->is64Bit())) {
    4004           0 :     return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
    4005             :   }
    4006             :   return 0;
    4007             : }
    4008             : 
    4009             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4010           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4011             :     return 0;
    4012           0 :   if ((Subtarget->is64Bit())) {
    4013           0 :     return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
    4014             :   }
    4015             :   return 0;
    4016             : }
    4017             : 
    4018           0 : unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4019           0 :   switch (VT.SimpleTy) {
    4020           0 :   case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    4021           0 :   case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4022           0 :   case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4023             :   default: return 0;
    4024             :   }
    4025             : }
    4026             : 
    4027             : // FastEmit functions for X86ISD::PHMINPOS.
    4028             : 
    4029           0 : unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4030           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4031             :     return 0;
    4032           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    4033           0 :     return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4034             :   }
    4035           0 :   if ((Subtarget->hasAVX())) {
    4036           0 :     return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4037             :   }
    4038             :   return 0;
    4039             : }
    4040             : 
    4041             : unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4042           0 :   switch (VT.SimpleTy) {
    4043           0 :   case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4044             :   default: return 0;
    4045             :   }
    4046             : }
    4047             : 
    4048             : // FastEmit functions for X86ISD::RCP14.
    4049             : 
    4050             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4051           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4052             :     return 0;
    4053           0 :   if ((Subtarget->hasVLX())) {
    4054           0 :     return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4055             :   }
    4056             :   return 0;
    4057             : }
    4058             : 
    4059             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4060           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    4061             :     return 0;
    4062           0 :   if ((Subtarget->hasVLX())) {
    4063           0 :     return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4064             :   }
    4065             :   return 0;
    4066             : }
    4067             : 
    4068             : unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4069           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    4070             :     return 0;
    4071           0 :   if ((Subtarget->hasAVX512())) {
    4072           0 :     return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4073             :   }
    4074             :   return 0;
    4075             : }
    4076             : 
    4077             : unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4078           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4079             :     return 0;
    4080           0 :   if ((Subtarget->hasVLX())) {
    4081           0 :     return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4082             :   }
    4083             :   return 0;
    4084             : }
    4085             : 
    4086             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4087           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    4088             :     return 0;
    4089           0 :   if ((Subtarget->hasVLX())) {
    4090           0 :     return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4091             :   }
    4092             :   return 0;
    4093             : }
    4094             : 
    4095             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4096           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    4097             :     return 0;
    4098           0 :   if ((Subtarget->hasAVX512())) {
    4099           0 :     return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4100             :   }
    4101             :   return 0;
    4102             : }
    4103             : 
    4104           0 : unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4105           0 :   switch (VT.SimpleTy) {
    4106           0 :   case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4107           0 :   case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    4108           0 :   case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    4109           0 :   case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4110           0 :   case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    4111           0 :   case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    4112             :   default: return 0;
    4113             :   }
    4114             : }
    4115             : 
    4116             : // FastEmit functions for X86ISD::RSQRT14.
    4117             : 
    4118             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4119           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4120             :     return 0;
    4121           0 :   if ((Subtarget->hasVLX())) {
    4122           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4123             :   }
    4124             :   return 0;
    4125             : }
    4126             : 
    4127             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4128           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    4129             :     return 0;
    4130           0 :   if ((Subtarget->hasVLX())) {
    4131           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4132             :   }
    4133             :   return 0;
    4134             : }
    4135             : 
    4136             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4137           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    4138             :     return 0;
    4139           0 :   if ((Subtarget->hasAVX512())) {
    4140           0 :     return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4141             :   }
    4142             :   return 0;
    4143             : }
    4144             : 
    4145             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4146           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4147             :     return 0;
    4148           0 :   if ((Subtarget->hasVLX())) {
    4149           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4150             :   }
    4151             :   return 0;
    4152             : }
    4153             : 
    4154             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4155           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    4156             :     return 0;
    4157           0 :   if ((Subtarget->hasVLX())) {
    4158           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4159             :   }
    4160             :   return 0;
    4161             : }
    4162             : 
    4163             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4164           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    4165             :     return 0;
    4166           0 :   if ((Subtarget->hasAVX512())) {
    4167           0 :     return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4168             :   }
    4169             :   return 0;
    4170             : }
    4171             : 
    4172           0 : unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4173           0 :   switch (VT.SimpleTy) {
    4174           0 :   case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4175           0 :   case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    4176           0 :   case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    4177           0 :   case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4178           0 :   case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    4179           0 :   case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    4180             :   default: return 0;
    4181             :   }
    4182             : }
    4183             : 
    4184             : // FastEmit functions for X86ISD::SEG_ALLOCA.
    4185             : 
    4186           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4187           0 :   if (RetVT.SimpleTy != MVT::i32)
    4188             :     return 0;
    4189           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    4190           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    4191             :   }
    4192             :   return 0;
    4193             : }
    4194             : 
    4195             : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4196           0 :   if (RetVT.SimpleTy != MVT::i64)
    4197             :     return 0;
    4198           0 :   if ((Subtarget->is64Bit())) {
    4199           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    4200             :   }
    4201             :   return 0;
    4202             : }
    4203             : 
    4204           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4205           0 :   switch (VT.SimpleTy) {
    4206           0 :   case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4207           0 :   case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4208             :   default: return 0;
    4209             :   }
    4210             : }
    4211             : 
    4212             : // FastEmit functions for X86ISD::VBROADCAST.
    4213             : 
    4214           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4215           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4216           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4217             :   }
    4218             :   return 0;
    4219             : }
    4220             : 
    4221           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4222           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4223           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4224             :   }
    4225             :   return 0;
    4226             : }
    4227             : 
    4228             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4229           0 :   if ((Subtarget->hasAVX512())) {
    4230           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4231             :   }
    4232             :   return 0;
    4233             : }
    4234             : 
    4235           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4236           0 : switch (RetVT.SimpleTy) {
    4237           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    4238           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
    4239           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
    4240             :   default: return 0;
    4241             : }
    4242             : }
    4243             : 
    4244           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4245           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4246           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4247             :   }
    4248             :   return 0;
    4249             : }
    4250             : 
    4251           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4252           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4253           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4254             :   }
    4255             :   return 0;
    4256             : }
    4257             : 
    4258             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4259           0 :   if ((Subtarget->hasAVX512())) {
    4260           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4261             :   }
    4262             :   return 0;
    4263             : }
    4264             : 
    4265           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4266           0 : switch (RetVT.SimpleTy) {
    4267           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
    4268           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
    4269           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
    4270             :   default: return 0;
    4271             : }
    4272             : }
    4273             : 
    4274           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4275           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4276           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4277             :   }
    4278           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4279           0 :     return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4280             :   }
    4281             :   return 0;
    4282             : }
    4283             : 
    4284           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    4285           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4286           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4287             :   }
    4288           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4289           0 :     return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4290             :   }
    4291             :   return 0;
    4292             : }
    4293             : 
    4294             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
    4295           0 :   if ((Subtarget->hasBWI())) {
    4296           0 :     return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4297             :   }
    4298             :   return 0;
    4299             : }
    4300             : 
    4301           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4302           0 : switch (RetVT.SimpleTy) {
    4303           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
    4304           0 :   case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
    4305           0 :   case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
    4306             :   default: return 0;
    4307             : }
    4308             : }
    4309             : 
    4310           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4311           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4312           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4313             :   }
    4314           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4315           0 :     return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4316             :   }
    4317             :   return 0;
    4318             : }
    4319             : 
    4320           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4321           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4322           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4323             :   }
    4324           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4325           0 :     return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4326             :   }
    4327             :   return 0;
    4328             : }
    4329             : 
    4330             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    4331           0 :   if ((Subtarget->hasBWI())) {
    4332           0 :     return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4333             :   }
    4334             :   return 0;
    4335             : }
    4336             : 
    4337           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4338           0 : switch (RetVT.SimpleTy) {
    4339           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
    4340           0 :   case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
    4341           0 :   case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
    4342             :   default: return 0;
    4343             : }
    4344             : }
    4345             : 
    4346           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4347           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4348           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4349             :   }
    4350           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4351           0 :     return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4352             :   }
    4353             :   return 0;
    4354             : }
    4355             : 
    4356           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4357           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4358           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4359             :   }
    4360           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4361           0 :     return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4362             :   }
    4363             :   return 0;
    4364             : }
    4365             : 
    4366             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4367           0 :   if ((Subtarget->hasAVX512())) {
    4368           0 :     return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4369             :   }
    4370             :   return 0;
    4371             : }
    4372             : 
    4373           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4374           0 : switch (RetVT.SimpleTy) {
    4375           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
    4376           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
    4377           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
    4378             :   default: return 0;
    4379             : }
    4380             : }
    4381             : 
    4382           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4383           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4384           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4385             :   }
    4386           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4387           0 :     return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4388             :   }
    4389             :   return 0;
    4390             : }
    4391             : 
    4392           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4393           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4394           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4395             :   }
    4396           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4397           0 :     return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4398             :   }
    4399             :   return 0;
    4400             : }
    4401             : 
    4402             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4403           0 :   if ((Subtarget->hasAVX512())) {
    4404           0 :     return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4405             :   }
    4406             :   return 0;
    4407             : }
    4408             : 
    4409           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4410           0 : switch (RetVT.SimpleTy) {
    4411           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
    4412           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
    4413           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
    4414             :   default: return 0;
    4415             : }
    4416             : }
    4417             : 
    4418           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    4419           0 :   if ((Subtarget->hasVLX())) {
    4420           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4421             :   }
    4422           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4423           0 :     return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4424             :   }
    4425             :   return 0;
    4426             : }
    4427             : 
    4428           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    4429           0 :   if ((Subtarget->hasVLX())) {
    4430           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4431             :   }
    4432           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4433           0 :     return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4434             :   }
    4435             :   return 0;
    4436             : }
    4437             : 
    4438             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
    4439           0 :   if ((Subtarget->hasAVX512())) {
    4440           0 :     return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4441             :   }
    4442             :   return 0;
    4443             : }
    4444             : 
    4445           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4446           0 : switch (RetVT.SimpleTy) {
    4447           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
    4448           0 :   case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
    4449           0 :   case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
    4450             :   default: return 0;
    4451             : }
    4452             : }
    4453             : 
    4454           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    4455           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4456           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4457             :   }
    4458           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4459           0 :     return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4460             :   }
    4461             :   return 0;
    4462             : }
    4463             : 
    4464           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    4465           0 :   if ((Subtarget->hasVLX())) {
    4466           0 :     return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4467             :   }
    4468           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4469           0 :     return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4470             :   }
    4471             :   return 0;
    4472             : }
    4473             : 
    4474             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    4475           0 :   if ((Subtarget->hasAVX512())) {
    4476           0 :     return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4477             :   }
    4478             :   return 0;
    4479             : }
    4480             : 
    4481           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4482           0 : switch (RetVT.SimpleTy) {
    4483           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill);
    4484           0 :   case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
    4485           0 :   case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
    4486             :   default: return 0;
    4487             : }
    4488             : }
    4489             : 
    4490           0 : unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4491           0 :   switch (VT.SimpleTy) {
    4492           0 :   case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4493           0 :   case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4494           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4495           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4496           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4497           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4498           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4499           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4500             :   default: return 0;
    4501             :   }
    4502             : }
    4503             : 
    4504             : // FastEmit functions for X86ISD::VBROADCASTM.
    4505             : 
    4506           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4507           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4508           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4509             :   }
    4510             :   return 0;
    4511             : }
    4512             : 
    4513           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4514           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4515           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4516             :   }
    4517             :   return 0;
    4518             : }
    4519             : 
    4520             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4521           0 :   if ((Subtarget->hasCDI())) {
    4522           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4523             :   }
    4524             :   return 0;
    4525             : }
    4526             : 
    4527           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4528           0 : switch (RetVT.SimpleTy) {
    4529           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
    4530           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
    4531           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    4532             :   default: return 0;
    4533             : }
    4534             : }
    4535             : 
    4536           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4537           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4538           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4539             :   }
    4540             :   return 0;
    4541             : }
    4542             : 
    4543           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4544           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4545           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4546             :   }
    4547             :   return 0;
    4548             : }
    4549             : 
    4550             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4551           0 :   if ((Subtarget->hasCDI())) {
    4552           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4553             :   }
    4554             :   return 0;
    4555             : }
    4556             : 
    4557           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4558           0 : switch (RetVT.SimpleTy) {
    4559           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
    4560           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
    4561           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    4562             :   default: return 0;
    4563             : }
    4564             : }
    4565             : 
    4566           0 : unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4567           0 :   switch (VT.SimpleTy) {
    4568           0 :   case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    4569           0 :   case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    4570             :   default: return 0;
    4571             :   }
    4572             : }
    4573             : 
    4574             : // FastEmit functions for X86ISD::VFPEXT.
    4575             : 
    4576           0 : unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4577           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4578             :     return 0;
    4579           0 :   if ((Subtarget->hasVLX())) {
    4580           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4581             :   }
    4582           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4583           0 :     return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4584             :   }
    4585           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4586           0 :     return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4587             :   }
    4588             :   return 0;
    4589             : }
    4590             : 
    4591             : unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4592           0 :   switch (VT.SimpleTy) {
    4593           0 :   case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4594             :   default: return 0;
    4595             :   }
    4596             : }
    4597             : 
    4598             : // FastEmit functions for X86ISD::VFPROUND.
    4599             : 
    4600           0 : unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4601           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4602             :     return 0;
    4603           0 :   if ((Subtarget->hasVLX())) {
    4604           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4605             :   }
    4606           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4607           0 :     return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4608             :   }
    4609           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4610           0 :     return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4611             :   }
    4612             :   return 0;
    4613             : }
    4614             : 
    4615             : unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4616           0 :   switch (VT.SimpleTy) {
    4617           0 :   case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4618             :   default: return 0;
    4619             :   }
    4620             : }
    4621             : 
    4622             : // FastEmit functions for X86ISD::VSEXT.
    4623             : 
    4624           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4625           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4626           0 :     return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4627             :   }
    4628           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4629           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4630             :   }
    4631             :   return 0;
    4632             : }
    4633             : 
    4634           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4635           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4636           0 :     return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4637             :   }
    4638           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4639           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4640             :   }
    4641             :   return 0;
    4642             : }
    4643             : 
    4644             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4645           0 :   if ((Subtarget->hasAVX512())) {
    4646           0 :     return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4647             :   }
    4648             :   return 0;
    4649             : }
    4650             : 
    4651           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4652           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4653           0 :     return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4654             :   }
    4655           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4656           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4657             :   }
    4658             :   return 0;
    4659             : }
    4660             : 
    4661             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4662           0 :   if ((Subtarget->hasAVX512())) {
    4663           0 :     return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4664             :   }
    4665             :   return 0;
    4666             : }
    4667             : 
    4668           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4669           0 : switch (RetVT.SimpleTy) {
    4670           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    4671           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    4672           0 :   case MVT::v16i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    4673           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    4674           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    4675             :   default: return 0;
    4676             : }
    4677             : }
    4678             : 
    4679             : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4680           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    4681             :     return 0;
    4682           0 :   if ((Subtarget->hasBWI())) {
    4683           0 :     return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4684             :   }
    4685             :   return 0;
    4686             : }
    4687             : 
    4688           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4689           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4690           0 :     return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4691             :   }
    4692           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4693           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4694             :   }
    4695             :   return 0;
    4696             : }
    4697             : 
    4698           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4699           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4700           0 :     return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4701             :   }
    4702           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4703           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4704             :   }
    4705             :   return 0;
    4706             : }
    4707             : 
    4708             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4709           0 :   if ((Subtarget->hasAVX512())) {
    4710           0 :     return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4711             :   }
    4712             :   return 0;
    4713             : }
    4714             : 
    4715           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4716           0 : switch (RetVT.SimpleTy) {
    4717           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    4718           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    4719           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    4720             :   default: return 0;
    4721             : }
    4722             : }
    4723             : 
    4724             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4725           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    4726             :     return 0;
    4727           0 :   if ((Subtarget->hasAVX512())) {
    4728           0 :     return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4729             :   }
    4730             :   return 0;
    4731             : }
    4732             : 
    4733           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4734           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    4735             :     return 0;
    4736           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4737           0 :     return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4738             :   }
    4739           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4740           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4741             :   }
    4742             :   return 0;
    4743             : }
    4744             : 
    4745             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4746           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    4747             :     return 0;
    4748           0 :   if ((Subtarget->hasAVX512())) {
    4749           0 :     return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4750             :   }
    4751             :   return 0;
    4752             : }
    4753             : 
    4754           0 : unsigned fastEmit_X86ISD_VSEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4755           0 :   switch (VT.SimpleTy) {
    4756           0 :   case MVT::v16i8: return fastEmit_X86ISD_VSEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4757           0 :   case MVT::v32i8: return fastEmit_X86ISD_VSEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    4758           0 :   case MVT::v8i16: return fastEmit_X86ISD_VSEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4759           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4760           0 :   case MVT::v4i32: return fastEmit_X86ISD_VSEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4761           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4762             :   default: return 0;
    4763             :   }
    4764             : }
    4765             : 
    4766             : // FastEmit functions for X86ISD::VTRUNC.
    4767             : 
    4768           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4769           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4770             :     return 0;
    4771           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4772           0 :     return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4773             :   }
    4774             :   return 0;
    4775             : }
    4776             : 
    4777           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4778           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4779           0 :     return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4780             :   }
    4781             :   return 0;
    4782             : }
    4783             : 
    4784           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4785           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4786           0 :     return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4787             :   }
    4788             :   return 0;
    4789             : }
    4790             : 
    4791           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4792           0 : switch (RetVT.SimpleTy) {
    4793           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4794           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4795             :   default: return 0;
    4796             : }
    4797             : }
    4798             : 
    4799           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4800           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4801             :     return 0;
    4802           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4803           0 :     return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4804             :   }
    4805             :   return 0;
    4806             : }
    4807             : 
    4808           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4809           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4810           0 :     return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4811             :   }
    4812             :   return 0;
    4813             : }
    4814             : 
    4815           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4816           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4817           0 :     return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4818             :   }
    4819             :   return 0;
    4820             : }
    4821             : 
    4822           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4823           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4824           0 :     return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4825             :   }
    4826             :   return 0;
    4827             : }
    4828             : 
    4829           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4830           0 : switch (RetVT.SimpleTy) {
    4831           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4832           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4833           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4834             :   default: return 0;
    4835             : }
    4836             : }
    4837             : 
    4838           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4839           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4840           0 :     return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4841             :   }
    4842             :   return 0;
    4843             : }
    4844             : 
    4845           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4846           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4847           0 :     return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4848             :   }
    4849             :   return 0;
    4850             : }
    4851             : 
    4852           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4853           0 : switch (RetVT.SimpleTy) {
    4854           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4855           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4856             :   default: return 0;
    4857             : }
    4858             : }
    4859             : 
    4860             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4861           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4862             :     return 0;
    4863           0 :   if ((Subtarget->hasAVX512())) {
    4864           0 :     return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4865             :   }
    4866             :   return 0;
    4867             : }
    4868             : 
    4869           0 : unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4870           0 :   switch (VT.SimpleTy) {
    4871           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4872           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4873           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4874           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4875           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4876           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4877             :   default: return 0;
    4878             :   }
    4879             : }
    4880             : 
    4881             : // FastEmit functions for X86ISD::VTRUNCS.
    4882             : 
    4883           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4884           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4885             :     return 0;
    4886           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4887           0 :     return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4888             :   }
    4889             :   return 0;
    4890             : }
    4891             : 
    4892           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4893           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4894             :     return 0;
    4895           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4896           0 :     return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4897             :   }
    4898             :   return 0;
    4899             : }
    4900             : 
    4901             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4902           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4903             :     return 0;
    4904           0 :   if ((Subtarget->hasBWI())) {
    4905           0 :     return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4906             :   }
    4907             :   return 0;
    4908             : }
    4909             : 
    4910           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4911           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4912           0 :     return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4913             :   }
    4914             :   return 0;
    4915             : }
    4916             : 
    4917           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4918           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4919           0 :     return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4920             :   }
    4921             :   return 0;
    4922             : }
    4923             : 
    4924           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4925           0 : switch (RetVT.SimpleTy) {
    4926           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4927           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4928             :   default: return 0;
    4929             : }
    4930             : }
    4931             : 
    4932           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4933           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4934           0 :     return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4935             :   }
    4936             :   return 0;
    4937             : }
    4938             : 
    4939           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4940           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4941           0 :     return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4942             :   }
    4943             :   return 0;
    4944             : }
    4945             : 
    4946           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4947           0 : switch (RetVT.SimpleTy) {
    4948           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4949           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4950             :   default: return 0;
    4951             : }
    4952             : }
    4953             : 
    4954             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4955           0 :   if ((Subtarget->hasAVX512())) {
    4956           0 :     return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4957             :   }
    4958             :   return 0;
    4959             : }
    4960             : 
    4961             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4962           0 :   if ((Subtarget->hasAVX512())) {
    4963           0 :     return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4964             :   }
    4965             :   return 0;
    4966             : }
    4967             : 
    4968           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4969           0 : switch (RetVT.SimpleTy) {
    4970           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4971           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4972             :   default: return 0;
    4973             : }
    4974             : }
    4975             : 
    4976           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4977           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4978           0 :     return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4979             :   }
    4980             :   return 0;
    4981             : }
    4982             : 
    4983           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4984           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4985           0 :     return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4986             :   }
    4987             :   return 0;
    4988             : }
    4989             : 
    4990           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4991           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4992           0 :     return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4993             :   }
    4994             :   return 0;
    4995             : }
    4996             : 
    4997           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4998           0 : switch (RetVT.SimpleTy) {
    4999           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    5000           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    5001           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    5002             :   default: return 0;
    5003             : }
    5004             : }
    5005             : 
    5006           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5007           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5008           0 :     return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5009             :   }
    5010             :   return 0;
    5011             : }
    5012             : 
    5013           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5014           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5015           0 :     return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5016             :   }
    5017             :   return 0;
    5018             : }
    5019             : 
    5020           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5021           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5022           0 :     return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5023             :   }
    5024             :   return 0;
    5025             : }
    5026             : 
    5027           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5028           0 : switch (RetVT.SimpleTy) {
    5029           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    5030           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    5031           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    5032             :   default: return 0;
    5033             : }
    5034             : }
    5035             : 
    5036             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5037           0 :   if ((Subtarget->hasAVX512())) {
    5038           0 :     return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5039             :   }
    5040             :   return 0;
    5041             : }
    5042             : 
    5043             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5044           0 :   if ((Subtarget->hasAVX512())) {
    5045           0 :     return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5046             :   }
    5047             :   return 0;
    5048             : }
    5049             : 
    5050             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5051           0 :   if ((Subtarget->hasAVX512())) {
    5052           0 :     return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5053             :   }
    5054             :   return 0;
    5055             : }
    5056             : 
    5057           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5058           0 : switch (RetVT.SimpleTy) {
    5059           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    5060           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    5061           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    5062             :   default: return 0;
    5063             : }
    5064             : }
    5065             : 
    5066           0 : unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5067           0 :   switch (VT.SimpleTy) {
    5068           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5069           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5070           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    5071           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5072           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5073           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    5074           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5075           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    5076           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    5077             :   default: return 0;
    5078             :   }
    5079             : }
    5080             : 
    5081             : // FastEmit functions for X86ISD::VTRUNCUS.
    5082             : 
    5083           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5084           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5085             :     return 0;
    5086           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5087           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5088             :   }
    5089             :   return 0;
    5090             : }
    5091             : 
    5092           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5093           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5094             :     return 0;
    5095           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5096           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5097             :   }
    5098             :   return 0;
    5099             : }
    5100             : 
    5101             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5102           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    5103             :     return 0;
    5104           0 :   if ((Subtarget->hasBWI())) {
    5105           0 :     return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5106             :   }
    5107             :   return 0;
    5108             : }
    5109             : 
    5110           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5111           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5112           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5113             :   }
    5114             :   return 0;
    5115             : }
    5116             : 
    5117           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5118           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5119           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5120             :   }
    5121             :   return 0;
    5122             : }
    5123             : 
    5124           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5125           0 : switch (RetVT.SimpleTy) {
    5126           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    5127           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    5128             :   default: return 0;
    5129             : }
    5130             : }
    5131             : 
    5132           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5133           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5134           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5135             :   }
    5136             :   return 0;
    5137             : }
    5138             : 
    5139           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5140           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5141           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5142             :   }
    5143             :   return 0;
    5144             : }
    5145             : 
    5146           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5147           0 : switch (RetVT.SimpleTy) {
    5148           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    5149           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    5150             :   default: return 0;
    5151             : }
    5152             : }
    5153             : 
    5154             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5155           0 :   if ((Subtarget->hasAVX512())) {
    5156           0 :     return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5157             :   }
    5158             :   return 0;
    5159             : }
    5160             : 
    5161             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    5162           0 :   if ((Subtarget->hasAVX512())) {
    5163           0 :     return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5164             :   }
    5165             :   return 0;
    5166             : }
    5167             : 
    5168           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5169           0 : switch (RetVT.SimpleTy) {
    5170           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    5171           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    5172             :   default: return 0;
    5173             : }
    5174             : }
    5175             : 
    5176           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5177           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5178           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5179             :   }
    5180             :   return 0;
    5181             : }
    5182             : 
    5183           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5184           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5185           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5186             :   }
    5187             :   return 0;
    5188             : }
    5189             : 
    5190           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5191           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5192           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5193             :   }
    5194             :   return 0;
    5195             : }
    5196             : 
    5197           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5198           0 : switch (RetVT.SimpleTy) {
    5199           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    5200           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    5201           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    5202             :   default: return 0;
    5203             : }
    5204             : }
    5205             : 
    5206           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5207           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5208           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5209             :   }
    5210             :   return 0;
    5211             : }
    5212             : 
    5213           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5214           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5215           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5216             :   }
    5217             :   return 0;
    5218             : }
    5219             : 
    5220           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5221           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5222           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5223             :   }
    5224             :   return 0;
    5225             : }
    5226             : 
    5227           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5228           0 : switch (RetVT.SimpleTy) {
    5229           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    5230           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    5231           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    5232             :   default: return 0;
    5233             : }
    5234             : }
    5235             : 
    5236             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5237           0 :   if ((Subtarget->hasAVX512())) {
    5238           0 :     return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5239             :   }
    5240             :   return 0;
    5241             : }
    5242             : 
    5243             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5244           0 :   if ((Subtarget->hasAVX512())) {
    5245           0 :     return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5246             :   }
    5247             :   return 0;
    5248             : }
    5249             : 
    5250             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5251           0 :   if ((Subtarget->hasAVX512())) {
    5252           0 :     return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5253             :   }
    5254             :   return 0;
    5255             : }
    5256             : 
    5257           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5258           0 : switch (RetVT.SimpleTy) {
    5259           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    5260           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    5261           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    5262             :   default: return 0;
    5263             : }
    5264             : }
    5265             : 
    5266           0 : unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5267           0 :   switch (VT.SimpleTy) {
    5268           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5269           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5270           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    5271           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5272           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5273           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    5274           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5275           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    5276           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    5277             :   default: return 0;
    5278             :   }
    5279             : }
    5280             : 
    5281             : // FastEmit functions for X86ISD::VZEXT.
    5282             : 
    5283           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    5284           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5285           0 :     return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5286             :   }
    5287           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5288           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5289             :   }
    5290             :   return 0;
    5291             : }
    5292             : 
    5293           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5294           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5295           0 :     return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5296             :   }
    5297           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5298           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5299             :   }
    5300             :   return 0;
    5301             : }
    5302             : 
    5303             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    5304           0 :   if ((Subtarget->hasAVX512())) {
    5305           0 :     return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5306             :   }
    5307             :   return 0;
    5308             : }
    5309             : 
    5310           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5311           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5312           0 :     return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5313             :   }
    5314           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5315           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5316             :   }
    5317             :   return 0;
    5318             : }
    5319             : 
    5320             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5321           0 :   if ((Subtarget->hasAVX512())) {
    5322           0 :     return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5323             :   }
    5324             :   return 0;
    5325             : }
    5326             : 
    5327           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5328           0 : switch (RetVT.SimpleTy) {
    5329           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    5330           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    5331           0 :   case MVT::v16i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    5332           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    5333           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    5334             :   default: return 0;
    5335             : }
    5336             : }
    5337             : 
    5338             : unsigned fastEmit_X86ISD_VZEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5339           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5340             :     return 0;
    5341           0 :   if ((Subtarget->hasBWI())) {
    5342           0 :     return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5343             :   }
    5344             :   return 0;
    5345             : }
    5346             : 
    5347           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5348           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5349           0 :     return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5350             :   }
    5351           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5352           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5353             :   }
    5354             :   return 0;
    5355             : }
    5356             : 
    5357           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5358           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5359           0 :     return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5360             :   }
    5361           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5362           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5363             :   }
    5364             :   return 0;
    5365             : }
    5366             : 
    5367             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5368           0 :   if ((Subtarget->hasAVX512())) {
    5369           0 :     return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5370             :   }
    5371             :   return 0;
    5372             : }
    5373             : 
    5374           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5375           0 : switch (RetVT.SimpleTy) {
    5376           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    5377           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    5378           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    5379             :   default: return 0;
    5380             : }
    5381             : }
    5382             : 
    5383             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5384           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    5385             :     return 0;
    5386           0 :   if ((Subtarget->hasAVX512())) {
    5387           0 :     return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5388             :   }
    5389             :   return 0;
    5390             : }
    5391             : 
    5392           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5393           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    5394             :     return 0;
    5395           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5396           0 :     return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5397             :   }
    5398           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5399           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5400             :   }
    5401             :   return 0;
    5402             : }
    5403             : 
    5404             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5405           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5406             :     return 0;
    5407           0 :   if ((Subtarget->hasAVX512())) {
    5408           0 :     return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5409             :   }
    5410             :   return 0;
    5411             : }
    5412             : 
    5413           0 : unsigned fastEmit_X86ISD_VZEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5414           0 :   switch (VT.SimpleTy) {
    5415           0 :   case MVT::v16i8: return fastEmit_X86ISD_VZEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    5416           0 :   case MVT::v32i8: return fastEmit_X86ISD_VZEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    5417           0 :   case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5418           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5419           0 :   case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5420           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5421             :   default: return 0;
    5422             :   }
    5423             : }
    5424             : 
    5425             : // FastEmit functions for X86ISD::VZEXT_MOVL.
    5426             : 
    5427           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5428           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5429             :     return 0;
    5430           0 :   if ((Subtarget->hasAVX512())) {
    5431           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5432             :   }
    5433           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5434           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5435             :   }
    5436           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5437           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5438             :   }
    5439             :   return 0;
    5440             : }
    5441             : 
    5442           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5443           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5444             :     return 0;
    5445           0 :   if ((Subtarget->hasAVX512())) {
    5446           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5447             :   }
    5448           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5449           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5450             :   }
    5451           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5452           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5453             :   }
    5454             :   return 0;
    5455             : }
    5456             : 
    5457           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5458           0 :   switch (VT.SimpleTy) {
    5459           0 :   case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5460           0 :   case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    5461             :   default: return 0;
    5462             :   }
    5463             : }
    5464             : 
    5465             : // FastEmit functions for X86ISD::WIN_ALLOCA.
    5466             : 
    5467           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5468           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5469             :     return 0;
    5470           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    5471           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    5472             :   }
    5473             :   return 0;
    5474             : }
    5475             : 
    5476             : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5477           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5478             :     return 0;
    5479           0 :   if ((Subtarget->is64Bit())) {
    5480           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    5481             :   }
    5482             :   return 0;
    5483             : }
    5484             : 
    5485           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5486           0 :   switch (VT.SimpleTy) {
    5487           0 :   case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    5488           0 :   case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    5489             :   default: return 0;
    5490             :   }
    5491             : }
    5492             : 
    5493             : // Top-level FastEmit function.
    5494             : 
    5495        6869 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
    5496        6869 :   switch (Opcode) {
    5497           0 :   case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
    5498          14 :   case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5499        4024 :   case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
    5500           0 :   case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    5501           0 :   case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
    5502           0 :   case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
    5503           0 :   case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
    5504           0 :   case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill);
    5505           0 :   case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill);
    5506           0 :   case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
    5507           0 :   case ISD::FCOS: return fastEmit_ISD_FCOS_r(VT, RetVT, Op0, Op0IsKill);
    5508          42 :   case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
    5509           0 :   case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5510           0 :   case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
    5511           0 :   case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
    5512           0 :   case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
    5513           0 :   case ISD::FSIN: return fastEmit_ISD_FSIN_r(VT, RetVT, Op0, Op0IsKill);
    5514           0 :   case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5515           1 :   case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill);
    5516        1164 :   case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5517           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5518          48 :   case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5519         213 :   case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
    5520           0 :   case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5521        1363 :   case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5522           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5523           0 :   case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5524           0 :   case X86ISD::COMPRESS: return fastEmit_X86ISD_COMPRESS_r(VT, RetVT, Op0, Op0IsKill);
    5525           0 :   case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill);
    5526           0 :   case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5527           0 :   case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5528           0 :   case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill);
    5529           0 :   case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
    5530           0 :   case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5531           0 :   case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5532           0 :   case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
    5533           0 :   case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill);
    5534           0 :   case X86ISD::EXPAND: return fastEmit_X86ISD_EXPAND_r(VT, RetVT, Op0, Op0IsKill);
    5535           0 :   case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill);
    5536           0 :   case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5537           0 :   case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill);
    5538           0 :   case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill);
    5539           0 :   case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill);
    5540           0 :   case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill);
    5541           0 :   case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill);
    5542           0 :   case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill);
    5543           0 :   case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill);
    5544           0 :   case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    5545           0 :   case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5546           0 :   case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0, Op0IsKill);
    5547           0 :   case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0, Op0IsKill);
    5548           0 :   case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0, Op0IsKill);
    5549           0 :   case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5550           0 :   case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill);
    5551           0 :   case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill);
    5552           0 :   case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
    5553           0 :   case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
    5554           0 :   case X86ISD::VSEXT: return fastEmit_X86ISD_VSEXT_r(VT, RetVT, Op0, Op0IsKill);
    5555           0 :   case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill);
    5556           0 :   case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill);
    5557           0 :   case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill);
    5558           0 :   case X86ISD::VZEXT: return fastEmit_X86ISD_VZEXT_r(VT, RetVT, Op0, Op0IsKill);
    5559           0 :   case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill);
    5560           0 :   case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5561             :   default: return 0;
    5562             :   }
    5563             : }
    5564             : 
    5565             : // FastEmit functions for ISD::ADD.
    5566             : 
    5567             : unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5568           1 :   if (RetVT.SimpleTy != MVT::i8)
    5569             :     return 0;
    5570           1 :   return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5571             : }
    5572             : 
    5573             : unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5574           1 :   if (RetVT.SimpleTy != MVT::i16)
    5575             :     return 0;
    5576           1 :   return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5577             : }
    5578             : 
    5579             : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5580         852 :   if (RetVT.SimpleTy != MVT::i32)
    5581             :     return 0;
    5582         852 :   return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5583             : }
    5584             : 
    5585             : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5586        1039 :   if (RetVT.SimpleTy != MVT::i64)
    5587             :     return 0;
    5588        1039 :   return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5589             : }
    5590             : 
    5591           6 : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5592           6 :   if (RetVT.SimpleTy != MVT::v16i8)
    5593             :     return 0;
    5594           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5595           2 :     return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5596             :   }
    5597           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5598           2 :     return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5599             :   }
    5600           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5601           2 :     return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5602             :   }
    5603             :   return 0;
    5604             : }
    5605             : 
    5606           2 : unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5607           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    5608             :     return 0;
    5609           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5610           0 :     return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5611             :   }
    5612           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5613           2 :     return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5614             :   }
    5615             :   return 0;
    5616             : }
    5617             : 
    5618             : unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5619           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    5620             :     return 0;
    5621           0 :   if ((Subtarget->hasBWI())) {
    5622           0 :     return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5623             :   }
    5624             :   return 0;
    5625             : }
    5626             : 
    5627           6 : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5628           6 :   if (RetVT.SimpleTy != MVT::v8i16)
    5629             :     return 0;
    5630           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5631           2 :     return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5632             :   }
    5633           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5634           2 :     return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5635             :   }
    5636           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5637           2 :     return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5638             :   }
    5639             :   return 0;
    5640             : }
    5641             : 
    5642           2 : unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5643           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    5644             :     return 0;
    5645           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5646           0 :     return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5647             :   }
    5648           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5649           2 :     return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5650             :   }
    5651             :   return 0;
    5652             : }
    5653             : 
    5654             : unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5655           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5656             :     return 0;
    5657           0 :   if ((Subtarget->hasBWI())) {
    5658           0 :     return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5659             :   }
    5660             :   return 0;
    5661             : }
    5662             : 
    5663          22 : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5664          22 :   if (RetVT.SimpleTy != MVT::v4i32)
    5665             :     return 0;
    5666          44 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5667           4 :     return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5668             :   }
    5669          18 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5670           6 :     return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5671             :   }
    5672          24 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5673          12 :     return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5674             :   }
    5675             :   return 0;
    5676             : }
    5677             : 
    5678           3 : unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5679           3 :   if (RetVT.SimpleTy != MVT::v8i32)
    5680             :     return 0;
    5681           6 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5682           1 :     return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5683             :   }
    5684           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5685           2 :     return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5686             :   }
    5687             :   return 0;
    5688             : }
    5689             : 
    5690             : unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5691           2 :   if (RetVT.SimpleTy != MVT::v16i32)
    5692             :     return 0;
    5693           4 :   if ((Subtarget->hasAVX512())) {
    5694           2 :     return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5695             :   }
    5696             :   return 0;
    5697             : }
    5698             : 
    5699           6 : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5700           6 :   if (RetVT.SimpleTy != MVT::v2i64)
    5701             :     return 0;
    5702          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5703           2 :     return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5704             :   }
    5705           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5706           2 :     return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5707             :   }
    5708           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5709           2 :     return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5710             :   }
    5711             :   return 0;
    5712             : }
    5713             : 
    5714           4 : unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5715           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    5716             :     return 0;
    5717           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5718           2 :     return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5719             :   }
    5720           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5721           2 :     return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5722             :   }
    5723             :   return 0;
    5724             : }
    5725             : 
    5726             : unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5727           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5728             :     return 0;
    5729           0 :   if ((Subtarget->hasAVX512())) {
    5730           0 :     return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5731             :   }
    5732             :   return 0;
    5733             : }
    5734             : 
    5735        1946 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5736        1946 :   switch (VT.SimpleTy) {
    5737           1 :   case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5738           1 :   case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5739         852 :   case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5740        1039 :   case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5741           6 :   case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5742           2 :   case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5743           0 :   case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5744           6 :   case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5745           2 :   case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5746           0 :   case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5747          22 :   case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5748           3 :   case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5749           2 :   case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5750           6 :   case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5751           4 :   case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5752           0 :   case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5753             :   default: return 0;
    5754             :   }
    5755             : }
    5756             : 
    5757             : // FastEmit functions for ISD::AND.
    5758             : 
    5759             : unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5760           4 :   if (RetVT.SimpleTy != MVT::i8)
    5761             :     return 0;
    5762           4 :   return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5763             : }
    5764             : 
    5765             : unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5766           0 :   if (RetVT.SimpleTy != MVT::i16)
    5767             :     return 0;
    5768           0 :   return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5769             : }
    5770             : 
    5771             : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5772         121 :   if (RetVT.SimpleTy != MVT::i32)
    5773             :     return 0;
    5774         121 :   return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5775             : }
    5776             : 
    5777             : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5778          14 :   if (RetVT.SimpleTy != MVT::i64)
    5779             :     return 0;
    5780          14 :   return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5781             : }
    5782             : 
    5783             : unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5784           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    5785             :     return 0;
    5786           0 :   if ((Subtarget->hasDQI())) {
    5787           0 :     return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5788             :   }
    5789             :   return 0;
    5790             : }
    5791             : 
    5792             : unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5793           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    5794             :     return 0;
    5795           0 :   if ((Subtarget->hasAVX512())) {
    5796           0 :     return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5797             :   }
    5798             :   return 0;
    5799             : }
    5800             : 
    5801             : unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5802           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    5803             :     return 0;
    5804           0 :   if ((Subtarget->hasBWI())) {
    5805           0 :     return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5806             :   }
    5807             :   return 0;
    5808             : }
    5809             : 
    5810             : unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5811           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    5812             :     return 0;
    5813           0 :   if ((Subtarget->hasBWI())) {
    5814           0 :     return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5815             :   }
    5816             :   return 0;
    5817             : }
    5818             : 
    5819          16 : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5820          16 :   if (RetVT.SimpleTy != MVT::v2i64)
    5821             :     return 0;
    5822          32 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5823           4 :     return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5824             :   }
    5825          12 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5826           4 :     return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5827             :   }
    5828          16 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5829           8 :     return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5830             :   }
    5831             :   return 0;
    5832             : }
    5833             : 
    5834          12 : unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5835          12 :   if (RetVT.SimpleTy != MVT::v4i64)
    5836             :     return 0;
    5837          24 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    5838           8 :     return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5839             :   }
    5840           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5841           0 :     return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5842             :   }
    5843           8 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5844           4 :     return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5845             :   }
    5846             :   return 0;
    5847             : }
    5848             : 
    5849             : unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5850           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5851             :     return 0;
    5852           0 :   if ((Subtarget->hasAVX512())) {
    5853           0 :     return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5854             :   }
    5855             :   return 0;
    5856             : }
    5857             : 
    5858         191 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5859         191 :   switch (VT.SimpleTy) {
    5860           4 :   case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5861           0 :   case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5862         121 :   case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5863          14 :   case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5864           0 :   case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5865           0 :   case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5866           0 :   case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5867           0 :   case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5868          16 :   case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5869          12 :   case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5870           0 :   case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5871             :   default: return 0;
    5872             :   }
    5873             : }
    5874             : 
    5875             : // FastEmit functions for ISD::FADD.
    5876             : 
    5877          23 : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5878          23 :   if (RetVT.SimpleTy != MVT::f32)
    5879             :     return 0;
    5880          46 :   if ((Subtarget->hasAVX512())) {
    5881           2 :     return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5882             :   }
    5883          21 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5884          19 :     return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5885             :   }
    5886           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5887           2 :     return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5888             :   }
    5889           0 :   if ((!Subtarget->hasSSE1())) {
    5890           0 :     return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5891             :   }
    5892             :   return 0;
    5893             : }
    5894             : 
    5895          23 : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5896          23 :   if (RetVT.SimpleTy != MVT::f64)
    5897             :     return 0;
    5898          46 :   if ((Subtarget->hasAVX512())) {
    5899           2 :     return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5900             :   }
    5901          21 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5902          19 :     return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5903             :   }
    5904           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5905           2 :     return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5906             :   }
    5907           0 :   if ((!Subtarget->hasSSE2())) {
    5908           0 :     return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5909             :   }
    5910             :   return 0;
    5911             : }
    5912             : 
    5913             : unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5914           0 :   if (RetVT.SimpleTy != MVT::f80)
    5915             :     return 0;
    5916           0 :   return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5917             : }
    5918             : 
    5919           6 : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5920           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    5921             :     return 0;
    5922          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5923           2 :     return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5924             :   }
    5925           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5926           2 :     return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5927             :   }
    5928           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5929           2 :     return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5930             :   }
    5931             :   return 0;
    5932             : }
    5933             : 
    5934           2 : unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5935           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5936             :     return 0;
    5937           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5938           0 :     return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5939             :   }
    5940           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5941           2 :     return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5942             :   }
    5943             :   return 0;
    5944             : }
    5945             : 
    5946             : unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5947           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5948             :     return 0;
    5949           0 :   if ((Subtarget->hasAVX512())) {
    5950           0 :     return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5951             :   }
    5952             :   return 0;
    5953             : }
    5954             : 
    5955          22 : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5956          22 :   if (RetVT.SimpleTy != MVT::v2f64)
    5957             :     return 0;
    5958          44 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5959           4 :     return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5960             :   }
    5961          18 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5962           6 :     return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5963             :   }
    5964          24 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5965          12 :     return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5966             :   }
    5967             :   return 0;
    5968             : }
    5969             : 
    5970          14 : unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5971          14 :   if (RetVT.SimpleTy != MVT::v4f64)
    5972             :     return 0;
    5973          28 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5974           2 :     return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5975             :   }
    5976          12 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5977          12 :     return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5978             :   }
    5979             :   return 0;
    5980             : }
    5981             : 
    5982             : unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5983           8 :   if (RetVT.SimpleTy != MVT::v8f64)
    5984             :     return 0;
    5985          16 :   if ((Subtarget->hasAVX512())) {
    5986           8 :     return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5987             :   }
    5988             :   return 0;
    5989             : }
    5990             : 
    5991          98 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5992          98 :   switch (VT.SimpleTy) {
    5993          23 :   case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5994          23 :   case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5995           0 :   case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5996           6 :   case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5997           2 :   case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5998           0 :   case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5999          22 :   case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6000          14 :   case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6001           8 :   case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6002             :   default: return 0;
    6003             :   }
    6004             : }
    6005             : 
    6006             : // FastEmit functions for ISD::FDIV.
    6007             : 
    6008           6 : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6009           6 :   if (RetVT.SimpleTy != MVT::f32)
    6010             :     return 0;
    6011          12 :   if ((Subtarget->hasAVX512())) {
    6012           0 :     return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6013             :   }
    6014           6 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6015           6 :     return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6016             :   }
    6017           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6018           0 :     return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6019             :   }
    6020           0 :   if ((!Subtarget->hasSSE1())) {
    6021           0 :     return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6022             :   }
    6023             :   return 0;
    6024             : }
    6025             : 
    6026           0 : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6027           0 :   if (RetVT.SimpleTy != MVT::f64)
    6028             :     return 0;
    6029           0 :   if ((Subtarget->hasAVX512())) {
    6030           0 :     return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6031             :   }
    6032           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6033           0 :     return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6034             :   }
    6035           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6036           0 :     return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6037             :   }
    6038           0 :   if ((!Subtarget->hasSSE2())) {
    6039           0 :     return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6040             :   }
    6041             :   return 0;
    6042             : }
    6043             : 
    6044             : unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6045           0 :   if (RetVT.SimpleTy != MVT::f80)
    6046             :     return 0;
    6047           0 :   return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6048             : }
    6049             : 
    6050           6 : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6051           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    6052             :     return 0;
    6053          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6054           2 :     return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6055             :   }
    6056           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6057           2 :     return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6058             :   }
    6059           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6060           2 :     return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6061             :   }
    6062             :   return 0;
    6063             : }
    6064             : 
    6065           2 : unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6066           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6067             :     return 0;
    6068           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6069           0 :     return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6070             :   }
    6071           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6072           2 :     return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6073             :   }
    6074             :   return 0;
    6075             : }
    6076             : 
    6077             : unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6078           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6079             :     return 0;
    6080           0 :   if ((Subtarget->hasAVX512())) {
    6081           0 :     return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6082             :   }
    6083             :   return 0;
    6084             : }
    6085             : 
    6086           6 : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6087           6 :   if (RetVT.SimpleTy != MVT::v2f64)
    6088             :     return 0;
    6089          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6090           2 :     return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6091             :   }
    6092           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6093           2 :     return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6094             :   }
    6095           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6096           2 :     return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6097             :   }
    6098             :   return 0;
    6099             : }
    6100             : 
    6101           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6102           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6103             :     return 0;
    6104           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6105           0 :     return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6106             :   }
    6107           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6108           2 :     return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6109             :   }
    6110             :   return 0;
    6111             : }
    6112             : 
    6113             : unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6114           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6115             :     return 0;
    6116           0 :   if ((Subtarget->hasAVX512())) {
    6117           0 :     return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6118             :   }
    6119             :   return 0;
    6120             : }
    6121             : 
    6122          22 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6123          22 :   switch (VT.SimpleTy) {
    6124           6 :   case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6125           0 :   case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6126           0 :   case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6127           6 :   case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6128           2 :   case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6129           0 :   case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6130           6 :   case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6131           2 :   case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6132           0 :   case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6133             :   default: return 0;
    6134             :   }
    6135             : }
    6136             : 
    6137             : // FastEmit functions for ISD::FMUL.
    6138             : 
    6139           7 : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6140           7 :   if (RetVT.SimpleTy != MVT::f32)
    6141             :     return 0;
    6142          14 :   if ((Subtarget->hasAVX512())) {
    6143           0 :     return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6144             :   }
    6145           7 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6146           7 :     return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6147             :   }
    6148           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6149           0 :     return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6150             :   }
    6151           0 :   if ((!Subtarget->hasSSE1())) {
    6152           0 :     return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6153             :   }
    6154             :   return 0;
    6155             : }
    6156             : 
    6157           5 : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6158           5 :   if (RetVT.SimpleTy != MVT::f64)
    6159             :     return 0;
    6160          10 :   if ((Subtarget->hasAVX512())) {
    6161           0 :     return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6162             :   }
    6163           5 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6164           5 :     return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6165             :   }
    6166           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6167           0 :     return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6168             :   }
    6169           0 :   if ((!Subtarget->hasSSE2())) {
    6170           0 :     return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6171             :   }
    6172             :   return 0;
    6173             : }
    6174             : 
    6175             : unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6176           0 :   if (RetVT.SimpleTy != MVT::f80)
    6177             :     return 0;
    6178           0 :   return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6179             : }
    6180             : 
    6181           6 : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6182           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    6183             :     return 0;
    6184          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6185           2 :     return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6186             :   }
    6187           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6188           2 :     return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6189             :   }
    6190           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6191           2 :     return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6192             :   }
    6193             :   return 0;
    6194             : }
    6195             : 
    6196           2 : unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6197           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6198             :     return 0;
    6199           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6200           0 :     return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6201             :   }
    6202           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6203           2 :     return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6204             :   }
    6205             :   return 0;
    6206             : }
    6207             : 
    6208             : unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6209           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6210             :     return 0;
    6211           0 :   if ((Subtarget->hasAVX512())) {
    6212           0 :     return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6213             :   }
    6214             :   return 0;
    6215             : }
    6216             : 
    6217           6 : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6218           6 :   if (RetVT.SimpleTy != MVT::v2f64)
    6219             :     return 0;
    6220          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6221           2 :     return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6222             :   }
    6223           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6224           2 :     return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6225             :   }
    6226           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6227           2 :     return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6228             :   }
    6229             :   return 0;
    6230             : }
    6231             : 
    6232           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6233           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6234             :     return 0;
    6235           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6236           0 :     return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6237             :   }
    6238           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6239           2 :     return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6240             :   }
    6241             :   return 0;
    6242             : }
    6243             : 
    6244             : unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6245           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6246             :     return 0;
    6247           0 :   if ((Subtarget->hasAVX512())) {
    6248           0 :     return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6249             :   }
    6250             :   return 0;
    6251             : }
    6252             : 
    6253          28 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6254          28 :   switch (VT.SimpleTy) {
    6255           7 :   case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6256           5 :   case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6257           0 :   case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6258           6 :   case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6259           2 :   case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6260           0 :   case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6261           6 :   case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6262           2 :   case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6263           0 :   case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6264             :   default: return 0;
    6265             :   }
    6266             : }
    6267             : 
    6268             : // FastEmit functions for ISD::FSUB.
    6269             : 
    6270           0 : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6271           0 :   if (RetVT.SimpleTy != MVT::f32)
    6272             :     return 0;
    6273           0 :   if ((Subtarget->hasAVX512())) {
    6274           0 :     return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6275             :   }
    6276           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6277           0 :     return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6278             :   }
    6279           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6280           0 :     return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6281             :   }
    6282           0 :   if ((!Subtarget->hasSSE1())) {
    6283           0 :     return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6284             :   }
    6285             :   return 0;
    6286             : }
    6287             : 
    6288          26 : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6289          26 :   if (RetVT.SimpleTy != MVT::f64)
    6290             :     return 0;
    6291          52 :   if ((Subtarget->hasAVX512())) {
    6292           0 :     return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6293             :   }
    6294          26 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6295          26 :     return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6296             :   }
    6297           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6298           0 :     return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6299             :   }
    6300           0 :   if ((!Subtarget->hasSSE2())) {
    6301           0 :     return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6302             :   }
    6303             :   return 0;
    6304             : }
    6305             : 
    6306             : unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6307           0 :   if (RetVT.SimpleTy != MVT::f80)
    6308             :     return 0;
    6309           0 :   return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6310             : }
    6311             : 
    6312           6 : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6313           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    6314             :     return 0;
    6315          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6316           2 :     return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6317             :   }
    6318           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6319           2 :     return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6320             :   }
    6321           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6322           2 :     return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6323             :   }
    6324             :   return 0;
    6325             : }
    6326             : 
    6327           2 : unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6328           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6329             :     return 0;
    6330           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6331           0 :     return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6332             :   }
    6333           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6334           2 :     return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6335             :   }
    6336             :   return 0;
    6337             : }
    6338             : 
    6339             : unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6340           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6341             :     return 0;
    6342           0 :   if ((Subtarget->hasAVX512())) {
    6343           0 :     return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6344             :   }
    6345             :   return 0;
    6346             : }
    6347             : 
    6348           6 : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6349           6 :   if (RetVT.SimpleTy != MVT::v2f64)
    6350             :     return 0;
    6351          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6352           2 :     return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6353             :   }
    6354           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6355           2 :     return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6356             :   }
    6357           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6358           2 :     return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6359             :   }
    6360             :   return 0;
    6361             : }
    6362             : 
    6363           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6364           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6365             :     return 0;
    6366           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6367           0 :     return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6368             :   }
    6369           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6370           2 :     return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6371             :   }
    6372             :   return 0;
    6373             : }
    6374             : 
    6375             : unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6376           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6377             :     return 0;
    6378           0 :   if ((Subtarget->hasAVX512())) {
    6379           0 :     return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6380             :   }
    6381             :   return 0;
    6382             : }
    6383             : 
    6384          42 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6385          42 :   switch (VT.SimpleTy) {
    6386           0 :   case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6387          26 :   case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6388           0 :   case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6389           6 :   case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6390           2 :   case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6391           0 :   case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6392           6 :   case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6393           2 :   case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6394           0 :   case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6395             :   default: return 0;
    6396             :   }
    6397             : }
    6398             : 
    6399             : // FastEmit functions for ISD::MUL.
    6400             : 
    6401           2 : unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6402           2 :   if (RetVT.SimpleTy != MVT::i8)
    6403             :     return 0;
    6404           4 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
    6405           2 :   return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill);
    6406             : }
    6407             : 
    6408             : unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6409           0 :   if (RetVT.SimpleTy != MVT::i16)
    6410             :     return 0;
    6411           0 :   return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6412             : }
    6413             : 
    6414             : unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6415          17 :   if (RetVT.SimpleTy != MVT::i32)
    6416             :     return 0;
    6417          17 :   return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6418             : }
    6419             : 
    6420             : unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6421          64 :   if (RetVT.SimpleTy != MVT::i64)
    6422             :     return 0;
    6423          64 :   return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6424             : }
    6425             : 
    6426           6 : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6427           6 :   if (RetVT.SimpleTy != MVT::v8i16)
    6428             :     return 0;
    6429           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6430           2 :     return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6431             :   }
    6432           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6433           2 :     return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6434             :   }
    6435           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6436           2 :     return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6437             :   }
    6438             :   return 0;
    6439             : }
    6440             : 
    6441           2 : unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6442           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    6443             :     return 0;
    6444           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6445           0 :     return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6446             :   }
    6447           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6448           2 :     return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6449             :   }
    6450             :   return 0;
    6451             : }
    6452             : 
    6453             : unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6454           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6455             :     return 0;
    6456           0 :   if ((Subtarget->hasBWI())) {
    6457           0 :     return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6458             :   }
    6459             :   return 0;
    6460             : }
    6461             : 
    6462           6 : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6463           6 :   if (RetVT.SimpleTy != MVT::v4i32)
    6464             :     return 0;
    6465          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6466           2 :     return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6467             :   }
    6468           4 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6469           2 :     return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6470             :   }
    6471           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6472           2 :     return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6473             :   }
    6474             :   return 0;
    6475             : }
    6476             : 
    6477           2 : unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6478           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    6479             :     return 0;
    6480           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6481           0 :     return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6482             :   }
    6483           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6484           2 :     return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6485             :   }
    6486             :   return 0;
    6487             : }
    6488             : 
    6489             : unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6490           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6491             :     return 0;
    6492           0 :   if ((Subtarget->hasAVX512())) {
    6493           0 :     return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6494             :   }
    6495             :   return 0;
    6496             : }
    6497             : 
    6498          12 : unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6499          12 :   if (RetVT.SimpleTy != MVT::v2i64)
    6500             :     return 0;
    6501          12 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6502           4 :     return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6503             :   }
    6504             :   return 0;
    6505             : }
    6506             : 
    6507           4 : unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6508           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    6509             :     return 0;
    6510           4 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6511           0 :     return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6512             :   }
    6513             :   return 0;
    6514             : }
    6515             : 
    6516             : unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6517           4 :   if (RetVT.SimpleTy != MVT::v8i64)
    6518             :     return 0;
    6519           4 :   if ((Subtarget->hasDQI())) {
    6520           0 :     return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6521             :   }
    6522             :   return 0;
    6523             : }
    6524             : 
    6525         119 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6526         119 :   switch (VT.SimpleTy) {
    6527           2 :   case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6528           0 :   case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6529          17 :   case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6530          64 :   case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6531           6 :   case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6532           2 :   case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6533           0 :   case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6534           6 :   case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6535           2 :   case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6536           0 :   case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6537          12 :   case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6538           4 :   case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6539           4 :   case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6540             :   default: return 0;
    6541             :   }
    6542             : }
    6543             : 
    6544             : // FastEmit functions for ISD::MULHS.
    6545             : 
    6546           0 : unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6547           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6548             :     return 0;
    6549           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6550           0 :     return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6551             :   }
    6552           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6553           0 :     return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6554             :   }
    6555           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6556           0 :     return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6557             :   }
    6558             :   return 0;
    6559             : }
    6560             : 
    6561           0 : unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6562           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6563             :     return 0;
    6564           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6565           0 :     return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6566             :   }
    6567           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6568           0 :     return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6569             :   }
    6570             :   return 0;
    6571             : }
    6572             : 
    6573             : unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6574           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6575             :     return 0;
    6576           0 :   if ((Subtarget->hasBWI())) {
    6577           0 :     return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6578             :   }
    6579             :   return 0;
    6580             : }
    6581             : 
    6582           0 : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6583           0 :   switch (VT.SimpleTy) {
    6584           0 :   case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6585           0 :   case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6586           0 :   case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6587             :   default: return 0;
    6588             :   }
    6589             : }
    6590             : 
    6591             : // FastEmit functions for ISD::MULHU.
    6592             : 
    6593           0 : unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6594           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6595             :     return 0;
    6596           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6597           0 :     return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6598             :   }
    6599           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6600           0 :     return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6601             :   }
    6602           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6603           0 :     return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6604             :   }
    6605             :   return 0;
    6606             : }
    6607             : 
    6608           0 : unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6609           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6610             :     return 0;
    6611           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6612           0 :     return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6613             :   }
    6614           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6615           0 :     return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6616             :   }
    6617             :   return 0;
    6618             : }
    6619             : 
    6620             : unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6621           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6622             :     return 0;
    6623           0 :   if ((Subtarget->hasBWI())) {
    6624           0 :     return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6625             :   }
    6626             :   return 0;
    6627             : }
    6628             : 
    6629           0 : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6630           0 :   switch (VT.SimpleTy) {
    6631           0 :   case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6632           0 :   case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6633           0 :   case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6634             :   default: return 0;
    6635             :   }
    6636             : }
    6637             : 
    6638             : // FastEmit functions for ISD::OR.
    6639             : 
    6640             : unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6641           4 :   if (RetVT.SimpleTy != MVT::i8)
    6642             :     return 0;
    6643           4 :   return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6644             : }
    6645             : 
    6646             : unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6647          14 :   if (RetVT.SimpleTy != MVT::i16)
    6648             :     return 0;
    6649          14 :   return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6650             : }
    6651             : 
    6652             : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6653         123 :   if (RetVT.SimpleTy != MVT::i32)
    6654             :     return 0;
    6655         123 :   return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6656             : }
    6657             : 
    6658             : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6659          12 :   if (RetVT.SimpleTy != MVT::i64)
    6660             :     return 0;
    6661          12 :   return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6662             : }
    6663             : 
    6664             : unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6665           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    6666             :     return 0;
    6667           0 :   if ((Subtarget->hasDQI())) {
    6668           0 :     return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6669             :   }
    6670             :   return 0;
    6671             : }
    6672             : 
    6673             : unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6674           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    6675             :     return 0;
    6676           0 :   if ((Subtarget->hasAVX512())) {
    6677           0 :     return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6678             :   }
    6679             :   return 0;
    6680             : }
    6681             : 
    6682             : unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6683           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    6684             :     return 0;
    6685           0 :   if ((Subtarget->hasBWI())) {
    6686           0 :     return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6687             :   }
    6688             :   return 0;
    6689             : }
    6690             : 
    6691             : unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6692           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    6693             :     return 0;
    6694           0 :   if ((Subtarget->hasBWI())) {
    6695           0 :     return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6696             :   }
    6697             :   return 0;
    6698             : }
    6699             : 
    6700           8 : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6701           8 :   if (RetVT.SimpleTy != MVT::v2i64)
    6702             :     return 0;
    6703          16 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6704           2 :     return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6705             :   }
    6706           6 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6707           2 :     return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6708             :   }
    6709           8 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6710           4 :     return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6711             :   }
    6712             :   return 0;
    6713             : }
    6714             : 
    6715           6 : unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6716           6 :   if (RetVT.SimpleTy != MVT::v4i64)
    6717             :     return 0;
    6718          12 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    6719           4 :     return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6720             :   }
    6721           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6722           0 :     return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6723             :   }
    6724           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6725           2 :     return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6726             :   }
    6727             :   return 0;
    6728             : }
    6729             : 
    6730             : unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6731           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6732             :     return 0;
    6733           0 :   if ((Subtarget->hasAVX512())) {
    6734           0 :     return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6735             :   }
    6736             :   return 0;
    6737             : }
    6738             : 
    6739         179 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6740         179 :   switch (VT.SimpleTy) {
    6741           4 :   case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6742          14 :   case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6743         123 :   case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6744          12 :   case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6745           0 :   case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6746           0 :   case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6747           0 :   case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6748           0 :   case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6749           8 :   case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6750           6 :   case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6751           0 :   case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6752             :   default: return 0;
    6753             :   }
    6754             : }
    6755             : 
    6756             : // FastEmit functions for ISD::ROTL.
    6757             : 
    6758           0 : unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6759           0 :   if (RetVT.SimpleTy != MVT::i8)
    6760             :     return 0;
    6761           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6762           0 :   return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6763             : }
    6764             : 
    6765             : unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6766           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6767             :     return 0;
    6768           0 :   if ((Subtarget->hasXOP())) {
    6769           0 :     return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6770             :   }
    6771             :   return 0;
    6772             : }
    6773             : 
    6774             : unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6775           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6776             :     return 0;
    6777           0 :   if ((Subtarget->hasXOP())) {
    6778           0 :     return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6779             :   }
    6780             :   return 0;
    6781             : }
    6782             : 
    6783           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6784           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6785             :     return 0;
    6786           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6787           0 :     return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6788             :   }
    6789           0 :   if ((Subtarget->hasXOP())) {
    6790           0 :     return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6791             :   }
    6792             :   return 0;
    6793             : }
    6794             : 
    6795           0 : unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6796           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6797             :     return 0;
    6798           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6799           0 :     return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6800             :   }
    6801             :   return 0;
    6802             : }
    6803             : 
    6804             : unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6805           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6806             :     return 0;
    6807           0 :   if ((Subtarget->hasAVX512())) {
    6808           0 :     return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6809             :   }
    6810             :   return 0;
    6811             : }
    6812             : 
    6813           0 : unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6814           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6815             :     return 0;
    6816           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6817           0 :     return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6818             :   }
    6819           0 :   if ((Subtarget->hasXOP())) {
    6820           0 :     return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6821             :   }
    6822             :   return 0;
    6823             : }
    6824             : 
    6825           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6826           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6827             :     return 0;
    6828           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6829           0 :     return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6830             :   }
    6831             :   return 0;
    6832             : }
    6833             : 
    6834             : unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6835           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6836             :     return 0;
    6837           0 :   if ((Subtarget->hasAVX512())) {
    6838           0 :     return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6839             :   }
    6840             :   return 0;
    6841             : }
    6842             : 
    6843           0 : unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6844           0 :   switch (VT.SimpleTy) {
    6845           0 :   case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6846           0 :   case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6847           0 :   case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6848           0 :   case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6849           0 :   case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6850           0 :   case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6851           0 :   case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6852           0 :   case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6853           0 :   case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6854             :   default: return 0;
    6855             :   }
    6856             : }
    6857             : 
    6858             : // FastEmit functions for ISD::ROTR.
    6859             : 
    6860           0 : unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6861           0 :   if (RetVT.SimpleTy != MVT::i8)
    6862             :     return 0;
    6863           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6864           0 :   return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6865             : }
    6866             : 
    6867           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6868           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6869             :     return 0;
    6870           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6871           0 :     return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6872             :   }
    6873             :   return 0;
    6874             : }
    6875             : 
    6876           0 : unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6877           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6878             :     return 0;
    6879           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6880           0 :     return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6881             :   }
    6882             :   return 0;
    6883             : }
    6884             : 
    6885             : unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6886           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6887             :     return 0;
    6888           0 :   if ((Subtarget->hasAVX512())) {
    6889           0 :     return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6890             :   }
    6891             :   return 0;
    6892             : }
    6893             : 
    6894           0 : unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6895           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6896             :     return 0;
    6897           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6898           0 :     return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6899             :   }
    6900             :   return 0;
    6901             : }
    6902             : 
    6903           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6904           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6905             :     return 0;
    6906           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6907           0 :     return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6908             :   }
    6909             :   return 0;
    6910             : }
    6911             : 
    6912             : unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6913           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6914             :     return 0;
    6915           0 :   if ((Subtarget->hasAVX512())) {
    6916           0 :     return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6917             :   }
    6918             :   return 0;
    6919             : }
    6920             : 
    6921           0 : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6922           0 :   switch (VT.SimpleTy) {
    6923           0 :   case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6924           0 :   case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6925           0 :   case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6926           0 :   case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6927           0 :   case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6928           0 :   case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6929           0 :   case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6930             :   default: return 0;
    6931             :   }
    6932             : }
    6933             : 
    6934             : // FastEmit functions for ISD::SHL.
    6935             : 
    6936           3 : unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6937           3 :   if (RetVT.SimpleTy != MVT::i8)
    6938             :     return 0;
    6939           6 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6940           3 :   return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6941             : }
    6942             : 
    6943           0 : unsigned fastEmit_ISD_SHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6944           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6945             :     return 0;
    6946           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6947           0 :     return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6948             :   }
    6949             :   return 0;
    6950             : }
    6951             : 
    6952           0 : unsigned fastEmit_ISD_SHL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6953           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6954             :     return 0;
    6955           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6956           0 :     return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6957             :   }
    6958             :   return 0;
    6959             : }
    6960             : 
    6961             : unsigned fastEmit_ISD_SHL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6962           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6963             :     return 0;
    6964           0 :   if ((Subtarget->hasBWI())) {
    6965           0 :     return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6966             :   }
    6967             :   return 0;
    6968             : }
    6969             : 
    6970           0 : unsigned fastEmit_ISD_SHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6971           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6972             :     return 0;
    6973           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6974           0 :     return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6975             :   }
    6976           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6977           0 :     return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6978             :   }
    6979             :   return 0;
    6980             : }
    6981             : 
    6982           0 : unsigned fastEmit_ISD_SHL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6983           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6984             :     return 0;
    6985           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6986           0 :     return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6987             :   }
    6988           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6989           0 :     return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6990             :   }
    6991             :   return 0;
    6992             : }
    6993             : 
    6994             : unsigned fastEmit_ISD_SHL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6995           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6996             :     return 0;
    6997           0 :   if ((Subtarget->hasAVX512())) {
    6998           0 :     return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6999             :   }
    7000             :   return 0;
    7001             : }
    7002             : 
    7003           0 : unsigned fastEmit_ISD_SHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7004           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7005             :     return 0;
    7006           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7007           0 :     return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7008             :   }
    7009           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7010           0 :     return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7011             :   }
    7012             :   return 0;
    7013             : }
    7014             : 
    7015           0 : unsigned fastEmit_ISD_SHL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7016           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7017             :     return 0;
    7018           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7019           0 :     return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7020             :   }
    7021           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7022           0 :     return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7023             :   }
    7024             :   return 0;
    7025             : }
    7026             : 
    7027             : unsigned fastEmit_ISD_SHL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7028           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7029             :     return 0;
    7030           0 :   if ((Subtarget->hasAVX512())) {
    7031           0 :     return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7032             :   }
    7033             :   return 0;
    7034             : }
    7035             : 
    7036          82 : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7037          82 :   switch (VT.SimpleTy) {
    7038           3 :   case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7039           0 :   case MVT::v8i16: return fastEmit_ISD_SHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7040           0 :   case MVT::v16i16: return fastEmit_ISD_SHL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7041           0 :   case MVT::v32i16: return fastEmit_ISD_SHL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7042           0 :   case MVT::v4i32: return fastEmit_ISD_SHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7043           0 :   case MVT::v8i32: return fastEmit_ISD_SHL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7044           0 :   case MVT::v16i32: return fastEmit_ISD_SHL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7045           0 :   case MVT::v2i64: return fastEmit_ISD_SHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7046           0 :   case MVT::v4i64: return fastEmit_ISD_SHL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7047           0 :   case MVT::v8i64: return fastEmit_ISD_SHL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7048             :   default: return 0;
    7049             :   }
    7050             : }
    7051             : 
    7052             : // FastEmit functions for ISD::SMAX.
    7053             : 
    7054           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7055           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7056             :     return 0;
    7057           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7058           0 :     return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7059             :   }
    7060           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7061           0 :     return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7062             :   }
    7063           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7064           0 :     return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7065             :   }
    7066             :   return 0;
    7067             : }
    7068             : 
    7069           0 : unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7070           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7071             :     return 0;
    7072           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7073           0 :     return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7074             :   }
    7075           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7076           0 :     return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7077             :   }
    7078             :   return 0;
    7079             : }
    7080             : 
    7081             : unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7082           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7083             :     return 0;
    7084           0 :   if ((Subtarget->hasBWI())) {
    7085           0 :     return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7086             :   }
    7087             :   return 0;
    7088             : }
    7089             : 
    7090           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7091           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7092             :     return 0;
    7093           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7094           0 :     return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7095             :   }
    7096           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7097           0 :     return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7098             :   }
    7099           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7100           0 :     return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7101             :   }
    7102             :   return 0;
    7103             : }
    7104             : 
    7105           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7106           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7107             :     return 0;
    7108           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7109           0 :     return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7110             :   }
    7111           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7112           0 :     return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7113             :   }
    7114             :   return 0;
    7115             : }
    7116             : 
    7117             : unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7118           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7119             :     return 0;
    7120           0 :   if ((Subtarget->hasBWI())) {
    7121           0 :     return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7122             :   }
    7123             :   return 0;
    7124             : }
    7125             : 
    7126           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7127           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7128             :     return 0;
    7129           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7130           0 :     return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7131             :   }
    7132           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7133           0 :     return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7134             :   }
    7135           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7136           0 :     return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7137             :   }
    7138             :   return 0;
    7139             : }
    7140             : 
    7141           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7142           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7143             :     return 0;
    7144           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7145           0 :     return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7146             :   }
    7147           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7148           0 :     return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7149             :   }
    7150             :   return 0;
    7151             : }
    7152             : 
    7153             : unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7154           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7155             :     return 0;
    7156           0 :   if ((Subtarget->hasAVX512())) {
    7157           0 :     return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7158             :   }
    7159             :   return 0;
    7160             : }
    7161             : 
    7162           0 : unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7163           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7164             :     return 0;
    7165           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7166           0 :     return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7167             :   }
    7168             :   return 0;
    7169             : }
    7170             : 
    7171           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7172           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7173             :     return 0;
    7174           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7175           0 :     return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7176             :   }
    7177             :   return 0;
    7178             : }
    7179             : 
    7180             : unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7181           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7182             :     return 0;
    7183           0 :   if ((Subtarget->hasAVX512())) {
    7184           0 :     return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7185             :   }
    7186             :   return 0;
    7187             : }
    7188             : 
    7189           0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7190           0 :   switch (VT.SimpleTy) {
    7191           0 :   case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7192           0 :   case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7193           0 :   case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7194           0 :   case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7195           0 :   case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7196           0 :   case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7197           0 :   case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7198           0 :   case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7199           0 :   case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7200           0 :   case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7201           0 :   case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7202           0 :   case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7203             :   default: return 0;
    7204             :   }
    7205             : }
    7206             : 
    7207             : // FastEmit functions for ISD::SMIN.
    7208             : 
    7209           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7210           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7211             :     return 0;
    7212           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7213           0 :     return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7214             :   }
    7215           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7216           0 :     return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7217             :   }
    7218           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7219           0 :     return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7220             :   }
    7221             :   return 0;
    7222             : }
    7223             : 
    7224           0 : unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7225           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7226             :     return 0;
    7227           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7228           0 :     return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7229             :   }
    7230           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7231           0 :     return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7232             :   }
    7233             :   return 0;
    7234             : }
    7235             : 
    7236             : unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7237           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7238             :     return 0;
    7239           0 :   if ((Subtarget->hasBWI())) {
    7240           0 :     return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7241             :   }
    7242             :   return 0;
    7243             : }
    7244             : 
    7245           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7246           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7247             :     return 0;
    7248           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7249           0 :     return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7250             :   }
    7251           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7252           0 :     return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7253             :   }
    7254           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7255           0 :     return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7256             :   }
    7257             :   return 0;
    7258             : }
    7259             : 
    7260           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7261           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7262             :     return 0;
    7263           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7264           0 :     return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7265             :   }
    7266           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7267           0 :     return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7268             :   }
    7269             :   return 0;
    7270             : }
    7271             : 
    7272             : unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7273           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7274             :     return 0;
    7275           0 :   if ((Subtarget->hasBWI())) {
    7276           0 :     return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7277             :   }
    7278             :   return 0;
    7279             : }
    7280             : 
    7281           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7282           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7283             :     return 0;
    7284           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7285           0 :     return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7286             :   }
    7287           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7288           0 :     return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7289             :   }
    7290           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7291           0 :     return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7292             :   }
    7293             :   return 0;
    7294             : }
    7295             : 
    7296           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7297           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7298             :     return 0;
    7299           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7300           0 :     return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7301             :   }
    7302           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7303           0 :     return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7304             :   }
    7305             :   return 0;
    7306             : }
    7307             : 
    7308             : unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7309           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7310             :     return 0;
    7311           0 :   if ((Subtarget->hasAVX512())) {
    7312           0 :     return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7313             :   }
    7314             :   return 0;
    7315             : }
    7316             : 
    7317           0 : unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7318           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7319             :     return 0;
    7320           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7321           0 :     return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7322             :   }
    7323             :   return 0;
    7324             : }
    7325             : 
    7326           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7327           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7328             :     return 0;
    7329           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7330           0 :     return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7331             :   }
    7332             :   return 0;
    7333             : }
    7334             : 
    7335             : unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7336           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7337             :     return 0;
    7338           0 :   if ((Subtarget->hasAVX512())) {
    7339           0 :     return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7340             :   }
    7341             :   return 0;
    7342             : }
    7343             : 
    7344           0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7345           0 :   switch (VT.SimpleTy) {
    7346           0 :   case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7347           0 :   case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7348           0 :   case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7349           0 :   case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7350           0 :   case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7351           0 :   case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7352           0 :   case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7353           0 :   case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7354           0 :   case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7355           0 :   case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7356           0 :   case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7357           0 :   case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7358             :   default: return 0;
    7359             :   }
    7360             : }
    7361             : 
    7362             : // FastEmit functions for ISD::SRA.
    7363             : 
    7364           1 : unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7365           1 :   if (RetVT.SimpleTy != MVT::i8)
    7366             :     return 0;
    7367           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7368           1 :   return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7369             : }
    7370             : 
    7371           0 : unsigned fastEmit_ISD_SRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7372           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7373             :     return 0;
    7374           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7375           0 :     return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7376             :   }
    7377             :   return 0;
    7378             : }
    7379             : 
    7380           0 : unsigned fastEmit_ISD_SRA_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7381           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7382             :     return 0;
    7383           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7384           0 :     return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7385             :   }
    7386             :   return 0;
    7387             : }
    7388             : 
    7389             : unsigned fastEmit_ISD_SRA_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7390           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7391             :     return 0;
    7392           0 :   if ((Subtarget->hasBWI())) {
    7393           0 :     return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7394             :   }
    7395             :   return 0;
    7396             : }
    7397             : 
    7398           0 : unsigned fastEmit_ISD_SRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7399           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7400             :     return 0;
    7401           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7402           0 :     return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7403             :   }
    7404           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7405           0 :     return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7406             :   }
    7407             :   return 0;
    7408             : }
    7409             : 
    7410           0 : unsigned fastEmit_ISD_SRA_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7411           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7412             :     return 0;
    7413           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7414           0 :     return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7415             :   }
    7416           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7417           0 :     return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7418             :   }
    7419             :   return 0;
    7420             : }
    7421             : 
    7422             : unsigned fastEmit_ISD_SRA_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7423           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7424             :     return 0;
    7425           0 :   if ((Subtarget->hasAVX512())) {
    7426           0 :     return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7427             :   }
    7428             :   return 0;
    7429             : }
    7430             : 
    7431           0 : unsigned fastEmit_ISD_SRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7432           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7433             :     return 0;
    7434           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7435           0 :     return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7436             :   }
    7437             :   return 0;
    7438             : }
    7439             : 
    7440           0 : unsigned fastEmit_ISD_SRA_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7441           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7442             :     return 0;
    7443           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7444           0 :     return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7445             :   }
    7446             :   return 0;
    7447             : }
    7448             : 
    7449             : unsigned fastEmit_ISD_SRA_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7450           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7451             :     return 0;
    7452           0 :   if ((Subtarget->hasAVX512())) {
    7453           0 :     return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7454             :   }
    7455             :   return 0;
    7456             : }
    7457             : 
    7458           5 : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7459           5 :   switch (VT.SimpleTy) {
    7460           1 :   case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7461           0 :   case MVT::v8i16: return fastEmit_ISD_SRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7462           0 :   case MVT::v16i16: return fastEmit_ISD_SRA_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7463           0 :   case MVT::v32i16: return fastEmit_ISD_SRA_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7464           0 :   case MVT::v4i32: return fastEmit_ISD_SRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7465           0 :   case MVT::v8i32: return fastEmit_ISD_SRA_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7466           0 :   case MVT::v16i32: return fastEmit_ISD_SRA_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7467           0 :   case MVT::v2i64: return fastEmit_ISD_SRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7468           0 :   case MVT::v4i64: return fastEmit_ISD_SRA_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7469           0 :   case MVT::v8i64: return fastEmit_ISD_SRA_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7470             :   default: return 0;
    7471             :   }
    7472             : }
    7473             : 
    7474             : // FastEmit functions for ISD::SRL.
    7475             : 
    7476           1 : unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7477           1 :   if (RetVT.SimpleTy != MVT::i8)
    7478             :     return 0;
    7479           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7480           1 :   return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7481             : }
    7482             : 
    7483           0 : unsigned fastEmit_ISD_SRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7484           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7485             :     return 0;
    7486           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7487           0 :     return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7488             :   }
    7489             :   return 0;
    7490             : }
    7491             : 
    7492           0 : unsigned fastEmit_ISD_SRL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7493           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7494             :     return 0;
    7495           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7496           0 :     return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7497             :   }
    7498             :   return 0;
    7499             : }
    7500             : 
    7501             : unsigned fastEmit_ISD_SRL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7502           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7503             :     return 0;
    7504           0 :   if ((Subtarget->hasBWI())) {
    7505           0 :     return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7506             :   }
    7507             :   return 0;
    7508             : }
    7509             : 
    7510           0 : unsigned fastEmit_ISD_SRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7511           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7512             :     return 0;
    7513           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7514           0 :     return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7515             :   }
    7516           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7517           0 :     return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7518             :   }
    7519             :   return 0;
    7520             : }
    7521             : 
    7522           0 : unsigned fastEmit_ISD_SRL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7523           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7524             :     return 0;
    7525           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7526           0 :     return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7527             :   }
    7528           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7529           0 :     return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7530             :   }
    7531             :   return 0;
    7532             : }
    7533             : 
    7534             : unsigned fastEmit_ISD_SRL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7535           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7536             :     return 0;
    7537           0 :   if ((Subtarget->hasAVX512())) {
    7538           0 :     return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7539             :   }
    7540             :   return 0;
    7541             : }
    7542             : 
    7543           0 : unsigned fastEmit_ISD_SRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7544           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7545             :     return 0;
    7546           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7547           0 :     return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7548             :   }
    7549           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7550           0 :     return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7551             :   }
    7552             :   return 0;
    7553             : }
    7554             : 
    7555           0 : unsigned fastEmit_ISD_SRL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7556           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7557             :     return 0;
    7558           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7559           0 :     return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7560             :   }
    7561           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7562           0 :     return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7563             :   }
    7564             :   return 0;
    7565             : }
    7566             : 
    7567             : unsigned fastEmit_ISD_SRL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7568           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7569             :     return 0;
    7570           0 :   if ((Subtarget->hasAVX512())) {
    7571           0 :     return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7572             :   }
    7573             :   return 0;
    7574             : }
    7575             : 
    7576           4 : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7577           4 :   switch (VT.SimpleTy) {
    7578           1 :   case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7579           0 :   case MVT::v8i16: return fastEmit_ISD_SRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7580           0 :   case MVT::v16i16: return fastEmit_ISD_SRL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7581           0 :   case MVT::v32i16: return fastEmit_ISD_SRL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7582           0 :   case MVT::v4i32: return fastEmit_ISD_SRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7583           0 :   case MVT::v8i32: return fastEmit_ISD_SRL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7584           0 :   case MVT::v16i32: return fastEmit_ISD_SRL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7585           0 :   case MVT::v2i64: return fastEmit_ISD_SRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7586           0 :   case MVT::v4i64: return fastEmit_ISD_SRL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7587           0 :   case MVT::v8i64: return fastEmit_ISD_SRL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7588             :   default: return 0;
    7589             :   }
    7590             : }
    7591             : 
    7592             : // FastEmit functions for ISD::SUB.
    7593             : 
    7594             : unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7595           1 :   if (RetVT.SimpleTy != MVT::i8)
    7596             :     return 0;
    7597           1 :   return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7598             : }
    7599             : 
    7600             : unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7601           0 :   if (RetVT.SimpleTy != MVT::i16)
    7602             :     return 0;
    7603           0 :   return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7604             : }
    7605             : 
    7606             : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7607          97 :   if (RetVT.SimpleTy != MVT::i32)
    7608             :     return 0;
    7609          97 :   return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7610             : }
    7611             : 
    7612             : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7613        1090 :   if (RetVT.SimpleTy != MVT::i64)
    7614             :     return 0;
    7615        1090 :   return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7616             : }
    7617             : 
    7618           6 : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7619           6 :   if (RetVT.SimpleTy != MVT::v16i8)
    7620             :     return 0;
    7621           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7622           2 :     return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7623             :   }
    7624           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7625           2 :     return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7626             :   }
    7627           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7628           2 :     return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7629             :   }
    7630             :   return 0;
    7631             : }
    7632             : 
    7633           2 : unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7634           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    7635             :     return 0;
    7636           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7637           0 :     return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7638             :   }
    7639           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7640           2 :     return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7641             :   }
    7642             :   return 0;
    7643             : }
    7644             : 
    7645             : unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7646           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7647             :     return 0;
    7648           0 :   if ((Subtarget->hasBWI())) {
    7649           0 :     return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7650             :   }
    7651             :   return 0;
    7652             : }
    7653             : 
    7654           6 : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7655           6 :   if (RetVT.SimpleTy != MVT::v8i16)
    7656             :     return 0;
    7657           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7658           2 :     return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7659             :   }
    7660           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7661           2 :     return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7662             :   }
    7663           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7664           2 :     return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7665             :   }
    7666             :   return 0;
    7667             : }
    7668             : 
    7669           2 : unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7670           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    7671             :     return 0;
    7672           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7673           0 :     return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7674             :   }
    7675           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7676           2 :     return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7677             :   }
    7678             :   return 0;
    7679             : }
    7680             : 
    7681             : unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7682           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7683             :     return 0;
    7684           0 :   if ((Subtarget->hasBWI())) {
    7685           0 :     return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7686             :   }
    7687             :   return 0;
    7688             : }
    7689             : 
    7690           6 : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7691           6 :   if (RetVT.SimpleTy != MVT::v4i32)
    7692             :     return 0;
    7693          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7694           2 :     return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7695             :   }
    7696           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7697           2 :     return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7698             :   }
    7699           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7700           2 :     return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7701             :   }
    7702             :   return 0;
    7703             : }
    7704             : 
    7705           2 : unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7706           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    7707             :     return 0;
    7708           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7709           0 :     return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7710             :   }
    7711           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7712           2 :     return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7713             :   }
    7714             :   return 0;
    7715             : }
    7716             : 
    7717             : unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7718           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7719             :     return 0;
    7720           0 :   if ((Subtarget->hasAVX512())) {
    7721           0 :     return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7722             :   }
    7723             :   return 0;
    7724             : }
    7725             : 
    7726           6 : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7727           6 :   if (RetVT.SimpleTy != MVT::v2i64)
    7728             :     return 0;
    7729          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7730           2 :     return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7731             :   }
    7732           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7733           2 :     return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7734             :   }
    7735           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7736           2 :     return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7737             :   }
    7738             :   return 0;
    7739             : }
    7740             : 
    7741           2 : unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7742           2 :   if (RetVT.SimpleTy != MVT::v4i64)
    7743             :     return 0;
    7744           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7745           0 :     return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7746             :   }
    7747           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7748           2 :     return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7749             :   }
    7750             :   return 0;
    7751             : }
    7752             : 
    7753             : unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7754           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7755             :     return 0;
    7756           0 :   if ((Subtarget->hasAVX512())) {
    7757           0 :     return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7758             :   }
    7759             :   return 0;
    7760             : }
    7761             : 
    7762        1220 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7763        1220 :   switch (VT.SimpleTy) {
    7764           1 :   case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7765           0 :   case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7766          97 :   case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7767        1090 :   case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7768           6 :   case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7769           2 :   case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7770           0 :   case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7771           6 :   case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7772           2 :   case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7773           0 :   case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7774           6 :   case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7775           2 :   case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7776           0 :   case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7777           6 :   case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7778           2 :   case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7779           0 :   case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7780             :   default: return 0;
    7781             :   }
    7782             : }
    7783             : 
    7784             : // FastEmit functions for ISD::UMAX.
    7785             : 
    7786           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7787           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7788             :     return 0;
    7789           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7790           0 :     return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7791             :   }
    7792           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7793           0 :     return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7794             :   }
    7795           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7796           0 :     return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7797             :   }
    7798             :   return 0;
    7799             : }
    7800             : 
    7801           0 : unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7802           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7803             :     return 0;
    7804           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7805           0 :     return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7806             :   }
    7807           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7808           0 :     return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7809             :   }
    7810             :   return 0;
    7811             : }
    7812             : 
    7813             : unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7814           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7815             :     return 0;
    7816           0 :   if ((Subtarget->hasBWI())) {
    7817           0 :     return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7818             :   }
    7819             :   return 0;
    7820             : }
    7821             : 
    7822           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7823           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7824             :     return 0;
    7825           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7826           0 :     return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7827             :   }
    7828           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7829           0 :     return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7830             :   }
    7831           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7832           0 :     return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7833             :   }
    7834             :   return 0;
    7835             : }
    7836             : 
    7837           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7838           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7839             :     return 0;
    7840           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7841           0 :     return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7842             :   }
    7843           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7844           0 :     return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7845             :   }
    7846             :   return 0;
    7847             : }
    7848             : 
    7849             : unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7850           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7851             :     return 0;
    7852           0 :   if ((Subtarget->hasBWI())) {
    7853           0 :     return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7854             :   }
    7855             :   return 0;
    7856             : }
    7857             : 
    7858           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7859           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7860             :     return 0;
    7861           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7862           0 :     return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7863             :   }
    7864           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7865           0 :     return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7866             :   }
    7867           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7868           0 :     return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7869             :   }
    7870             :   return 0;
    7871             : }
    7872             : 
    7873           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7874           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7875             :     return 0;
    7876           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7877           0 :     return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7878             :   }
    7879           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7880           0 :     return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7881             :   }
    7882             :   return 0;
    7883             : }
    7884             : 
    7885             : unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7886           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7887             :     return 0;
    7888           0 :   if ((Subtarget->hasAVX512())) {
    7889           0 :     return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7890             :   }
    7891             :   return 0;
    7892             : }
    7893             : 
    7894           0 : unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7895           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7896             :     return 0;
    7897           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7898           0 :     return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7899             :   }
    7900             :   return 0;
    7901             : }
    7902             : 
    7903           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7904           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7905             :     return 0;
    7906           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7907           0 :     return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7908             :   }
    7909             :   return 0;
    7910             : }
    7911             : 
    7912             : unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7913           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7914             :     return 0;
    7915           0 :   if ((Subtarget->hasAVX512())) {
    7916           0 :     return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7917             :   }
    7918             :   return 0;
    7919             : }
    7920             : 
    7921           0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7922           0 :   switch (VT.SimpleTy) {
    7923           0 :   case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7924           0 :   case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7925           0 :   case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7926           0 :   case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7927           0 :   case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7928           0 :   case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7929           0 :   case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7930           0 :   case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7931           0 :   case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7932           0 :   case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7933           0 :   case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7934           0 :   case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7935             :   default: return 0;
    7936             :   }
    7937             : }
    7938             : 
    7939             : // FastEmit functions for ISD::UMIN.
    7940             : 
    7941           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7942           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7943             :     return 0;
    7944           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7945           0 :     return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7946             :   }
    7947           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7948           0 :     return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7949             :   }
    7950           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7951           0 :     return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7952             :   }
    7953             :   return 0;
    7954             : }
    7955             : 
    7956           0 : unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7957           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7958             :     return 0;
    7959           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7960           0 :     return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7961             :   }
    7962           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7963           0 :     return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7964             :   }
    7965             :   return 0;
    7966             : }
    7967             : 
    7968             : unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7969           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7970             :     return 0;
    7971           0 :   if ((Subtarget->hasBWI())) {
    7972           0 :     return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7973             :   }
    7974             :   return 0;
    7975             : }
    7976             : 
    7977           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7978           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7979             :     return 0;
    7980           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7981           0 :     return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7982             :   }
    7983           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7984           0 :     return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7985             :   }
    7986           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7987           0 :     return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7988             :   }
    7989             :   return 0;
    7990             : }
    7991             : 
    7992           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7993           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7994             :     return 0;
    7995           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7996           0 :     return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7997             :   }
    7998           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7999           0 :     return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8000             :   }
    8001             :   return 0;
    8002             : }
    8003             : 
    8004             : unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8005           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8006             :     return 0;
    8007           0 :   if ((Subtarget->hasBWI())) {
    8008           0 :     return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8009             :   }
    8010             :   return 0;
    8011             : }
    8012             : 
    8013           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8014           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    8015             :     return 0;
    8016           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8017           0 :     return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8018             :   }
    8019           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    8020           0 :     return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8021             :   }
    8022           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8023           0 :     return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8024             :   }
    8025             :   return 0;
    8026             : }
    8027             : 
    8028           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8029           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    8030             :     return 0;
    8031           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8032           0 :     return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8033             :   }
    8034           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8035           0 :     return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8036             :   }
    8037             :   return 0;
    8038             : }
    8039             : 
    8040             : unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8041           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    8042             :     return 0;
    8043           0 :   if ((Subtarget->hasAVX512())) {
    8044           0 :     return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8045             :   }
    8046             :   return 0;
    8047             : }
    8048             : 
    8049           0 : unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8050           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    8051             :     return 0;
    8052           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8053           0 :     return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8054             :   }
    8055             :   return 0;
    8056             : }
    8057             : 
    8058           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8059           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    8060             :     return 0;
    8061           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8062           0 :     return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8063             :   }
    8064             :   return 0;
    8065             : }
    8066             : 
    8067             : unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8068           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8069             :     return 0;
    8070           0 :   if ((Subtarget->hasAVX512())) {
    8071           0 :     return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8072             :   }
    8073             :   return 0;
    8074             : }
    8075             : 
    8076           0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8077           0 :   switch (VT.SimpleTy) {
    8078           0 :   case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8079           0 :   case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8080           0 :   case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8081           0 :   case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8082           0 :   case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8083           0 :   case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8084           0 :   case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8085           0 :   case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8086           0 :   case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8087           0 :   case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8088           0 :   case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8089           0 :   case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8090             :   default: return 0;
    8091             :   }
    8092             : }
    8093             : 
    8094             : // FastEmit functions for ISD::XOR.
    8095             : 
    8096             : unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8097           0 :   if (RetVT.SimpleTy != MVT::i8)
    8098             :     return 0;
    8099           0 :   return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8100             : }
    8101             : 
    8102             : unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8103           0 :   if (RetVT.SimpleTy != MVT::i16)
    8104             :     return 0;
    8105           0 :   return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8106             : }
    8107             : 
    8108             : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8109          17 :   if (RetVT.SimpleTy != MVT::i32)
    8110             :     return 0;
    8111          17 :   return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8112             : }
    8113             : 
    8114             : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8115           7 :   if (RetVT.SimpleTy != MVT::i64)
    8116             :     return 0;
    8117           7 :   return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8118             : }
    8119             : 
    8120             : unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8121           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    8122             :     return 0;
    8123           0 :   if ((Subtarget->hasDQI())) {
    8124           0 :     return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8125             :   }
    8126             :   return 0;
    8127             : }
    8128             : 
    8129             : unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8130           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    8131             :     return 0;
    8132           0 :   if ((Subtarget->hasAVX512())) {
    8133           0 :     return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8134             :   }
    8135             :   return 0;
    8136             : }
    8137             : 
    8138             : unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8139           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    8140             :     return 0;
    8141           0 :   if ((Subtarget->hasBWI())) {
    8142           0 :     return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8143             :   }
    8144             :   return 0;
    8145             : }
    8146             : 
    8147             : unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8148           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    8149             :     return 0;
    8150           0 :   if ((Subtarget->hasBWI())) {
    8151           0 :     return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8152             :   }
    8153             :   return 0;
    8154             : }
    8155             : 
    8156           6 : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8157           6 :   if (RetVT.SimpleTy != MVT::v2i64)
    8158             :     return 0;
    8159          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8160           2 :     return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8161             :   }
    8162           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8163           2 :     return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8164             :   }
    8165           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8166           2 :     return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8167             :   }
    8168             :   return 0;
    8169             : }
    8170             : 
    8171           4 : unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8172           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    8173             :     return 0;
    8174           8 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    8175           2 :     return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8176             :   }
    8177           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8178           0 :     return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8179             :   }
    8180           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8181           2 :     return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8182             :   }
    8183             :   return 0;
    8184             : }
    8185             : 
    8186             : unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8187           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8188             :     return 0;
    8189           0 :   if ((Subtarget->hasAVX512())) {
    8190           0 :     return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8191             :   }
    8192             :   return 0;
    8193             : }
    8194             : 
    8195          46 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8196          46 :   switch (VT.SimpleTy) {
    8197           0 :   case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8198           0 :   case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8199          17 :   case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8200           7 :   case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8201           0 :   case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8202           0 :   case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8203           0 :   case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8204           0 :   case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8205           6 :   case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8206           4 :   case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8207           0 :   case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8208             :   default: return 0;
    8209             :   }
    8210             : }
    8211             : 
    8212             : // FastEmit functions for X86ISD::ADDS.
    8213             : 
    8214           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8215           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8216             :     return 0;
    8217           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8218           0 :     return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8219             :   }
    8220           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8221           0 :     return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8222             :   }
    8223           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8224           0 :     return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8225             :   }
    8226             :   return 0;
    8227             : }
    8228             : 
    8229           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8230           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8231             :     return 0;
    8232           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8233           0 :     return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8234             :   }
    8235           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8236           0 :     return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8237             :   }
    8238             :   return 0;
    8239             : }
    8240             : 
    8241             : unsigned fastEmit_X86ISD_ADDS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8242           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8243             :     return 0;
    8244           0 :   if ((Subtarget->hasBWI())) {
    8245           0 :     return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8246             :   }
    8247             :   return 0;
    8248             : }
    8249             : 
    8250           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8251           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8252             :     return 0;
    8253           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8254           0 :     return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8255             :   }
    8256           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8257           0 :     return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8258             :   }
    8259           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8260           0 :     return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8261             :   }
    8262             :   return 0;
    8263             : }
    8264             : 
    8265           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8266           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8267             :     return 0;
    8268           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8269           0 :     return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8270             :   }
    8271           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8272           0 :     return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8273             :   }
    8274             :   return 0;
    8275             : }
    8276             : 
    8277             : unsigned fastEmit_X86ISD_ADDS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8278           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8279             :     return 0;
    8280           0 :   if ((Subtarget->hasBWI())) {
    8281           0 :     return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8282             :   }
    8283             :   return 0;
    8284             : }
    8285             : 
    8286           0 : unsigned fastEmit_X86ISD_ADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8287           0 :   switch (VT.SimpleTy) {
    8288           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8289           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8290           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8291           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8292           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8293           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8294             :   default: return 0;
    8295             :   }
    8296             : }
    8297             : 
    8298             : // FastEmit functions for X86ISD::ADDSUB.
    8299             : 
    8300           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8301           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8302             :     return 0;
    8303           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8304           0 :     return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8305             :   }
    8306           0 :   if ((Subtarget->hasAVX())) {
    8307           0 :     return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8308             :   }
    8309             :   return 0;
    8310             : }
    8311             : 
    8312             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8313           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8314             :     return 0;
    8315           0 :   if ((Subtarget->hasAVX())) {
    8316           0 :     return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8317             :   }
    8318             :   return 0;
    8319             : }
    8320             : 
    8321           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8322           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8323             :     return 0;
    8324           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8325           0 :     return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8326             :   }
    8327           0 :   if ((Subtarget->hasAVX())) {
    8328           0 :     return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8329             :   }
    8330             :   return 0;
    8331             : }
    8332             : 
    8333             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8334           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8335             :     return 0;
    8336           0 :   if ((Subtarget->hasAVX())) {
    8337           0 :     return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8338             :   }
    8339             :   return 0;
    8340             : }
    8341             : 
    8342           0 : unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8343           0 :   switch (VT.SimpleTy) {
    8344           0 :   case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8345           0 :   case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8346           0 :   case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8347           0 :   case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8348             :   default: return 0;
    8349             :   }
    8350             : }
    8351             : 
    8352             : // FastEmit functions for X86ISD::ADDUS.
    8353             : 
    8354           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8355           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8356             :     return 0;
    8357           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8358           0 :     return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8359             :   }
    8360           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8361           0 :     return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8362             :   }
    8363           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8364           0 :     return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8365             :   }
    8366             :   return 0;
    8367             : }
    8368             : 
    8369           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8370           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8371             :     return 0;
    8372           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8373           0 :     return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8374             :   }
    8375           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8376           0 :     return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8377             :   }
    8378             :   return 0;
    8379             : }
    8380             : 
    8381             : unsigned fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8382           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8383             :     return 0;
    8384           0 :   if ((Subtarget->hasBWI())) {
    8385           0 :     return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8386             :   }
    8387             :   return 0;
    8388             : }
    8389             : 
    8390           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8391           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8392             :     return 0;
    8393           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8394           0 :     return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8395             :   }
    8396           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8397           0 :     return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8398             :   }
    8399           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8400           0 :     return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8401             :   }
    8402             :   return 0;
    8403             : }
    8404             : 
    8405           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8406           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8407             :     return 0;
    8408           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8409           0 :     return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8410             :   }
    8411           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8412           0 :     return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8413             :   }
    8414             :   return 0;
    8415             : }
    8416             : 
    8417             : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8418           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8419             :     return 0;
    8420           0 :   if ((Subtarget->hasBWI())) {
    8421           0 :     return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8422             :   }
    8423             :   return 0;
    8424             : }
    8425             : 
    8426           0 : unsigned fastEmit_X86ISD_ADDUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8427           0 :   switch (VT.SimpleTy) {
    8428           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8429           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8430           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8431           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8432           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8433           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8434             :   default: return 0;
    8435             :   }
    8436             : }
    8437             : 
    8438             : // FastEmit functions for X86ISD::ANDNP.
    8439             : 
    8440           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8441           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    8442             :     return 0;
    8443           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8444           0 :     return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8445             :   }
    8446           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8447           0 :     return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8448             :   }
    8449           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8450           0 :     return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8451             :   }
    8452             :   return 0;
    8453             : }
    8454             : 
    8455           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8456           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    8457             :     return 0;
    8458           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    8459           0 :     return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8460             :   }
    8461           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8462           0 :     return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8463             :   }
    8464           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8465           0 :     return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8466             :   }
    8467             :   return 0;
    8468             : }
    8469             : 
    8470             : unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8471           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8472             :     return 0;
    8473           0 :   if ((Subtarget->hasAVX512())) {
    8474           0 :     return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8475             :   }
    8476             :   return 0;
    8477             : }
    8478             : 
    8479           0 : unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8480           0 :   switch (VT.SimpleTy) {
    8481           0 :   case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8482           0 :   case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8483           0 :   case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8484             :   default: return 0;
    8485             :   }
    8486             : }
    8487             : 
    8488             : // FastEmit functions for X86ISD::AVG.
    8489             : 
    8490           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8491           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8492             :     return 0;
    8493           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8494           0 :     return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8495             :   }
    8496           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8497           0 :     return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8498             :   }
    8499           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8500           0 :     return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8501             :   }
    8502             :   return 0;
    8503             : }
    8504             : 
    8505           0 : unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8506           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8507             :     return 0;
    8508           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8509           0 :     return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8510             :   }
    8511           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8512           0 :     return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8513             :   }
    8514             :   return 0;
    8515             : }
    8516             : 
    8517             : unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8518           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8519             :     return 0;
    8520           0 :   if ((Subtarget->hasBWI())) {
    8521           0 :     return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8522             :   }
    8523             :   return 0;
    8524             : }
    8525             : 
    8526           0 : unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8527           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8528             :     return 0;
    8529           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8530           0 :     return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8531             :   }
    8532           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8533           0 :     return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8534             :   }
    8535           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8536           0 :     return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8537             :   }
    8538             :   return 0;
    8539             : }
    8540             : 
    8541           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8542           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8543             :     return 0;
    8544           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8545           0 :     return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8546             :   }
    8547           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8548           0 :     return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8549             :   }
    8550             :   return 0;
    8551             : }
    8552             : 
    8553             : unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8554           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8555             :     return 0;
    8556           0 :   if ((Subtarget->hasBWI())) {
    8557           0 :     return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8558             :   }
    8559             :   return 0;
    8560             : }
    8561             : 
    8562           0 : unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8563           0 :   switch (VT.SimpleTy) {
    8564           0 :   case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8565           0 :   case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8566           0 :   case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8567           0 :   case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8568           0 :   case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8569           0 :   case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8570             :   default: return 0;
    8571             :   }
    8572             : }
    8573             : 
    8574             : // FastEmit functions for X86ISD::BEXTR.
    8575             : 
    8576             : unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8577           0 :   if (RetVT.SimpleTy != MVT::i32)
    8578             :     return 0;
    8579           0 :   if ((Subtarget->hasBMI())) {
    8580           0 :     return fastEmitInst_rr(X86::BEXTR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8581             :   }
    8582             :   return 0;
    8583             : }
    8584             : 
    8585             : unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8586           0 :   if (RetVT.SimpleTy != MVT::i64)
    8587             :     return 0;
    8588           0 :   if ((Subtarget->hasBMI())) {
    8589           0 :     return fastEmitInst_rr(X86::BEXTR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8590             :   }
    8591             :   return 0;
    8592             : }
    8593             : 
    8594           0 : unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8595           0 :   switch (VT.SimpleTy) {
    8596           0 :   case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8597           0 :   case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8598             :   default: return 0;
    8599             :   }
    8600             : }
    8601             : 
    8602             : // FastEmit functions for X86ISD::BT.
    8603             : 
    8604             : unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8605           0 :   if (RetVT.SimpleTy != MVT::i32)
    8606             :     return 0;
    8607           0 :   return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8608             : }
    8609             : 
    8610             : unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8611           0 :   if (RetVT.SimpleTy != MVT::i32)
    8612             :     return 0;
    8613           0 :   return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8614             : }
    8615             : 
    8616             : unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8617           0 :   if (RetVT.SimpleTy != MVT::i32)
    8618             :     return 0;
    8619           0 :   return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8620             : }
    8621             : 
    8622           0 : unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8623           0 :   switch (VT.SimpleTy) {
    8624           0 :   case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8625           0 :   case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8626           0 :   case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8627             :   default: return 0;
    8628             :   }
    8629             : }
    8630             : 
    8631             : // FastEmit functions for X86ISD::CMP.
    8632             : 
    8633             : unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8634           0 :   if (RetVT.SimpleTy != MVT::i32)
    8635             :     return 0;
    8636           0 :   return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8637             : }
    8638             : 
    8639             : unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8640           0 :   if (RetVT.SimpleTy != MVT::i32)
    8641             :     return 0;
    8642           0 :   return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8643             : }
    8644             : 
    8645             : unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8646           0 :   if (RetVT.SimpleTy != MVT::i32)
    8647             :     return 0;
    8648           0 :   return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8649             : }
    8650             : 
    8651             : unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8652           0 :   if (RetVT.SimpleTy != MVT::i32)
    8653             :     return 0;
    8654           0 :   return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8655             : }
    8656             : 
    8657           0 : unsigned fastEmit_X86ISD_CMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8658           0 :   if (RetVT.SimpleTy != MVT::i32)
    8659             :     return 0;
    8660           0 :   if ((Subtarget->hasAVX512())) {
    8661           0 :     return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8662             :   }
    8663           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8664           0 :     return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8665             :   }
    8666           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8667           0 :     return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8668             :   }
    8669           0 :   if ((!Subtarget->hasSSE1())) {
    8670           0 :     return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8671             :   }
    8672             :   return 0;
    8673             : }
    8674             : 
    8675           0 : unsigned fastEmit_X86ISD_CMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8676           0 :   if (RetVT.SimpleTy != MVT::i32)
    8677             :     return 0;
    8678           0 :   if ((Subtarget->hasAVX512())) {
    8679           0 :     return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8680             :   }
    8681           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8682           0 :     return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8683             :   }
    8684           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8685           0 :     return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8686             :   }
    8687           0 :   if ((!Subtarget->hasSSE2())) {
    8688           0 :     return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8689             :   }
    8690             :   return 0;
    8691             : }
    8692             : 
    8693             : unsigned fastEmit_X86ISD_CMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8694           0 :   if (RetVT.SimpleTy != MVT::i32)
    8695             :     return 0;
    8696           0 :   return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8697             : }
    8698             : 
    8699           0 : unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8700           0 :   switch (VT.SimpleTy) {
    8701           0 :   case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8702           0 :   case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8703           0 :   case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8704           0 :   case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8705           0 :   case MVT::f32: return fastEmit_X86ISD_CMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8706           0 :   case MVT::f64: return fastEmit_X86ISD_CMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8707           0 :   case MVT::f80: return fastEmit_X86ISD_CMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8708             :   default: return 0;
    8709             :   }
    8710             : }
    8711             : 
    8712             : // FastEmit functions for X86ISD::COMI.
    8713             : 
    8714           0 : unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8715           0 :   if (RetVT.SimpleTy != MVT::i32)
    8716             :     return 0;
    8717           0 :   if ((Subtarget->hasAVX512())) {
    8718           0 :     return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8719             :   }
    8720           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8721           0 :     return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8722             :   }
    8723           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8724           0 :     return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8725             :   }
    8726             :   return 0;
    8727             : }
    8728             : 
    8729           0 : unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8730           0 :   if (RetVT.SimpleTy != MVT::i32)
    8731             :     return 0;
    8732           0 :   if ((Subtarget->hasAVX512())) {
    8733           0 :     return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8734             :   }
    8735           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8736           0 :     return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8737             :   }
    8738           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8739           0 :     return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8740             :   }
    8741             :   return 0;
    8742             : }
    8743             : 
    8744           0 : unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8745           0 :   switch (VT.SimpleTy) {
    8746           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8747           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8748             :   default: return 0;
    8749             :   }
    8750             : }
    8751             : 
    8752             : // FastEmit functions for X86ISD::FAND.
    8753             : 
    8754             : unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8755           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8756             :     return 0;
    8757           0 :   return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8758             : }
    8759             : 
    8760             : unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8761           0 :   switch (VT.SimpleTy) {
    8762           0 :   case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8763             :   default: return 0;
    8764             :   }
    8765             : }
    8766             : 
    8767             : // FastEmit functions for X86ISD::FANDN.
    8768             : 
    8769             : unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8770           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8771             :     return 0;
    8772           0 :   return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8773             : }
    8774             : 
    8775             : unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8776           0 :   switch (VT.SimpleTy) {
    8777           0 :   case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8778             :   default: return 0;
    8779             :   }
    8780             : }
    8781             : 
    8782             : // FastEmit functions for X86ISD::FHADD.
    8783             : 
    8784           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8785           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8786             :     return 0;
    8787           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8788           0 :     return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8789             :   }
    8790           0 :   if ((Subtarget->hasAVX())) {
    8791           0 :     return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8792             :   }
    8793             :   return 0;
    8794             : }
    8795             : 
    8796             : unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8797           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8798             :     return 0;
    8799           0 :   if ((Subtarget->hasAVX())) {
    8800           0 :     return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8801             :   }
    8802             :   return 0;
    8803             : }
    8804             : 
    8805           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8806           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8807             :     return 0;
    8808           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8809           0 :     return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8810             :   }
    8811           0 :   if ((Subtarget->hasAVX())) {
    8812           0 :     return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8813             :   }
    8814             :   return 0;
    8815             : }
    8816             : 
    8817             : unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8818           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8819             :     return 0;
    8820           0 :   if ((Subtarget->hasAVX())) {
    8821           0 :     return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8822             :   }
    8823             :   return 0;
    8824             : }
    8825             : 
    8826           0 : unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8827           0 :   switch (VT.SimpleTy) {
    8828           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8829           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8830           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8831           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8832             :   default: return 0;
    8833             :   }
    8834             : }
    8835             : 
    8836             : // FastEmit functions for X86ISD::FHSUB.
    8837             : 
    8838           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8839           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8840             :     return 0;
    8841           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8842           0 :     return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8843             :   }
    8844           0 :   if ((Subtarget->hasAVX())) {
    8845           0 :     return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8846             :   }
    8847             :   return 0;
    8848             : }
    8849             : 
    8850             : unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8851           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8852             :     return 0;
    8853           0 :   if ((Subtarget->hasAVX())) {
    8854           0 :     return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8855             :   }
    8856             :   return 0;
    8857             : }
    8858             : 
    8859           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8860           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8861             :     return 0;
    8862           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8863           0 :     return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8864             :   }
    8865           0 :   if ((Subtarget->hasAVX())) {
    8866           0 :     return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8867             :   }
    8868             :   return 0;
    8869             : }
    8870             : 
    8871             : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8872           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8873             :     return 0;
    8874           0 :   if ((Subtarget->hasAVX())) {
    8875           0 :     return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8876             :   }
    8877             :   return 0;
    8878             : }
    8879             : 
    8880           0 : unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8881           0 :   switch (VT.SimpleTy) {
    8882           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8883           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8884           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8885           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8886             :   default: return 0;
    8887             :   }
    8888             : }
    8889             : 
    8890             : // FastEmit functions for X86ISD::FMAX.
    8891             : 
    8892           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8893           0 :   if (RetVT.SimpleTy != MVT::f32)
    8894             :     return 0;
    8895           0 :   if ((Subtarget->hasAVX512())) {
    8896           0 :     return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8897             :   }
    8898           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8899           0 :     return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8900             :   }
    8901           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8902           0 :     return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8903             :   }
    8904             :   return 0;
    8905             : }
    8906             : 
    8907           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8908           0 :   if (RetVT.SimpleTy != MVT::f64)
    8909             :     return 0;
    8910           0 :   if ((Subtarget->hasAVX512())) {
    8911           0 :     return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8912             :   }
    8913           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8914           0 :     return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8915             :   }
    8916           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8917           0 :     return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8918             :   }
    8919             :   return 0;
    8920             : }
    8921             : 
    8922           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8923           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8924             :     return 0;
    8925           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8926           0 :     return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8927             :   }
    8928           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8929           0 :     return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8930             :   }
    8931           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8932           0 :     return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8933             :   }
    8934             :   return 0;
    8935             : }
    8936             : 
    8937           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8938           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8939             :     return 0;
    8940           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8941           0 :     return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8942             :   }
    8943           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8944           0 :     return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8945             :   }
    8946             :   return 0;
    8947             : }
    8948             : 
    8949             : unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8950           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8951             :     return 0;
    8952           0 :   if ((Subtarget->hasAVX512())) {
    8953           0 :     return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8954             :   }
    8955             :   return 0;
    8956             : }
    8957             : 
    8958           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8959           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8960             :     return 0;
    8961           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8962           0 :     return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8963             :   }
    8964           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8965           0 :     return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8966             :   }
    8967           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8968           0 :     return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8969             :   }
    8970             :   return 0;
    8971             : }
    8972             : 
    8973           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8974           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8975             :     return 0;
    8976           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8977           0 :     return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8978             :   }
    8979           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8980           0 :     return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8981             :   }
    8982             :   return 0;
    8983             : }
    8984             : 
    8985             : unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8986           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8987             :     return 0;
    8988           0 :   if ((Subtarget->hasAVX512())) {
    8989           0 :     return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8990             :   }
    8991             :   return 0;
    8992             : }
    8993             : 
    8994           0 : unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8995           0 :   switch (VT.SimpleTy) {
    8996           0 :   case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8997           0 :   case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8998           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8999           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9000           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9001           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9002           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9003           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9004             :   default: return 0;
    9005             :   }
    9006             : }
    9007             : 
    9008             : // FastEmit functions for X86ISD::FMAXC.
    9009             : 
    9010           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9011           0 :   if (RetVT.SimpleTy != MVT::f32)
    9012             :     return 0;
    9013           0 :   if ((Subtarget->hasAVX512())) {
    9014           0 :     return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9015             :   }
    9016           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9017           0 :     return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9018             :   }
    9019           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9020           0 :     return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9021             :   }
    9022             :   return 0;
    9023             : }
    9024             : 
    9025           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9026           0 :   if (RetVT.SimpleTy != MVT::f64)
    9027             :     return 0;
    9028           0 :   if ((Subtarget->hasAVX512())) {
    9029           0 :     return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9030             :   }
    9031           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9032           0 :     return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9033             :   }
    9034           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9035           0 :     return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9036             :   }
    9037             :   return 0;
    9038             : }
    9039             : 
    9040           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9041           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9042             :     return 0;
    9043           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9044           0 :     return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9045             :   }
    9046           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9047           0 :     return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9048             :   }
    9049           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9050           0 :     return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9051             :   }
    9052             :   return 0;
    9053             : }
    9054             : 
    9055           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9056           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9057             :     return 0;
    9058           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9059           0 :     return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9060             :   }
    9061           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9062           0 :     return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9063             :   }
    9064             :   return 0;
    9065             : }
    9066             : 
    9067             : unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9068           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9069             :     return 0;
    9070           0 :   if ((Subtarget->hasAVX512())) {
    9071           0 :     return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9072             :   }
    9073             :   return 0;
    9074             : }
    9075             : 
    9076           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9077           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9078             :     return 0;
    9079           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9080           0 :     return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9081             :   }
    9082           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9083           0 :     return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9084             :   }
    9085           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9086           0 :     return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9087             :   }
    9088             :   return 0;
    9089             : }
    9090             : 
    9091           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9092           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9093             :     return 0;
    9094           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9095           0 :     return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9096             :   }
    9097           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9098           0 :     return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9099             :   }
    9100             :   return 0;
    9101             : }
    9102             : 
    9103             : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9104           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9105             :     return 0;
    9106           0 :   if ((Subtarget->hasAVX512())) {
    9107           0 :     return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9108             :   }
    9109             :   return 0;
    9110             : }
    9111             : 
    9112           0 : unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9113           0 :   switch (VT.SimpleTy) {
    9114           0 :   case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9115           0 :   case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9116           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9117           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9118           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9119           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9120           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9121           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9122             :   default: return 0;
    9123             :   }
    9124             : }
    9125             : 
    9126             : // FastEmit functions for X86ISD::FMAXS.
    9127             : 
    9128           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9129           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9130             :     return 0;
    9131           0 :   if ((Subtarget->hasAVX512())) {
    9132           0 :     return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9133             :   }
    9134           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9135           0 :     return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9136             :   }
    9137           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9138           0 :     return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9139             :   }
    9140             :   return 0;
    9141             : }
    9142             : 
    9143           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9144           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9145             :     return 0;
    9146           0 :   if ((Subtarget->hasAVX512())) {
    9147           0 :     return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9148             :   }
    9149           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9150           0 :     return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9151             :   }
    9152           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9153           0 :     return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9154             :   }
    9155             :   return 0;
    9156             : }
    9157             : 
    9158           0 : unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9159           0 :   switch (VT.SimpleTy) {
    9160           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9161           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9162             :   default: return 0;
    9163             :   }
    9164             : }
    9165             : 
    9166             : // FastEmit functions for X86ISD::FMIN.
    9167             : 
    9168           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9169           0 :   if (RetVT.SimpleTy != MVT::f32)
    9170             :     return 0;
    9171           0 :   if ((Subtarget->hasAVX512())) {
    9172           0 :     return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9173             :   }
    9174           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9175           0 :     return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9176             :   }
    9177           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9178           0 :     return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9179             :   }
    9180             :   return 0;
    9181             : }
    9182             : 
    9183           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9184           0 :   if (RetVT.SimpleTy != MVT::f64)
    9185             :     return 0;
    9186           0 :   if ((Subtarget->hasAVX512())) {
    9187           0 :     return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9188             :   }
    9189           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9190           0 :     return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9191             :   }
    9192           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9193           0 :     return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9194             :   }
    9195             :   return 0;
    9196             : }
    9197             : 
    9198           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9199           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9200             :     return 0;
    9201           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9202           0 :     return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9203             :   }
    9204           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9205           0 :     return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9206             :   }
    9207           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9208           0 :     return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9209             :   }
    9210             :   return 0;
    9211             : }
    9212             : 
    9213           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9214           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9215             :     return 0;
    9216           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9217           0 :     return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9218             :   }
    9219           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9220           0 :     return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9221             :   }
    9222             :   return 0;
    9223             : }
    9224             : 
    9225             : unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9226           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9227             :     return 0;
    9228           0 :   if ((Subtarget->hasAVX512())) {
    9229           0 :     return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9230             :   }
    9231             :   return 0;
    9232             : }
    9233             : 
    9234           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9235           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9236             :     return 0;
    9237           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9238           0 :     return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9239             :   }
    9240           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9241           0 :     return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9242             :   }
    9243           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9244           0 :     return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9245             :   }
    9246             :   return 0;
    9247             : }
    9248             : 
    9249           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9250           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9251             :     return 0;
    9252           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9253           0 :     return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9254             :   }
    9255           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9256           0 :     return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9257             :   }
    9258             :   return 0;
    9259             : }
    9260             : 
    9261             : unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9262           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9263             :     return 0;
    9264           0 :   if ((Subtarget->hasAVX512())) {
    9265           0 :     return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9266             :   }
    9267             :   return 0;
    9268             : }
    9269             : 
    9270           0 : unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9271           0 :   switch (VT.SimpleTy) {
    9272           0 :   case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9273           0 :   case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9274           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9275           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9276           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9277           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9278           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9279           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9280             :   default: return 0;
    9281             :   }
    9282             : }
    9283             : 
    9284             : // FastEmit functions for X86ISD::FMINC.
    9285             : 
    9286           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9287           0 :   if (RetVT.SimpleTy != MVT::f32)
    9288             :     return 0;
    9289           0 :   if ((Subtarget->hasAVX512())) {
    9290           0 :     return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9291             :   }
    9292           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9293           0 :     return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9294             :   }
    9295           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9296           0 :     return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9297             :   }
    9298             :   return 0;
    9299             : }
    9300             : 
    9301           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9302           0 :   if (RetVT.SimpleTy != MVT::f64)
    9303             :     return 0;
    9304           0 :   if ((Subtarget->hasAVX512())) {
    9305           0 :     return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9306             :   }
    9307           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9308           0 :     return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9309             :   }
    9310           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9311           0 :     return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9312             :   }
    9313             :   return 0;
    9314             : }
    9315             : 
    9316           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9317           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9318             :     return 0;
    9319           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9320           0 :     return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9321             :   }
    9322           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9323           0 :     return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9324             :   }
    9325           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9326           0 :     return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9327             :   }
    9328             :   return 0;
    9329             : }
    9330             : 
    9331           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9332           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9333             :     return 0;
    9334           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9335           0 :     return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9336             :   }
    9337           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9338           0 :     return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9339             :   }
    9340             :   return 0;
    9341             : }
    9342             : 
    9343             : unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9344           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9345             :     return 0;
    9346           0 :   if ((Subtarget->hasAVX512())) {
    9347           0 :     return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9348             :   }
    9349             :   return 0;
    9350             : }
    9351             : 
    9352           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9353           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9354             :     return 0;
    9355           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9356           0 :     return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9357             :   }
    9358           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9359           0 :     return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9360             :   }
    9361           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9362           0 :     return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9363             :   }
    9364             :   return 0;
    9365             : }
    9366             : 
    9367           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9368           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9369             :     return 0;
    9370           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9371           0 :     return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9372             :   }
    9373           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9374           0 :     return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9375             :   }
    9376             :   return 0;
    9377             : }
    9378             : 
    9379             : unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9380           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9381             :     return 0;
    9382           0 :   if ((Subtarget->hasAVX512())) {
    9383           0 :     return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9384             :   }
    9385             :   return 0;
    9386             : }
    9387             : 
    9388           0 : unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9389           0 :   switch (VT.SimpleTy) {
    9390           0 :   case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9391           0 :   case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9392           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9393           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9394           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9395           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9396           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9397           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9398             :   default: return 0;
    9399             :   }
    9400             : }
    9401             : 
    9402             : // FastEmit functions for X86ISD::FMINS.
    9403             : 
    9404           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9405           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9406             :     return 0;
    9407           0 :   if ((Subtarget->hasAVX512())) {
    9408           0 :     return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9409             :   }
    9410           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9411           0 :     return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9412             :   }
    9413           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9414           0 :     return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9415             :   }
    9416             :   return 0;
    9417             : }
    9418             : 
    9419           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9420           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9421             :     return 0;
    9422           0 :   if ((Subtarget->hasAVX512())) {
    9423           0 :     return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9424             :   }
    9425           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9426           0 :     return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9427             :   }
    9428           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9429           0 :     return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9430             :   }
    9431             :   return 0;
    9432             : }
    9433             : 
    9434           0 : unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9435           0 :   switch (VT.SimpleTy) {
    9436           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9437           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9438             :   default: return 0;
    9439             :   }
    9440             : }
    9441             : 
    9442             : // FastEmit functions for X86ISD::FOR.
    9443             : 
    9444             : unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9445           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9446             :     return 0;
    9447           0 :   return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9448             : }
    9449             : 
    9450             : unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9451           0 :   switch (VT.SimpleTy) {
    9452           0 :   case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9453             :   default: return 0;
    9454             :   }
    9455             : }
    9456             : 
    9457             : // FastEmit functions for X86ISD::FXOR.
    9458             : 
    9459             : unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9460           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9461             :     return 0;
    9462           0 :   return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9463             : }
    9464             : 
    9465             : unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9466           0 :   switch (VT.SimpleTy) {
    9467           0 :   case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9468             :   default: return 0;
    9469             :   }
    9470             : }
    9471             : 
    9472             : // FastEmit functions for X86ISD::GF2P8MULB.
    9473             : 
    9474           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9475           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    9476             :     return 0;
    9477           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
    9478           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9479             :   }
    9480           0 :   if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9481           0 :     return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9482             :   }
    9483           0 :   if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9484           0 :     return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9485             :   }
    9486             :   return 0;
    9487             : }
    9488             : 
    9489           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9490           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    9491             :     return 0;
    9492           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
    9493           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9494             :   }
    9495           0 :   if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9496           0 :     return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9497             :   }
    9498             :   return 0;
    9499             : }
    9500             : 
    9501           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9502           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    9503             :     return 0;
    9504           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasBWI()) && (Subtarget->hasGFNI())) {
    9505           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9506             :   }
    9507             :   return 0;
    9508             : }
    9509             : 
    9510           0 : unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9511           0 :   switch (VT.SimpleTy) {
    9512           0 :   case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9513           0 :   case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9514           0 :   case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9515             :   default: return 0;
    9516             :   }
    9517             : }
    9518             : 
    9519             : // FastEmit functions for X86ISD::HADD.
    9520             : 
    9521           0 : unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9522           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9523             :     return 0;
    9524           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9525           0 :     return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9526             :   }
    9527           0 :   if ((Subtarget->hasAVX())) {
    9528           0 :     return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9529             :   }
    9530             :   return 0;
    9531             : }
    9532             : 
    9533             : unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9534           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9535             :     return 0;
    9536           0 :   if ((Subtarget->hasAVX2())) {
    9537           0 :     return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9538             :   }
    9539             :   return 0;
    9540             : }
    9541             : 
    9542           0 : unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9543           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9544             :     return 0;
    9545           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9546           0 :     return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9547             :   }
    9548           0 :   if ((Subtarget->hasAVX())) {
    9549           0 :     return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9550             :   }
    9551             :   return 0;
    9552             : }
    9553             : 
    9554             : unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9555           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    9556             :     return 0;
    9557           0 :   if ((Subtarget->hasAVX2())) {
    9558           0 :     return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9559             :   }
    9560             :   return 0;
    9561             : }
    9562             : 
    9563           0 : unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9564           0 :   switch (VT.SimpleTy) {
    9565           0 :   case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9566           0 :   case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9567           0 :   case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9568           0 :   case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9569             :   default: return 0;
    9570             :   }
    9571             : }
    9572             : 
    9573             : // FastEmit functions for X86ISD::HSUB.
    9574             : 
    9575           0 : unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9576           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9577             :     return 0;
    9578           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9579           0 :     return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);<