LCOV - code coverage report
Current view: top level - build-llvm/lib/Target/X86 - X86GenFastISel.inc (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 980 6996 14.0 %
Date: 2018-10-17 09:37:48 Functions: 136 892 15.2 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
       2             : |*                                                                            *|
       3             : |* "Fast" Instruction Selector for the X86 target                             *|
       4             : |*                                                                            *|
       5             : |* Automatically generated file, do not edit!                                 *|
       6             : |*                                                                            *|
       7             : \*===----------------------------------------------------------------------===*/
       8             : 
       9             : 
      10             : // FastEmit Immediate Predicate functions.
      11             : static bool Predicate_i16immSExt8(int64_t Imm) {
      12             :  return isInt<8>(Imm); 
      13             : }
      14             : static bool Predicate_i32immSExt8(int64_t Imm) {
      15             :  return isInt<8>(Imm); 
      16             : }
      17             : static bool Predicate_i64immSExt8(int64_t Imm) {
      18             :  return isInt<8>(Imm); 
      19             : }
      20             : static bool Predicate_i64immSExt32(int64_t Imm) {
      21             :  return isInt<32>(Imm); 
      22             : }
      23             : static bool Predicate_AndMask64(int64_t Imm) {
      24             : 
      25             :   return isMask_64(Imm) && !isUInt<32>(Imm);
      26             : 
      27             : }
      28             : static bool Predicate_BTRMask64(int64_t Imm) {
      29             : 
      30             :   return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
      31             : 
      32             : }
      33             : static bool Predicate_BTCBTSMask64(int64_t Imm) {
      34             : 
      35             :   return !isInt<32>(Imm) && isPowerOf2_64(Imm);
      36             : 
      37             : }
      38             : 
      39             : 
      40             : // FastEmit functions for ISD::ABS.
      41             : 
      42           0 : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      43           0 :   if (RetVT.SimpleTy != MVT::v16i8)
      44             :     return 0;
      45           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      46           0 :     return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      47             :   }
      48           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      49           0 :     return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      50             :   }
      51           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      52           0 :     return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      53             :   }
      54             :   return 0;
      55             : }
      56             : 
      57           0 : unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      58           0 :   if (RetVT.SimpleTy != MVT::v32i8)
      59             :     return 0;
      60           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      61           0 :     return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
      62             :   }
      63           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      64           0 :     return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
      65             :   }
      66             :   return 0;
      67             : }
      68             : 
      69             : unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      70           0 :   if (RetVT.SimpleTy != MVT::v64i8)
      71             :     return 0;
      72           0 :   if ((Subtarget->hasBWI())) {
      73           0 :     return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
      74             :   }
      75             :   return 0;
      76             : }
      77             : 
      78           0 : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      79           0 :   if (RetVT.SimpleTy != MVT::v8i16)
      80             :     return 0;
      81           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      82           0 :     return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      83             :   }
      84           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      85           0 :     return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      86             :   }
      87           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      88           0 :     return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      89             :   }
      90             :   return 0;
      91             : }
      92             : 
      93           0 : unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      94           0 :   if (RetVT.SimpleTy != MVT::v16i16)
      95             :     return 0;
      96           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      97           0 :     return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
      98             :   }
      99           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
     100           0 :     return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     101             :   }
     102             :   return 0;
     103             : }
     104             : 
     105             : unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     106           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     107             :     return 0;
     108           0 :   if ((Subtarget->hasBWI())) {
     109           0 :     return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     110             :   }
     111             :   return 0;
     112             : }
     113             : 
     114           0 : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     115           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     116             :     return 0;
     117           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     118           0 :     return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     119             :   }
     120           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
     121           0 :     return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     122             :   }
     123           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     124           0 :     return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     125             :   }
     126             :   return 0;
     127             : }
     128             : 
     129           0 : unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     130           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     131             :     return 0;
     132           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     133           0 :     return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     134             :   }
     135           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
     136           0 :     return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     137             :   }
     138             :   return 0;
     139             : }
     140             : 
     141             : unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     142           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     143             :     return 0;
     144           0 :   if ((Subtarget->hasAVX512())) {
     145           0 :     return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     146             :   }
     147             :   return 0;
     148             : }
     149             : 
     150           0 : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     151           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     152             :     return 0;
     153           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     154           0 :     return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     155             :   }
     156             :   return 0;
     157             : }
     158             : 
     159           0 : unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     160           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     161             :     return 0;
     162           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     163           0 :     return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     164             :   }
     165             :   return 0;
     166             : }
     167             : 
     168             : unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     169           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     170             :     return 0;
     171           0 :   if ((Subtarget->hasAVX512())) {
     172           0 :     return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     173             :   }
     174             :   return 0;
     175             : }
     176             : 
     177           0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     178           0 :   switch (VT.SimpleTy) {
     179           0 :   case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     180           0 :   case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     181           0 :   case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     182           0 :   case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     183           0 :   case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     184           0 :   case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     185           0 :   case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     186           0 :   case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     187           0 :   case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     188           0 :   case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     189           0 :   case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     190           0 :   case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     191             :   default: return 0;
     192             :   }
     193             : }
     194             : 
     195             : // FastEmit functions for ISD::ANY_EXTEND.
     196             : 
     197             : unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     198          29 :   if (RetVT.SimpleTy != MVT::i32)
     199             :     return 0;
     200          29 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
     201             : }
     202             : 
     203             : unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     204          32 :   switch (VT.SimpleTy) {
     205          29 :   case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
     206             :   default: return 0;
     207             :   }
     208             : }
     209             : 
     210             : // FastEmit functions for ISD::BITCAST.
     211             : 
     212          23 : unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     213          23 :   if (RetVT.SimpleTy != MVT::f32)
     214             :     return 0;
     215          22 :   if ((Subtarget->hasAVX512())) {
     216           0 :     return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
     217             :   }
     218          11 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     219          11 :     return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     220             :   }
     221           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     222           0 :     return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     223             :   }
     224             :   return 0;
     225             : }
     226             : 
     227         135 : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     228         270 :   if ((Subtarget->hasAVX512())) {
     229           0 :     return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
     230             :   }
     231         135 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     232         135 :     return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     233             :   }
     234           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     235           0 :     return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     236             :   }
     237             :   return 0;
     238             : }
     239             : 
     240             : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     241           0 :   if ((Subtarget->hasMMX())) {
     242           0 :     return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
     243             :   }
     244             :   return 0;
     245             : }
     246             : 
     247         138 : unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     248         138 : switch (RetVT.SimpleTy) {
     249         135 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
     250           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill);
     251             :   default: return 0;
     252             : }
     253             : }
     254             : 
     255          26 : unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     256          26 :   if (RetVT.SimpleTy != MVT::i32)
     257             :     return 0;
     258          52 :   if ((Subtarget->hasAVX512())) {
     259           2 :     return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     260             :   }
     261          24 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     262          22 :     return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     263             :   }
     264           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     265           2 :     return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     266             :   }
     267             :   return 0;
     268             : }
     269             : 
     270         136 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     271         272 :   if ((Subtarget->hasAVX512())) {
     272           0 :     return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     273             :   }
     274         136 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     275         135 :     return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     276             :   }
     277           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     278           1 :     return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     279             :   }
     280             :   return 0;
     281             : }
     282             : 
     283           0 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     284           0 :   if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
     285           0 :     return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
     286             :   }
     287             :   return 0;
     288             : }
     289             : 
     290         136 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     291         136 : switch (RetVT.SimpleTy) {
     292         136 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     293           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill);
     294             :   default: return 0;
     295             : }
     296             : }
     297             : 
     298             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     299           0 :   if ((Subtarget->hasMMX())) {
     300           0 :     return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     301             :   }
     302             :   return 0;
     303             : }
     304             : 
     305           0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     306           0 :   if ((Subtarget->hasMMX()) && (Subtarget->hasSSE2())) {
     307           0 :     return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
     308             :   }
     309             :   return 0;
     310             : }
     311             : 
     312           0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     313           0 : switch (RetVT.SimpleTy) {
     314           0 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
     315           0 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
     316             :   default: return 0;
     317             : }
     318             : }
     319             : 
     320        4267 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     321        4267 :   switch (VT.SimpleTy) {
     322          23 :   case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
     323         138 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
     324          26 :   case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
     325         136 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
     326           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
     327             :   default: return 0;
     328             :   }
     329             : }
     330             : 
     331             : // FastEmit functions for ISD::BRIND.
     332             : 
     333             : unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     334           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     335             :     return 0;
     336           0 :   if ((!Subtarget->is64Bit())) {
     337           0 :     return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
     338             :   }
     339             :   return 0;
     340             : }
     341             : 
     342             : unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     343           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     344             :     return 0;
     345           0 :   if ((!Subtarget->is64Bit())) {
     346           0 :     return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     347             :   }
     348             :   return 0;
     349             : }
     350             : 
     351             : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     352           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     353             :     return 0;
     354           0 :   if ((Subtarget->is64Bit())) {
     355           0 :     return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     356             :   }
     357             :   return 0;
     358             : }
     359             : 
     360           0 : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     361           0 :   switch (VT.SimpleTy) {
     362           0 :   case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
     363           0 :   case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
     364           0 :   case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
     365             :   default: return 0;
     366             :   }
     367             : }
     368             : 
     369             : // FastEmit functions for ISD::BSWAP.
     370             : 
     371             : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     372           0 :   if (RetVT.SimpleTy != MVT::i32)
     373             :     return 0;
     374           0 :   return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     375             : }
     376             : 
     377             : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     378           0 :   if (RetVT.SimpleTy != MVT::i64)
     379             :     return 0;
     380           0 :   return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     381             : }
     382             : 
     383           0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     384           0 :   switch (VT.SimpleTy) {
     385           0 :   case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     386           0 :   case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     387             :   default: return 0;
     388             :   }
     389             : }
     390             : 
     391             : // FastEmit functions for ISD::CTLZ.
     392             : 
     393             : unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     394           0 :   if (RetVT.SimpleTy != MVT::i16)
     395             :     return 0;
     396           0 :   if ((Subtarget->hasLZCNT())) {
     397           0 :     return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     398             :   }
     399             :   return 0;
     400             : }
     401             : 
     402             : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     403           0 :   if (RetVT.SimpleTy != MVT::i32)
     404             :     return 0;
     405           0 :   if ((Subtarget->hasLZCNT())) {
     406           0 :     return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     407             :   }
     408             :   return 0;
     409             : }
     410             : 
     411             : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     412           0 :   if (RetVT.SimpleTy != MVT::i64)
     413             :     return 0;
     414           0 :   if ((Subtarget->hasLZCNT())) {
     415           0 :     return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     416             :   }
     417             :   return 0;
     418             : }
     419             : 
     420           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     421           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     422             :     return 0;
     423           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     424           0 :     return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     425             :   }
     426             :   return 0;
     427             : }
     428             : 
     429           0 : unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     430           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     431             :     return 0;
     432           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     433           0 :     return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     434             :   }
     435             :   return 0;
     436             : }
     437             : 
     438             : unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     439           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     440             :     return 0;
     441           0 :   if ((Subtarget->hasCDI())) {
     442           0 :     return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     443             :   }
     444             :   return 0;
     445             : }
     446             : 
     447           0 : unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     448           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     449             :     return 0;
     450           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     451           0 :     return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     452             :   }
     453             :   return 0;
     454             : }
     455             : 
     456           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     457           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     458             :     return 0;
     459           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     460           0 :     return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     461             :   }
     462             :   return 0;
     463             : }
     464             : 
     465             : unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     466           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     467             :     return 0;
     468           0 :   if ((Subtarget->hasCDI())) {
     469           0 :     return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     470             :   }
     471             :   return 0;
     472             : }
     473             : 
     474           0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     475           0 :   switch (VT.SimpleTy) {
     476           0 :   case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     477           0 :   case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     478           0 :   case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     479           0 :   case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     480           0 :   case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     481           0 :   case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     482           0 :   case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     483           0 :   case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     484           0 :   case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     485             :   default: return 0;
     486             :   }
     487             : }
     488             : 
     489             : // FastEmit functions for ISD::CTPOP.
     490             : 
     491             : unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     492           0 :   if (RetVT.SimpleTy != MVT::i16)
     493             :     return 0;
     494           0 :   if ((Subtarget->hasPOPCNT())) {
     495           0 :     return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     496             :   }
     497             :   return 0;
     498             : }
     499             : 
     500             : unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     501           0 :   if (RetVT.SimpleTy != MVT::i32)
     502             :     return 0;
     503           0 :   if ((Subtarget->hasPOPCNT())) {
     504           0 :     return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     505             :   }
     506             :   return 0;
     507             : }
     508             : 
     509             : unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     510           0 :   if (RetVT.SimpleTy != MVT::i64)
     511             :     return 0;
     512           0 :   if ((Subtarget->hasPOPCNT())) {
     513           0 :     return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     514             :   }
     515             :   return 0;
     516             : }
     517             : 
     518           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     519           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     520             :     return 0;
     521           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     522           0 :     return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     523             :   }
     524             :   return 0;
     525             : }
     526             : 
     527           0 : unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     528           0 :   if (RetVT.SimpleTy != MVT::v32i8)
     529             :     return 0;
     530           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     531           0 :     return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     532             :   }
     533             :   return 0;
     534             : }
     535             : 
     536             : unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     537           0 :   if (RetVT.SimpleTy != MVT::v64i8)
     538             :     return 0;
     539           0 :   if ((Subtarget->hasBITALG())) {
     540           0 :     return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     541             :   }
     542             :   return 0;
     543             : }
     544             : 
     545           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     546           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     547             :     return 0;
     548           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     549           0 :     return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     550             :   }
     551             :   return 0;
     552             : }
     553             : 
     554           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     555           0 :   if (RetVT.SimpleTy != MVT::v16i16)
     556             :     return 0;
     557           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     558           0 :     return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     559             :   }
     560             :   return 0;
     561             : }
     562             : 
     563             : unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     564           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     565             :     return 0;
     566           0 :   if ((Subtarget->hasBITALG())) {
     567           0 :     return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     568             :   }
     569             :   return 0;
     570             : }
     571             : 
     572           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     573           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     574             :     return 0;
     575           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     576           0 :     return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     577             :   }
     578             :   return 0;
     579             : }
     580             : 
     581           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     582           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     583             :     return 0;
     584           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     585           0 :     return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     586             :   }
     587             :   return 0;
     588             : }
     589             : 
     590             : unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     591           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     592             :     return 0;
     593           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     594           0 :     return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     595             :   }
     596             :   return 0;
     597             : }
     598             : 
     599           0 : unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     600           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     601             :     return 0;
     602           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     603           0 :     return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     604             :   }
     605             :   return 0;
     606             : }
     607             : 
     608           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     609           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     610             :     return 0;
     611           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     612           0 :     return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     613             :   }
     614             :   return 0;
     615             : }
     616             : 
     617             : unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     618           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     619             :     return 0;
     620           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     621           0 :     return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     622             :   }
     623             :   return 0;
     624             : }
     625             : 
     626           0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     627           0 :   switch (VT.SimpleTy) {
     628           0 :   case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
     629           0 :   case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     630           0 :   case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     631           0 :   case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     632           0 :   case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     633           0 :   case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     634           0 :   case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     635           0 :   case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     636           0 :   case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     637           0 :   case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     638           0 :   case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     639           0 :   case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     640           0 :   case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     641           0 :   case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     642           0 :   case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     643             :   default: return 0;
     644             :   }
     645             : }
     646             : 
     647             : // FastEmit functions for ISD::CTTZ.
     648             : 
     649             : unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     650           0 :   if (RetVT.SimpleTy != MVT::i16)
     651             :     return 0;
     652           0 :   if ((Subtarget->hasBMI())) {
     653           0 :     return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     654             :   }
     655             :   return 0;
     656             : }
     657             : 
     658             : unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     659           0 :   if (RetVT.SimpleTy != MVT::i32)
     660             :     return 0;
     661           0 :   if ((Subtarget->hasBMI())) {
     662           0 :     return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     663             :   }
     664             :   return 0;
     665             : }
     666             : 
     667             : unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     668           0 :   if (RetVT.SimpleTy != MVT::i64)
     669             :     return 0;
     670           0 :   if ((Subtarget->hasBMI())) {
     671           0 :     return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     672             :   }
     673             :   return 0;
     674             : }
     675             : 
     676           0 : unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     677           0 :   switch (VT.SimpleTy) {
     678           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     679           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     680           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     681             :   default: return 0;
     682             :   }
     683             : }
     684             : 
     685             : // FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
     686             : 
     687             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     688           0 :   if (RetVT.SimpleTy != MVT::i16)
     689             :     return 0;
     690           0 :   return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     691             : }
     692             : 
     693             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     694           0 :   if (RetVT.SimpleTy != MVT::i32)
     695             :     return 0;
     696           0 :   return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     697             : }
     698             : 
     699             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     700           0 :   if (RetVT.SimpleTy != MVT::i64)
     701             :     return 0;
     702           0 :   return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     703             : }
     704             : 
     705           0 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     706           0 :   switch (VT.SimpleTy) {
     707           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
     708           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
     709           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
     710             :   default: return 0;
     711             :   }
     712             : }
     713             : 
     714             : // FastEmit functions for ISD::FABS.
     715             : 
     716             : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     717           0 :   if (RetVT.SimpleTy != MVT::f32)
     718             :     return 0;
     719           0 :   if ((!Subtarget->hasSSE1())) {
     720           0 :     return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     721             :   }
     722             :   return 0;
     723             : }
     724             : 
     725             : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     726           0 :   if (RetVT.SimpleTy != MVT::f64)
     727             :     return 0;
     728           0 :   if ((!Subtarget->hasSSE2())) {
     729           0 :     return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     730             :   }
     731             :   return 0;
     732             : }
     733             : 
     734             : unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     735           0 :   if (RetVT.SimpleTy != MVT::f80)
     736             :     return 0;
     737           0 :   return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     738             : }
     739             : 
     740           0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     741           0 :   switch (VT.SimpleTy) {
     742           0 :   case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     743           0 :   case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     744           0 :   case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     745             :   default: return 0;
     746             :   }
     747             : }
     748             : 
     749             : // FastEmit functions for ISD::FCOS.
     750             : 
     751             : unsigned fastEmit_ISD_FCOS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     752           0 :   if (RetVT.SimpleTy != MVT::f32)
     753             :     return 0;
     754           0 :   if ((!Subtarget->hasSSE1())) {
     755           0 :     return fastEmitInst_r(X86::COS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     756             :   }
     757             :   return 0;
     758             : }
     759             : 
     760             : unsigned fastEmit_ISD_FCOS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     761           0 :   if (RetVT.SimpleTy != MVT::f64)
     762             :     return 0;
     763           0 :   if ((!Subtarget->hasSSE2())) {
     764           0 :     return fastEmitInst_r(X86::COS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     765             :   }
     766             :   return 0;
     767             : }
     768             : 
     769             : unsigned fastEmit_ISD_FCOS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     770           0 :   if (RetVT.SimpleTy != MVT::f80)
     771             :     return 0;
     772           0 :   return fastEmitInst_r(X86::COS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     773             : }
     774             : 
     775           0 : unsigned fastEmit_ISD_FCOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     776           0 :   switch (VT.SimpleTy) {
     777           0 :   case MVT::f32: return fastEmit_ISD_FCOS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     778           0 :   case MVT::f64: return fastEmit_ISD_FCOS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     779           0 :   case MVT::f80: return fastEmit_ISD_FCOS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     780             :   default: return 0;
     781             :   }
     782             : }
     783             : 
     784             : // FastEmit functions for ISD::FNEG.
     785             : 
     786             : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     787          11 :   if (RetVT.SimpleTy != MVT::f32)
     788             :     return 0;
     789          22 :   if ((!Subtarget->hasSSE1())) {
     790           0 :     return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     791             :   }
     792             :   return 0;
     793             : }
     794             : 
     795             : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     796         135 :   if (RetVT.SimpleTy != MVT::f64)
     797             :     return 0;
     798         270 :   if ((!Subtarget->hasSSE2())) {
     799           0 :     return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     800             :   }
     801             :   return 0;
     802             : }
     803             : 
     804             : unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     805           2 :   if (RetVT.SimpleTy != MVT::f80)
     806             :     return 0;
     807           2 :   return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     808             : }
     809             : 
     810         188 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     811         188 :   switch (VT.SimpleTy) {
     812          11 :   case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
     813         135 :   case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
     814           2 :   case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill);
     815             :   default: return 0;
     816             :   }
     817             : }
     818             : 
     819             : // FastEmit functions for ISD::FP_EXTEND.
     820             : 
     821           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     822           0 :   if (RetVT.SimpleTy != MVT::f64)
     823             :     return 0;
     824           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     825           0 :     return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     826             :   }
     827             :   return 0;
     828             : }
     829             : 
     830           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     831           0 :   if (RetVT.SimpleTy != MVT::v4f64)
     832             :     return 0;
     833           0 :   if ((Subtarget->hasVLX())) {
     834           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     835             :   }
     836           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     837           0 :     return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     838             :   }
     839             :   return 0;
     840             : }
     841             : 
     842             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     843           0 :   if (RetVT.SimpleTy != MVT::v8f64)
     844             :     return 0;
     845           0 :   if ((Subtarget->hasAVX512())) {
     846           0 :     return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     847             :   }
     848             :   return 0;
     849             : }
     850             : 
     851           0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     852           0 :   switch (VT.SimpleTy) {
     853           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
     854           0 :   case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     855           0 :   case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
     856             :   default: return 0;
     857             :   }
     858             : }
     859             : 
     860             : // FastEmit functions for ISD::FP_ROUND.
     861             : 
     862           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     863           0 :   if (RetVT.SimpleTy != MVT::f32)
     864             :     return 0;
     865           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     866           0 :     return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     867             :   }
     868             :   return 0;
     869             : }
     870             : 
     871           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     872           0 :   if (RetVT.SimpleTy != MVT::v4f32)
     873             :     return 0;
     874           0 :   if ((Subtarget->hasVLX())) {
     875           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     876             :   }
     877           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     878           0 :     return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
     879             :   }
     880             :   return 0;
     881             : }
     882             : 
     883             : unsigned fastEmit_ISD_FP_ROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     884           0 :   if (RetVT.SimpleTy != MVT::v8f32)
     885             :     return 0;
     886           0 :   if ((Subtarget->hasAVX512())) {
     887           0 :     return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
     888             :   }
     889             :   return 0;
     890             : }
     891             : 
     892           0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     893           0 :   switch (VT.SimpleTy) {
     894           0 :   case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
     895           0 :   case MVT::v4f64: return fastEmit_ISD_FP_ROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
     896           0 :   case MVT::v8f64: return fastEmit_ISD_FP_ROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
     897             :   default: return 0;
     898             :   }
     899             : }
     900             : 
     901             : // FastEmit functions for ISD::FP_TO_SINT.
     902             : 
     903           9 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     904          18 :   if ((Subtarget->hasAVX512())) {
     905           0 :     return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     906             :   }
     907           9 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     908           9 :     return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     909             :   }
     910           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     911           0 :     return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     912             :   }
     913             :   return 0;
     914             : }
     915             : 
     916           4 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     917           8 :   if ((Subtarget->hasAVX512())) {
     918           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     919             :   }
     920           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     921           4 :     return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     922             :   }
     923           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     924           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     925             :   }
     926             :   return 0;
     927             : }
     928             : 
     929          13 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     930          13 : switch (RetVT.SimpleTy) {
     931           9 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
     932           4 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
     933             :   default: return 0;
     934             : }
     935             : }
     936             : 
     937          69 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     938         138 :   if ((Subtarget->hasAVX512())) {
     939           0 :     return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     940             :   }
     941          69 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     942          69 :     return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     943             :   }
     944           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     945           0 :     return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     946             :   }
     947             :   return 0;
     948             : }
     949             : 
     950          34 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     951          68 :   if ((Subtarget->hasAVX512())) {
     952           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     953             :   }
     954          34 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     955          34 :     return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     956             :   }
     957           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     958           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     959             :   }
     960             :   return 0;
     961             : }
     962             : 
     963         103 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     964         103 : switch (RetVT.SimpleTy) {
     965          69 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
     966          34 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     967             :   default: return 0;
     968             : }
     969             : }
     970             : 
     971           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     972           0 :   if ((Subtarget->hasVLX())) {
     973           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     974             :   }
     975           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     976           0 :     return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     977             :   }
     978           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     979           0 :     return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     980             :   }
     981             :   return 0;
     982             : }
     983             : 
     984           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
     985           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
     986           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     987             :   }
     988             :   return 0;
     989             : }
     990             : 
     991           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     992           0 : switch (RetVT.SimpleTy) {
     993           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
     994           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
     995             :   default: return 0;
     996             : }
     997             : }
     998             : 
     999           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1000           0 :   if ((Subtarget->hasVLX())) {
    1001           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1002             :   }
    1003           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1004           0 :     return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1005             :   }
    1006             :   return 0;
    1007             : }
    1008             : 
    1009             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1010           0 :   if ((Subtarget->hasDQI())) {
    1011           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1012             :   }
    1013             :   return 0;
    1014             : }
    1015             : 
    1016           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1017           0 : switch (RetVT.SimpleTy) {
    1018           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1019           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1020             :   default: return 0;
    1021             : }
    1022             : }
    1023             : 
    1024             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1025           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1026             :     return 0;
    1027           0 :   if ((Subtarget->hasAVX512())) {
    1028           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1029             :   }
    1030             :   return 0;
    1031             : }
    1032             : 
    1033           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1034           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1035             :     return 0;
    1036           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1037           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1038             :   }
    1039             :   return 0;
    1040             : }
    1041             : 
    1042           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1043           0 :   if ((Subtarget->hasVLX())) {
    1044           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1045             :   }
    1046           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1047           0 :     return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1048             :   }
    1049             :   return 0;
    1050             : }
    1051             : 
    1052           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1053           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1054           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1055             :   }
    1056             :   return 0;
    1057             : }
    1058             : 
    1059           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1060           0 : switch (RetVT.SimpleTy) {
    1061           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1062           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1063             :   default: return 0;
    1064             : }
    1065             : }
    1066             : 
    1067             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1068           0 :   if ((Subtarget->hasAVX512())) {
    1069           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1070             :   }
    1071             :   return 0;
    1072             : }
    1073             : 
    1074             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1075           0 :   if ((Subtarget->hasDQI())) {
    1076           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1077             :   }
    1078             :   return 0;
    1079             : }
    1080             : 
    1081           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1082           0 : switch (RetVT.SimpleTy) {
    1083           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1084           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1085             :   default: return 0;
    1086             : }
    1087             : }
    1088             : 
    1089         120 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1090         120 :   switch (VT.SimpleTy) {
    1091          13 :   case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1092         103 :   case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1093           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1094           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1095           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1096           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1097           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1098           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1099             :   default: return 0;
    1100             :   }
    1101             : }
    1102             : 
    1103             : // FastEmit functions for ISD::FP_TO_UINT.
    1104             : 
    1105             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1106           0 :   if ((Subtarget->hasAVX512())) {
    1107           0 :     return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1108             :   }
    1109             :   return 0;
    1110             : }
    1111             : 
    1112             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1113           0 :   if ((Subtarget->hasAVX512())) {
    1114           0 :     return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1115             :   }
    1116             :   return 0;
    1117             : }
    1118             : 
    1119           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1120           0 : switch (RetVT.SimpleTy) {
    1121           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    1122           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    1123             :   default: return 0;
    1124             : }
    1125             : }
    1126             : 
    1127             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1128           0 :   if ((Subtarget->hasAVX512())) {
    1129           0 :     return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1130             :   }
    1131             :   return 0;
    1132             : }
    1133             : 
    1134             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1135           0 :   if ((Subtarget->hasAVX512())) {
    1136           0 :     return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1137             :   }
    1138             :   return 0;
    1139             : }
    1140             : 
    1141           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1142           0 : switch (RetVT.SimpleTy) {
    1143           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    1144           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    1145             :   default: return 0;
    1146             : }
    1147             : }
    1148             : 
    1149             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1150           0 :   if ((Subtarget->hasVLX())) {
    1151           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1152             :   }
    1153             :   return 0;
    1154             : }
    1155             : 
    1156           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1157           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1158           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1159             :   }
    1160             :   return 0;
    1161             : }
    1162             : 
    1163           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1164           0 : switch (RetVT.SimpleTy) {
    1165           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    1166           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    1167             :   default: return 0;
    1168             : }
    1169             : }
    1170             : 
    1171             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1172           0 :   if ((Subtarget->hasVLX())) {
    1173           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1174             :   }
    1175             :   return 0;
    1176             : }
    1177             : 
    1178             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1179           0 :   if ((Subtarget->hasDQI())) {
    1180           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1181             :   }
    1182             :   return 0;
    1183             : }
    1184             : 
    1185           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1186           0 : switch (RetVT.SimpleTy) {
    1187           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1188           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1189             :   default: return 0;
    1190             : }
    1191             : }
    1192             : 
    1193             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1194           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1195             :     return 0;
    1196           0 :   if ((Subtarget->hasAVX512())) {
    1197           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1198             :   }
    1199             :   return 0;
    1200             : }
    1201             : 
    1202           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1203           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1204             :     return 0;
    1205           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1206           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1207             :   }
    1208             :   return 0;
    1209             : }
    1210             : 
    1211             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1212           0 :   if ((Subtarget->hasVLX())) {
    1213           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1214             :   }
    1215             :   return 0;
    1216             : }
    1217             : 
    1218           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1219           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1220           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1221             :   }
    1222             :   return 0;
    1223             : }
    1224             : 
    1225           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1226           0 : switch (RetVT.SimpleTy) {
    1227           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1228           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1229             :   default: return 0;
    1230             : }
    1231             : }
    1232             : 
    1233             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1234           0 :   if ((Subtarget->hasAVX512())) {
    1235           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1236             :   }
    1237             :   return 0;
    1238             : }
    1239             : 
    1240             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1241           0 :   if ((Subtarget->hasDQI())) {
    1242           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1243             :   }
    1244             :   return 0;
    1245             : }
    1246             : 
    1247           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1248           0 : switch (RetVT.SimpleTy) {
    1249           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1250           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1251             :   default: return 0;
    1252             : }
    1253             : }
    1254             : 
    1255           0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1256           0 :   switch (VT.SimpleTy) {
    1257           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1258           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1259           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1260           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1261           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1262           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1263           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1264           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1265             :   default: return 0;
    1266             :   }
    1267             : }
    1268             : 
    1269             : // FastEmit functions for ISD::FSIN.
    1270             : 
    1271             : unsigned fastEmit_ISD_FSIN_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1272           0 :   if (RetVT.SimpleTy != MVT::f32)
    1273             :     return 0;
    1274           0 :   if ((!Subtarget->hasSSE1())) {
    1275           0 :     return fastEmitInst_r(X86::SIN_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1276             :   }
    1277             :   return 0;
    1278             : }
    1279             : 
    1280             : unsigned fastEmit_ISD_FSIN_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1281           0 :   if (RetVT.SimpleTy != MVT::f64)
    1282             :     return 0;
    1283           0 :   if ((!Subtarget->hasSSE2())) {
    1284           0 :     return fastEmitInst_r(X86::SIN_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1285             :   }
    1286             :   return 0;
    1287             : }
    1288             : 
    1289             : unsigned fastEmit_ISD_FSIN_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1290           0 :   if (RetVT.SimpleTy != MVT::f80)
    1291             :     return 0;
    1292           0 :   return fastEmitInst_r(X86::SIN_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1293             : }
    1294             : 
    1295           0 : unsigned fastEmit_ISD_FSIN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1296           0 :   switch (VT.SimpleTy) {
    1297           0 :   case MVT::f32: return fastEmit_ISD_FSIN_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1298           0 :   case MVT::f64: return fastEmit_ISD_FSIN_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1299           0 :   case MVT::f80: return fastEmit_ISD_FSIN_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1300             :   default: return 0;
    1301             :   }
    1302             : }
    1303             : 
    1304             : // FastEmit functions for ISD::FSQRT.
    1305             : 
    1306           0 : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1307           0 :   if (RetVT.SimpleTy != MVT::f32)
    1308             :     return 0;
    1309           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1310           0 :     return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    1311             :   }
    1312           0 :   if ((!Subtarget->hasSSE1())) {
    1313           0 :     return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1314             :   }
    1315             :   return 0;
    1316             : }
    1317             : 
    1318           0 : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1319           0 :   if (RetVT.SimpleTy != MVT::f64)
    1320             :     return 0;
    1321           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1322           0 :     return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
    1323             :   }
    1324           0 :   if ((!Subtarget->hasSSE2())) {
    1325           0 :     return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1326             :   }
    1327             :   return 0;
    1328             : }
    1329             : 
    1330             : unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1331           0 :   if (RetVT.SimpleTy != MVT::f80)
    1332             :     return 0;
    1333           0 :   return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1334             : }
    1335             : 
    1336           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1337           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1338             :     return 0;
    1339           0 :   if ((Subtarget->hasVLX())) {
    1340           0 :     return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1341             :   }
    1342           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1343           0 :     return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1344             :   }
    1345           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1346           0 :     return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1347             :   }
    1348             :   return 0;
    1349             : }
    1350             : 
    1351           0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1352           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    1353             :     return 0;
    1354           0 :   if ((Subtarget->hasVLX())) {
    1355           0 :     return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1356             :   }
    1357           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1358           0 :     return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1359             :   }
    1360             :   return 0;
    1361             : }
    1362             : 
    1363             : unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1364           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1365             :     return 0;
    1366           0 :   if ((Subtarget->hasAVX512())) {
    1367           0 :     return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1368             :   }
    1369             :   return 0;
    1370             : }
    1371             : 
    1372           0 : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1373           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1374             :     return 0;
    1375           0 :   if ((Subtarget->hasVLX())) {
    1376           0 :     return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1377             :   }
    1378           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1379           0 :     return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1380             :   }
    1381           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1382           0 :     return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1383             :   }
    1384             :   return 0;
    1385             : }
    1386             : 
    1387           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1388           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    1389             :     return 0;
    1390           0 :   if ((Subtarget->hasVLX())) {
    1391           0 :     return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1392             :   }
    1393           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1394           0 :     return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1395             :   }
    1396             :   return 0;
    1397             : }
    1398             : 
    1399             : unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1400           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    1401             :     return 0;
    1402           0 :   if ((Subtarget->hasAVX512())) {
    1403           0 :     return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1404             :   }
    1405             :   return 0;
    1406             : }
    1407             : 
    1408           0 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1409           0 :   switch (VT.SimpleTy) {
    1410           0 :   case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1411           0 :   case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1412           0 :   case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1413           0 :   case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1414           0 :   case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1415           0 :   case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1416           0 :   case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1417           0 :   case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1418           0 :   case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1419             :   default: return 0;
    1420             :   }
    1421             : }
    1422             : 
    1423             : // FastEmit functions for ISD::SCALAR_TO_VECTOR.
    1424             : 
    1425           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1426           2 :   if ((Subtarget->hasAVX512())) {
    1427           0 :     return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1428             :   }
    1429           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1430           0 :     return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1431             :   }
    1432           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1433           1 :     return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1434             :   }
    1435             :   return 0;
    1436             : }
    1437             : 
    1438             : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
    1439           0 :   if ((Subtarget->hasMMX())) {
    1440           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    1441             :   }
    1442             :   return 0;
    1443             : }
    1444             : 
    1445           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1446           1 : switch (RetVT.SimpleTy) {
    1447           1 :   case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    1448           0 :   case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill);
    1449             :   default: return 0;
    1450             : }
    1451             : }
    1452             : 
    1453           0 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1454           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1455             :     return 0;
    1456           0 :   if ((Subtarget->hasAVX512())) {
    1457           0 :     return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1458             :   }
    1459           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1460           0 :     return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1461             :   }
    1462           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1463           0 :     return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1464             :   }
    1465             :   return 0;
    1466             : }
    1467             : 
    1468           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1469           1 :   switch (VT.SimpleTy) {
    1470           1 :   case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1471           0 :   case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1472             :   default: return 0;
    1473             :   }
    1474             : }
    1475             : 
    1476             : // FastEmit functions for ISD::SIGN_EXTEND.
    1477             : 
    1478             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1479       26427 :   return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    1480             : }
    1481             : 
    1482             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1483         107 :   return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
    1484             : }
    1485             : 
    1486       26545 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1487       26545 : switch (RetVT.SimpleTy) {
    1488       26427 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill);
    1489         107 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill);
    1490             :   default: return 0;
    1491             : }
    1492             : }
    1493             : 
    1494             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1495         583 :   return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    1496             : }
    1497             : 
    1498             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1499          32 :   return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
    1500             : }
    1501             : 
    1502         615 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1503         615 : switch (RetVT.SimpleTy) {
    1504         583 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill);
    1505          32 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill);
    1506             :   default: return 0;
    1507             : }
    1508             : }
    1509             : 
    1510             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1511       29055 :   if (RetVT.SimpleTy != MVT::i64)
    1512             :     return 0;
    1513       29055 :   if ((Subtarget->is64Bit())) {
    1514       29055 :     return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
    1515             :   }
    1516             :   return 0;
    1517             : }
    1518             : 
    1519          28 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1520          28 :   if (RetVT.SimpleTy != MVT::v2i64)
    1521             :     return 0;
    1522          28 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1523          28 :     return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1524             :   }
    1525             :   return 0;
    1526             : }
    1527             : 
    1528          30 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1529          30 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1530          30 :     return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1531             :   }
    1532             :   return 0;
    1533             : }
    1534             : 
    1535           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1536           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1537           0 :     return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1538             :   }
    1539             :   return 0;
    1540             : }
    1541             : 
    1542          30 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1543          30 : switch (RetVT.SimpleTy) {
    1544          30 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
    1545           0 :   case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill);
    1546             :   default: return 0;
    1547             : }
    1548             : }
    1549             : 
    1550           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1551           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1552           6 :     return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1553             :   }
    1554             :   return 0;
    1555             : }
    1556             : 
    1557           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1558           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1559           0 :     return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1560             :   }
    1561             :   return 0;
    1562             : }
    1563             : 
    1564             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1565           0 :   if ((Subtarget->hasDQI())) {
    1566           0 :     return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1567             :   }
    1568             :   return 0;
    1569             : }
    1570             : 
    1571           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1572           6 : switch (RetVT.SimpleTy) {
    1573           6 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
    1574           0 :   case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill);
    1575           0 :   case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    1576             :   default: return 0;
    1577             : }
    1578             : }
    1579             : 
    1580           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1581           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1582           6 :     return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1583             :   }
    1584             :   return 0;
    1585             : }
    1586             : 
    1587           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    1588           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1589           0 :     return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1590             :   }
    1591             :   return 0;
    1592             : }
    1593             : 
    1594             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    1595           0 :   if ((Subtarget->hasDQI())) {
    1596           0 :     return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1597             :   }
    1598             :   return 0;
    1599             : }
    1600             : 
    1601           6 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1602           6 : switch (RetVT.SimpleTy) {
    1603           6 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
    1604           0 :   case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill);
    1605           0 :   case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    1606             :   default: return 0;
    1607             : }
    1608             : }
    1609             : 
    1610           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    1611           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1612           0 :     return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1613             :   }
    1614             :   return 0;
    1615             : }
    1616             : 
    1617             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    1618           0 :   if ((Subtarget->hasBWI())) {
    1619           0 :     return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1620             :   }
    1621             :   return 0;
    1622             : }
    1623             : 
    1624           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1625           0 : switch (RetVT.SimpleTy) {
    1626           0 :   case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
    1627           0 :   case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
    1628             :   default: return 0;
    1629             : }
    1630             : }
    1631             : 
    1632             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1633           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    1634             :     return 0;
    1635           0 :   if ((Subtarget->hasBWI())) {
    1636           0 :     return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1637             :   }
    1638             :   return 0;
    1639             : }
    1640             : 
    1641       56291 : unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1642       56291 :   switch (VT.SimpleTy) {
    1643       26545 :   case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    1644         615 :   case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    1645       29055 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1646          28 :   case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill);
    1647          30 :   case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill);
    1648           6 :   case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    1649           6 :   case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    1650           0 :   case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill);
    1651           0 :   case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill);
    1652             :   default: return 0;
    1653             :   }
    1654             : }
    1655             : 
    1656             : // FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
    1657             : 
    1658           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1659           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1660           0 :     return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1661             :   }
    1662           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    1663           0 :     return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1664             :   }
    1665           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1666           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1667             :   }
    1668             :   return 0;
    1669             : }
    1670             : 
    1671           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1672           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1673           0 :     return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1674             :   }
    1675           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1676           0 :     return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1677             :   }
    1678           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1679           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1680             :   }
    1681             :   return 0;
    1682             : }
    1683             : 
    1684           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1685           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1686           0 :     return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1687             :   }
    1688           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1689           0 :     return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1690             :   }
    1691           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1692           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1693             :   }
    1694             :   return 0;
    1695             : }
    1696             : 
    1697           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1698           0 : switch (RetVT.SimpleTy) {
    1699           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1700           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1701           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1702             :   default: return 0;
    1703             : }
    1704             : }
    1705             : 
    1706           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1707           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1708           0 :     return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1709             :   }
    1710           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1711           0 :     return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1712             :   }
    1713           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1714           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1715             :   }
    1716             :   return 0;
    1717             : }
    1718             : 
    1719           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1720           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1721           0 :     return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1722             :   }
    1723           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1724           0 :     return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1725             :   }
    1726           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1727           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1728             :   }
    1729             :   return 0;
    1730             : }
    1731             : 
    1732           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1733           0 : switch (RetVT.SimpleTy) {
    1734           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    1735           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    1736             :   default: return 0;
    1737             : }
    1738             : }
    1739             : 
    1740           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1741           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1742             :     return 0;
    1743           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1744           0 :     return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1745             :   }
    1746           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1747           0 :     return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1748             :   }
    1749           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1750           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1751             :   }
    1752             :   return 0;
    1753             : }
    1754             : 
    1755           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1756           0 :   switch (VT.SimpleTy) {
    1757           0 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1758           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1759           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1760             :   default: return 0;
    1761             :   }
    1762             : }
    1763             : 
    1764             : // FastEmit functions for ISD::SINT_TO_FP.
    1765             : 
    1766          46 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1767          92 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1768          40 :     return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1769             :   }
    1770             :   return 0;
    1771             : }
    1772             : 
    1773         972 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1774        1944 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1775         964 :     return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1776             :   }
    1777             :   return 0;
    1778             : }
    1779             : 
    1780        1018 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1781        1018 : switch (RetVT.SimpleTy) {
    1782          46 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    1783         972 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    1784             :   default: return 0;
    1785             : }
    1786             : }
    1787             : 
    1788           9 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1789          18 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1790           3 :     return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1791             :   }
    1792             :   return 0;
    1793             : }
    1794             : 
    1795          48 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1796          96 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1797          42 :     return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1798             :   }
    1799             :   return 0;
    1800             : }
    1801             : 
    1802          73 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1803          73 : switch (RetVT.SimpleTy) {
    1804           9 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    1805          48 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    1806             :   default: return 0;
    1807             : }
    1808             : }
    1809             : 
    1810           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1811           6 :   if ((Subtarget->hasVLX())) {
    1812           2 :     return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1813             :   }
    1814           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1815           2 :     return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1816             :   }
    1817           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1818           2 :     return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1819             :   }
    1820             :   return 0;
    1821             : }
    1822             : 
    1823           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1824           2 :   if ((Subtarget->hasVLX())) {
    1825           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1826             :   }
    1827           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1828           2 :     return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1829             :   }
    1830             :   return 0;
    1831             : }
    1832             : 
    1833           8 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1834           8 : switch (RetVT.SimpleTy) {
    1835           6 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    1836           2 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    1837             :   default: return 0;
    1838             : }
    1839             : }
    1840             : 
    1841           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1842           2 :   if ((Subtarget->hasVLX())) {
    1843           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1844             :   }
    1845           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1846           2 :     return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1847             :   }
    1848             :   return 0;
    1849             : }
    1850             : 
    1851             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1852           0 :   if ((Subtarget->hasAVX512())) {
    1853           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1854             :   }
    1855             :   return 0;
    1856             : }
    1857             : 
    1858           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1859           2 : switch (RetVT.SimpleTy) {
    1860           2 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    1861           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    1862             :   default: return 0;
    1863             : }
    1864             : }
    1865             : 
    1866             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1867           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1868             :     return 0;
    1869           0 :   if ((Subtarget->hasAVX512())) {
    1870           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1871             :   }
    1872             :   return 0;
    1873             : }
    1874             : 
    1875           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1876           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1877             :     return 0;
    1878           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1879           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1880             :   }
    1881             :   return 0;
    1882             : }
    1883             : 
    1884           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1885           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1886           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1887             :   }
    1888             :   return 0;
    1889             : }
    1890             : 
    1891           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1892           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1893           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1894             :   }
    1895             :   return 0;
    1896             : }
    1897             : 
    1898           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1899           0 : switch (RetVT.SimpleTy) {
    1900           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    1901           0 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    1902             :   default: return 0;
    1903             : }
    1904             : }
    1905             : 
    1906             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1907           0 :   if ((Subtarget->hasDQI())) {
    1908           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1909             :   }
    1910             :   return 0;
    1911             : }
    1912             : 
    1913             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1914           0 :   if ((Subtarget->hasDQI())) {
    1915           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1916             :   }
    1917             :   return 0;
    1918             : }
    1919             : 
    1920           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1921           0 : switch (RetVT.SimpleTy) {
    1922           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    1923           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    1924             :   default: return 0;
    1925             : }
    1926             : }
    1927             : 
    1928        1101 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1929        1101 :   switch (VT.SimpleTy) {
    1930        1018 :   case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1931          73 :   case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1932           8 :   case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1933           2 :   case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    1934           0 :   case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    1935           0 :   case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1936           0 :   case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    1937           0 :   case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    1938             :   default: return 0;
    1939             :   }
    1940             : }
    1941             : 
    1942             : // FastEmit functions for ISD::TRUNCATE.
    1943             : 
    1944             : unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1945           3 :   if (RetVT.SimpleTy != MVT::i8)
    1946             :     return 0;
    1947           3 :   if ((Subtarget->is64Bit())) {
    1948           2 :     return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
    1949             :   }
    1950             :   return 0;
    1951             : }
    1952             : 
    1953             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1954        2196 :   if ((Subtarget->is64Bit())) {
    1955        4384 :     return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1956             :   }
    1957             :   return 0;
    1958             : }
    1959             : 
    1960             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1961        1569 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1962             : }
    1963             : 
    1964        3765 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1965        3765 : switch (RetVT.SimpleTy) {
    1966        2196 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill);
    1967        1569 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill);
    1968             :   default: return 0;
    1969             : }
    1970             : }
    1971             : 
    1972             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1973        3091 :   return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1974             : }
    1975             : 
    1976             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1977          18 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1978             : }
    1979             : 
    1980             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1981        5766 :   return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
    1982             : }
    1983             : 
    1984        8875 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1985        8875 : switch (RetVT.SimpleTy) {
    1986        3091 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill);
    1987          18 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill);
    1988        5766 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill);
    1989             :   default: return 0;
    1990             : }
    1991             : }
    1992             : 
    1993           6 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1994           6 :   if (RetVT.SimpleTy != MVT::v16i8)
    1995             :     return 0;
    1996           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1997           4 :     return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1998             :   }
    1999             :   return 0;
    2000             : }
    2001             : 
    2002             : unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2003           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    2004             :     return 0;
    2005           2 :   if ((Subtarget->hasBWI())) {
    2006           2 :     return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2007             :   }
    2008             :   return 0;
    2009             : }
    2010             : 
    2011           6 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2012           6 :   if (RetVT.SimpleTy != MVT::v8i16)
    2013             :     return 0;
    2014          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2015           4 :     return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2016             :   }
    2017             :   return 0;
    2018             : }
    2019             : 
    2020             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2021           2 :   if ((Subtarget->hasAVX512())) {
    2022           2 :     return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2023             :   }
    2024             :   return 0;
    2025             : }
    2026             : 
    2027             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    2028           0 :   if ((Subtarget->hasAVX512())) {
    2029           0 :     return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2030             :   }
    2031             :   return 0;
    2032             : }
    2033             : 
    2034           2 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2035           2 : switch (RetVT.SimpleTy) {
    2036           2 :   case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    2037           0 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    2038             :   default: return 0;
    2039             : }
    2040             : }
    2041             : 
    2042           4 : unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2043           4 :   if (RetVT.SimpleTy != MVT::v4i32)
    2044             :     return 0;
    2045           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2046           2 :     return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2047             :   }
    2048             :   return 0;
    2049             : }
    2050             : 
    2051             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2052           2 :   if ((Subtarget->hasAVX512())) {
    2053           2 :     return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2054             :   }
    2055             :   return 0;
    2056             : }
    2057             : 
    2058             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2059           2 :   if ((Subtarget->hasAVX512())) {
    2060           2 :     return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2061             :   }
    2062             :   return 0;
    2063             : }
    2064             : 
    2065           8 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2066           8 : switch (RetVT.SimpleTy) {
    2067           2 :   case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    2068           2 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    2069             :   default: return 0;
    2070             : }
    2071             : }
    2072             : 
    2073       12674 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2074       12674 :   switch (VT.SimpleTy) {
    2075           3 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2076        3765 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2077        8875 :   case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2078           6 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2079           2 :   case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2080           6 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2081           2 :   case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2082           4 :   case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2083           8 :   case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2084             :   default: return 0;
    2085             :   }
    2086             : }
    2087             : 
    2088             : // FastEmit functions for ISD::UINT_TO_FP.
    2089             : 
    2090             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2091           0 :   if ((Subtarget->hasVLX())) {
    2092           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2093             :   }
    2094             :   return 0;
    2095             : }
    2096             : 
    2097             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2098           0 :   if ((Subtarget->hasVLX())) {
    2099           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2100             :   }
    2101             :   return 0;
    2102             : }
    2103             : 
    2104           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2105           0 : switch (RetVT.SimpleTy) {
    2106           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    2107           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    2108             :   default: return 0;
    2109             : }
    2110             : }
    2111             : 
    2112             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2113           0 :   if ((Subtarget->hasVLX())) {
    2114           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2115             :   }
    2116             :   return 0;
    2117             : }
    2118             : 
    2119             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2120           0 :   if ((Subtarget->hasAVX512())) {
    2121           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2122             :   }
    2123             :   return 0;
    2124             : }
    2125             : 
    2126           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2127           0 : switch (RetVT.SimpleTy) {
    2128           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    2129           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    2130             :   default: return 0;
    2131             : }
    2132             : }
    2133             : 
    2134             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2135           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2136             :     return 0;
    2137           0 :   if ((Subtarget->hasAVX512())) {
    2138           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2139             :   }
    2140             :   return 0;
    2141             : }
    2142             : 
    2143           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2144           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2145             :     return 0;
    2146           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2147           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2148             :   }
    2149             :   return 0;
    2150             : }
    2151             : 
    2152           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2153           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2154           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2155             :   }
    2156             :   return 0;
    2157             : }
    2158             : 
    2159           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2160           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2161           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2162             :   }
    2163             :   return 0;
    2164             : }
    2165             : 
    2166           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2167           0 : switch (RetVT.SimpleTy) {
    2168           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    2169           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    2170             :   default: return 0;
    2171             : }
    2172             : }
    2173             : 
    2174             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2175           0 :   if ((Subtarget->hasDQI())) {
    2176           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2177             :   }
    2178             :   return 0;
    2179             : }
    2180             : 
    2181             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2182           0 :   if ((Subtarget->hasDQI())) {
    2183           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2184             :   }
    2185             :   return 0;
    2186             : }
    2187             : 
    2188           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2189           0 : switch (RetVT.SimpleTy) {
    2190           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    2191           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    2192             :   default: return 0;
    2193             : }
    2194             : }
    2195             : 
    2196           0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2197           0 :   switch (VT.SimpleTy) {
    2198           0 :   case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2199           0 :   case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2200           0 :   case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2201           0 :   case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2202           0 :   case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2203           0 :   case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2204             :   default: return 0;
    2205             :   }
    2206             : }
    2207             : 
    2208             : // FastEmit functions for ISD::ZERO_EXTEND.
    2209             : 
    2210             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2211       86979 :   if (RetVT.SimpleTy != MVT::i32)
    2212             :     return 0;
    2213       49446 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    2214             : }
    2215             : 
    2216             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2217        5905 :   if (RetVT.SimpleTy != MVT::i32)
    2218             :     return 0;
    2219        5835 :   return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    2220             : }
    2221             : 
    2222      127672 : unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2223      127672 :   switch (VT.SimpleTy) {
    2224       86979 :   case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    2225        5905 :   case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2226             :   default: return 0;
    2227             :   }
    2228             : }
    2229             : 
    2230             : // FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
    2231             : 
    2232           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2233           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2234           0 :     return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2235             :   }
    2236           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    2237           0 :     return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2238             :   }
    2239           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2240           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2241             :   }
    2242             :   return 0;
    2243             : }
    2244             : 
    2245           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2246           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2247           0 :     return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2248             :   }
    2249           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2250           0 :     return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2251             :   }
    2252           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2253           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2254             :   }
    2255             :   return 0;
    2256             : }
    2257             : 
    2258           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2259           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2260           0 :     return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2261             :   }
    2262           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2263           0 :     return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2264             :   }
    2265           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2266           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2267             :   }
    2268             :   return 0;
    2269             : }
    2270             : 
    2271           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2272           0 : switch (RetVT.SimpleTy) {
    2273           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    2274           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    2275           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    2276             :   default: return 0;
    2277             : }
    2278             : }
    2279             : 
    2280           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2281           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2282           0 :     return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2283             :   }
    2284           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2285           0 :     return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2286             :   }
    2287           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2288           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2289             :   }
    2290             :   return 0;
    2291             : }
    2292             : 
    2293           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2294           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2295           0 :     return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2296             :   }
    2297           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2298           0 :     return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2299             :   }
    2300           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2301           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2302             :   }
    2303             :   return 0;
    2304             : }
    2305             : 
    2306           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2307           0 : switch (RetVT.SimpleTy) {
    2308           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    2309           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    2310             :   default: return 0;
    2311             : }
    2312             : }
    2313             : 
    2314           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2315           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2316             :     return 0;
    2317           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2318           0 :     return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2319             :   }
    2320           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2321           0 :     return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2322             :   }
    2323           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2324           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2325             :   }
    2326             :   return 0;
    2327             : }
    2328             : 
    2329           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2330           0 :   switch (VT.SimpleTy) {
    2331           0 :   case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2332           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2333           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2334             :   default: return 0;
    2335             :   }
    2336             : }
    2337             : 
    2338             : // FastEmit functions for X86ISD::CALL.
    2339             : 
    2340             : unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2341           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2342             :     return 0;
    2343           0 :   if ((!Subtarget->is64Bit())) {
    2344           0 :     return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
    2345             :   }
    2346             :   return 0;
    2347             : }
    2348             : 
    2349           0 : unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2350           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2351             :     return 0;
    2352           0 :   if ((!Subtarget->is64Bit()) && (Subtarget->useRetpolineIndirectCalls())) {
    2353           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL32, &X86::GR32RegClass, Op0, Op0IsKill);
    2354             :   }
    2355           0 :   if ((!Subtarget->is64Bit()) && (!Subtarget->useRetpolineIndirectCalls())) {
    2356           0 :     return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
    2357             :   }
    2358             :   return 0;
    2359             : }
    2360             : 
    2361           0 : unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2362           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2363             :     return 0;
    2364           0 :   if ((Subtarget->is64Bit()) && (Subtarget->useRetpolineIndirectCalls())) {
    2365           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL64, &X86::GR64RegClass, Op0, Op0IsKill);
    2366             :   }
    2367           0 :   if ((Subtarget->is64Bit()) && (!Subtarget->useRetpolineIndirectCalls())) {
    2368           0 :     return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
    2369             :   }
    2370             :   return 0;
    2371             : }
    2372             : 
    2373           0 : unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2374           0 :   switch (VT.SimpleTy) {
    2375           0 :   case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2376           0 :   case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2377           0 :   case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2378             :   default: return 0;
    2379             :   }
    2380             : }
    2381             : 
    2382             : // FastEmit functions for X86ISD::COMPRESS.
    2383             : 
    2384           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2385           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2386             :     return 0;
    2387           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2388           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2389             :   }
    2390             :   return 0;
    2391             : }
    2392             : 
    2393           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2394           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    2395             :     return 0;
    2396           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2397           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2398             :   }
    2399             :   return 0;
    2400             : }
    2401             : 
    2402             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2403           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    2404             :     return 0;
    2405           0 :   if ((Subtarget->hasVBMI2())) {
    2406           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2407             :   }
    2408             :   return 0;
    2409             : }
    2410             : 
    2411           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2412           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    2413             :     return 0;
    2414           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2415           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2416             :   }
    2417             :   return 0;
    2418             : }
    2419             : 
    2420           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2421           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    2422             :     return 0;
    2423           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2424           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2425             :   }
    2426             :   return 0;
    2427             : }
    2428             : 
    2429             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2430           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    2431             :     return 0;
    2432           0 :   if ((Subtarget->hasVBMI2())) {
    2433           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2434             :   }
    2435             :   return 0;
    2436             : }
    2437             : 
    2438           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2439           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2440             :     return 0;
    2441           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2442           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2443             :   }
    2444             :   return 0;
    2445             : }
    2446             : 
    2447           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2448           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2449             :     return 0;
    2450           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2451           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2452             :   }
    2453             :   return 0;
    2454             : }
    2455             : 
    2456             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2457           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2458             :     return 0;
    2459           0 :   if ((Subtarget->hasAVX512())) {
    2460           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2461             :   }
    2462             :   return 0;
    2463             : }
    2464             : 
    2465           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2466           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2467             :     return 0;
    2468           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2469           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2470             :   }
    2471             :   return 0;
    2472             : }
    2473             : 
    2474           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2475           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2476             :     return 0;
    2477           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2478           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2479             :   }
    2480             :   return 0;
    2481             : }
    2482             : 
    2483             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2484           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2485             :     return 0;
    2486           0 :   if ((Subtarget->hasAVX512())) {
    2487           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2488             :   }
    2489             :   return 0;
    2490             : }
    2491             : 
    2492           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2493           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2494             :     return 0;
    2495           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2496           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2497             :   }
    2498             :   return 0;
    2499             : }
    2500             : 
    2501           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2502           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    2503             :     return 0;
    2504           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2505           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2506             :   }
    2507             :   return 0;
    2508             : }
    2509             : 
    2510             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2511           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2512             :     return 0;
    2513           0 :   if ((Subtarget->hasAVX512())) {
    2514           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2515             :   }
    2516             :   return 0;
    2517             : }
    2518             : 
    2519           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2520           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2521             :     return 0;
    2522           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2523           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2524             :   }
    2525             :   return 0;
    2526             : }
    2527             : 
    2528           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2529           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    2530             :     return 0;
    2531           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2532           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2533             :   }
    2534             :   return 0;
    2535             : }
    2536             : 
    2537             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2538           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    2539             :     return 0;
    2540           0 :   if ((Subtarget->hasAVX512())) {
    2541           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2542             :   }
    2543             :   return 0;
    2544             : }
    2545             : 
    2546           0 : unsigned fastEmit_X86ISD_COMPRESS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2547           0 :   switch (VT.SimpleTy) {
    2548           0 :   case MVT::v16i8: return fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2549           0 :   case MVT::v32i8: return fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    2550           0 :   case MVT::v64i8: return fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    2551           0 :   case MVT::v8i16: return fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2552           0 :   case MVT::v16i16: return fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2553           0 :   case MVT::v32i16: return fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2554           0 :   case MVT::v4i32: return fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2555           0 :   case MVT::v8i32: return fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2556           0 :   case MVT::v16i32: return fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2557           0 :   case MVT::v2i64: return fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2558           0 :   case MVT::v4i64: return fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2559           0 :   case MVT::v8i64: return fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2560           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2561           0 :   case MVT::v8f32: return fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2562           0 :   case MVT::v16f32: return fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2563           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2564           0 :   case MVT::v4f64: return fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2565           0 :   case MVT::v8f64: return fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2566             :   default: return 0;
    2567             :   }
    2568             : }
    2569             : 
    2570             : // FastEmit functions for X86ISD::CONFLICT.
    2571             : 
    2572           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2573           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2574             :     return 0;
    2575           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2576           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2577             :   }
    2578             :   return 0;
    2579             : }
    2580             : 
    2581           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2582           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2583             :     return 0;
    2584           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2585           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2586             :   }
    2587             :   return 0;
    2588             : }
    2589             : 
    2590             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2591           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2592             :     return 0;
    2593           0 :   if ((Subtarget->hasCDI())) {
    2594           0 :     return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2595             :   }
    2596             :   return 0;
    2597             : }
    2598             : 
    2599           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2600           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2601             :     return 0;
    2602           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2603           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2604             :   }
    2605             :   return 0;
    2606             : }
    2607             : 
    2608           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2609           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2610             :     return 0;
    2611           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2612           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2613             :   }
    2614             :   return 0;
    2615             : }
    2616             : 
    2617             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2618           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2619             :     return 0;
    2620           0 :   if ((Subtarget->hasCDI())) {
    2621           0 :     return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2622             :   }
    2623             :   return 0;
    2624             : }
    2625             : 
    2626           0 : unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2627           0 :   switch (VT.SimpleTy) {
    2628           0 :   case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2629           0 :   case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2630           0 :   case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2631           0 :   case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2632           0 :   case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2633           0 :   case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2634             :   default: return 0;
    2635             :   }
    2636             : }
    2637             : 
    2638             : // FastEmit functions for X86ISD::CVTP2SI.
    2639             : 
    2640           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2641           0 :   if ((Subtarget->hasVLX())) {
    2642           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2643             :   }
    2644           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2645           0 :     return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2646             :   }
    2647           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2648           0 :     return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2649             :   }
    2650             :   return 0;
    2651             : }
    2652             : 
    2653           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2654           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2655           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2656             :   }
    2657             :   return 0;
    2658             : }
    2659             : 
    2660           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2661           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2662           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2663             :   }
    2664             :   return 0;
    2665             : }
    2666             : 
    2667           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2668           0 : switch (RetVT.SimpleTy) {
    2669           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2670           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2671           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2672             :   default: return 0;
    2673             : }
    2674             : }
    2675             : 
    2676           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2677           0 :   if ((Subtarget->hasVLX())) {
    2678           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2679             :   }
    2680           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2681           0 :     return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    2682             :   }
    2683             :   return 0;
    2684             : }
    2685             : 
    2686             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2687           0 :   if ((Subtarget->hasDQI())) {
    2688           0 :     return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2689             :   }
    2690             :   return 0;
    2691             : }
    2692             : 
    2693           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2694           0 : switch (RetVT.SimpleTy) {
    2695           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2696           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2697             :   default: return 0;
    2698             : }
    2699             : }
    2700             : 
    2701             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2702           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2703             :     return 0;
    2704           0 :   if ((Subtarget->hasAVX512())) {
    2705           0 :     return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2706             :   }
    2707             :   return 0;
    2708             : }
    2709             : 
    2710           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2711           0 :   if ((Subtarget->hasVLX())) {
    2712           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2713             :   }
    2714           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2715           0 :     return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2716             :   }
    2717           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2718           0 :     return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2719             :   }
    2720             :   return 0;
    2721             : }
    2722             : 
    2723           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2724           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2725           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2726             :   }
    2727             :   return 0;
    2728             : }
    2729             : 
    2730           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2731           0 : switch (RetVT.SimpleTy) {
    2732           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2733           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2734             :   default: return 0;
    2735             : }
    2736             : }
    2737             : 
    2738           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2739           0 :   if ((Subtarget->hasVLX())) {
    2740           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2741             :   }
    2742           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2743           0 :     return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2744             :   }
    2745             :   return 0;
    2746             : }
    2747             : 
    2748           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2749           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2750           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2751             :   }
    2752             :   return 0;
    2753             : }
    2754             : 
    2755           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2756           0 : switch (RetVT.SimpleTy) {
    2757           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2758           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2759             :   default: return 0;
    2760             : }
    2761             : }
    2762             : 
    2763             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2764           0 :   if ((Subtarget->hasAVX512())) {
    2765           0 :     return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2766             :   }
    2767             :   return 0;
    2768             : }
    2769             : 
    2770             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2771           0 :   if ((Subtarget->hasDQI())) {
    2772           0 :     return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2773             :   }
    2774             :   return 0;
    2775             : }
    2776             : 
    2777           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2778           0 : switch (RetVT.SimpleTy) {
    2779           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2780           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2781             :   default: return 0;
    2782             : }
    2783             : }
    2784             : 
    2785           0 : unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2786           0 :   switch (VT.SimpleTy) {
    2787           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2788           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2789           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2790           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2791           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2792           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2793             :   default: return 0;
    2794             :   }
    2795             : }
    2796             : 
    2797             : // FastEmit functions for X86ISD::CVTP2UI.
    2798             : 
    2799             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2800           0 :   if ((Subtarget->hasVLX())) {
    2801           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2802             :   }
    2803             :   return 0;
    2804             : }
    2805             : 
    2806           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2807           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2808           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2809             :   }
    2810             :   return 0;
    2811             : }
    2812             : 
    2813           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2814           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2815           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2816             :   }
    2817             :   return 0;
    2818             : }
    2819             : 
    2820           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2821           0 : switch (RetVT.SimpleTy) {
    2822           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2823           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2824           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2825             :   default: return 0;
    2826             : }
    2827             : }
    2828             : 
    2829             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2830           0 :   if ((Subtarget->hasVLX())) {
    2831           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2832             :   }
    2833             :   return 0;
    2834             : }
    2835             : 
    2836             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2837           0 :   if ((Subtarget->hasDQI())) {
    2838           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2839             :   }
    2840             :   return 0;
    2841             : }
    2842             : 
    2843           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2844           0 : switch (RetVT.SimpleTy) {
    2845           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2846           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2847             :   default: return 0;
    2848             : }
    2849             : }
    2850             : 
    2851             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2852           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2853             :     return 0;
    2854           0 :   if ((Subtarget->hasAVX512())) {
    2855           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2856             :   }
    2857             :   return 0;
    2858             : }
    2859             : 
    2860             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2861           0 :   if ((Subtarget->hasVLX())) {
    2862           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2863             :   }
    2864             :   return 0;
    2865             : }
    2866             : 
    2867           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2868           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2869           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2870             :   }
    2871             :   return 0;
    2872             : }
    2873             : 
    2874           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2875           0 : switch (RetVT.SimpleTy) {
    2876           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2877           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2878             :   default: return 0;
    2879             : }
    2880             : }
    2881             : 
    2882             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2883           0 :   if ((Subtarget->hasVLX())) {
    2884           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2885             :   }
    2886             :   return 0;
    2887             : }
    2888             : 
    2889           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2890           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2891           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2892             :   }
    2893             :   return 0;
    2894             : }
    2895             : 
    2896           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2897           0 : switch (RetVT.SimpleTy) {
    2898           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2899           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2900             :   default: return 0;
    2901             : }
    2902             : }
    2903             : 
    2904             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2905           0 :   if ((Subtarget->hasAVX512())) {
    2906           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2907             :   }
    2908             :   return 0;
    2909             : }
    2910             : 
    2911             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2912           0 :   if ((Subtarget->hasDQI())) {
    2913           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2914             :   }
    2915             :   return 0;
    2916             : }
    2917             : 
    2918           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2919           0 : switch (RetVT.SimpleTy) {
    2920           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2921           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2922             :   default: return 0;
    2923             : }
    2924             : }
    2925             : 
    2926           0 : unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2927           0 :   switch (VT.SimpleTy) {
    2928           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2929           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2930           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2931           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2932           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2933           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2934             :   default: return 0;
    2935             :   }
    2936             : }
    2937             : 
    2938             : // FastEmit functions for X86ISD::CVTPH2PS.
    2939             : 
    2940           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2941           0 :   if ((Subtarget->hasVLX())) {
    2942           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2943             :   }
    2944           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2945           0 :     return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2946             :   }
    2947             :   return 0;
    2948             : }
    2949             : 
    2950           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2951           0 :   if ((Subtarget->hasVLX())) {
    2952           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2953             :   }
    2954           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2955           0 :     return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    2956             :   }
    2957             :   return 0;
    2958             : }
    2959             : 
    2960           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2961           0 : switch (RetVT.SimpleTy) {
    2962           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
    2963           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
    2964             :   default: return 0;
    2965             : }
    2966             : }
    2967             : 
    2968             : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2969           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2970             :     return 0;
    2971           0 :   if ((Subtarget->hasAVX512())) {
    2972           0 :     return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2973             :   }
    2974             :   return 0;
    2975             : }
    2976             : 
    2977           0 : unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2978           0 :   switch (VT.SimpleTy) {
    2979           0 :   case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2980           0 :   case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2981             :   default: return 0;
    2982             :   }
    2983             : }
    2984             : 
    2985             : // FastEmit functions for X86ISD::CVTS2SI.
    2986             : 
    2987           0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    2988           0 :   if ((Subtarget->hasAVX512())) {
    2989           0 :     return fastEmitInst_r(X86::VCVTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    2990             :   }
    2991           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    2992           0 :     return fastEmitInst_r(X86::CVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    2993             :   }
    2994           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    2995           0 :     return fastEmitInst_r(X86::VCVTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    2996             :   }
    2997             :   return 0;
    2998             : }
    2999             : 
    3000           0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3001           0 :   if ((Subtarget->hasAVX512())) {
    3002           0 :     return fastEmitInst_r(X86::VCVTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3003             :   }
    3004           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3005           0 :     return fastEmitInst_r(X86::CVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3006             :   }
    3007           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    3008           0 :     return fastEmitInst_r(X86::VCVTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3009             :   }
    3010             :   return 0;
    3011             : }
    3012             : 
    3013           0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3014           0 : switch (RetVT.SimpleTy) {
    3015           0 :   case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
    3016           0 :   case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
    3017             :   default: return 0;
    3018             : }
    3019             : }
    3020             : 
    3021           0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3022           0 :   if ((Subtarget->hasAVX512())) {
    3023           0 :     return fastEmitInst_r(X86::VCVTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3024             :   }
    3025           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3026           0 :     return fastEmitInst_r(X86::CVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3027             :   }
    3028           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    3029           0 :     return fastEmitInst_r(X86::VCVTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3030             :   }
    3031             :   return 0;
    3032             : }
    3033             : 
    3034           0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3035           0 :   if ((Subtarget->hasAVX512())) {
    3036           0 :     return fastEmitInst_r(X86::VCVTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3037             :   }
    3038           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3039           0 :     return fastEmitInst_r(X86::CVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3040             :   }
    3041           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    3042           0 :     return fastEmitInst_r(X86::VCVTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3043             :   }
    3044             :   return 0;
    3045             : }
    3046             : 
    3047           0 : unsigned fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3048           0 : switch (RetVT.SimpleTy) {
    3049           0 :   case MVT::i32: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
    3050           0 :   case MVT::i64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
    3051             :   default: return 0;
    3052             : }
    3053             : }
    3054             : 
    3055           0 : unsigned fastEmit_X86ISD_CVTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3056           0 :   switch (VT.SimpleTy) {
    3057           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3058           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3059             :   default: return 0;
    3060             :   }
    3061             : }
    3062             : 
    3063             : // FastEmit functions for X86ISD::CVTS2UI.
    3064             : 
    3065             : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3066           0 :   if ((Subtarget->hasAVX512())) {
    3067           0 :     return fastEmitInst_r(X86::VCVTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3068             :   }
    3069             :   return 0;
    3070             : }
    3071             : 
    3072             : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3073           0 :   if ((Subtarget->hasAVX512())) {
    3074           0 :     return fastEmitInst_r(X86::VCVTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3075             :   }
    3076             :   return 0;
    3077             : }
    3078             : 
    3079           0 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3080           0 : switch (RetVT.SimpleTy) {
    3081           0 :   case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
    3082           0 :   case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
    3083             :   default: return 0;
    3084             : }
    3085             : }
    3086             : 
    3087             : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3088           0 :   if ((Subtarget->hasAVX512())) {
    3089           0 :     return fastEmitInst_r(X86::VCVTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3090             :   }
    3091             :   return 0;
    3092             : }
    3093             : 
    3094             : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3095           0 :   if ((Subtarget->hasAVX512())) {
    3096           0 :     return fastEmitInst_r(X86::VCVTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3097             :   }
    3098             :   return 0;
    3099             : }
    3100             : 
    3101           0 : unsigned fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3102           0 : switch (RetVT.SimpleTy) {
    3103           0 :   case MVT::i32: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
    3104           0 :   case MVT::i64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
    3105             :   default: return 0;
    3106             : }
    3107             : }
    3108             : 
    3109           0 : unsigned fastEmit_X86ISD_CVTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3110           0 :   switch (VT.SimpleTy) {
    3111           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3112           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3113             :   default: return 0;
    3114             :   }
    3115             : }
    3116             : 
    3117             : // FastEmit functions for X86ISD::CVTSI2P.
    3118             : 
    3119           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3120           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3121             :     return 0;
    3122           0 :   if ((Subtarget->hasVLX())) {
    3123           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3124             :   }
    3125           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3126           0 :     return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3127             :   }
    3128           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3129           0 :     return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3130             :   }
    3131             :   return 0;
    3132             : }
    3133             : 
    3134           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3135           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3136             :     return 0;
    3137           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3138           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3139             :   }
    3140             :   return 0;
    3141             : }
    3142             : 
    3143           0 : unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3144           0 :   switch (VT.SimpleTy) {
    3145           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3146           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3147             :   default: return 0;
    3148             :   }
    3149             : }
    3150             : 
    3151             : // FastEmit functions for X86ISD::CVTTP2SI.
    3152             : 
    3153           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3154           0 :   if ((Subtarget->hasVLX())) {
    3155           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3156             :   }
    3157           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3158           0 :     return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3159             :   }
    3160           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3161           0 :     return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3162             :   }
    3163             :   return 0;
    3164             : }
    3165             : 
    3166           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3167           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3168           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3169             :   }
    3170             :   return 0;
    3171             : }
    3172             : 
    3173           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3174           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3175           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3176             :   }
    3177             :   return 0;
    3178             : }
    3179             : 
    3180           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3181           0 : switch (RetVT.SimpleTy) {
    3182           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    3183           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    3184           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    3185             :   default: return 0;
    3186             : }
    3187             : }
    3188             : 
    3189           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3190           0 :   if ((Subtarget->hasVLX())) {
    3191           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3192             :   }
    3193           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3194           0 :     return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3195             :   }
    3196             :   return 0;
    3197             : }
    3198             : 
    3199             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3200           0 :   if ((Subtarget->hasDQI())) {
    3201           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3202             :   }
    3203             :   return 0;
    3204             : }
    3205             : 
    3206           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3207           0 : switch (RetVT.SimpleTy) {
    3208           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    3209           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    3210             :   default: return 0;
    3211             : }
    3212             : }
    3213             : 
    3214             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3215           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3216             :     return 0;
    3217           0 :   if ((Subtarget->hasAVX512())) {
    3218           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3219             :   }
    3220             :   return 0;
    3221             : }
    3222             : 
    3223           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3224           0 :   if ((Subtarget->hasVLX())) {
    3225           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3226             :   }
    3227           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3228           0 :     return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3229             :   }
    3230           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3231           0 :     return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3232             :   }
    3233             :   return 0;
    3234             : }
    3235             : 
    3236           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3237           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3238           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3239             :   }
    3240             :   return 0;
    3241             : }
    3242             : 
    3243           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3244           0 : switch (RetVT.SimpleTy) {
    3245           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    3246           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    3247             :   default: return 0;
    3248             : }
    3249             : }
    3250             : 
    3251           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3252           0 :   if ((Subtarget->hasVLX())) {
    3253           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3254             :   }
    3255           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3256           0 :     return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3257             :   }
    3258             :   return 0;
    3259             : }
    3260             : 
    3261           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3262           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3263           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3264             :   }
    3265             :   return 0;
    3266             : }
    3267             : 
    3268           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3269           0 : switch (RetVT.SimpleTy) {
    3270           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    3271           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    3272             :   default: return 0;
    3273             : }
    3274             : }
    3275             : 
    3276             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3277           0 :   if ((Subtarget->hasAVX512())) {
    3278           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3279             :   }
    3280             :   return 0;
    3281             : }
    3282             : 
    3283             : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3284           0 :   if ((Subtarget->hasDQI())) {
    3285           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3286             :   }
    3287             :   return 0;
    3288             : }
    3289             : 
    3290           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3291           0 : switch (RetVT.SimpleTy) {
    3292           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    3293           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    3294             :   default: return 0;
    3295             : }
    3296             : }
    3297             : 
    3298           0 : unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3299           0 :   switch (VT.SimpleTy) {
    3300           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3301           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3302           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3303           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3304           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3305           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3306             :   default: return 0;
    3307             :   }
    3308             : }
    3309             : 
    3310             : // FastEmit functions for X86ISD::CVTTP2UI.
    3311             : 
    3312             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3313           0 :   if ((Subtarget->hasVLX())) {
    3314           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3315             :   }
    3316             :   return 0;
    3317             : }
    3318             : 
    3319           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3320           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3321           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3322             :   }
    3323             :   return 0;
    3324             : }
    3325             : 
    3326           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3327           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3328           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3329             :   }
    3330             :   return 0;
    3331             : }
    3332             : 
    3333           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3334           0 : switch (RetVT.SimpleTy) {
    3335           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    3336           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    3337           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    3338             :   default: return 0;
    3339             : }
    3340             : }
    3341             : 
    3342             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3343           0 :   if ((Subtarget->hasVLX())) {
    3344           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3345             :   }
    3346             :   return 0;
    3347             : }
    3348             : 
    3349             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3350           0 :   if ((Subtarget->hasDQI())) {
    3351           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3352             :   }
    3353             :   return 0;
    3354             : }
    3355             : 
    3356           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3357           0 : switch (RetVT.SimpleTy) {
    3358           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    3359           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    3360             :   default: return 0;
    3361             : }
    3362             : }
    3363             : 
    3364             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3365           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3366             :     return 0;
    3367           0 :   if ((Subtarget->hasAVX512())) {
    3368           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3369             :   }
    3370             :   return 0;
    3371             : }
    3372             : 
    3373             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3374           0 :   if ((Subtarget->hasVLX())) {
    3375           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3376             :   }
    3377             :   return 0;
    3378             : }
    3379             : 
    3380           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3381           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3382           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3383             :   }
    3384             :   return 0;
    3385             : }
    3386             : 
    3387           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3388           0 : switch (RetVT.SimpleTy) {
    3389           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    3390           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    3391             :   default: return 0;
    3392             : }
    3393             : }
    3394             : 
    3395             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3396           0 :   if ((Subtarget->hasVLX())) {
    3397           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3398             :   }
    3399             :   return 0;
    3400             : }
    3401             : 
    3402           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3403           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3404           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3405             :   }
    3406             :   return 0;
    3407             : }
    3408             : 
    3409           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3410           0 : switch (RetVT.SimpleTy) {
    3411           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    3412           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    3413             :   default: return 0;
    3414             : }
    3415             : }
    3416             : 
    3417             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3418           0 :   if ((Subtarget->hasAVX512())) {
    3419           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3420             :   }
    3421             :   return 0;
    3422             : }
    3423             : 
    3424             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3425           0 :   if ((Subtarget->hasDQI())) {
    3426           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3427             :   }
    3428             :   return 0;
    3429             : }
    3430             : 
    3431           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3432           0 : switch (RetVT.SimpleTy) {
    3433           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    3434           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    3435             :   default: return 0;
    3436             : }
    3437             : }
    3438             : 
    3439           0 : unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3440           0 :   switch (VT.SimpleTy) {
    3441           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3442           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3443           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3444           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3445           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3446           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3447             :   default: return 0;
    3448             :   }
    3449             : }
    3450             : 
    3451             : // FastEmit functions for X86ISD::CVTTS2SI.
    3452             : 
    3453           0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3454           0 :   if ((Subtarget->hasAVX512())) {
    3455           0 :     return fastEmitInst_r(X86::VCVTTSS2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3456             :   }
    3457           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3458           0 :     return fastEmitInst_r(X86::CVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3459             :   }
    3460           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    3461           0 :     return fastEmitInst_r(X86::VCVTTSS2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3462             :   }
    3463             :   return 0;
    3464             : }
    3465             : 
    3466           0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3467           0 :   if ((Subtarget->hasAVX512())) {
    3468           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3469             :   }
    3470           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3471           0 :     return fastEmitInst_r(X86::CVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3472             :   }
    3473           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    3474           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3475             :   }
    3476             :   return 0;
    3477             : }
    3478             : 
    3479           0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3480           0 : switch (RetVT.SimpleTy) {
    3481           0 :   case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
    3482           0 :   case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
    3483             :   default: return 0;
    3484             : }
    3485             : }
    3486             : 
    3487           0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3488           0 :   if ((Subtarget->hasAVX512())) {
    3489           0 :     return fastEmitInst_r(X86::VCVTTSD2SIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3490             :   }
    3491           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3492           0 :     return fastEmitInst_r(X86::CVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3493             :   }
    3494           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    3495           0 :     return fastEmitInst_r(X86::VCVTTSD2SIrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3496             :   }
    3497             :   return 0;
    3498             : }
    3499             : 
    3500           0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3501           0 :   if ((Subtarget->hasAVX512())) {
    3502           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3503             :   }
    3504           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3505           0 :     return fastEmitInst_r(X86::CVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3506             :   }
    3507           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    3508           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64rr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3509             :   }
    3510             :   return 0;
    3511             : }
    3512             : 
    3513           0 : unsigned fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3514           0 : switch (RetVT.SimpleTy) {
    3515           0 :   case MVT::i32: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
    3516           0 :   case MVT::i64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
    3517             :   default: return 0;
    3518             : }
    3519             : }
    3520             : 
    3521           0 : unsigned fastEmit_X86ISD_CVTTS2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3522           0 :   switch (VT.SimpleTy) {
    3523           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTS2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3524           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTS2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3525             :   default: return 0;
    3526             :   }
    3527             : }
    3528             : 
    3529             : // FastEmit functions for X86ISD::CVTTS2UI.
    3530             : 
    3531             : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3532           0 :   if ((Subtarget->hasAVX512())) {
    3533           0 :     return fastEmitInst_r(X86::VCVTTSS2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3534             :   }
    3535             :   return 0;
    3536             : }
    3537             : 
    3538             : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3539           0 :   if ((Subtarget->hasAVX512())) {
    3540           0 :     return fastEmitInst_r(X86::VCVTTSS2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3541             :   }
    3542             :   return 0;
    3543             : }
    3544             : 
    3545           0 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3546           0 : switch (RetVT.SimpleTy) {
    3547           0 :   case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i32_r(Op0, Op0IsKill);
    3548           0 :   case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_MVT_i64_r(Op0, Op0IsKill);
    3549             :   default: return 0;
    3550             : }
    3551             : }
    3552             : 
    3553             : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    3554           0 :   if ((Subtarget->hasAVX512())) {
    3555           0 :     return fastEmitInst_r(X86::VCVTTSD2USIZrr_Int, &X86::GR32RegClass, Op0, Op0IsKill);
    3556             :   }
    3557             :   return 0;
    3558             : }
    3559             : 
    3560             : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    3561           0 :   if ((Subtarget->hasAVX512())) {
    3562           0 :     return fastEmitInst_r(X86::VCVTTSD2USI64Zrr_Int, &X86::GR64RegClass, Op0, Op0IsKill);
    3563             :   }
    3564             :   return 0;
    3565             : }
    3566             : 
    3567           0 : unsigned fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3568           0 : switch (RetVT.SimpleTy) {
    3569           0 :   case MVT::i32: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i32_r(Op0, Op0IsKill);
    3570           0 :   case MVT::i64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_MVT_i64_r(Op0, Op0IsKill);
    3571             :   default: return 0;
    3572             : }
    3573             : }
    3574             : 
    3575           0 : unsigned fastEmit_X86ISD_CVTTS2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3576           0 :   switch (VT.SimpleTy) {
    3577           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTS2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3578           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTS2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3579             :   default: return 0;
    3580             :   }
    3581             : }
    3582             : 
    3583             : // FastEmit functions for X86ISD::CVTUI2P.
    3584             : 
    3585             : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3586           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3587             :     return 0;
    3588           0 :   if ((Subtarget->hasVLX())) {
    3589           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3590             :   }
    3591             :   return 0;
    3592             : }
    3593             : 
    3594           0 : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3595           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3596             :     return 0;
    3597           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3598           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3599             :   }
    3600             :   return 0;
    3601             : }
    3602             : 
    3603           0 : unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3604           0 :   switch (VT.SimpleTy) {
    3605           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3606           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3607             :   default: return 0;
    3608             :   }
    3609             : }
    3610             : 
    3611             : // FastEmit functions for X86ISD::EH_RETURN.
    3612             : 
    3613             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3614           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3615             :     return 0;
    3616           0 :   return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
    3617             : }
    3618             : 
    3619             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3620           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3621             :     return 0;
    3622           0 :   return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
    3623             : }
    3624             : 
    3625           0 : unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3626           0 :   switch (VT.SimpleTy) {
    3627           0 :   case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3628           0 :   case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3629             :   default: return 0;
    3630             :   }
    3631             : }
    3632             : 
    3633             : // FastEmit functions for X86ISD::EXPAND.
    3634             : 
    3635           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3636           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    3637             :     return 0;
    3638           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3639           0 :     return fastEmitInst_r(X86::VPEXPANDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3640             :   }
    3641             :   return 0;
    3642             : }
    3643             : 
    3644           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3645           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    3646             :     return 0;
    3647           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3648           0 :     return fastEmitInst_r(X86::VPEXPANDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3649             :   }
    3650             :   return 0;
    3651             : }
    3652             : 
    3653             : unsigned fastEmit_X86ISD_EXPAND_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3654           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    3655             :     return 0;
    3656           0 :   if ((Subtarget->hasVBMI2())) {
    3657           0 :     return fastEmitInst_r(X86::VPEXPANDBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3658             :   }
    3659             :   return 0;
    3660             : }
    3661             : 
    3662           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3663           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3664             :     return 0;
    3665           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3666           0 :     return fastEmitInst_r(X86::VPEXPANDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3667             :   }
    3668             :   return 0;
    3669             : }
    3670             : 
    3671           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3672           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    3673             :     return 0;
    3674           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3675           0 :     return fastEmitInst_r(X86::VPEXPANDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3676             :   }
    3677             :   return 0;
    3678             : }
    3679             : 
    3680             : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3681           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    3682             :     return 0;
    3683           0 :   if ((Subtarget->hasVBMI2())) {
    3684           0 :     return fastEmitInst_r(X86::VPEXPANDWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3685             :   }
    3686             :   return 0;
    3687             : }
    3688             : 
    3689           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3690           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3691             :     return 0;
    3692           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3693           0 :     return fastEmitInst_r(X86::VPEXPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3694             :   }
    3695             :   return 0;
    3696             : }
    3697             : 
    3698           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3699           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3700             :     return 0;
    3701           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3702           0 :     return fastEmitInst_r(X86::VPEXPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3703             :   }
    3704             :   return 0;
    3705             : }
    3706             : 
    3707             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3708           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3709             :     return 0;
    3710           0 :   if ((Subtarget->hasAVX512())) {
    3711           0 :     return fastEmitInst_r(X86::VPEXPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3712             :   }
    3713             :   return 0;
    3714             : }
    3715             : 
    3716           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3717           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3718             :     return 0;
    3719           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3720           0 :     return fastEmitInst_r(X86::VPEXPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3721             :   }
    3722             :   return 0;
    3723             : }
    3724             : 
    3725           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3726           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    3727             :     return 0;
    3728           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3729           0 :     return fastEmitInst_r(X86::VPEXPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3730             :   }
    3731             :   return 0;
    3732             : }
    3733             : 
    3734             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3735           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    3736             :     return 0;
    3737           0 :   if ((Subtarget->hasAVX512())) {
    3738           0 :     return fastEmitInst_r(X86::VPEXPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3739             :   }
    3740             :   return 0;
    3741             : }
    3742             : 
    3743           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3744           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3745             :     return 0;
    3746           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3747           0 :     return fastEmitInst_r(X86::VEXPANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3748             :   }
    3749             :   return 0;
    3750             : }
    3751             : 
    3752           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3753           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3754             :     return 0;
    3755           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3756           0 :     return fastEmitInst_r(X86::VEXPANDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3757             :   }
    3758             :   return 0;
    3759             : }
    3760             : 
    3761             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3762           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3763             :     return 0;
    3764           0 :   if ((Subtarget->hasAVX512())) {
    3765           0 :     return fastEmitInst_r(X86::VEXPANDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3766             :   }
    3767             :   return 0;
    3768             : }
    3769             : 
    3770           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3771           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3772             :     return 0;
    3773           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3774           0 :     return fastEmitInst_r(X86::VEXPANDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3775             :   }
    3776             :   return 0;
    3777             : }
    3778             : 
    3779           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3780           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3781             :     return 0;
    3782           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3783           0 :     return fastEmitInst_r(X86::VEXPANDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3784             :   }
    3785             :   return 0;
    3786             : }
    3787             : 
    3788             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3789           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3790             :     return 0;
    3791           0 :   if ((Subtarget->hasAVX512())) {
    3792           0 :     return fastEmitInst_r(X86::VEXPANDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3793             :   }
    3794             :   return 0;
    3795             : }
    3796             : 
    3797           0 : unsigned fastEmit_X86ISD_EXPAND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3798           0 :   switch (VT.SimpleTy) {
    3799           0 :   case MVT::v16i8: return fastEmit_X86ISD_EXPAND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3800           0 :   case MVT::v32i8: return fastEmit_X86ISD_EXPAND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3801           0 :   case MVT::v64i8: return fastEmit_X86ISD_EXPAND_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    3802           0 :   case MVT::v8i16: return fastEmit_X86ISD_EXPAND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3803           0 :   case MVT::v16i16: return fastEmit_X86ISD_EXPAND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    3804           0 :   case MVT::v32i16: return fastEmit_X86ISD_EXPAND_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    3805           0 :   case MVT::v4i32: return fastEmit_X86ISD_EXPAND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3806           0 :   case MVT::v8i32: return fastEmit_X86ISD_EXPAND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3807           0 :   case MVT::v16i32: return fastEmit_X86ISD_EXPAND_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    3808           0 :   case MVT::v2i64: return fastEmit_X86ISD_EXPAND_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3809           0 :   case MVT::v4i64: return fastEmit_X86ISD_EXPAND_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    3810           0 :   case MVT::v8i64: return fastEmit_X86ISD_EXPAND_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    3811           0 :   case MVT::v4f32: return fastEmit_X86ISD_EXPAND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3812           0 :   case MVT::v8f32: return fastEmit_X86ISD_EXPAND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3813           0 :   case MVT::v16f32: return fastEmit_X86ISD_EXPAND_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3814           0 :   case MVT::v2f64: return fastEmit_X86ISD_EXPAND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3815           0 :   case MVT::v4f64: return fastEmit_X86ISD_EXPAND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3816           0 :   case MVT::v8f64: return fastEmit_X86ISD_EXPAND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3817             :   default: return 0;
    3818             :   }
    3819             : }
    3820             : 
    3821             : // FastEmit functions for X86ISD::FRCP.
    3822             : 
    3823           0 : unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3824           0 :   if (RetVT.SimpleTy != MVT::f32)
    3825             :     return 0;
    3826           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3827           0 :     return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3828             :   }
    3829             :   return 0;
    3830             : }
    3831             : 
    3832           0 : unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3833           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3834             :     return 0;
    3835           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3836           0 :     return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3837             :   }
    3838           0 :   if ((Subtarget->hasAVX())) {
    3839           0 :     return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3840             :   }
    3841             :   return 0;
    3842             : }
    3843             : 
    3844             : unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3845           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3846             :     return 0;
    3847           0 :   if ((Subtarget->hasAVX())) {
    3848           0 :     return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3849             :   }
    3850             :   return 0;
    3851             : }
    3852             : 
    3853           0 : unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3854           0 :   switch (VT.SimpleTy) {
    3855           0 :   case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3856           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3857           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3858             :   default: return 0;
    3859             :   }
    3860             : }
    3861             : 
    3862             : // FastEmit functions for X86ISD::FRSQRT.
    3863             : 
    3864           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3865           0 :   if (RetVT.SimpleTy != MVT::f32)
    3866             :     return 0;
    3867           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3868           0 :     return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3869             :   }
    3870             :   return 0;
    3871             : }
    3872             : 
    3873           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3874           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3875             :     return 0;
    3876           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3877           0 :     return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3878             :   }
    3879           0 :   if ((Subtarget->hasAVX())) {
    3880           0 :     return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3881             :   }
    3882             :   return 0;
    3883             : }
    3884             : 
    3885             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3886           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3887             :     return 0;
    3888           0 :   if ((Subtarget->hasAVX())) {
    3889           0 :     return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3890             :   }
    3891             :   return 0;
    3892             : }
    3893             : 
    3894           0 : unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3895           0 :   switch (VT.SimpleTy) {
    3896           0 :   case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3897           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3898           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3899             :   default: return 0;
    3900             :   }
    3901             : }
    3902             : 
    3903             : // FastEmit functions for X86ISD::MMX_MOVD2W.
    3904             : 
    3905             : unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3906           0 :   if (RetVT.SimpleTy != MVT::i32)
    3907             :     return 0;
    3908           0 :   if ((Subtarget->hasMMX())) {
    3909           0 :     return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
    3910             :   }
    3911             :   return 0;
    3912             : }
    3913             : 
    3914           0 : unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3915           0 :   switch (VT.SimpleTy) {
    3916           0 :   case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
    3917             :   default: return 0;
    3918             :   }
    3919             : }
    3920             : 
    3921             : // FastEmit functions for X86ISD::MMX_MOVW2D.
    3922             : 
    3923             : unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3924           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3925             :     return 0;
    3926           0 :   if ((Subtarget->hasMMX())) {
    3927           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    3928             :   }
    3929             :   return 0;
    3930             : }
    3931             : 
    3932           0 : unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3933           0 :   switch (VT.SimpleTy) {
    3934           0 :   case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3935             :   default: return 0;
    3936             :   }
    3937             : }
    3938             : 
    3939             : // FastEmit functions for X86ISD::MOVDDUP.
    3940             : 
    3941           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3942           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3943             :     return 0;
    3944           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3945           0 :     return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3946             :   }
    3947           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3948           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3949             :   }
    3950             :   return 0;
    3951             : }
    3952             : 
    3953           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3954           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3955             :     return 0;
    3956           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3957           0 :     return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3958             :   }
    3959           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3960           0 :     return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3961             :   }
    3962             :   return 0;
    3963             : }
    3964             : 
    3965             : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3966           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3967             :     return 0;
    3968           0 :   if ((Subtarget->hasAVX512())) {
    3969           0 :     return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3970             :   }
    3971             :   return 0;
    3972             : }
    3973             : 
    3974           0 : unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3975           0 :   switch (VT.SimpleTy) {
    3976           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3977           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3978           0 :   case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3979             :   default: return 0;
    3980             :   }
    3981             : }
    3982             : 
    3983             : // FastEmit functions for X86ISD::MOVDQ2Q.
    3984             : 
    3985             : unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3986           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3987             :     return 0;
    3988           0 :   return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
    3989             : }
    3990             : 
    3991             : unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3992           0 :   switch (VT.SimpleTy) {
    3993           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3994             :   default: return 0;
    3995             :   }
    3996             : }
    3997             : 
    3998             : // FastEmit functions for X86ISD::MOVMSK.
    3999             : 
    4000           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4001           0 :   if (RetVT.SimpleTy != MVT::i32)
    4002             :     return 0;
    4003           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4004           0 :     return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4005             :   }
    4006           0 :   if ((Subtarget->hasAVX())) {
    4007           0 :     return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4008             :   }
    4009             :   return 0;
    4010             : }
    4011             : 
    4012             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4013           0 :   if (RetVT.SimpleTy != MVT::i32)
    4014             :     return 0;
    4015           0 :   if ((Subtarget->hasAVX2())) {
    4016           0 :     return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4017             :   }
    4018             :   return 0;
    4019             : }
    4020             : 
    4021           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4022           0 :   if (RetVT.SimpleTy != MVT::i32)
    4023             :     return 0;
    4024           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4025           0 :     return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4026             :   }
    4027           0 :   if ((Subtarget->hasAVX())) {
    4028           0 :     return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4029             :   }
    4030             :   return 0;
    4031             : }
    4032             : 
    4033             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4034           0 :   if (RetVT.SimpleTy != MVT::i32)
    4035             :     return 0;
    4036           0 :   if ((Subtarget->hasAVX())) {
    4037           0 :     return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4038             :   }
    4039             :   return 0;
    4040             : }
    4041             : 
    4042           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4043           0 :   if (RetVT.SimpleTy != MVT::i32)
    4044             :     return 0;
    4045           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4046           0 :     return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4047             :   }
    4048           0 :   if ((Subtarget->hasAVX())) {
    4049           0 :     return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4050             :   }
    4051             :   return 0;
    4052             : }
    4053             : 
    4054             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4055           0 :   if (RetVT.SimpleTy != MVT::i32)
    4056             :     return 0;
    4057           0 :   if ((Subtarget->hasAVX())) {
    4058           0 :     return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4059             :   }
    4060             :   return 0;
    4061             : }
    4062             : 
    4063           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4064           0 :   if (RetVT.SimpleTy != MVT::i32)
    4065             :     return 0;
    4066           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    4067           0 :     return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4068             :   }
    4069           0 :   if ((Subtarget->hasAVX())) {
    4070           0 :     return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4071             :   }
    4072             :   return 0;
    4073             : }
    4074             : 
    4075             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4076           0 :   if (RetVT.SimpleTy != MVT::i32)
    4077             :     return 0;
    4078           0 :   if ((Subtarget->hasAVX())) {
    4079           0 :     return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4080             :   }
    4081             :   return 0;
    4082             : }
    4083             : 
    4084           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4085           0 :   if (RetVT.SimpleTy != MVT::i32)
    4086             :     return 0;
    4087           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4088           0 :     return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4089             :   }
    4090           0 :   if ((Subtarget->hasAVX())) {
    4091           0 :     return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4092             :   }
    4093             :   return 0;
    4094             : }
    4095             : 
    4096             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4097           0 :   if (RetVT.SimpleTy != MVT::i32)
    4098             :     return 0;
    4099           0 :   if ((Subtarget->hasAVX())) {
    4100           0 :     return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    4101             :   }
    4102             :   return 0;
    4103             : }
    4104             : 
    4105           0 : unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4106           0 :   switch (VT.SimpleTy) {
    4107           0 :   case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4108           0 :   case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    4109           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVMSK_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4110           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVMSK_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4111           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVMSK_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4112           0 :   case MVT::v4i64: return fastEmit_X86ISD_MOVMSK_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4113           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4114           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    4115           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4116           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    4117             :   default: return 0;
    4118             :   }
    4119             : }
    4120             : 
    4121             : // FastEmit functions for X86ISD::MOVSHDUP.
    4122             : 
    4123           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4124           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4125             :     return 0;
    4126           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    4127           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4128             :   }
    4129           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4130           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4131             :   }
    4132             :   return 0;
    4133             : }
    4134             : 
    4135           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4136           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    4137             :     return 0;
    4138           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4139           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4140             :   }
    4141             :   return 0;
    4142             : }
    4143             : 
    4144           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4145           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4146             :     return 0;
    4147           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4148           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4149             :   }
    4150           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    4151           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4152             :   }
    4153           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4154           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4155             :   }
    4156             :   return 0;
    4157             : }
    4158             : 
    4159           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4160           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    4161             :     return 0;
    4162           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4163           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4164             :   }
    4165           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4166           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4167             :   }
    4168             :   return 0;
    4169             : }
    4170             : 
    4171             : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4172           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    4173             :     return 0;
    4174           0 :   if ((Subtarget->hasAVX512())) {
    4175           0 :     return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4176             :   }
    4177             :   return 0;
    4178             : }
    4179             : 
    4180           0 : unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4181           0 :   switch (VT.SimpleTy) {
    4182           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4183           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4184           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4185           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    4186           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    4187             :   default: return 0;
    4188             :   }
    4189             : }
    4190             : 
    4191             : // FastEmit functions for X86ISD::MOVSLDUP.
    4192             : 
    4193           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4194           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    4195             :     return 0;
    4196           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    4197           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4198             :   }
    4199           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4200           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4201             :   }
    4202             :   return 0;
    4203             : }
    4204             : 
    4205           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4206           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    4207             :     return 0;
    4208           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4209           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4210             :   }
    4211             :   return 0;
    4212             : }
    4213             : 
    4214           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4215           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4216             :     return 0;
    4217           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4218           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4219             :   }
    4220           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    4221           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4222             :   }
    4223           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4224           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4225             :   }
    4226             :   return 0;
    4227             : }
    4228             : 
    4229           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4230           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    4231             :     return 0;
    4232           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4233           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4234             :   }
    4235           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4236           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4237             :   }
    4238             :   return 0;
    4239             : }
    4240             : 
    4241             : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4242           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    4243             :     return 0;
    4244           0 :   if ((Subtarget->hasAVX512())) {
    4245           0 :     return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4246             :   }
    4247             :   return 0;
    4248             : }
    4249             : 
    4250           0 : unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4251           0 :   switch (VT.SimpleTy) {
    4252           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4253           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4254           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4255           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    4256           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    4257             :   default: return 0;
    4258             :   }
    4259             : }
    4260             : 
    4261             : // FastEmit functions for X86ISD::NT_BRIND.
    4262             : 
    4263             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4264           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4265             :     return 0;
    4266           0 :   if ((!Subtarget->is64Bit())) {
    4267           0 :     return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
    4268             :   }
    4269             :   return 0;
    4270             : }
    4271             : 
    4272             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4273           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4274             :     return 0;
    4275           0 :   if ((!Subtarget->is64Bit())) {
    4276           0 :     return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
    4277             :   }
    4278             :   return 0;
    4279             : }
    4280             : 
    4281             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4282           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4283             :     return 0;
    4284           0 :   if ((Subtarget->is64Bit())) {
    4285           0 :     return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
    4286             :   }
    4287             :   return 0;
    4288             : }
    4289             : 
    4290           0 : unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4291           0 :   switch (VT.SimpleTy) {
    4292           0 :   case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    4293           0 :   case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4294           0 :   case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4295             :   default: return 0;
    4296             :   }
    4297             : }
    4298             : 
    4299             : // FastEmit functions for X86ISD::NT_CALL.
    4300             : 
    4301             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4302           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4303             :     return 0;
    4304           0 :   if ((!Subtarget->is64Bit())) {
    4305           0 :     return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
    4306             :   }
    4307             :   return 0;
    4308             : }
    4309             : 
    4310             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4311           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4312             :     return 0;
    4313           0 :   if ((!Subtarget->is64Bit())) {
    4314           0 :     return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
    4315             :   }
    4316             :   return 0;
    4317             : }
    4318             : 
    4319             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4320           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4321             :     return 0;
    4322           0 :   if ((Subtarget->is64Bit())) {
    4323           0 :     return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
    4324             :   }
    4325             :   return 0;
    4326             : }
    4327             : 
    4328           0 : unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4329           0 :   switch (VT.SimpleTy) {
    4330           0 :   case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    4331           0 :   case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4332           0 :   case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4333             :   default: return 0;
    4334             :   }
    4335             : }
    4336             : 
    4337             : // FastEmit functions for X86ISD::PHMINPOS.
    4338             : 
    4339           0 : unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4340           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    4341             :     return 0;
    4342           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    4343           0 :     return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4344             :   }
    4345           0 :   if ((Subtarget->hasAVX())) {
    4346           0 :     return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4347             :   }
    4348             :   return 0;
    4349             : }
    4350             : 
    4351             : unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4352           0 :   switch (VT.SimpleTy) {
    4353           0 :   case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4354             :   default: return 0;
    4355             :   }
    4356             : }
    4357             : 
    4358             : // FastEmit functions for X86ISD::RCP14.
    4359             : 
    4360             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4361           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4362             :     return 0;
    4363           0 :   if ((Subtarget->hasVLX())) {
    4364           0 :     return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4365             :   }
    4366             :   return 0;
    4367             : }
    4368             : 
    4369             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4370           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    4371             :     return 0;
    4372           0 :   if ((Subtarget->hasVLX())) {
    4373           0 :     return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4374             :   }
    4375             :   return 0;
    4376             : }
    4377             : 
    4378             : unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4379           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    4380             :     return 0;
    4381           0 :   if ((Subtarget->hasAVX512())) {
    4382           0 :     return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4383             :   }
    4384             :   return 0;
    4385             : }
    4386             : 
    4387             : unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4388           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4389             :     return 0;
    4390           0 :   if ((Subtarget->hasVLX())) {
    4391           0 :     return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4392             :   }
    4393             :   return 0;
    4394             : }
    4395             : 
    4396             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4397           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    4398             :     return 0;
    4399           0 :   if ((Subtarget->hasVLX())) {
    4400           0 :     return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4401             :   }
    4402             :   return 0;
    4403             : }
    4404             : 
    4405             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4406           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    4407             :     return 0;
    4408           0 :   if ((Subtarget->hasAVX512())) {
    4409           0 :     return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4410             :   }
    4411             :   return 0;
    4412             : }
    4413             : 
    4414           0 : unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4415           0 :   switch (VT.SimpleTy) {
    4416           0 :   case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4417           0 :   case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    4418           0 :   case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    4419           0 :   case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4420           0 :   case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    4421           0 :   case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    4422             :   default: return 0;
    4423             :   }
    4424             : }
    4425             : 
    4426             : // FastEmit functions for X86ISD::RSQRT14.
    4427             : 
    4428             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4429           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4430             :     return 0;
    4431           0 :   if ((Subtarget->hasVLX())) {
    4432           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4433             :   }
    4434             :   return 0;
    4435             : }
    4436             : 
    4437             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4438           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    4439             :     return 0;
    4440           0 :   if ((Subtarget->hasVLX())) {
    4441           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4442             :   }
    4443             :   return 0;
    4444             : }
    4445             : 
    4446             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4447           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    4448             :     return 0;
    4449           0 :   if ((Subtarget->hasAVX512())) {
    4450           0 :     return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4451             :   }
    4452             :   return 0;
    4453             : }
    4454             : 
    4455             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4456           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4457             :     return 0;
    4458           0 :   if ((Subtarget->hasVLX())) {
    4459           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4460             :   }
    4461             :   return 0;
    4462             : }
    4463             : 
    4464             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4465           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    4466             :     return 0;
    4467           0 :   if ((Subtarget->hasVLX())) {
    4468           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4469             :   }
    4470             :   return 0;
    4471             : }
    4472             : 
    4473             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4474           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    4475             :     return 0;
    4476           0 :   if ((Subtarget->hasAVX512())) {
    4477           0 :     return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4478             :   }
    4479             :   return 0;
    4480             : }
    4481             : 
    4482           0 : unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4483           0 :   switch (VT.SimpleTy) {
    4484           0 :   case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4485           0 :   case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    4486           0 :   case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    4487           0 :   case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4488           0 :   case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    4489           0 :   case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    4490             :   default: return 0;
    4491             :   }
    4492             : }
    4493             : 
    4494             : // FastEmit functions for X86ISD::SEG_ALLOCA.
    4495             : 
    4496           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4497           0 :   if (RetVT.SimpleTy != MVT::i32)
    4498             :     return 0;
    4499           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    4500           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    4501             :   }
    4502             :   return 0;
    4503             : }
    4504             : 
    4505             : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4506           0 :   if (RetVT.SimpleTy != MVT::i64)
    4507             :     return 0;
    4508           0 :   if ((Subtarget->is64Bit())) {
    4509           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    4510             :   }
    4511             :   return 0;
    4512             : }
    4513             : 
    4514           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4515           0 :   switch (VT.SimpleTy) {
    4516           0 :   case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4517           0 :   case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4518             :   default: return 0;
    4519             :   }
    4520             : }
    4521             : 
    4522             : // FastEmit functions for X86ISD::VBROADCAST.
    4523             : 
    4524           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4525           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4526           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4527             :   }
    4528             :   return 0;
    4529             : }
    4530             : 
    4531           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4532           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4533           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4534             :   }
    4535             :   return 0;
    4536             : }
    4537             : 
    4538             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4539           0 :   if ((Subtarget->hasAVX512())) {
    4540           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4541             :   }
    4542             :   return 0;
    4543             : }
    4544             : 
    4545           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4546           0 : switch (RetVT.SimpleTy) {
    4547           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    4548           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
    4549           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
    4550             :   default: return 0;
    4551             : }
    4552             : }
    4553             : 
    4554           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4555           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4556           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4557             :   }
    4558             :   return 0;
    4559             : }
    4560             : 
    4561           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4562           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4563           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4564             :   }
    4565             :   return 0;
    4566             : }
    4567             : 
    4568             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4569           0 :   if ((Subtarget->hasAVX512())) {
    4570           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4571             :   }
    4572             :   return 0;
    4573             : }
    4574             : 
    4575           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4576           0 : switch (RetVT.SimpleTy) {
    4577           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
    4578           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
    4579           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
    4580             :   default: return 0;
    4581             : }
    4582             : }
    4583             : 
    4584           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4585           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4586           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4587             :   }
    4588           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4589           0 :     return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4590             :   }
    4591             :   return 0;
    4592             : }
    4593             : 
    4594           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    4595           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4596           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4597             :   }
    4598           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4599           0 :     return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4600             :   }
    4601             :   return 0;
    4602             : }
    4603             : 
    4604             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
    4605           0 :   if ((Subtarget->hasBWI())) {
    4606           0 :     return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4607             :   }
    4608             :   return 0;
    4609             : }
    4610             : 
    4611           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4612           0 : switch (RetVT.SimpleTy) {
    4613           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
    4614           0 :   case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
    4615           0 :   case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
    4616             :   default: return 0;
    4617             : }
    4618             : }
    4619             : 
    4620           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4621           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4622           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4623             :   }
    4624           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4625           0 :     return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4626             :   }
    4627             :   return 0;
    4628             : }
    4629             : 
    4630           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4631           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4632           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4633             :   }
    4634           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4635           0 :     return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4636             :   }
    4637             :   return 0;
    4638             : }
    4639             : 
    4640             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    4641           0 :   if ((Subtarget->hasBWI())) {
    4642           0 :     return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4643             :   }
    4644             :   return 0;
    4645             : }
    4646             : 
    4647           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4648           0 : switch (RetVT.SimpleTy) {
    4649           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
    4650           0 :   case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
    4651           0 :   case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
    4652             :   default: return 0;
    4653             : }
    4654             : }
    4655             : 
    4656           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4657           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4658           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4659             :   }
    4660           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4661           0 :     return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4662             :   }
    4663             :   return 0;
    4664             : }
    4665             : 
    4666           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4667           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4668           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4669             :   }
    4670           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4671           0 :     return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4672             :   }
    4673             :   return 0;
    4674             : }
    4675             : 
    4676             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4677           0 :   if ((Subtarget->hasAVX512())) {
    4678           0 :     return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4679             :   }
    4680             :   return 0;
    4681             : }
    4682             : 
    4683           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4684           0 : switch (RetVT.SimpleTy) {
    4685           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
    4686           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
    4687           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
    4688             :   default: return 0;
    4689             : }
    4690             : }
    4691             : 
    4692           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4693           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4694           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4695             :   }
    4696           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4697           0 :     return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4698             :   }
    4699             :   return 0;
    4700             : }
    4701             : 
    4702           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4703           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4704           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4705             :   }
    4706           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4707           0 :     return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4708             :   }
    4709             :   return 0;
    4710             : }
    4711             : 
    4712             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4713           0 :   if ((Subtarget->hasAVX512())) {
    4714           0 :     return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4715             :   }
    4716             :   return 0;
    4717             : }
    4718             : 
    4719           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4720           0 : switch (RetVT.SimpleTy) {
    4721           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
    4722           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
    4723           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
    4724             :   default: return 0;
    4725             : }
    4726             : }
    4727             : 
    4728           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    4729           0 :   if ((Subtarget->hasVLX())) {
    4730           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4731             :   }
    4732           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4733           0 :     return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4734             :   }
    4735             :   return 0;
    4736             : }
    4737             : 
    4738           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    4739           0 :   if ((Subtarget->hasVLX())) {
    4740           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4741             :   }
    4742           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4743           0 :     return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4744             :   }
    4745             :   return 0;
    4746             : }
    4747             : 
    4748             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
    4749           0 :   if ((Subtarget->hasAVX512())) {
    4750           0 :     return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4751             :   }
    4752             :   return 0;
    4753             : }
    4754             : 
    4755           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4756           0 : switch (RetVT.SimpleTy) {
    4757           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
    4758           0 :   case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
    4759           0 :   case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
    4760             :   default: return 0;
    4761             : }
    4762             : }
    4763             : 
    4764           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    4765           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4766           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4767             :   }
    4768           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4769           0 :     return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4770             :   }
    4771             :   return 0;
    4772             : }
    4773             : 
    4774           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    4775           0 :   if ((Subtarget->hasVLX())) {
    4776           0 :     return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4777             :   }
    4778           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4779           0 :     return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4780             :   }
    4781             :   return 0;
    4782             : }
    4783             : 
    4784             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    4785           0 :   if ((Subtarget->hasAVX512())) {
    4786           0 :     return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4787             :   }
    4788             :   return 0;
    4789             : }
    4790             : 
    4791           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4792           0 : switch (RetVT.SimpleTy) {
    4793           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill);
    4794           0 :   case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
    4795           0 :   case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
    4796             :   default: return 0;
    4797             : }
    4798             : }
    4799             : 
    4800           0 : unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4801           0 :   switch (VT.SimpleTy) {
    4802           0 :   case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4803           0 :   case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4804           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4805           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4806           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4807           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4808           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4809           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4810             :   default: return 0;
    4811             :   }
    4812             : }
    4813             : 
    4814             : // FastEmit functions for X86ISD::VBROADCASTM.
    4815             : 
    4816           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4817           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4818           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4819             :   }
    4820             :   return 0;
    4821             : }
    4822             : 
    4823           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4824           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4825           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4826             :   }
    4827             :   return 0;
    4828             : }
    4829             : 
    4830             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4831           0 :   if ((Subtarget->hasCDI())) {
    4832           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4833             :   }
    4834             :   return 0;
    4835             : }
    4836             : 
    4837           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4838           0 : switch (RetVT.SimpleTy) {
    4839           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
    4840           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
    4841           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    4842             :   default: return 0;
    4843             : }
    4844             : }
    4845             : 
    4846           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4847           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4848           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4849             :   }
    4850             :   return 0;
    4851             : }
    4852             : 
    4853           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4854           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4855           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4856             :   }
    4857             :   return 0;
    4858             : }
    4859             : 
    4860             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4861           0 :   if ((Subtarget->hasCDI())) {
    4862           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4863             :   }
    4864             :   return 0;
    4865             : }
    4866             : 
    4867           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4868           0 : switch (RetVT.SimpleTy) {
    4869           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
    4870           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
    4871           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    4872             :   default: return 0;
    4873             : }
    4874             : }
    4875             : 
    4876           0 : unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4877           0 :   switch (VT.SimpleTy) {
    4878           0 :   case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    4879           0 :   case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    4880             :   default: return 0;
    4881             :   }
    4882             : }
    4883             : 
    4884             : // FastEmit functions for X86ISD::VFPEXT.
    4885             : 
    4886           0 : unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4887           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4888             :     return 0;
    4889           0 :   if ((Subtarget->hasVLX())) {
    4890           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4891             :   }
    4892           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4893           0 :     return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4894             :   }
    4895           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4896           0 :     return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4897             :   }
    4898             :   return 0;
    4899             : }
    4900             : 
    4901             : unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4902           0 :   switch (VT.SimpleTy) {
    4903           0 :   case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4904             :   default: return 0;
    4905             :   }
    4906             : }
    4907             : 
    4908             : // FastEmit functions for X86ISD::VFPROUND.
    4909             : 
    4910           0 : unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4911           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4912             :     return 0;
    4913           0 :   if ((Subtarget->hasVLX())) {
    4914           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4915             :   }
    4916           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4917           0 :     return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4918             :   }
    4919           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4920           0 :     return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4921             :   }
    4922             :   return 0;
    4923             : }
    4924             : 
    4925             : unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4926           0 :   switch (VT.SimpleTy) {
    4927           0 :   case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4928             :   default: return 0;
    4929             :   }
    4930             : }
    4931             : 
    4932             : // FastEmit functions for X86ISD::VSEXT.
    4933             : 
    4934           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4935           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4936           0 :     return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4937             :   }
    4938           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4939           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4940             :   }
    4941             :   return 0;
    4942             : }
    4943             : 
    4944           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4945           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4946           0 :     return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4947             :   }
    4948           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4949           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4950             :   }
    4951             :   return 0;
    4952             : }
    4953             : 
    4954             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4955           0 :   if ((Subtarget->hasAVX512())) {
    4956           0 :     return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4957             :   }
    4958             :   return 0;
    4959             : }
    4960             : 
    4961           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4962           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4963           0 :     return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4964             :   }
    4965           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4966           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4967             :   }
    4968             :   return 0;
    4969             : }
    4970             : 
    4971             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4972           0 :   if ((Subtarget->hasAVX512())) {
    4973           0 :     return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4974             :   }
    4975             :   return 0;
    4976             : }
    4977             : 
    4978           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4979           0 : switch (RetVT.SimpleTy) {
    4980           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    4981           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    4982           0 :   case MVT::v16i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    4983           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    4984           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    4985             :   default: return 0;
    4986             : }
    4987             : }
    4988             : 
    4989             : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4990           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    4991             :     return 0;
    4992           0 :   if ((Subtarget->hasBWI())) {
    4993           0 :     return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4994             :   }
    4995             :   return 0;
    4996             : }
    4997             : 
    4998           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4999           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5000           0 :     return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5001             :   }
    5002           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5003           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5004             :   }
    5005             :   return 0;
    5006             : }
    5007             : 
    5008           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5009           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5010           0 :     return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5011             :   }
    5012           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5013           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5014             :   }
    5015             :   return 0;
    5016             : }
    5017             : 
    5018             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5019           0 :   if ((Subtarget->hasAVX512())) {
    5020           0 :     return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5021             :   }
    5022             :   return 0;
    5023             : }
    5024             : 
    5025           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5026           0 : switch (RetVT.SimpleTy) {
    5027           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    5028           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    5029           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    5030             :   default: return 0;
    5031             : }
    5032             : }
    5033             : 
    5034             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5035           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    5036             :     return 0;
    5037           0 :   if ((Subtarget->hasAVX512())) {
    5038           0 :     return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5039             :   }
    5040             :   return 0;
    5041             : }
    5042             : 
    5043           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5044           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    5045             :     return 0;
    5046           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5047           0 :     return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5048             :   }
    5049           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5050           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5051             :   }
    5052             :   return 0;
    5053             : }
    5054             : 
    5055             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5056           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5057             :     return 0;
    5058           0 :   if ((Subtarget->hasAVX512())) {
    5059           0 :     return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5060             :   }
    5061             :   return 0;
    5062             : }
    5063             : 
    5064           0 : unsigned fastEmit_X86ISD_VSEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5065           0 :   switch (VT.SimpleTy) {
    5066           0 :   case MVT::v16i8: return fastEmit_X86ISD_VSEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    5067           0 :   case MVT::v32i8: return fastEmit_X86ISD_VSEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    5068           0 :   case MVT::v8i16: return fastEmit_X86ISD_VSEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5069           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5070           0 :   case MVT::v4i32: return fastEmit_X86ISD_VSEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5071           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5072             :   default: return 0;
    5073             :   }
    5074             : }
    5075             : 
    5076             : // FastEmit functions for X86ISD::VTRUNC.
    5077             : 
    5078           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5079           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5080             :     return 0;
    5081           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5082           0 :     return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5083             :   }
    5084             :   return 0;
    5085             : }
    5086             : 
    5087           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5088           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5089           0 :     return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5090             :   }
    5091             :   return 0;
    5092             : }
    5093             : 
    5094           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5095           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5096           0 :     return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5097             :   }
    5098             :   return 0;
    5099             : }
    5100             : 
    5101           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5102           0 : switch (RetVT.SimpleTy) {
    5103           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    5104           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    5105             :   default: return 0;
    5106             : }
    5107             : }
    5108             : 
    5109           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5110           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5111             :     return 0;
    5112           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5113           0 :     return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5114             :   }
    5115             :   return 0;
    5116             : }
    5117             : 
    5118           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5119           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5120           0 :     return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5121             :   }
    5122             :   return 0;
    5123             : }
    5124             : 
    5125           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5126           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5127           0 :     return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5128             :   }
    5129             :   return 0;
    5130             : }
    5131             : 
    5132           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5133           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5134           0 :     return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5135             :   }
    5136             :   return 0;
    5137             : }
    5138             : 
    5139           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5140           0 : switch (RetVT.SimpleTy) {
    5141           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    5142           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    5143           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    5144             :   default: return 0;
    5145             : }
    5146             : }
    5147             : 
    5148           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5149           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5150           0 :     return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5151             :   }
    5152             :   return 0;
    5153             : }
    5154             : 
    5155           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5156           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5157           0 :     return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5158             :   }
    5159             :   return 0;
    5160             : }
    5161             : 
    5162           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5163           0 : switch (RetVT.SimpleTy) {
    5164           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    5165           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    5166             :   default: return 0;
    5167             : }
    5168             : }
    5169             : 
    5170             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5171           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5172             :     return 0;
    5173           0 :   if ((Subtarget->hasAVX512())) {
    5174           0 :     return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5175             :   }
    5176             :   return 0;
    5177             : }
    5178             : 
    5179           0 : unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5180           0 :   switch (VT.SimpleTy) {
    5181           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5182           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5183           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5184           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5185           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    5186           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    5187             :   default: return 0;
    5188             :   }
    5189             : }
    5190             : 
    5191             : // FastEmit functions for X86ISD::VTRUNCS.
    5192             : 
    5193           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5194           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5195             :     return 0;
    5196           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5197           0 :     return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5198             :   }
    5199             :   return 0;
    5200             : }
    5201             : 
    5202           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5203           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5204             :     return 0;
    5205           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5206           0 :     return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5207             :   }
    5208             :   return 0;
    5209             : }
    5210             : 
    5211             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5212           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    5213             :     return 0;
    5214           0 :   if ((Subtarget->hasBWI())) {
    5215           0 :     return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5216             :   }
    5217             :   return 0;
    5218             : }
    5219             : 
    5220           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5221           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5222           0 :     return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5223             :   }
    5224             :   return 0;
    5225             : }
    5226             : 
    5227           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5228           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5229           0 :     return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5230             :   }
    5231             :   return 0;
    5232             : }
    5233             : 
    5234           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5235           0 : switch (RetVT.SimpleTy) {
    5236           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    5237           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    5238             :   default: return 0;
    5239             : }
    5240             : }
    5241             : 
    5242           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5243           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5244           0 :     return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5245             :   }
    5246             :   return 0;
    5247             : }
    5248             : 
    5249           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5250           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5251           0 :     return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5252             :   }
    5253             :   return 0;
    5254             : }
    5255             : 
    5256           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5257           0 : switch (RetVT.SimpleTy) {
    5258           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    5259           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    5260             :   default: return 0;
    5261             : }
    5262             : }
    5263             : 
    5264             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5265           0 :   if ((Subtarget->hasAVX512())) {
    5266           0 :     return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5267             :   }
    5268             :   return 0;
    5269             : }
    5270             : 
    5271             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    5272           0 :   if ((Subtarget->hasAVX512())) {
    5273           0 :     return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5274             :   }
    5275             :   return 0;
    5276             : }
    5277             : 
    5278           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5279           0 : switch (RetVT.SimpleTy) {
    5280           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    5281           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    5282             :   default: return 0;
    5283             : }
    5284             : }
    5285             : 
    5286           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5287           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5288           0 :     return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5289             :   }
    5290             :   return 0;
    5291             : }
    5292             : 
    5293           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5294           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5295           0 :     return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5296             :   }
    5297             :   return 0;
    5298             : }
    5299             : 
    5300           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5301           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5302           0 :     return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5303             :   }
    5304             :   return 0;
    5305             : }
    5306             : 
    5307           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5308           0 : switch (RetVT.SimpleTy) {
    5309           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    5310           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    5311           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    5312             :   default: return 0;
    5313             : }
    5314             : }
    5315             : 
    5316           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5317           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5318           0 :     return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5319             :   }
    5320             :   return 0;
    5321             : }
    5322             : 
    5323           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5324           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5325           0 :     return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5326             :   }
    5327             :   return 0;
    5328             : }
    5329             : 
    5330           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5331           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5332           0 :     return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5333             :   }
    5334             :   return 0;
    5335             : }
    5336             : 
    5337           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5338           0 : switch (RetVT.SimpleTy) {
    5339           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    5340           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    5341           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    5342             :   default: return 0;
    5343             : }
    5344             : }
    5345             : 
    5346             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5347           0 :   if ((Subtarget->hasAVX512())) {
    5348           0 :     return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5349             :   }
    5350             :   return 0;
    5351             : }
    5352             : 
    5353             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5354           0 :   if ((Subtarget->hasAVX512())) {
    5355           0 :     return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5356             :   }
    5357             :   return 0;
    5358             : }
    5359             : 
    5360             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5361           0 :   if ((Subtarget->hasAVX512())) {
    5362           0 :     return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5363             :   }
    5364             :   return 0;
    5365             : }
    5366             : 
    5367           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5368           0 : switch (RetVT.SimpleTy) {
    5369           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    5370           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    5371           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    5372             :   default: return 0;
    5373             : }
    5374             : }
    5375             : 
    5376           0 : unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5377           0 :   switch (VT.SimpleTy) {
    5378           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5379           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5380           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    5381           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5382           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5383           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    5384           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5385           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    5386           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    5387             :   default: return 0;
    5388             :   }
    5389             : }
    5390             : 
    5391             : // FastEmit functions for X86ISD::VTRUNCUS.
    5392             : 
    5393           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5394           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5395             :     return 0;
    5396           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5397           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5398             :   }
    5399             :   return 0;
    5400             : }
    5401             : 
    5402           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5403           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    5404             :     return 0;
    5405           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5406           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5407             :   }
    5408             :   return 0;
    5409             : }
    5410             : 
    5411             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5412           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    5413             :     return 0;
    5414           0 :   if ((Subtarget->hasBWI())) {
    5415           0 :     return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5416             :   }
    5417             :   return 0;
    5418             : }
    5419             : 
    5420           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5421           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5422           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5423             :   }
    5424             :   return 0;
    5425             : }
    5426             : 
    5427           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5428           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5429           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5430             :   }
    5431             :   return 0;
    5432             : }
    5433             : 
    5434           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5435           0 : switch (RetVT.SimpleTy) {
    5436           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    5437           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    5438             :   default: return 0;
    5439             : }
    5440             : }
    5441             : 
    5442           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5443           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5444           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5445             :   }
    5446             :   return 0;
    5447             : }
    5448             : 
    5449           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5450           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5451           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5452             :   }
    5453             :   return 0;
    5454             : }
    5455             : 
    5456           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5457           0 : switch (RetVT.SimpleTy) {
    5458           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    5459           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    5460             :   default: return 0;
    5461             : }
    5462             : }
    5463             : 
    5464             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5465           0 :   if ((Subtarget->hasAVX512())) {
    5466           0 :     return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5467             :   }
    5468             :   return 0;
    5469             : }
    5470             : 
    5471             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    5472           0 :   if ((Subtarget->hasAVX512())) {
    5473           0 :     return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5474             :   }
    5475             :   return 0;
    5476             : }
    5477             : 
    5478           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5479           0 : switch (RetVT.SimpleTy) {
    5480           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    5481           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    5482             :   default: return 0;
    5483             : }
    5484             : }
    5485             : 
    5486           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5487           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5488           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5489             :   }
    5490             :   return 0;
    5491             : }
    5492             : 
    5493           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5494           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5495           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5496             :   }
    5497             :   return 0;
    5498             : }
    5499             : 
    5500           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5501           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5502           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5503             :   }
    5504             :   return 0;
    5505             : }
    5506             : 
    5507           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5508           0 : switch (RetVT.SimpleTy) {
    5509           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    5510           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    5511           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    5512             :   default: return 0;
    5513             : }
    5514             : }
    5515             : 
    5516           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5517           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5518           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5519             :   }
    5520             :   return 0;
    5521             : }
    5522             : 
    5523           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5524           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5525           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5526             :   }
    5527             :   return 0;
    5528             : }
    5529             : 
    5530           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    5531           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5532           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5533             :   }
    5534             :   return 0;
    5535             : }
    5536             : 
    5537           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5538           0 : switch (RetVT.SimpleTy) {
    5539           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    5540           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    5541           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    5542             :   default: return 0;
    5543             : }
    5544             : }
    5545             : 
    5546             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5547           0 :   if ((Subtarget->hasAVX512())) {
    5548           0 :     return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5549             :   }
    5550             :   return 0;
    5551             : }
    5552             : 
    5553             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5554           0 :   if ((Subtarget->hasAVX512())) {
    5555           0 :     return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5556             :   }
    5557             :   return 0;
    5558             : }
    5559             : 
    5560             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5561           0 :   if ((Subtarget->hasAVX512())) {
    5562           0 :     return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5563             :   }
    5564             :   return 0;
    5565             : }
    5566             : 
    5567           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5568           0 : switch (RetVT.SimpleTy) {
    5569           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    5570           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    5571           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    5572             :   default: return 0;
    5573             : }
    5574             : }
    5575             : 
    5576           0 : unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5577           0 :   switch (VT.SimpleTy) {
    5578           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5579           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5580           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    5581           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5582           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5583           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    5584           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5585           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    5586           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    5587             :   default: return 0;
    5588             :   }
    5589             : }
    5590             : 
    5591             : // FastEmit functions for X86ISD::VZEXT.
    5592             : 
    5593           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    5594           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5595           0 :     return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5596             :   }
    5597           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5598           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5599             :   }
    5600             :   return 0;
    5601             : }
    5602             : 
    5603           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5604           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5605           0 :     return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5606             :   }
    5607           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5608           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5609             :   }
    5610             :   return 0;
    5611             : }
    5612             : 
    5613             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    5614           0 :   if ((Subtarget->hasAVX512())) {
    5615           0 :     return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5616             :   }
    5617             :   return 0;
    5618             : }
    5619             : 
    5620           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5621           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5622           0 :     return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5623             :   }
    5624           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5625           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5626             :   }
    5627             :   return 0;
    5628             : }
    5629             : 
    5630             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5631           0 :   if ((Subtarget->hasAVX512())) {
    5632           0 :     return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5633             :   }
    5634             :   return 0;
    5635             : }
    5636             : 
    5637           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5638           0 : switch (RetVT.SimpleTy) {
    5639           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    5640           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    5641           0 :   case MVT::v16i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    5642           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    5643           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    5644             :   default: return 0;
    5645             : }
    5646             : }
    5647             : 
    5648             : unsigned fastEmit_X86ISD_VZEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5649           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5650             :     return 0;
    5651           0 :   if ((Subtarget->hasBWI())) {
    5652           0 :     return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5653             :   }
    5654             :   return 0;
    5655             : }
    5656             : 
    5657           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5658           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5659           0 :     return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5660             :   }
    5661           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5662           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5663             :   }
    5664             :   return 0;
    5665             : }
    5666             : 
    5667           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5668           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5669           0 :     return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5670             :   }
    5671           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5672           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5673             :   }
    5674             :   return 0;
    5675             : }
    5676             : 
    5677             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5678           0 :   if ((Subtarget->hasAVX512())) {
    5679           0 :     return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5680             :   }
    5681             :   return 0;
    5682             : }
    5683             : 
    5684           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5685           0 : switch (RetVT.SimpleTy) {
    5686           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    5687           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    5688           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    5689             :   default: return 0;
    5690             : }
    5691             : }
    5692             : 
    5693             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5694           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    5695             :     return 0;
    5696           0 :   if ((Subtarget->hasAVX512())) {
    5697           0 :     return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5698             :   }
    5699             :   return 0;
    5700             : }
    5701             : 
    5702           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5703           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    5704             :     return 0;
    5705           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5706           0 :     return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5707             :   }
    5708           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5709           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5710             :   }
    5711             :   return 0;
    5712             : }
    5713             : 
    5714             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5715           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5716             :     return 0;
    5717           0 :   if ((Subtarget->hasAVX512())) {
    5718           0 :     return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5719             :   }
    5720             :   return 0;
    5721             : }
    5722             : 
    5723           0 : unsigned fastEmit_X86ISD_VZEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5724           0 :   switch (VT.SimpleTy) {
    5725           0 :   case MVT::v16i8: return fastEmit_X86ISD_VZEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    5726           0 :   case MVT::v32i8: return fastEmit_X86ISD_VZEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    5727           0 :   case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5728           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5729           0 :   case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5730           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5731             :   default: return 0;
    5732             :   }
    5733             : }
    5734             : 
    5735             : // FastEmit functions for X86ISD::VZEXT_MOVL.
    5736             : 
    5737           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5738           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5739             :     return 0;
    5740           0 :   if ((Subtarget->hasAVX512())) {
    5741           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5742             :   }
    5743           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5744           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5745             :   }
    5746           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5747           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5748             :   }
    5749             :   return 0;
    5750             : }
    5751             : 
    5752           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5753           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5754             :     return 0;
    5755           0 :   if ((Subtarget->hasAVX512())) {
    5756           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5757             :   }
    5758           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5759           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5760             :   }
    5761           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5762           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5763             :   }
    5764             :   return 0;
    5765             : }
    5766             : 
    5767           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5768           0 :   switch (VT.SimpleTy) {
    5769           0 :   case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5770           0 :   case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    5771             :   default: return 0;
    5772             :   }
    5773             : }
    5774             : 
    5775             : // FastEmit functions for X86ISD::WIN_ALLOCA.
    5776             : 
    5777           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5778           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5779             :     return 0;
    5780           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    5781           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    5782             :   }
    5783             :   return 0;
    5784             : }
    5785             : 
    5786             : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5787           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5788             :     return 0;
    5789           0 :   if ((Subtarget->is64Bit())) {
    5790           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    5791             :   }
    5792             :   return 0;
    5793             : }
    5794             : 
    5795           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5796           0 :   switch (VT.SimpleTy) {
    5797           0 :   case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    5798           0 :   case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    5799             :   default: return 0;
    5800             :   }
    5801             : }
    5802             : 
    5803             : // Top-level FastEmit function.
    5804             : 
    5805      202346 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
    5806      202346 :   switch (Opcode) {
    5807           0 :   case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
    5808          32 :   case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5809        4267 :   case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
    5810           0 :   case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    5811           0 :   case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
    5812           0 :   case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
    5813           0 :   case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
    5814           0 :   case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill);
    5815           0 :   case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill);
    5816           0 :   case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
    5817           0 :   case ISD::FCOS: return fastEmit_ISD_FCOS_r(VT, RetVT, Op0, Op0IsKill);
    5818         188 :   case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
    5819           0 :   case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5820           0 :   case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
    5821         120 :   case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
    5822           0 :   case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
    5823           0 :   case ISD::FSIN: return fastEmit_ISD_FSIN_r(VT, RetVT, Op0, Op0IsKill);
    5824           0 :   case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5825           1 :   case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill);
    5826       56291 :   case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5827           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5828        1101 :   case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5829       12674 :   case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
    5830           0 :   case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5831      127672 :   case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5832           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5833           0 :   case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5834           0 :   case X86ISD::COMPRESS: return fastEmit_X86ISD_COMPRESS_r(VT, RetVT, Op0, Op0IsKill);
    5835           0 :   case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill);
    5836           0 :   case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5837           0 :   case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5838           0 :   case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill);
    5839           0 :   case X86ISD::CVTS2SI: return fastEmit_X86ISD_CVTS2SI_r(VT, RetVT, Op0, Op0IsKill);
    5840           0 :   case X86ISD::CVTS2UI: return fastEmit_X86ISD_CVTS2UI_r(VT, RetVT, Op0, Op0IsKill);
    5841           0 :   case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
    5842           0 :   case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5843           0 :   case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5844           0 :   case X86ISD::CVTTS2SI: return fastEmit_X86ISD_CVTTS2SI_r(VT, RetVT, Op0, Op0IsKill);
    5845           0 :   case X86ISD::CVTTS2UI: return fastEmit_X86ISD_CVTTS2UI_r(VT, RetVT, Op0, Op0IsKill);
    5846           0 :   case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
    5847           0 :   case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill);
    5848           0 :   case X86ISD::EXPAND: return fastEmit_X86ISD_EXPAND_r(VT, RetVT, Op0, Op0IsKill);
    5849           0 :   case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill);
    5850           0 :   case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5851           0 :   case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill);
    5852           0 :   case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill);
    5853           0 :   case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill);
    5854           0 :   case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill);
    5855           0 :   case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill);
    5856           0 :   case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill);
    5857           0 :   case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill);
    5858           0 :   case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    5859           0 :   case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5860           0 :   case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0, Op0IsKill);
    5861           0 :   case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0, Op0IsKill);
    5862           0 :   case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0, Op0IsKill);
    5863           0 :   case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5864           0 :   case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill);
    5865           0 :   case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill);
    5866           0 :   case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
    5867           0 :   case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
    5868           0 :   case X86ISD::VSEXT: return fastEmit_X86ISD_VSEXT_r(VT, RetVT, Op0, Op0IsKill);
    5869           0 :   case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill);
    5870           0 :   case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill);
    5871           0 :   case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill);
    5872           0 :   case X86ISD::VZEXT: return fastEmit_X86ISD_VZEXT_r(VT, RetVT, Op0, Op0IsKill);
    5873           0 :   case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill);
    5874           0 :   case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5875             :   default: return 0;
    5876             :   }
    5877             : }
    5878             : 
    5879             : // FastEmit functions for ISD::ADD.
    5880             : 
    5881             : unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5882           1 :   if (RetVT.SimpleTy != MVT::i8)
    5883             :     return 0;
    5884           1 :   return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5885             : }
    5886             : 
    5887             : unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5888           1 :   if (RetVT.SimpleTy != MVT::i16)
    5889             :     return 0;
    5890           1 :   return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5891             : }
    5892             : 
    5893             : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5894        2641 :   if (RetVT.SimpleTy != MVT::i32)
    5895             :     return 0;
    5896        2641 :   return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5897             : }
    5898             : 
    5899             : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5900       80905 :   if (RetVT.SimpleTy != MVT::i64)
    5901             :     return 0;
    5902       80905 :   return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5903             : }
    5904             : 
    5905           6 : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5906           6 :   if (RetVT.SimpleTy != MVT::v16i8)
    5907             :     return 0;
    5908           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5909           2 :     return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5910             :   }
    5911           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5912           2 :     return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5913             :   }
    5914           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5915           2 :     return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5916             :   }
    5917             :   return 0;
    5918             : }
    5919             : 
    5920           2 : unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5921           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    5922             :     return 0;
    5923           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5924           0 :     return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5925             :   }
    5926           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5927           2 :     return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5928             :   }
    5929             :   return 0;
    5930             : }
    5931             : 
    5932             : unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5933           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    5934             :     return 0;
    5935           0 :   if ((Subtarget->hasBWI())) {
    5936           0 :     return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5937             :   }
    5938             :   return 0;
    5939             : }
    5940             : 
    5941           6 : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5942           6 :   if (RetVT.SimpleTy != MVT::v8i16)
    5943             :     return 0;
    5944           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5945           2 :     return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5946             :   }
    5947           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5948           2 :     return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5949             :   }
    5950           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5951           2 :     return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5952             :   }
    5953             :   return 0;
    5954             : }
    5955             : 
    5956           2 : unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5957           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    5958             :     return 0;
    5959           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5960           0 :     return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5961             :   }
    5962           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5963           2 :     return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5964             :   }
    5965             :   return 0;
    5966             : }
    5967             : 
    5968             : unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5969           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5970             :     return 0;
    5971           0 :   if ((Subtarget->hasBWI())) {
    5972           0 :     return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5973             :   }
    5974             :   return 0;
    5975             : }
    5976             : 
    5977          22 : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5978          22 :   if (RetVT.SimpleTy != MVT::v4i32)
    5979             :     return 0;
    5980          44 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5981           4 :     return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5982             :   }
    5983          18 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5984           6 :     return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5985             :   }
    5986          12 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5987          12 :     return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5988             :   }
    5989             :   return 0;
    5990             : }
    5991             : 
    5992           3 : unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5993           3 :   if (RetVT.SimpleTy != MVT::v8i32)
    5994             :     return 0;
    5995           6 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5996           1 :     return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5997             :   }
    5998           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5999           2 :     return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6000             :   }
    6001             :   return 0;
    6002             : }
    6003             : 
    6004             : unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6005           2 :   if (RetVT.SimpleTy != MVT::v16i32)
    6006             :     return 0;
    6007           4 :   if ((Subtarget->hasAVX512())) {
    6008           2 :     return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6009             :   }
    6010             :   return 0;
    6011             : }
    6012             : 
    6013           6 : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6014           6 :   if (RetVT.SimpleTy != MVT::v2i64)
    6015             :     return 0;
    6016          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6017           2 :     return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6018             :   }
    6019           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6020           2 :     return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6021             :   }
    6022           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6023           2 :     return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6024             :   }
    6025             :   return 0;
    6026             : }
    6027             : 
    6028           4 : unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6029           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    6030             :     return 0;
    6031           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6032           2 :     return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6033             :   }
    6034           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6035           2 :     return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6036             :   }
    6037             :   return 0;
    6038             : }
    6039             : 
    6040             : unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6041           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6042             :     return 0;
    6043           0 :   if ((Subtarget->hasAVX512())) {
    6044           0 :     return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6045             :   }
    6046             :   return 0;
    6047             : }
    6048             : 
    6049       83601 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6050       83601 :   switch (VT.SimpleTy) {
    6051           1 :   case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6052           1 :   case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6053        2641 :   case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6054       80905 :   case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6055           6 :   case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6056           2 :   case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6057           0 :   case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6058           6 :   case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6059           2 :   case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6060           0 :   case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6061          22 :   case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6062           3 :   case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6063           2 :   case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6064           6 :   case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6065           4 :   case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6066           0 :   case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6067             :   default: return 0;
    6068             :   }
    6069             : }
    6070             : 
    6071             : // FastEmit functions for ISD::AND.
    6072             : 
    6073             : unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6074          87 :   if (RetVT.SimpleTy != MVT::i8)
    6075             :     return 0;
    6076          87 :   return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6077             : }
    6078             : 
    6079             : unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6080           0 :   if (RetVT.SimpleTy != MVT::i16)
    6081             :     return 0;
    6082           0 :   return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6083             : }
    6084             : 
    6085             : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6086       10966 :   if (RetVT.SimpleTy != MVT::i32)
    6087             :     return 0;
    6088       10966 :   return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6089             : }
    6090             : 
    6091             : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6092       19846 :   if (RetVT.SimpleTy != MVT::i64)
    6093             :     return 0;
    6094       19846 :   return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6095             : }
    6096             : 
    6097             : unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6098           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    6099             :     return 0;
    6100           0 :   if ((Subtarget->hasDQI())) {
    6101           0 :     return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6102             :   }
    6103             :   return 0;
    6104             : }
    6105             : 
    6106             : unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6107           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    6108             :     return 0;
    6109           0 :   if ((Subtarget->hasAVX512())) {
    6110           0 :     return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6111             :   }
    6112             :   return 0;
    6113             : }
    6114             : 
    6115             : unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6116           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    6117             :     return 0;
    6118           0 :   if ((Subtarget->hasBWI())) {
    6119           0 :     return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6120             :   }
    6121             :   return 0;
    6122             : }
    6123             : 
    6124             : unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6125           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    6126             :     return 0;
    6127           0 :   if ((Subtarget->hasBWI())) {
    6128           0 :     return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6129             :   }
    6130             :   return 0;
    6131             : }
    6132             : 
    6133          16 : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6134          16 :   if (RetVT.SimpleTy != MVT::v2i64)
    6135             :     return 0;
    6136          32 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6137           4 :     return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6138             :   }
    6139          12 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6140           4 :     return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6141             :   }
    6142           8 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6143           8 :     return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6144             :   }
    6145             :   return 0;
    6146             : }
    6147             : 
    6148          12 : unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6149          12 :   if (RetVT.SimpleTy != MVT::v4i64)
    6150             :     return 0;
    6151          24 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    6152           8 :     return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6153             :   }
    6154           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6155           0 :     return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6156             :   }
    6157           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6158           4 :     return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6159             :   }
    6160             :   return 0;
    6161             : }
    6162             : 
    6163             : unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6164           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6165             :     return 0;
    6166           0 :   if ((Subtarget->hasAVX512())) {
    6167           0 :     return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6168             :   }
    6169             :   return 0;
    6170             : }
    6171             : 
    6172       30951 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6173       30951 :   switch (VT.SimpleTy) {
    6174          87 :   case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6175           0 :   case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6176       10966 :   case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6177       19846 :   case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6178           0 :   case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6179           0 :   case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6180           0 :   case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6181           0 :   case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6182          16 :   case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6183          12 :   case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6184           0 :   case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6185             :   default: return 0;
    6186             :   }
    6187             : }
    6188             : 
    6189             : // FastEmit functions for ISD::FADD.
    6190             : 
    6191         106 : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6192         106 :   if (RetVT.SimpleTy != MVT::f32)
    6193             :     return 0;
    6194         212 :   if ((Subtarget->hasAVX512())) {
    6195           2 :     return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6196             :   }
    6197         104 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6198         102 :     return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6199             :   }
    6200           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6201           2 :     return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6202             :   }
    6203           0 :   if ((!Subtarget->hasSSE1())) {
    6204           0 :     return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6205             :   }
    6206             :   return 0;
    6207             : }
    6208             : 
    6209        1709 : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6210        1709 :   if (RetVT.SimpleTy != MVT::f64)
    6211             :     return 0;
    6212        3418 :   if ((Subtarget->hasAVX512())) {
    6213           2 :     return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6214             :   }
    6215        1707 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6216        1705 :     return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6217             :   }
    6218           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6219           2 :     return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6220             :   }
    6221           0 :   if ((!Subtarget->hasSSE2())) {
    6222           0 :     return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6223             :   }
    6224             :   return 0;
    6225             : }
    6226             : 
    6227             : unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6228           0 :   if (RetVT.SimpleTy != MVT::f80)
    6229             :     return 0;
    6230           0 :   return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6231             : }
    6232             : 
    6233           6 : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6234           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    6235             :     return 0;
    6236          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6237           2 :     return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6238             :   }
    6239           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6240           2 :     return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6241             :   }
    6242           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6243           2 :     return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6244             :   }
    6245             :   return 0;
    6246             : }
    6247             : 
    6248           2 : unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6249           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6250             :     return 0;
    6251           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6252           0 :     return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6253             :   }
    6254           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6255           2 :     return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6256             :   }
    6257             :   return 0;
    6258             : }
    6259             : 
    6260             : unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6261           1 :   if (RetVT.SimpleTy != MVT::v16f32)
    6262             :     return 0;
    6263           2 :   if ((Subtarget->hasAVX512())) {
    6264           1 :     return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6265             :   }
    6266             :   return 0;
    6267             : }
    6268             : 
    6269          22 : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6270          22 :   if (RetVT.SimpleTy != MVT::v2f64)
    6271             :     return 0;
    6272          44 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6273           4 :     return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6274             :   }
    6275          18 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6276           6 :     return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6277             :   }
    6278          12 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6279          12 :     return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6280             :   }
    6281             :   return 0;
    6282             : }
    6283             : 
    6284          14 : unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6285          14 :   if (RetVT.SimpleTy != MVT::v4f64)
    6286             :     return 0;
    6287          28 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6288           2 :     return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6289             :   }
    6290          12 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6291          12 :     return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6292             :   }
    6293             :   return 0;
    6294             : }
    6295             : 
    6296             : unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6297           8 :   if (RetVT.SimpleTy != MVT::v8f64)
    6298             :     return 0;
    6299          16 :   if ((Subtarget->hasAVX512())) {
    6300           8 :     return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6301             :   }
    6302             :   return 0;
    6303             : }
    6304             : 
    6305        1868 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6306        1868 :   switch (VT.SimpleTy) {
    6307         106 :   case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6308        1709 :   case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6309           0 :   case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6310           6 :   case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6311           2 :   case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6312           1 :   case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6313          22 :   case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6314          14 :   case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6315           8 :   case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6316             :   default: return 0;
    6317             :   }
    6318             : }
    6319             : 
    6320             : // FastEmit functions for ISD::FDIV.
    6321             : 
    6322         679 : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6323         679 :   if (RetVT.SimpleTy != MVT::f32)
    6324             :     return 0;
    6325        1358 :   if ((Subtarget->hasAVX512())) {
    6326           0 :     return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6327             :   }
    6328         679 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6329         679 :     return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6330             :   }
    6331           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6332           0 :     return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6333             :   }
    6334           0 :   if ((!Subtarget->hasSSE1())) {
    6335           0 :     return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6336             :   }
    6337             :   return 0;
    6338             : }
    6339             : 
    6340        1615 : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6341        1615 :   if (RetVT.SimpleTy != MVT::f64)
    6342             :     return 0;
    6343        3230 :   if ((Subtarget->hasAVX512())) {
    6344           0 :     return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6345             :   }
    6346        1615 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6347        1615 :     return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6348             :   }
    6349           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6350           0 :     return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6351             :   }
    6352           0 :   if ((!Subtarget->hasSSE2())) {
    6353           0 :     return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6354             :   }
    6355             :   return 0;
    6356             : }
    6357             : 
    6358             : unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6359           0 :   if (RetVT.SimpleTy != MVT::f80)
    6360             :     return 0;
    6361           0 :   return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6362             : }
    6363             : 
    6364           6 : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6365           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    6366             :     return 0;
    6367          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6368           2 :     return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6369             :   }
    6370           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6371           2 :     return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6372             :   }
    6373           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6374           2 :     return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6375             :   }
    6376             :   return 0;
    6377             : }
    6378             : 
    6379           2 : unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6380           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6381             :     return 0;
    6382           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6383           0 :     return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6384             :   }
    6385           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6386           2 :     return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6387             :   }
    6388             :   return 0;
    6389             : }
    6390             : 
    6391             : unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6392           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6393             :     return 0;
    6394           0 :   if ((Subtarget->hasAVX512())) {
    6395           0 :     return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6396             :   }
    6397             :   return 0;
    6398             : }
    6399             : 
    6400           6 : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6401           6 :   if (RetVT.SimpleTy != MVT::v2f64)
    6402             :     return 0;
    6403          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6404           2 :     return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6405             :   }
    6406           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6407           2 :     return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6408             :   }
    6409           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6410           2 :     return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6411             :   }
    6412             :   return 0;
    6413             : }
    6414             : 
    6415           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6416           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6417             :     return 0;
    6418           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6419           0 :     return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6420             :   }
    6421           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6422           2 :     return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6423             :   }
    6424             :   return 0;
    6425             : }
    6426             : 
    6427             : unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6428           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6429             :     return 0;
    6430           0 :   if ((Subtarget->hasAVX512())) {
    6431           0 :     return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6432             :   }
    6433             :   return 0;
    6434             : }
    6435             : 
    6436        2310 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6437        2310 :   switch (VT.SimpleTy) {
    6438         679 :   case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6439        1615 :   case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6440           0 :   case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6441           6 :   case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6442           2 :   case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6443           0 :   case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6444           6 :   case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6445           2 :   case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6446           0 :   case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6447             :   default: return 0;
    6448             :   }
    6449             : }
    6450             : 
    6451             : // FastEmit functions for ISD::FMUL.
    6452             : 
    6453        1025 : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6454        1025 :   if (RetVT.SimpleTy != MVT::f32)
    6455             :     return 0;
    6456        2050 :   if ((Subtarget->hasAVX512())) {
    6457           0 :     return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6458             :   }
    6459        1025 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6460        1025 :     return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6461             :   }
    6462           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6463           0 :     return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6464             :   }
    6465           0 :   if ((!Subtarget->hasSSE1())) {
    6466           0 :     return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6467             :   }
    6468             :   return 0;
    6469             : }
    6470             : 
    6471        2495 : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6472        2495 :   if (RetVT.SimpleTy != MVT::f64)
    6473             :     return 0;
    6474        4990 :   if ((Subtarget->hasAVX512())) {
    6475           0 :     return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6476             :   }
    6477        2495 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6478        2495 :     return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6479             :   }
    6480           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6481           0 :     return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6482             :   }
    6483           0 :   if ((!Subtarget->hasSSE2())) {
    6484           0 :     return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6485             :   }
    6486             :   return 0;
    6487             : }
    6488             : 
    6489             : unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6490           0 :   if (RetVT.SimpleTy != MVT::f80)
    6491             :     return 0;
    6492           0 :   return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6493             : }
    6494             : 
    6495           6 : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6496           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    6497             :     return 0;
    6498          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6499           2 :     return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6500             :   }
    6501           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6502           2 :     return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6503             :   }
    6504           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6505           2 :     return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6506             :   }
    6507             :   return 0;
    6508             : }
    6509             : 
    6510           2 : unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6511           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6512             :     return 0;
    6513           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6514           0 :     return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6515             :   }
    6516           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6517           2 :     return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6518             :   }
    6519             :   return 0;
    6520             : }
    6521             : 
    6522             : unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6523           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6524             :     return 0;
    6525           0 :   if ((Subtarget->hasAVX512())) {
    6526           0 :     return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6527             :   }
    6528             :   return 0;
    6529             : }
    6530             : 
    6531           6 : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6532           6 :   if (RetVT.SimpleTy != MVT::v2f64)
    6533             :     return 0;
    6534          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6535           2 :     return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6536             :   }
    6537           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6538           2 :     return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6539             :   }
    6540           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6541           2 :     return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6542             :   }
    6543             :   return 0;
    6544             : }
    6545             : 
    6546           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6547           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6548             :     return 0;
    6549           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6550           0 :     return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6551             :   }
    6552           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6553           2 :     return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6554             :   }
    6555             :   return 0;
    6556             : }
    6557             : 
    6558             : unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6559           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6560             :     return 0;
    6561           0 :   if ((Subtarget->hasAVX512())) {
    6562           0 :     return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6563             :   }
    6564             :   return 0;
    6565             : }
    6566             : 
    6567        3536 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6568        3536 :   switch (VT.SimpleTy) {
    6569        1025 :   case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6570        2495 :   case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6571           0 :   case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6572           6 :   case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6573           2 :   case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6574           0 :   case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6575           6 :   case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6576           2 :   case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6577           0 :   case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6578             :   default: return 0;
    6579             :   }
    6580             : }
    6581             : 
    6582             : // FastEmit functions for ISD::FSUB.
    6583             : 
    6584         268 : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6585         268 :   if (RetVT.SimpleTy != MVT::f32)
    6586             :     return 0;
    6587         536 :   if ((Subtarget->hasAVX512())) {
    6588           0 :     return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6589             :   }
    6590         268 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6591         268 :     return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6592             :   }
    6593           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6594           0 :     return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6595             :   }
    6596           0 :   if ((!Subtarget->hasSSE1())) {
    6597           0 :     return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6598             :   }
    6599             :   return 0;
    6600             : }
    6601             : 
    6602        1744 : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6603        1744 :   if (RetVT.SimpleTy != MVT::f64)
    6604             :     return 0;
    6605        3488 :   if ((Subtarget->hasAVX512())) {
    6606           0 :     return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6607             :   }
    6608        1744 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6609        1744 :     return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6610             :   }
    6611           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6612           0 :     return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6613             :   }
    6614           0 :   if ((!Subtarget->hasSSE2())) {
    6615           0 :     return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6616             :   }
    6617             :   return 0;
    6618             : }
    6619             : 
    6620             : unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6621           0 :   if (RetVT.SimpleTy != MVT::f80)
    6622             :     return 0;
    6623           0 :   return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6624             : }
    6625             : 
    6626           6 : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6627           6 :   if (RetVT.SimpleTy != MVT::v4f32)
    6628             :     return 0;
    6629          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6630           2 :     return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6631             :   }
    6632           4 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6633           2 :     return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6634             :   }
    6635           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6636           2 :     return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6637             :   }
    6638             :   return 0;
    6639             : }
    6640             : 
    6641           2 : unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6642           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6643             :     return 0;
    6644           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6645           0 :     return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6646             :   }
    6647           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6648           2 :     return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6649             :   }
    6650             :   return 0;
    6651             : }
    6652             : 
    6653             : unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6654           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6655             :     return 0;
    6656           0 :   if ((Subtarget->hasAVX512())) {
    6657           0 :     return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6658             :   }
    6659             :   return 0;
    6660             : }
    6661             : 
    6662           6 : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6663           6 :   if (RetVT.SimpleTy != MVT::v2f64)
    6664             :     return 0;
    6665          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6666           2 :     return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6667             :   }
    6668           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6669           2 :     return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6670             :   }
    6671           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6672           2 :     return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6673             :   }
    6674             :   return 0;
    6675             : }
    6676             : 
    6677           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6678           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6679             :     return 0;
    6680           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6681           0 :     return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6682             :   }
    6683           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6684           2 :     return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6685             :   }
    6686             :   return 0;
    6687             : }
    6688             : 
    6689             : unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6690           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6691             :     return 0;
    6692           0 :   if ((Subtarget->hasAVX512())) {
    6693           0 :     return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6694             :   }
    6695             :   return 0;
    6696             : }
    6697             : 
    6698        2028 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6699        2028 :   switch (VT.SimpleTy) {
    6700         268 :   case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6701        1744 :   case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6702           0 :   case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6703           6 :   case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6704           2 :   case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6705           0 :   case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6706           6 :   case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6707           2 :   case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6708           0 :   case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6709             :   default: return 0;
    6710             :   }
    6711             : }
    6712             : 
    6713             : // FastEmit functions for ISD::MUL.
    6714             : 
    6715           2 : unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6716           2 :   if (RetVT.SimpleTy != MVT::i8)
    6717             :     return 0;
    6718           4 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
    6719           2 :   return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill);
    6720             : }
    6721             : 
    6722             : unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6723           0 :   if (RetVT.SimpleTy != MVT::i16)
    6724             :     return 0;
    6725           0 :   return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6726             : }
    6727             : 
    6728             : unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6729         357 :   if (RetVT.SimpleTy != MVT::i32)
    6730             :     return 0;
    6731         357 :   return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6732             : }
    6733             : 
    6734             : unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6735         928 :   if (RetVT.SimpleTy != MVT::i64)
    6736             :     return 0;
    6737         928 :   return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6738             : }
    6739             : 
    6740           6 : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6741           6 :   if (RetVT.SimpleTy != MVT::v8i16)
    6742             :     return 0;
    6743           6 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6744           2 :     return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6745             :   }
    6746           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6747           2 :     return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6748             :   }
    6749           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6750           2 :     return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6751             :   }
    6752             :   return 0;
    6753             : }
    6754             : 
    6755           2 : unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6756           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    6757             :     return 0;
    6758           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6759           0 :     return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6760             :   }
    6761           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6762           2 :     return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6763             :   }
    6764             :   return 0;
    6765             : }
    6766             : 
    6767             : unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6768           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6769             :     return 0;
    6770           0 :   if ((Subtarget->hasBWI())) {
    6771           0 :     return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6772             :   }
    6773             :   return 0;
    6774             : }
    6775             : 
    6776           6 : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6777           6 :   if (RetVT.SimpleTy != MVT::v4i32)
    6778             :     return 0;
    6779          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6780           2 :     return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6781             :   }
    6782           4 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6783           2 :     return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6784             :   }
    6785           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6786           2 :     return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6787             :   }
    6788             :   return 0;
    6789             : }
    6790             : 
    6791           2 : unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6792           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    6793             :     return 0;
    6794           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6795           0 :     return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6796             :   }
    6797           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6798           2 :     return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6799             :   }
    6800             :   return 0;
    6801             : }
    6802             : 
    6803             : unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6804           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6805             :     return 0;
    6806           0 :   if ((Subtarget->hasAVX512())) {
    6807           0 :     return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6808             :   }
    6809             :   return 0;
    6810             : }
    6811             : 
    6812          12 : unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6813          12 :   if (RetVT.SimpleTy != MVT::v2i64)
    6814             :     return 0;
    6815          12 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6816           4 :     return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6817             :   }
    6818             :   return 0;
    6819             : }
    6820             : 
    6821           4 : unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6822           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    6823             :     return 0;
    6824           4 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6825           0 :     return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6826             :   }
    6827             :   return 0;
    6828             : }
    6829             : 
    6830             : unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6831           4 :   if (RetVT.SimpleTy != MVT::v8i64)
    6832             :     return 0;
    6833           4 :   if ((Subtarget->hasDQI())) {
    6834           0 :     return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6835             :   }
    6836             :   return 0;
    6837             : }
    6838             : 
    6839        1323 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6840        1323 :   switch (VT.SimpleTy) {
    6841           2 :   case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6842           0 :   case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6843         357 :   case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6844         928 :   case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6845           6 :   case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6846           2 :   case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6847           0 :   case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6848           6 :   case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6849           2 :   case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6850           0 :   case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6851          12 :   case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6852           4 :   case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6853           4 :   case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6854             :   default: return 0;
    6855             :   }
    6856             : }
    6857             : 
    6858             : // FastEmit functions for ISD::MULHS.
    6859             : 
    6860           0 : unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6861           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6862             :     return 0;
    6863           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6864           0 :     return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6865             :   }
    6866           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6867           0 :     return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6868             :   }
    6869           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6870           0 :     return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6871             :   }
    6872             :   return 0;
    6873             : }
    6874             : 
    6875           0 : unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6876           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6877             :     return 0;
    6878           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6879           0 :     return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6880             :   }
    6881           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6882           0 :     return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6883             :   }
    6884             :   return 0;
    6885             : }
    6886             : 
    6887             : unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6888           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6889             :     return 0;
    6890           0 :   if ((Subtarget->hasBWI())) {
    6891           0 :     return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6892             :   }
    6893             :   return 0;
    6894             : }
    6895             : 
    6896           0 : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6897           0 :   switch (VT.SimpleTy) {
    6898           0 :   case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6899           0 :   case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6900           0 :   case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6901             :   default: return 0;
    6902             :   }
    6903             : }
    6904             : 
    6905             : // FastEmit functions for ISD::MULHU.
    6906             : 
    6907           0 : unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6908           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6909             :     return 0;
    6910           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6911           0 :     return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6912             :   }
    6913           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6914           0 :     return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6915             :   }
    6916           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6917           0 :     return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6918             :   }
    6919             :   return 0;
    6920             : }
    6921             : 
    6922           0 : unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6923           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6924             :     return 0;
    6925           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6926           0 :     return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6927             :   }
    6928           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6929           0 :     return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6930             :   }
    6931             :   return 0;
    6932             : }
    6933             : 
    6934             : unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6935           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6936             :     return 0;
    6937           0 :   if ((Subtarget->hasBWI())) {
    6938           0 :     return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6939             :   }
    6940             :   return 0;
    6941             : }
    6942             : 
    6943           0 : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6944           0 :   switch (VT.SimpleTy) {
    6945           0 :   case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6946           0 :   case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6947           0 :   case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6948             :   default: return 0;
    6949             :   }
    6950             : }
    6951             : 
    6952             : // FastEmit functions for ISD::OR.
    6953             : 
    6954             : unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6955          99 :   if (RetVT.SimpleTy != MVT::i8)
    6956             :     return 0;
    6957          99 :   return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6958             : }
    6959             : 
    6960             : unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6961           2 :   if (RetVT.SimpleTy != MVT::i16)
    6962             :     return 0;
    6963           2 :   return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6964             : }
    6965             : 
    6966             : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6967        5880 :   if (RetVT.SimpleTy != MVT::i32)
    6968             :     return 0;
    6969        5880 :   return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6970             : }
    6971             : 
    6972             : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6973        1983 :   if (RetVT.SimpleTy != MVT::i64)
    6974             :     return 0;
    6975        1983 :   return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6976             : }
    6977             : 
    6978             : unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6979           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    6980             :     return 0;
    6981           0 :   if ((Subtarget->hasDQI())) {
    6982           0 :     return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6983             :   }
    6984             :   return 0;
    6985             : }
    6986             : 
    6987             : unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6988           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    6989             :     return 0;
    6990           0 :   if ((Subtarget->hasAVX512())) {
    6991           0 :     return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6992             :   }
    6993             :   return 0;
    6994             : }
    6995             : 
    6996             : unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6997           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    6998             :     return 0;
    6999           0 :   if ((Subtarget->hasBWI())) {
    7000           0 :     return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7001             :   }
    7002             :   return 0;
    7003             : }
    7004             : 
    7005             : unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7006           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    7007             :     return 0;
    7008           0 :   if ((Subtarget->hasBWI())) {
    7009           0 :     return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7010             :   }
    7011             :   return 0;
    7012             : }
    7013             : 
    7014           8 : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7015           8 :   if (RetVT.SimpleTy != MVT::v2i64)
    7016             :     return 0;
    7017          16 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7018           2 :     return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7019             :   }
    7020           6 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7021           2 :     return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7022             :   }
    7023           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7024           4 :     return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7025             :   }
    7026             :   return 0;
    7027             : }
    7028             : 
    7029           6 : unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7030           6 :   if (RetVT.SimpleTy != MVT::v4i64)
    7031             :     return 0;
    7032          12 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    7033           4 :     return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7034             :   }
    7035           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7036           0 :     return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7037             :   }
    7038           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7039           2 :     return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7040             :   }
    7041             :   return 0;
    7042             : }
    7043             : 
    7044             : unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7045           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7046             :     return 0;
    7047           0 :   if ((Subtarget->hasAVX512())) {
    7048           0 :     return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7049             :   }
    7050             :   return 0;
    7051             : }
    7052             : 
    7053        7990 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7054        7990 :   switch (VT.SimpleTy) {
    7055          99 :   case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7056           2 :   case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7057        5880 :   case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7058        1983 :   case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7059           0 :   case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7060           0 :   case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7061           0 :   case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7062           0 :   case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7063           8 :   case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7064           6 :   case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7065           0 :   case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7066             :   default: return 0;
    7067             :   }
    7068             : }
    7069             : 
    7070             : // FastEmit functions for ISD::ROTL.
    7071             : 
    7072           0 : unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7073           0 :   if (RetVT.SimpleTy != MVT::i8)
    7074             :     return 0;
    7075           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7076           0 :   return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7077             : }
    7078             : 
    7079             : unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7080           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7081             :     return 0;
    7082           0 :   if ((Subtarget->hasXOP())) {
    7083           0 :     return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7084             :   }
    7085             :   return 0;
    7086             : }
    7087             : 
    7088             : unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7089           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7090             :     return 0;
    7091           0 :   if ((Subtarget->hasXOP())) {
    7092           0 :     return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7093             :   }
    7094             :   return 0;
    7095             : }
    7096             : 
    7097           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7098           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7099             :     return 0;
    7100           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7101           0 :     return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7102             :   }
    7103           0 :   if ((Subtarget->hasXOP())) {
    7104           0 :     return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7105             :   }
    7106             :   return 0;
    7107             : }
    7108             : 
    7109           0 : unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7110           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7111             :     return 0;
    7112           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7113           0 :     return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7114             :   }
    7115             :   return 0;
    7116             : }
    7117             : 
    7118             : unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7119           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7120             :     return 0;
    7121           0 :   if ((Subtarget->hasAVX512())) {
    7122           0 :     return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7123             :   }
    7124             :   return 0;
    7125             : }
    7126             : 
    7127           0 : unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7128           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7129             :     return 0;
    7130           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7131           0 :     return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7132             :   }
    7133           0 :   if ((Subtarget->hasXOP())) {
    7134           0 :     return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7135             :   }
    7136             :   return 0;
    7137             : }
    7138             : 
    7139           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7140           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7141             :     return 0;
    7142           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7143           0 :     return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7144             :   }
    7145             :   return 0;
    7146             : }
    7147             : 
    7148             : unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7149           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7150             :     return 0;
    7151           0 :   if ((Subtarget->hasAVX512())) {
    7152           0 :     return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7153             :   }
    7154             :   return 0;
    7155             : }
    7156             : 
    7157           0 : unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7158           0 :   switch (VT.SimpleTy) {
    7159           0 :   case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7160           0 :   case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7161           0 :   case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7162           0 :   case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7163           0 :   case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7164           0 :   case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7165           0 :   case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7166           0 :   case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7167           0 :   case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7168             :   default: return 0;
    7169             :   }
    7170             : }
    7171             : 
    7172             : // FastEmit functions for ISD::ROTR.
    7173             : 
    7174           0 : unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7175           0 :   if (RetVT.SimpleTy != MVT::i8)
    7176             :     return 0;
    7177           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7178           0 :   return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7179             : }
    7180             : 
    7181           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7182           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7183             :     return 0;
    7184           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7185           0 :     return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7186             :   }
    7187             :   return 0;
    7188             : }
    7189             : 
    7190           0 : unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7191           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7192             :     return 0;
    7193           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7194           0 :     return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7195             :   }
    7196             :   return 0;
    7197             : }
    7198             : 
    7199             : unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7200           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7201             :     return 0;
    7202           0 :   if ((Subtarget->hasAVX512())) {
    7203           0 :     return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7204             :   }
    7205             :   return 0;
    7206             : }
    7207             : 
    7208           0 : unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7209           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7210             :     return 0;
    7211           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7212           0 :     return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7213             :   }
    7214             :   return 0;
    7215             : }
    7216             : 
    7217           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7218           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7219             :     return 0;
    7220           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7221           0 :     return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7222             :   }
    7223             :   return 0;
    7224             : }
    7225             : 
    7226             : unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7227           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7228             :     return 0;
    7229           0 :   if ((Subtarget->hasAVX512())) {
    7230           0 :     return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7231             :   }
    7232             :   return 0;
    7233             : }
    7234             : 
    7235           0 : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7236           0 :   switch (VT.SimpleTy) {
    7237           0 :   case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7238           0 :   case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7239           0 :   case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7240           0 :   case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7241           0 :   case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7242           0 :   case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7243           0 :   case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7244             :   default: return 0;
    7245             :   }
    7246             : }
    7247             : 
    7248             : // FastEmit functions for ISD::SHL.
    7249             : 
    7250           3 : unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7251           3 :   if (RetVT.SimpleTy != MVT::i8)
    7252             :     return 0;
    7253           6 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7254           3 :   return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7255             : }
    7256             : 
    7257           0 : unsigned fastEmit_ISD_SHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7258           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7259             :     return 0;
    7260           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7261           0 :     return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7262             :   }
    7263             :   return 0;
    7264             : }
    7265             : 
    7266           0 : unsigned fastEmit_ISD_SHL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7267           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7268             :     return 0;
    7269           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7270           0 :     return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7271             :   }
    7272             :   return 0;
    7273             : }
    7274             : 
    7275             : unsigned fastEmit_ISD_SHL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7276           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7277             :     return 0;
    7278           0 :   if ((Subtarget->hasBWI())) {
    7279           0 :     return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7280             :   }
    7281             :   return 0;
    7282             : }
    7283             : 
    7284           0 : unsigned fastEmit_ISD_SHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7285           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7286             :     return 0;
    7287           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7288           0 :     return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7289             :   }
    7290           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7291           0 :     return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7292             :   }
    7293             :   return 0;
    7294             : }
    7295             : 
    7296           0 : unsigned fastEmit_ISD_SHL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7297           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7298             :     return 0;
    7299           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7300           0 :     return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7301             :   }
    7302           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7303           0 :     return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7304             :   }
    7305             :   return 0;
    7306             : }
    7307             : 
    7308             : unsigned fastEmit_ISD_SHL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7309           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7310             :     return 0;
    7311           0 :   if ((Subtarget->hasAVX512())) {
    7312           0 :     return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7313             :   }
    7314             :   return 0;
    7315             : }
    7316             : 
    7317           0 : unsigned fastEmit_ISD_SHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7318           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7319             :     return 0;
    7320           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7321           0 :     return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7322             :   }
    7323           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7324           0 :     return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7325             :   }
    7326             :   return 0;
    7327             : }
    7328             : 
    7329           0 : unsigned fastEmit_ISD_SHL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7330           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7331             :     return 0;
    7332           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7333           0 :     return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7334             :   }
    7335           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7336           0 :     return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7337             :   }
    7338             :   return 0;
    7339             : }
    7340             : 
    7341             : unsigned fastEmit_ISD_SHL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7342           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7343             :     return 0;
    7344           0 :   if ((Subtarget->hasAVX512())) {
    7345           0 :     return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7346             :   }
    7347             :   return 0;
    7348             : }
    7349             : 
    7350        3169 : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7351        3169 :   switch (VT.SimpleTy) {
    7352           3 :   case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7353           0 :   case MVT::v8i16: return fastEmit_ISD_SHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7354           0 :   case MVT::v16i16: return fastEmit_ISD_SHL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7355           0 :   case MVT::v32i16: return fastEmit_ISD_SHL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7356           0 :   case MVT::v4i32: return fastEmit_ISD_SHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7357           0 :   case MVT::v8i32: return fastEmit_ISD_SHL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7358           0 :   case MVT::v16i32: return fastEmit_ISD_SHL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7359           0 :   case MVT::v2i64: return fastEmit_ISD_SHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7360           0 :   case MVT::v4i64: return fastEmit_ISD_SHL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7361           0 :   case MVT::v8i64: return fastEmit_ISD_SHL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7362             :   default: return 0;
    7363             :   }
    7364             : }
    7365             : 
    7366             : // FastEmit functions for ISD::SMAX.
    7367             : 
    7368           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7369           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7370             :     return 0;
    7371           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7372           0 :     return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7373             :   }
    7374           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7375           0 :     return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7376             :   }
    7377           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7378           0 :     return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7379             :   }
    7380             :   return 0;
    7381             : }
    7382             : 
    7383           0 : unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7384           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7385             :     return 0;
    7386           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7387           0 :     return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7388             :   }
    7389           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7390           0 :     return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7391             :   }
    7392             :   return 0;
    7393             : }
    7394             : 
    7395             : unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7396           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7397             :     return 0;
    7398           0 :   if ((Subtarget->hasBWI())) {
    7399           0 :     return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7400             :   }
    7401             :   return 0;
    7402             : }
    7403             : 
    7404           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7405           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7406             :     return 0;
    7407           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7408           0 :     return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7409             :   }
    7410           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7411           0 :     return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7412             :   }
    7413           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7414           0 :     return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7415             :   }
    7416             :   return 0;
    7417             : }
    7418             : 
    7419           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7420           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7421             :     return 0;
    7422           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7423           0 :     return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7424             :   }
    7425           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7426           0 :     return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7427             :   }
    7428             :   return 0;
    7429             : }
    7430             : 
    7431             : unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7432           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7433             :     return 0;
    7434           0 :   if ((Subtarget->hasBWI())) {
    7435           0 :     return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7436             :   }
    7437             :   return 0;
    7438             : }
    7439             : 
    7440           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7441           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7442             :     return 0;
    7443           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7444           0 :     return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7445             :   }
    7446           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7447           0 :     return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7448             :   }
    7449           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7450           0 :     return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7451             :   }
    7452             :   return 0;
    7453             : }
    7454             : 
    7455           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7456           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7457             :     return 0;
    7458           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7459           0 :     return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7460             :   }
    7461           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7462           0 :     return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7463             :   }
    7464             :   return 0;
    7465             : }
    7466             : 
    7467             : unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7468           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7469             :     return 0;
    7470           0 :   if ((Subtarget->hasAVX512())) {
    7471           0 :     return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7472             :   }
    7473             :   return 0;
    7474             : }
    7475             : 
    7476           0 : unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7477           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7478             :     return 0;
    7479           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7480           0 :     return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7481             :   }
    7482             :   return 0;
    7483             : }
    7484             : 
    7485           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7486           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7487             :     return 0;
    7488           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7489           0 :     return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7490             :   }
    7491             :   return 0;
    7492             : }
    7493             : 
    7494             : unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7495           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7496             :     return 0;
    7497           0 :   if ((Subtarget->hasAVX512())) {
    7498           0 :     return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7499             :   }
    7500             :   return 0;
    7501             : }
    7502             : 
    7503           0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7504           0 :   switch (VT.SimpleTy) {
    7505           0 :   case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7506           0 :   case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7507           0 :   case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7508           0 :   case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7509           0 :   case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7510           0 :   case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7511           0 :   case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7512           0 :   case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7513           0 :   case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7514           0 :   case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7515           0 :   case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7516           0 :   case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7517             :   default: return 0;
    7518             :   }
    7519             : }
    7520             : 
    7521             : // FastEmit functions for ISD::SMIN.
    7522             : 
    7523           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7524           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7525             :     return 0;
    7526           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7527           0 :     return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7528             :   }
    7529           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7530           0 :     return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7531             :   }
    7532           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7533           0 :     return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7534             :   }
    7535             :   return 0;
    7536             : }
    7537             : 
    7538           0 : unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7539           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7540             :     return 0;
    7541           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7542           0 :     return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7543             :   }
    7544           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7545           0 :     return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7546             :   }
    7547             :   return 0;
    7548             : }
    7549             : 
    7550             : unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7551           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7552             :     return 0;
    7553           0 :   if ((Subtarget->hasBWI())) {
    7554           0 :     return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7555             :   }
    7556             :   return 0;
    7557             : }
    7558             : 
    7559           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7560           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7561             :     return 0;
    7562           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7563           0 :     return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7564             :   }
    7565           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7566           0 :     return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7567             :   }
    7568           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7569           0 :     return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7570             :   }
    7571             :   return 0;
    7572             : }
    7573             : 
    7574           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7575           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7576             :     return 0;
    7577           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7578           0 :     return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7579             :   }
    7580           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7581           0 :     return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7582             :   }
    7583             :   return 0;
    7584             : }
    7585             : 
    7586             : unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7587           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7588             :     return 0;
    7589           0 :   if ((Subtarget->hasBWI())) {
    7590           0 :     return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7591             :   }
    7592             :   return 0;
    7593             : }
    7594             : 
    7595           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7596           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7597             :     return 0;
    7598           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7599           0 :     return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7600             :   }
    7601           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7602           0 :     return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7603             :   }
    7604           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7605           0 :     return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7606             :   }
    7607             :   return 0;
    7608             : }
    7609             : 
    7610           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7611           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7612             :     return 0;
    7613           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7614           0 :     return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7615             :   }
    7616           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7617           0 :     return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7618             :   }
    7619             :   return 0;
    7620             : }
    7621             : 
    7622             : unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7623           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7624             :     return 0;
    7625           0 :   if ((Subtarget->hasAVX512())) {
    7626           0 :     return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7627             :   }
    7628             :   return 0;
    7629             : }
    7630             : 
    7631           0 : unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7632           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7633             :     return 0;
    7634           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7635           0 :     return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7636             :   }
    7637             :   return 0;
    7638             : }
    7639             : 
    7640           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7641           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7642             :     return 0;
    7643           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7644           0 :     return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7645             :   }
    7646             :   return 0;
    7647             : }
    7648             : 
    7649             : unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7650           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7651             :     return 0;
    7652           0 :   if ((Subtarget->hasAVX512())) {
    7653           0 :     return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7654             :   }
    7655             :   return 0;
    7656             : }
    7657             : 
    7658           0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7659           0 :   switch (VT.SimpleTy) {
    7660           0 :   case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7661           0 :   case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7662           0 :   case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7663           0 :   case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7664           0 :   case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7665           0 :   case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7666           0 :   case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7667           0 :   case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7668           0 :   case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7669           0 :   case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7670           0 :   case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7671           0 :   case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7672             :   default: return 0;
    7673             :   }
    7674             : }
    7675             : 
    7676             : // FastEmit functions for ISD::SRA.
    7677             : 
    7678           1 : unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7679           1 :   if (RetVT.SimpleTy != MVT::i8)
    7680             :     return 0;
    7681           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7682           1 :   return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7683             : }
    7684             : 
    7685           0 : unsigned fastEmit_ISD_SRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7686           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7687             :     return 0;
    7688           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7689           0 :     return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7690             :   }
    7691             :   return 0;
    7692             : }
    7693             : 
    7694           0 : unsigned fastEmit_ISD_SRA_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7695           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7696             :     return 0;
    7697           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7698           0 :     return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7699             :   }
    7700             :   return 0;
    7701             : }
    7702             : 
    7703             : unsigned fastEmit_ISD_SRA_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7704           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7705             :     return 0;
    7706           0 :   if ((Subtarget->hasBWI())) {
    7707           0 :     return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7708             :   }
    7709             :   return 0;
    7710             : }
    7711             : 
    7712           0 : unsigned fastEmit_ISD_SRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7713           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7714             :     return 0;
    7715           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7716           0 :     return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7717             :   }
    7718           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7719           0 :     return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7720             :   }
    7721             :   return 0;
    7722             : }
    7723             : 
    7724           0 : unsigned fastEmit_ISD_SRA_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7725           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7726             :     return 0;
    7727           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7728           0 :     return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7729             :   }
    7730           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7731           0 :     return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7732             :   }
    7733             :   return 0;
    7734             : }
    7735             : 
    7736             : unsigned fastEmit_ISD_SRA_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7737           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7738             :     return 0;
    7739           0 :   if ((Subtarget->hasAVX512())) {
    7740           0 :     return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7741             :   }
    7742             :   return 0;
    7743             : }
    7744             : 
    7745           0 : unsigned fastEmit_ISD_SRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7746           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7747             :     return 0;
    7748           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7749           0 :     return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7750             :   }
    7751             :   return 0;
    7752             : }
    7753             : 
    7754           0 : unsigned fastEmit_ISD_SRA_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7755           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7756             :     return 0;
    7757           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7758           0 :     return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7759             :   }
    7760             :   return 0;
    7761             : }
    7762             : 
    7763             : unsigned fastEmit_ISD_SRA_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7764           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7765             :     return 0;
    7766           0 :   if ((Subtarget->hasAVX512())) {
    7767           0 :     return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7768             :   }
    7769             :   return 0;
    7770             : }
    7771             : 
    7772          18 : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7773          18 :   switch (VT.SimpleTy) {
    7774           1 :   case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7775           0 :   case MVT::v8i16: return fastEmit_ISD_SRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7776           0 :   case MVT::v16i16: return fastEmit_ISD_SRA_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7777           0 :   case MVT::v32i16: return fastEmit_ISD_SRA_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7778           0 :   case MVT::v4i32: return fastEmit_ISD_SRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7779           0 :   case MVT::v8i32: return fastEmit_ISD_SRA_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7780           0 :   case MVT::v16i32: return fastEmit_ISD_SRA_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7781           0 :   case MVT::v2i64: return fastEmit_ISD_SRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7782           0 :   case MVT::v4i64: return fastEmit_ISD_SRA_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7783           0 :   case MVT::v8i64: return fastEmit_ISD_SRA_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7784             :   default: return 0;
    7785             :   }
    7786             : }
    7787             : 
    7788             : // FastEmit functions for ISD::SRL.
    7789             : 
    7790           1 : unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7791           1 :   if (RetVT.SimpleTy != MVT::i8)
    7792             :     return 0;
    7793           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7794           1 :   return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7795             : }
    7796             : 
    7797           0 : unsigned fastEmit_ISD_SRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7798           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7799             :     return 0;
    7800           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7801           0 :     return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7802             :   }
    7803             :   return 0;
    7804             : }
    7805             : 
    7806           0 : unsigned fastEmit_ISD_SRL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7807           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7808             :     return 0;
    7809           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7810           0 :     return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7811             :   }
    7812             :   return 0;
    7813             : }
    7814             : 
    7815             : unsigned fastEmit_ISD_SRL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7816           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7817             :     return 0;
    7818           0 :   if ((Subtarget->hasBWI())) {
    7819           0 :     return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7820             :   }
    7821             :   return 0;
    7822             : }
    7823             : 
    7824           0 : unsigned fastEmit_ISD_SRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7825           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7826             :     return 0;
    7827           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7828           0 :     return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7829             :   }
    7830           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7831           0 :     return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7832             :   }
    7833             :   return 0;
    7834             : }
    7835             : 
    7836           0 : unsigned fastEmit_ISD_SRL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7837           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7838             :     return 0;
    7839           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7840           0 :     return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7841             :   }
    7842           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7843           0 :     return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7844             :   }
    7845             :   return 0;
    7846             : }
    7847             : 
    7848             : unsigned fastEmit_ISD_SRL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7849           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7850             :     return 0;
    7851           0 :   if ((Subtarget->hasAVX512())) {
    7852           0 :     return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7853             :   }
    7854             :   return 0;
    7855             : }
    7856             : 
    7857           0 : unsigned fastEmit_ISD_SRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7858           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7859             :     return 0;
    7860           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7861           0 :     return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7862             :   }
    7863           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7864           0 :     return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7865             :   }
    7866             :   return 0;
    7867             : }
    7868             : 
    7869           0 : unsigned fastEmit_ISD_SRL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7870           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7871             :     return 0;
    7872           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7873           0 :     return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7874             :   }
    7875           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7876           0 :     return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7877             :   }
    7878             :   return 0;
    7879             : }
    7880             : 
    7881             : unsigned fastEmit_ISD_SRL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7882           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7883             :     return 0;
    7884           0 :   if ((Subtarget->hasAVX512())) {
    7885           0 :     return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7886             :   }
    7887             :   return 0;
    7888             : }
    7889             : 
    7890        3607 : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7891        3607 :   switch (VT.SimpleTy) {
    7892           1 :   case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7893           0 :   case MVT::v8i16: return fastEmit_ISD_SRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7894           0 :   case MVT::v16i16: return fastEmit_ISD_SRL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7895           0 :   case MVT::v32i16: return fastEmit_ISD_SRL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7896           0 :   case MVT::v4i32: return fastEmit_ISD_SRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7897           0 :   case MVT::v8i32: return fastEmit_ISD_SRL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7898           0 :   case MVT::v16i32: return fastEmit_ISD_SRL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7899           0 :   case MVT::v2i64: return fastEmit_ISD_SRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7900           0 :   case MVT::v4i64: return fastEmit_ISD_SRL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7901           0 :   case MVT::v8i64: return fastEmit_ISD_SRL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7902             :   default: return 0;
    7903             :   }
    7904             : }
    7905             : 
    7906             : // FastEmit functions for ISD::SUB.
    7907             : 
    7908             : unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7909           0 :   if (RetVT.SimpleTy != MVT::i8)
    7910             :     return 0;
    7911           0 :   return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7912             : }
    7913             : 
    7914             : unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7915           0 :   if (RetVT.SimpleTy != MVT::i16)
    7916             :     return 0;
    7917           0 :   return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7918             : }
    7919             : 
    7920             : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7921        4699 :   if (RetVT.SimpleTy != MVT::i32)
    7922             :     return 0;
    7923        4699 :   return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7924             : }
    7925             : 
    7926             : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7927       71550 :   if (RetVT.SimpleTy != MVT::i64)
    7928             :     return 0;
    7929       71550 :   return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7930             : }
    7931             : 
    7932          12 : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7933          12 :   if (RetVT.SimpleTy != MVT::v16i8)
    7934             :     return 0;
    7935          12 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7936           4 :     return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7937             :   }
    7938          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7939           4 :     return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7940             :   }
    7941           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7942           4 :     return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7943             :   }
    7944             :   return 0;
    7945             : }
    7946             : 
    7947           4 : unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7948           4 :   if (RetVT.SimpleTy != MVT::v32i8)
    7949             :     return 0;
    7950           4 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7951           0 :     return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7952             :   }
    7953           8 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7954           4 :     return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7955             :   }
    7956             :   return 0;
    7957             : }
    7958             : 
    7959             : unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7960           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7961             :     return 0;
    7962           0 :   if ((Subtarget->hasBWI())) {
    7963           0 :     return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7964             :   }
    7965             :   return 0;
    7966             : }
    7967             : 
    7968          12 : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7969          12 :   if (RetVT.SimpleTy != MVT::v8i16)
    7970             :     return 0;
    7971          12 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7972           4 :     return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7973             :   }
    7974          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7975           4 :     return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7976             :   }
    7977           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7978           4 :     return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7979             :   }
    7980             :   return 0;
    7981             : }
    7982             : 
    7983           4 : unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7984           4 :   if (RetVT.SimpleTy != MVT::v16i16)
    7985             :     return 0;
    7986           4 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7987           0 :     return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7988             :   }
    7989           8 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7990           4 :     return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7991             :   }
    7992             :   return 0;
    7993             : }
    7994             : 
    7995             : unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7996           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7997             :     return 0;
    7998           0 :   if ((Subtarget->hasBWI())) {
    7999           0 :     return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8000             :   }
    8001             :   return 0;
    8002             : }
    8003             : 
    8004           6 : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8005           6 :   if (RetVT.SimpleTy != MVT::v4i32)
    8006             :     return 0;
    8007          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8008           2 :     return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8009             :   }
    8010           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8011           2 :     return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8012             :   }
    8013           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8014           2 :     return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8015             :   }
    8016             :   return 0;
    8017             : }
    8018             : 
    8019           2 : unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8020           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    8021             :     return 0;
    8022           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8023           0 :     return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8024             :   }
    8025           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8026           2 :     return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8027             :   }
    8028             :   return 0;
    8029             : }
    8030             : 
    8031             : unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8032           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    8033             :     return 0;
    8034           0 :   if ((Subtarget->hasAVX512())) {
    8035           0 :     return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8036             :   }
    8037             :   return 0;
    8038             : }
    8039             : 
    8040           6 : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8041           6 :   if (RetVT.SimpleTy != MVT::v2i64)
    8042             :     return 0;
    8043          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8044           2 :     return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8045             :   }
    8046           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8047           2 :     return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8048             :   }
    8049           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8050           2 :     return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8051             :   }
    8052             :   return 0;
    8053             : }
    8054             : 
    8055           2 : unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8056           2 :   if (RetVT.SimpleTy != MVT::v4i64)
    8057             :     return 0;
    8058           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8059           0 :     return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8060             :   }
    8061           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8062           2 :     return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8063             :   }
    8064             :   return 0;
    8065             : }
    8066             : 
    8067             : unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8068           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8069             :     return 0;
    8070           0 :   if ((Subtarget->hasAVX512())) {
    8071           0 :     return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8072             :   }
    8073             :   return 0;
    8074             : }
    8075             : 
    8076       76297 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8077       76297 :   switch (VT.SimpleTy) {
    8078           0 :   case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8079           0 :   case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8080        4699 :   case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8081       71550 :   case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8082          12 :   case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8083           4 :   case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8084           0 :   case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8085          12 :   case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8086           4 :   case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8087           0 :   case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8088           6 :   case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8089           2 :   case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8090           0 :   case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8091           6 :   case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8092           2 :   case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8093           0 :   case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8094             :   default: return 0;
    8095             :   }
    8096             : }
    8097             : 
    8098             : // FastEmit functions for ISD::UMAX.
    8099             : 
    8100           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8101           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8102             :     return 0;
    8103           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8104           0 :     return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8105             :   }
    8106           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8107           0 :     return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8108             :   }
    8109           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8110           0 :     return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8111             :   }
    8112             :   return 0;
    8113             : }
    8114             : 
    8115           0 : unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8116           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8117             :     return 0;
    8118           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8119           0 :     return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8120             :   }
    8121           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8122           0 :     return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8123             :   }
    8124             :   return 0;
    8125             : }
    8126             : 
    8127             : unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8128           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8129             :     return 0;
    8130           0 :   if ((Subtarget->hasBWI())) {
    8131           0 :     return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8132             :   }
    8133             :   return 0;
    8134             : }
    8135             : 
    8136           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8137           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8138             :     return 0;
    8139           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8140           0 :     return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8141             :   }
    8142           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    8143           0 :     return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8144             :   }
    8145           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8146           0 :     return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8147             :   }
    8148             :   return 0;
    8149             : }
    8150             : 
    8151           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8152           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8153             :     return 0;
    8154           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8155           0 :     return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8156             :   }
    8157           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8158           0 :     return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8159             :   }
    8160             :   return 0;
    8161             : }
    8162             : 
    8163             : unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8164           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8165             :     return 0;
    8166           0 :   if ((Subtarget->hasBWI())) {
    8167           0 :     return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8168             :   }
    8169             :   return 0;
    8170             : }
    8171             : 
    8172           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8173           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    8174             :     return 0;
    8175           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8176           0 :     return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8177             :   }
    8178           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    8179           0 :     return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8180             :   }
    8181           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8182           0 :     return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8183             :   }
    8184             :   return 0;
    8185             : }
    8186             : 
    8187           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8188           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    8189             :     return 0;
    8190           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8191           0 :     return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8192             :   }
    8193           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8194           0 :     return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8195             :   }
    8196             :   return 0;
    8197             : }
    8198             : 
    8199             : unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8200           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    8201             :     return 0;
    8202           0 :   if ((Subtarget->hasAVX512())) {
    8203           0 :     return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8204             :   }
    8205             :   return 0;
    8206             : }
    8207             : 
    8208           0 : unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8209           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    8210             :     return 0;
    8211           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8212           0 :     return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8213             :   }
    8214             :   return 0;
    8215             : }
    8216             : 
    8217           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8218           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    8219             :     return 0;
    8220           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8221           0 :     return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8222             :   }
    8223             :   return 0;
    8224             : }
    8225             : 
    8226             : unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8227           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8228             :     return 0;
    8229           0 :   if ((Subtarget->hasAVX512())) {
    8230           0 :     return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8231             :   }
    8232             :   return 0;
    8233             : }
    8234             : 
    8235           0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8236           0 :   switch (VT.SimpleTy) {
    8237           0 :   case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8238           0 :   case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8239           0 :   case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8240           0 :   case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8241           0 :   case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8242           0 :   case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8243           0 :   case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8244           0 :   case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8245           0 :   case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8246           0 :   case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8247           0 :   case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8248           0 :   case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8249             :   default: return 0;
    8250             :   }
    8251             : }
    8252             : 
    8253             : // FastEmit functions for ISD::UMIN.
    8254             : 
    8255           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8256           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8257             :     return 0;
    8258           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8259           0 :     return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8260             :   }
    8261           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8262           0 :     return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8263             :   }
    8264           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8265           0 :     return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8266             :   }
    8267             :   return 0;
    8268             : }
    8269             : 
    8270           0 : unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8271           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8272             :     return 0;
    8273           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8274           0 :     return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8275             :   }
    8276           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8277           0 :     return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8278             :   }
    8279             :   return 0;
    8280             : }
    8281             : 
    8282             : unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8283           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8284             :     return 0;
    8285           0 :   if ((Subtarget->hasBWI())) {
    8286           0 :     return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8287             :   }
    8288             :   return 0;
    8289             : }
    8290             : 
    8291           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8292           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8293             :     return 0;
    8294           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8295           0 :     return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8296             :   }
    8297           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    8298           0 :     return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8299             :   }
    8300           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8301           0 :     return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8302             :   }
    8303             :   return 0;
    8304             : }
    8305             : 
    8306           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8307           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8308             :     return 0;
    8309           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8310           0 :     return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8311             :   }
    8312           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8313           0 :     return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8314             :   }
    8315             :   return 0;
    8316             : }
    8317             : 
    8318             : unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8319           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8320             :     return 0;
    8321           0 :   if ((Subtarget->hasBWI())) {
    8322           0 :     return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8323             :   }
    8324             :   return 0;
    8325             : }
    8326             : 
    8327           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8328           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    8329             :     return 0;
    8330           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8331           0 :     return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8332             :   }
    8333           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    8334           0 :     return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8335             :   }
    8336           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8337           0 :     return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8338             :   }
    8339             :   return 0;
    8340             : }
    8341             : 
    8342           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8343           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    8344             :     return 0;
    8345           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8346           0 :     return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8347             :   }
    8348           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8349           0 :     return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8350             :   }
    8351             :   return 0;
    8352             : }
    8353             : 
    8354             : unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8355           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    8356             :     return 0;
    8357           0 :   if ((Subtarget->hasAVX512())) {
    8358           0 :     return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8359             :   }
    8360             :   return 0;
    8361             : }
    8362             : 
    8363           0 : unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8364           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    8365             :     return 0;
    8366           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8367           0 :     return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8368             :   }
    8369             :   return 0;
    8370             : }
    8371             : 
    8372           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8373           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    8374             :     return 0;
    8375           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8376           0 :     return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8377             :   }
    8378             :   return 0;
    8379             : }
    8380             : 
    8381             : unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8382           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8383             :     return 0;
    8384           0 :   if ((Subtarget->hasAVX512())) {
    8385           0 :     return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8386             :   }
    8387             :   return 0;
    8388             : }
    8389             : 
    8390           0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8391           0 :   switch (VT.SimpleTy) {
    8392           0 :   case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8393           0 :   case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8394           0 :   case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8395           0 :   case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8396           0 :   case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8397           0 :   case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8398           0 :   case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8399           0 :   case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8400           0 :   case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8401           0 :   case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8402           0 :   case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8403           0 :   case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8404             :   default: return 0;
    8405             :   }
    8406             : }
    8407             : 
    8408             : // FastEmit functions for ISD::XOR.
    8409             : 
    8410             : unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8411           0 :   if (RetVT.SimpleTy != MVT::i8)
    8412             :     return 0;
    8413           0 :   return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8414             : }
    8415             : 
    8416             : unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8417           0 :   if (RetVT.SimpleTy != MVT::i16)
    8418             :     return 0;
    8419           0 :   return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8420             : }
    8421             : 
    8422             : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8423         267 :   if (RetVT.SimpleTy != MVT::i32)
    8424             :     return 0;
    8425         267 :   return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8426             : }
    8427             : 
    8428             : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8429        1304 :   if (RetVT.SimpleTy != MVT::i64)
    8430             :     return 0;
    8431        1304 :   return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8432             : }
    8433             : 
    8434             : unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8435           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    8436             :     return 0;
    8437           0 :   if ((Subtarget->hasDQI())) {
    8438           0 :     return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8439             :   }
    8440             :   return 0;
    8441             : }
    8442             : 
    8443             : unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8444           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    8445             :     return 0;
    8446           0 :   if ((Subtarget->hasAVX512())) {
    8447           0 :     return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8448             :   }
    8449             :   return 0;
    8450             : }
    8451             : 
    8452             : unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8453           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    8454             :     return 0;
    8455           0 :   if ((Subtarget->hasBWI())) {
    8456           0 :     return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8457             :   }
    8458             :   return 0;
    8459             : }
    8460             : 
    8461             : unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8462           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    8463             :     return 0;
    8464           0 :   if ((Subtarget->hasBWI())) {
    8465           0 :     return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8466             :   }
    8467             :   return 0;
    8468             : }
    8469             : 
    8470           6 : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8471           6 :   if (RetVT.SimpleTy != MVT::v2i64)
    8472             :     return 0;
    8473          12 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8474           2 :     return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8475             :   }
    8476           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8477           2 :     return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8478             :   }
    8479           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8480           2 :     return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8481             :   }
    8482             :   return 0;
    8483             : }
    8484             : 
    8485           4 : unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8486           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    8487             :     return 0;
    8488           8 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    8489           2 :     return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8490             :   }
    8491           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8492           0 :     return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8493             :   }
    8494           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8495           2 :     return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8496             :   }
    8497             :   return 0;
    8498             : }
    8499             : 
    8500             : unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8501           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8502             :     return 0;
    8503           0 :   if ((Subtarget->hasAVX512())) {
    8504           0 :     return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8505             :   }
    8506             :   return 0;
    8507             : }
    8508             : 
    8509        1593 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8510        1593 :   switch (VT.SimpleTy) {
    8511           0 :   case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8512           0 :   case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8513         267 :   case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8514        1304 :   case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8515           0 :   case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8516           0 :   case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8517           0 :   case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8518           0 :   case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8519           6 :   case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8520           4 :   case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8521           0 :   case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8522             :   default: return 0;
    8523             :   }
    8524             : }
    8525             : 
    8526             : // FastEmit functions for X86ISD::ADDS.
    8527             : 
    8528           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8529           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8530             :     return 0;
    8531           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8532           0 :     return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8533             :   }
    8534           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8535           0 :     return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8536             :   }
    8537           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8538           0 :     return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8539             :   }
    8540             :   return 0;
    8541             : }
    8542             : 
    8543           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8544           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8545             :     return 0;
    8546           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8547           0 :     return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8548             :   }
    8549           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8550           0 :     return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8551             :   }
    8552             :   return 0;
    8553             : }
    8554             : 
    8555             : unsigned fastEmit_X86ISD_ADDS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8556           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8557             :     return 0;
    8558           0 :   if ((Subtarget->hasBWI())) {
    8559           0 :     return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8560             :   }
    8561             :   return 0;
    8562             : }
    8563             : 
    8564           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8565           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8566             :     return 0;
    8567           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8568           0 :     return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8569             :   }
    8570           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8571           0 :     return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8572             :   }
    8573           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8574           0 :     return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8575             :   }
    8576             :   return 0;
    8577             : }
    8578             : 
    8579           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8580           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8581             :     return 0;
    8582           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8583           0 :     return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8584             :   }
    8585           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8586           0 :     return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8587             :   }
    8588             :   return 0;
    8589             : }
    8590             : 
    8591             : unsigned fastEmit_X86ISD_ADDS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8592           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8593             :     return 0;
    8594           0 :   if ((Subtarget->hasBWI())) {
    8595           0 :     return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8596             :   }
    8597             :   return 0;
    8598             : }
    8599             : 
    8600           0 : unsigned fastEmit_X86ISD_ADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8601           0 :   switch (VT.SimpleTy) {
    8602           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8603           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8604           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8605           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8606           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8607           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8608             :   default: return 0;
    8609             :   }
    8610             : }
    8611             : 
    8612             : // FastEmit functions for X86ISD::ADDSUB.
    8613             : 
    8614           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8615           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8616             :     return 0;
    8617           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8618           0 :     return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8619             :   }
    8620           0 :   if ((Subtarget->hasAVX())) {
    8621           0 :     return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8622             :   }
    8623             :   return 0;
    8624             : }
    8625             : 
    8626             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8627           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8628             :     return 0;
    8629           0 :   if ((Subtarget->hasAVX())) {
    8630           0 :     return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8631             :   }
    8632             :   return 0;
    8633             : }
    8634             : 
    8635           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8636           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8637             :     return 0;
    8638           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8639           0 :     return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8640             :   }
    8641           0 :   if ((Subtarget->hasAVX())) {
    8642           0 :     return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8643             :   }
    8644             :   return 0;
    8645             : }
    8646             : 
    8647             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8648           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8649             :     return 0;
    8650           0 :   if ((Subtarget->hasAVX())) {
    8651           0 :     return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8652             :   }
    8653             :   return 0;
    8654             : }
    8655             : 
    8656           0 : unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8657           0 :   switch (VT.SimpleTy) {
    8658           0 :   case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8659           0 :   case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8660           0 :   case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8661           0 :   case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8662             :   default: return 0;
    8663             :   }
    8664             : }
    8665             : 
    8666             : // FastEmit functions for X86ISD::ADDUS.
    8667             : 
    8668           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8669           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8670             :     return 0;
    8671           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8672           0 :     return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8673             :   }
    8674           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8675           0 :     return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8676             :   }
    8677           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8678           0 :     return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8679             :   }
    8680             :   return 0;
    8681             : }
    8682             : 
    8683           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8684           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8685             :     return 0;
    8686           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8687           0 :     return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8688             :   }
    8689           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8690           0 :     return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8691             :   }
    8692             :   return 0;
    8693             : }
    8694             : 
    8695             : unsigned fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8696           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8697             :     return 0;
    8698           0 :   if ((Subtarget->hasBWI())) {
    8699           0 :     return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8700             :   }
    8701             :   return 0;
    8702             : }
    8703             : 
    8704           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8705           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8706             :     return 0;
    8707           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8708           0 :     return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8709             :   }
    8710           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8711           0 :     return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8712             :   }
    8713           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8714           0 :     return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8715             :   }
    8716             :   return 0;
    8717             : }
    8718             : 
    8719           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8720           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8721             :     return 0;
    8722           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8723           0 :     return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8724             :   }
    8725           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8726           0 :     return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8727             :   }
    8728             :   return 0;
    8729             : }
    8730             : 
    8731             : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8732           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8733             :     return 0;
    8734           0 :   if ((Subtarget->hasBWI())) {
    8735           0 :     return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8736             :   }
    8737             :   return 0;
    8738             : }
    8739             : 
    8740           0 : unsigned fastEmit_X86ISD_ADDUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8741           0 :   switch (VT.SimpleTy) {
    8742           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8743           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8744           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8745           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8746           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8747           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8748             :   default: return 0;
    8749             :   }
    8750             : }
    8751             : 
    8752             : // FastEmit functions for X86ISD::ANDNP.
    8753             : 
    8754           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8755           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    8756             :     return 0;
    8757           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8758           0 :     return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8759             :   }
    8760           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8761           0 :     return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8762             :   }
    8763           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8764           0 :     return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8765             :   }
    8766             :   return 0;
    8767             : }
    8768             : 
    8769           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8770           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    8771             :     return 0;
    8772           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    8773           0 :     return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8774             :   }
    8775           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8776           0 :     return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8777             :   }
    8778           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8779           0 :     return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8780             :   }
    8781             :   return 0;
    8782             : }
    8783             : 
    8784             : unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8785           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8786             :     return 0;
    8787           0 :   if ((Subtarget->hasAVX512())) {
    8788           0 :     return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8789             :   }
    8790             :   return 0;
    8791             : }
    8792             : 
    8793           0 : unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8794           0 :   switch (VT.SimpleTy) {
    8795           0 :   case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8796           0 :   case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8797           0 :   case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8798             :   default: return 0;
    8799             :   }
    8800             : }
    8801             : 
    8802             : // FastEmit functions for X86ISD::AVG.
    8803             : 
    8804           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8805           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8806             :     return 0;
    8807           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8808           0 :     return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8809             :   }
    8810           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8811           0 :     return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8812             :   }
    8813           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8814           0 :     return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8815             :   }
    8816             :   return 0;
    8817             : }
    8818             : 
    8819           0 : unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8820           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8821             :     return 0;
    8822           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8823           0 :     return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8824             :   }
    8825           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8826           0 :     return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8827             :   }
    8828             :   return 0;
    8829             : }
    8830             : 
    8831             : unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8832           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8833             :     return 0;
    8834           0 :   if ((Subtarget->hasBWI())) {
    8835           0 :     return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8836             :   }
    8837             :   return 0;
    8838             : }
    8839             : 
    8840           0 : unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8841           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8842             :     return 0;
    8843           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8844           0 :     return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8845             :   }
    8846           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8847           0 :     return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8848             :   }
    8849           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8850           0 :     return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8851             :   }
    8852             :   return 0;
    8853             : }
    8854             : 
    8855           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8856           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8857             :     return 0;
    8858           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8859           0 :     return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8860             :   }
    8861           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8862           0 :     return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8863             :   }
    8864             :   return 0;
    8865             : }
    8866             : 
    8867             : unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8868           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8869             :     return 0;
    8870           0 :   if ((Subtarget->hasBWI())) {
    8871           0 :     return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8872             :   }
    8873             :   return 0;
    8874             : }
    8875             : 
    8876           0 : unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8877           0 :   switch (VT.SimpleTy) {
    8878           0 :   case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8879           0 :   case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8880           0 :   case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8881           0 :   case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8882           0 :   case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8883           0 :   case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8884             :   default: return 0;
    8885             :   }
    8886             : }
    8887             : 
    8888             : // FastEmit functions for X86ISD::BEXTR.
    8889             : 
    8890             : unsigned fastEmit_X86ISD_BEXTR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8891           0 :   if (RetVT.SimpleTy != MVT::i32)
    8892             :     return 0;
    8893           0 :   if ((Subtarget->hasBMI())) {
    8894           0 :     return fastEmitInst_rr(X86::BEXTR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8895             :   }
    8896             :   return 0;
    8897             : }
    8898             : 
    8899             : unsigned fastEmit_X86ISD_BEXTR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8900           0 :   if (RetVT.SimpleTy != MVT::i64)
    8901             :     return 0;
    8902           0 :   if ((Subtarget->hasBMI())) {
    8903           0 :     return fastEmitInst_rr(X86::BEXTR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8904             :   }
    8905             :   return 0;
    8906             : }
    8907             : 
    8908           0 : unsigned fastEmit_X86ISD_BEXTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8909           0 :   switch (VT.SimpleTy) {
    8910           0 :   case MVT::i32: return fastEmit_X86ISD_BEXTR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8911           0 :   case MVT::i64: return fastEmit_X86ISD_BEXTR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8912             :   default: return 0;
    8913             :   }
    8914             : }
    8915             : 
    8916             : // FastEmit functions for X86ISD::BT.
    8917             : 
    8918             : unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8919           0 :   if (RetVT.SimpleTy != MVT::i32)
    8920             :     return 0;
    8921           0 :   return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8922             : }
    8923             : 
    8924             : unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8925           0 :   if (RetVT.SimpleTy != MVT::i32)
    8926             :     return 0;
    8927           0 :   return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8928             : }
    8929             : 
    8930             : unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8931           0 :   if (RetVT.SimpleTy != MVT::i32)
    8932             :     return 0;
    8933           0 :   return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8934             : }
    8935             : 
    8936           0 : unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8937           0 :   switch (VT.SimpleTy) {
    8938           0 :   case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8939           0 :   case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8940           0 :   case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8941             :   default: return 0;
    8942             :   }
    8943             : }
    8944             : 
    8945             : // FastEmit functions for X86ISD::CMP.
    8946             : 
    8947             : unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8948           0 :   if (RetVT.SimpleTy != MVT::i32)
    8949             :     return 0;
    8950           0 :   return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8951             : }
    8952             : 
    8953             : unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8954           0 :   if (RetVT.SimpleTy != MVT::i32)
    8955             :     return 0;
    8956           0 :   return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8957             : }
    8958             : 
    8959             : unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8960           0 :   if (RetVT.SimpleTy != MVT::i32)
    8961             :     return 0;
    8962           0 :   return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8963             : }
    8964             : 
    8965             : unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8966           0 :   if (RetVT.SimpleTy != MVT::i32)
    8967             :     return 0;
    8968           0 :   return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8969             : }
    8970             : 
    8971           0 : unsigned fastEmit_X86ISD_CMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8972           0 :   if (RetVT.SimpleTy != MVT::i32)
    8973             :     return 0;
    8974           0 :   if ((Subtarget->hasAVX512())) {
    8975           0 :     return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8976             :   }
    8977           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8978           0 :     return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8979             :   }
    8980           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8981           0 :     return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8982             :   }
    8983           0 :   if ((!Subtarget->hasSSE1()) && (Subtarget->hasCMov())) {
    8984           0 :     return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8985             :   }
    8986             :   return 0;
    8987             : }
    8988             : 
    8989           0 : unsigned fastEmit_X86ISD_CMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8990           0 :   if (RetVT.SimpleTy != MVT::i32)
    8991             :     return 0;
    8992           0 :   if ((Subtarget->hasAVX512())) {
    8993           0 :     return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8994             :   }
    8995           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8996           0 :     return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8997             :   }
    8998           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8999           0 :     return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9000             :   }
    9001           0 :   if ((!Subtarget->hasSSE2()) && (Subtarget->hasCMov())) {
    9002           0 :     return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9003             :   }
    9004             :   return 0;
    9005             : }
    9006             : 
    9007           0 : unsigned fastEmit_X86ISD_CMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9008           0 :   if (RetVT.SimpleTy != MVT::i32)
    9009             :     return 0;
    9010           0 :   if ((Subtarget->hasCMov())) {
    9011           0 :     return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9012             :   }
    9013             :   return 0;
    9014             : }
    9015             : 
    9016           0 : unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9017           0 :   switch (VT.SimpleTy) {
    9018           0 :   case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9019           0 :   case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9020           0 :   case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9021           0 :   case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9022           0 :   case MVT::f32: return fastEmit_X86ISD_CMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9023           0 :   case MVT::f64: return fastEmit_X86ISD_CMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9024           0 :   case MVT::f80: return fastEmit_X86ISD_CMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9025             :   default: return 0;
    9026             :   }
    9027             : }
    9028             : 
    9029             : // FastEmit functions for X86ISD::COMI.
    9030             : 
    9031           0 : unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9032           0 :   if (RetVT.SimpleTy != MVT::i32)
    9033             :     return 0;
    9034           0 :   if ((Subtarget->hasAVX512())) {
    9035           0 :     return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9036             :   }
    9037           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9038           0 :     return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9039             :   }
    9040           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9041           0 :     return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9042             :   }
    9043             :   return 0;
    9044             : }
    9045             : 
    9046           0 : unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9047           0 :   if (RetVT.SimpleTy != MVT::i32)
    9048             :     return 0;
    9049           0 :   if ((Subtarget->hasAVX512())) {
    9050           0 :     return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9051             :   }
    9052           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9053           0 :     return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9054             :   }
    9055           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9056           0 :     return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9057             :   }
    9058             :   return 0;
    9059             : }
    9060             : 
    9061           0 : unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9062           0 :   switch (VT.SimpleTy) {
    9063           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9064           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9065             :   default: return 0;
    9066             :   }
    9067             : }
    9068             : 
    9069             : // FastEmit functions for X86ISD::FAND.
    9070             : 
    9071           0 : unsigned fastEmit_X86ISD_FAND_MVT_f128_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9072           0 :   if (RetVT.SimpleTy != MVT::f128)
    9073             :     return 0;
    9074           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9075           0 :     return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9076             :   }
    9077             :   return 0;
    9078             : }
    9079             : 
    9080             : unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9081           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9082             :     return 0;
    9083           0 :   return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9084             : }
    9085             : 
    9086           0 : unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9087           0 :   switch (VT.SimpleTy) {
    9088           0 :   case MVT::f128: return fastEmit_X86ISD_FAND_MVT_f128_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9089           0 :   case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9090             :   default: return 0;
    9091             :   }
    9092             : }
    9093             : 
    9094             : // FastEmit functions for X86ISD::FANDN.
    9095             : 
    9096             : unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9097           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9098             :     return 0;
    9099           0 :   return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9100             : }
    9101             : 
    9102             : unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9103           0 :   switch (VT.SimpleTy) {
    9104           0 :   case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9105             :   default: return 0;
    9106             :   }
    9107             : }
    9108             : 
    9109             : // FastEmit functions for X86ISD::FHADD.
    9110             : 
    9111           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9112           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9113             :     return 0;
    9114           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    9115           0 :     return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9116             :   }
    9117           0 :   if ((Subtarget->hasAVX())) {
    9118           0 :     return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9119             :   }
    9120             :   return 0;
    9121             : }
    9122             : 
    9123             : unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9124           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9125             :     return 0;
    9126           0 :   if ((Subtarget->hasAVX())) {
    9127           0 :     return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9128             :   }
    9129             :   return 0;
    9130             : }
    9131             : 
    9132           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9133           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9134             :     return 0;
    9135           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    9136           0 :     return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9137             :   }
    9138           0 :   if ((Subtarget->hasAVX())) {
    9139           0 :     return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9140             :   }
    9141             :   return 0;
    9142             : }
    9143             : 
    9144             : unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9145           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9146             :     return 0;
    9147           0 :   if ((Subtarget->hasAVX())) {
    9148           0 :     return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9149             :   }
    9150             :   return 0;
    9151             : }
    9152             : 
    9153           0 : unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9154           0 :   switch (VT.SimpleTy) {
    9155           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9156           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9157           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9158           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9159             :   default: return 0;
    9160             :   }
    9161             : }
    9162             : 
    9163             : // FastEmit functions for X86ISD::FHSUB.
    9164             : 
    9165           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9166           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9167             :     return 0;
    9168           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    9169           0 :     return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9170             :   }
    9171           0 :   if ((Subtarget->hasAVX())) {
    9172           0 :     return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9173             :   }
    9174             :   return 0;
    9175             : }
    9176             : 
    9177             : unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9178           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9179             :     return 0;
    9180           0 :   if ((Subtarget->hasAVX())) {
    9181           0 :     return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9182             :   }
    9183             :   return 0;
    9184             : }
    9185             : 
    9186           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9187           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9188             :     return 0;
    9189           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    9190           0 :     return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9191             :   }
    9192           0 :   if ((Subtarget->hasAVX())) {
    9193           0 :     return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9194             :   }
    9195             :   return 0;
    9196             : }
    9197             : 
    9198             : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9199           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9200             :     return 0;
    9201           0 :   if ((Subtarget->hasAVX())) {
    9202           0 :     return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9203             :   }
    9204             :   return 0;
    9205             : }
    9206             : 
    9207           0 : unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9208           0 :   switch (VT.SimpleTy) {
    9209           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9210           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9211           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9212           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9213             :   default: return 0;
    9214             :   }
    9215             : }
    9216             : 
    9217             : // FastEmit functions for X86ISD::FMAX.
    9218             : 
    9219           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9220           0 :   if (RetVT.SimpleTy != MVT::f32)
    9221             :     return 0;
    9222           0 :   if ((Subtarget->hasAVX512())) {
    9223           0 :     return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9224             :   }
    9225           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9226           0 :     return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9227             :   }
    9228           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9229           0 :     return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9230             :   }
    9231             :   return 0;
    9232             : }
    9233             : 
    9234           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9235           0 :   if (RetVT.SimpleTy != MVT::f64)
    9236             :     return 0;
    9237           0 :   if ((Subtarget->hasAVX512())) {
    9238           0 :     return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9239             :   }
    9240           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9241           0 :     return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9242             :   }
    9243           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9244           0 :     return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9245             :   }
    9246             :   return 0;
    9247             : }
    9248             : 
    9249           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9250           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9251             :     return 0;
    9252           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9253           0 :     return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9254             :   }
    9255           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9256           0 :     return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9257             :   }
    9258           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9259           0 :     return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9260             :   }
    9261             :   return 0;
    9262             : }
    9263             : 
    9264           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9265           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9266             :     return 0;
    9267           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9268           0 :     return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9269             :   }
    9270           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9271           0 :     return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9272             :   }
    9273             :   return 0;
    9274             : }
    9275             : 
    9276             : unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9277           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9278             :     return 0;
    9279           0 :   if ((Subtarget->hasAVX512())) {
    9280           0 :     return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9281             :   }
    9282             :   return 0;
    9283             : }
    9284             : 
    9285           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9286           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9287             :     return 0;
    9288           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9289           0 :     return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9290             :   }
    9291           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9292           0 :     return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9293             :   }
    9294           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9295           0 :     return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9296             :   }
    9297             :   return 0;
    9298             : }
    9299             : 
    9300           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9301           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9302             :     return 0;
    9303           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9304           0 :     return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9305             :   }
    9306           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9307           0 :     return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9308             :   }
    9309             :   return 0;
    9310             : }
    9311             : 
    9312             : unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9313           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9314             :     return 0;
    9315           0 :   if ((Subtarget->hasAVX512())) {
    9316           0 :     return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9317             :   }
    9318             :   return 0;
    9319             : }
    9320             : 
    9321           0 : unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9322           0 :   switch (VT.SimpleTy) {
    9323           0 :   case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9324           0 :   case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9325           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9326           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9327           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9328           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9329           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9330           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9331             :   default: return 0;
    9332             :   }
    9333             : }
    9334             : 
    9335             : // FastEmit functions for X86ISD::FMAXC.
    9336             : 
    9337           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9338           0 :   if (RetVT.SimpleTy != MVT::f32)
    9339             :     return 0;
    9340           0 :   if ((Subtarget->hasAVX512())) {
    9341           0 :     return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9342             :   }
    9343           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9344           0 :     return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9345             :   }
    9346           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9347           0 :     return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9348             :   }
    9349             :   return 0;
    9350             : }
    9351             : 
    9352           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9353           0 :   if (RetVT.SimpleTy != MVT::f64)
    9354             :     return 0;
    9355           0 :   if ((Subtarget->hasAVX512())) {
    9356           0 :     return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9357             :   }
    9358           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9359           0 :     return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9360             :   }
    9361           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9362           0 :     return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9363             :   }
    9364             :   return 0;
    9365             : }
    9366             : 
    9367           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9368           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9369             :     return 0;
    9370           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9371           0 :     return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9372             :   }
    9373           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9374           0 :     return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9375             :   }
    9376           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9377           0 :     return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9378             :   }
    9379             :   return 0;
    9380             : }
    9381             : 
    9382           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9383           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9384             :     return 0;
    9385           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9386           0 :     return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9387             :   }
    9388           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9389           0 :     return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9390             :   }
    9391             :   return 0;
    9392             : }
    9393             : 
    9394             : unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9395           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9396             :     return 0;
    9397           0 :   if ((Subtarget->hasAVX512())) {
    9398           0 :     return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9399             :   }
    9400             :   return 0;
    9401             : }
    9402             : 
    9403           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9404           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9405             :     return 0;
    9406           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9407           0 :     return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9408             :   }
    9409           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9410           0 :     return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9411             :   }
    9412           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9413           0 :     return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9414             :   }
    9415             :   return 0;
    9416             : }
    9417             : 
    9418           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9419           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9420             :     return 0;
    9421           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9422           0 :     return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9423             :   }
    9424           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9425           0 :     return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9426             :   }
    9427             :   return 0;
    9428             : }
    9429             : 
    9430             : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9431           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9432             :     return 0;
    9433           0 :   if ((Subtarget->hasAVX512())) {
    9434           0 :     return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9435             :   }
    9436             :   return 0;
    9437             : }
    9438             : 
    9439           0 : unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9440           0 :   switch (VT.SimpleTy) {
    9441           0 :   case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9442           0 :   case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9443           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9444           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9445           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9446           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9447           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9448           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9449             :   default: return 0;
    9450             :   }
    9451             : }
    9452             : 
    9453             : // FastEmit functions for X86ISD::FMAXS.
    9454             : 
    9455           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9456           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9457             :     return 0;
    9458           0 :   if ((Subtarget->hasAVX512())) {
    9459           0 :     return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9460             :   }
    9461           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9462           0 :     return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9463             :   }
    9464           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9465           0 :     return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9466             :   }
    9467             :   return 0;
    9468             : }
    9469             : 
    9470           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9471           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9472             :     return 0;
    9473           0 :   if ((Subtarget->hasAVX512())) {
    9474           0 :     return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9475             :   }
    9476           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9477           0 :     return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9478             :   }
    9479           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9480           0 :     return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9481             :   }
    9482             :   return 0;
    9483             : }
    9484             : 
    9485           0 : unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9486           0 :   switch (VT.SimpleTy) {
    9487           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9488           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9489             :   default: return 0;
    9490             :   }
    9491             : }
    9492             : 
    9493             : // FastEmit functions for X86ISD::FMIN.
    9494             : 
    9495           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9496           0 :   if (RetVT.SimpleTy != MVT::f32)
    9497             :     return 0;
    9498           0 :   if ((Subtarget->hasAVX512())) {
    9499           0 :     return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9500             :   }
    9501           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9502           0 :     return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9503             :   }
    9504           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9505           0 :     return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9506             :   }
    9507             :   return 0;
    9508             : }
    9509             : 
    9510           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9511           0 :   if (RetVT.SimpleTy != MVT::f64)
    9512             :     return 0;
    9513           0 :   if ((Subtarget->hasAVX512())) {
    9514           0 :     return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9515             :   }
    9516           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9517           0 :     return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9518             :   }
    9519           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9520           0 :     return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9521             :   }
    9522             :   return 0;
    9523             : }
    9524             : 
    9525           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9526           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9527             :     return 0;
    9528           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9529           0 :     return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9530             :   }
    9531           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9532           0 :     return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9533             :   }
    9534           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9535           0 :     return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9536             :   }
    9537             :   return 0;
    9538             : }
    9539             : 
    9540           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9541           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9542             :     return 0;
    9543           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9544           0 :     return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9545             :   }
    9546           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9547           0 :     return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9548             :   }
    9549             :   return 0;
    9550             : }
    9551             : 
    9552             : unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9553           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9554             :     return 0;
    9555           0 :   if ((Subtarget->hasAVX512())) {
    9556           0 :     return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9557             :   }
    9558             :   return 0;
    9559             : }
    9560             : 
    9561           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9562           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9563             :     return 0;
    9564           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9565           0 :     return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9566             :   }
    9567           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9568           0 :     return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9569             :   }
    9570           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9571           0 :     return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9572             :   }
    9573             :   return 0;
    9574             : }
    9575             : 
    9576           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9577           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9578             :     return 0;
    9579           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9580           0 :     return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9581             :   }
    9582           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9583           0 :     return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9584             :   }
    9585             :   return 0;
    9586             : }
    9587             : 
    9588             : unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9589           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9590             :     return 0;
    9591           0 :   if ((Subtarget->hasAVX512())) {
    9592           0 :     return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9593             :   }
    9594             :   return 0;
    9595             : }
    9596             : 
    9597           0 : unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9598           0 :   switch (VT.SimpleTy) {
    9599