LCOV - code coverage report
Current view: top level - build-llvm/lib/Target/X86 - X86GenFastISel.inc (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 708 6549 10.8 %
Date: 2017-09-14 15:23:50 Functions: 102 809 12.6 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
       2             : |*                                                                            *|
       3             : |* "Fast" Instruction Selector for the X86 target                             *|
       4             : |*                                                                            *|
       5             : |* Automatically generated file, do not edit!                                 *|
       6             : |*                                                                            *|
       7             : \*===----------------------------------------------------------------------===*/
       8             : 
       9             : 
      10             : // FastEmit Immediate Predicate functions.
      11             : static bool Predicate_i16immSExt8(int64_t Imm) {
      12          10 :  return isInt<8>(Imm); 
      13             : }
      14             : static bool Predicate_i32immSExt8(int64_t Imm) {
      15         220 :  return isInt<8>(Imm); 
      16             : }
      17             : static bool Predicate_i64immSExt8(int64_t Imm) {
      18          96 :  return isInt<8>(Imm); 
      19             : }
      20             : static bool Predicate_i8immZExt3(int64_t Imm) {
      21             : 
      22             :   return Imm >= 0 && Imm < 8;
      23             : 
      24             : }
      25             : static bool Predicate_i8immZExt5(int64_t Imm) {
      26             : 
      27             :   return Imm >= 0 && Imm < 32;
      28             : 
      29             : }
      30             : static bool Predicate_FROUND_NO_EXC(int64_t Imm) {
      31             :  return Imm == 8; 
      32             : }
      33             : static bool Predicate_FROUND_CURRENT(int64_t Imm) {
      34             : 
      35             :   return Imm == X86::STATIC_ROUNDING::CUR_DIRECTION;
      36             : 
      37             : }
      38             : static bool Predicate_AndMask64(int64_t Imm) {
      39             : 
      40             :   return isMask_64(Imm) && Imm > UINT32_MAX;
      41             : 
      42             : }
      43             : static bool Predicate_i64immSExt32(int64_t Imm) {
      44          27 :  return isInt<32>(Imm); 
      45             : }
      46             : 
      47             : 
      48             : // FastEmit functions for ISD::ABS.
      49             : 
      50           0 : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      51           0 :   if (RetVT.SimpleTy != MVT::v16i8)
      52             :     return 0;
      53           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      54           0 :     return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      55             :   }
      56           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      57           0 :     return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      58             :   }
      59           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      60           0 :     return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      61             :   }
      62             :   return 0;
      63             : }
      64             : 
      65           0 : unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      66           0 :   if (RetVT.SimpleTy != MVT::v32i8)
      67             :     return 0;
      68           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      69           0 :     return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
      70             :   }
      71           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      72           0 :     return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
      73             :   }
      74             :   return 0;
      75             : }
      76             : 
      77             : unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      78           0 :   if (RetVT.SimpleTy != MVT::v64i8)
      79             :     return 0;
      80           0 :   if ((Subtarget->hasBWI())) {
      81           0 :     return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
      82             :   }
      83             :   return 0;
      84             : }
      85             : 
      86           0 : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      87           0 :   if (RetVT.SimpleTy != MVT::v8i16)
      88             :     return 0;
      89           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      90           0 :     return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      91             :   }
      92           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      93           0 :     return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      94             :   }
      95           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      96           0 :     return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      97             :   }
      98             :   return 0;
      99             : }
     100             : 
     101           0 : unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     102           0 :   if (RetVT.SimpleTy != MVT::v16i16)
     103             :     return 0;
     104           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
     105           0 :     return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     106             :   }
     107           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
     108           0 :     return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     109             :   }
     110             :   return 0;
     111             : }
     112             : 
     113             : unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     114           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     115             :     return 0;
     116           0 :   if ((Subtarget->hasBWI())) {
     117           0 :     return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     118             :   }
     119             :   return 0;
     120             : }
     121             : 
     122           0 : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     123           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     124             :     return 0;
     125           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     126           0 :     return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     127             :   }
     128           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
     129           0 :     return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     130             :   }
     131           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     132           0 :     return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     133             :   }
     134             :   return 0;
     135             : }
     136             : 
     137           0 : unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     138           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     139             :     return 0;
     140           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     141           0 :     return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     142             :   }
     143           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
     144           0 :     return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     145             :   }
     146             :   return 0;
     147             : }
     148             : 
     149             : unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     150           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     151             :     return 0;
     152           0 :   if ((Subtarget->hasAVX512())) {
     153           0 :     return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     154             :   }
     155             :   return 0;
     156             : }
     157             : 
     158           0 : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     159           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     160             :     return 0;
     161           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     162           0 :     return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     163             :   }
     164             :   return 0;
     165             : }
     166             : 
     167           0 : unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     168           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     169             :     return 0;
     170           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     171           0 :     return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     172             :   }
     173             :   return 0;
     174             : }
     175             : 
     176             : unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     177           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     178             :     return 0;
     179           0 :   if ((Subtarget->hasAVX512())) {
     180           0 :     return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     181             :   }
     182             :   return 0;
     183             : }
     184             : 
     185           0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     186           0 :   switch (VT.SimpleTy) {
     187           0 :   case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     188           0 :   case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     189           0 :   case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     190           0 :   case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     191           0 :   case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     192           0 :   case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     193           0 :   case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     194           0 :   case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     195           0 :   case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     196           0 :   case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     197           0 :   case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     198           0 :   case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     199             :   default: return 0;
     200             :   }
     201             : }
     202             : 
     203             : // FastEmit functions for ISD::ANY_EXTEND.
     204             : 
     205             : unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     206           3 :   if (RetVT.SimpleTy != MVT::i32)
     207             :     return 0;
     208           3 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
     209             : }
     210             : 
     211             : unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     212           6 :   switch (VT.SimpleTy) {
     213           3 :   case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
     214             :   default: return 0;
     215             :   }
     216             : }
     217             : 
     218             : // FastEmit functions for ISD::BITCAST.
     219             : 
     220           1 : unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     221           1 :   if (RetVT.SimpleTy != MVT::f32)
     222             :     return 0;
     223           2 :   if ((Subtarget->hasAVX512())) {
     224           0 :     return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
     225             :   }
     226           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     227           1 :     return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     228             :   }
     229           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     230           0 :     return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     231             :   }
     232             :   return 0;
     233             : }
     234             : 
     235           2 : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     236           4 :   if ((Subtarget->hasAVX512())) {
     237           0 :     return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
     238             :   }
     239           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     240           2 :     return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     241             :   }
     242           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     243           0 :     return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     244             :   }
     245             :   return 0;
     246             : }
     247             : 
     248             : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     249           0 :   if ((Subtarget->hasMMX())) {
     250           0 :     return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
     251             :   }
     252             :   return 0;
     253             : }
     254             : 
     255           2 : unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     256           2 : switch (RetVT.SimpleTy) {
     257           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
     258           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill);
     259             :   default: return 0;
     260             : }
     261             : }
     262             : 
     263           1 : unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     264           1 :   if (RetVT.SimpleTy != MVT::i32)
     265             :     return 0;
     266           2 :   if ((Subtarget->hasAVX512())) {
     267           0 :     return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     268             :   }
     269           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     270           1 :     return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     271             :   }
     272           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     273           0 :     return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     274             :   }
     275             :   return 0;
     276             : }
     277             : 
     278           3 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     279           6 :   if ((Subtarget->hasAVX512())) {
     280           0 :     return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     281             :   }
     282           3 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     283           2 :     return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     284             :   }
     285           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     286           1 :     return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     287             :   }
     288             :   return 0;
     289             : }
     290             : 
     291             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     292           0 :   if ((Subtarget->hasSSE2())) {
     293           0 :     return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
     294             :   }
     295             :   return 0;
     296             : }
     297             : 
     298           3 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     299           3 : switch (RetVT.SimpleTy) {
     300           3 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     301           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill);
     302             :   default: return 0;
     303             : }
     304             : }
     305             : 
     306             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     307           0 :   if ((Subtarget->hasMMX())) {
     308           0 :     return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     309             :   }
     310             :   return 0;
     311             : }
     312             : 
     313             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     314           0 :   if ((Subtarget->hasSSE2())) {
     315           0 :     return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
     316             :   }
     317             :   return 0;
     318             : }
     319             : 
     320           0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     321           0 : switch (RetVT.SimpleTy) {
     322           0 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
     323           0 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
     324             :   default: return 0;
     325             : }
     326             : }
     327             : 
     328        1665 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     329        1665 :   switch (VT.SimpleTy) {
     330           1 :   case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
     331           2 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
     332           1 :   case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
     333           3 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
     334           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
     335             :   default: return 0;
     336             :   }
     337             : }
     338             : 
     339             : // FastEmit functions for ISD::BRIND.
     340             : 
     341             : unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     342           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     343             :     return 0;
     344           0 :   if ((!Subtarget->is64Bit())) {
     345           0 :     return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
     346             :   }
     347             :   return 0;
     348             : }
     349             : 
     350             : unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     351           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     352             :     return 0;
     353           0 :   if ((!Subtarget->is64Bit())) {
     354           0 :     return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     355             :   }
     356             :   return 0;
     357             : }
     358             : 
     359             : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     360           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     361             :     return 0;
     362           0 :   if ((Subtarget->is64Bit())) {
     363           0 :     return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     364             :   }
     365             :   return 0;
     366             : }
     367             : 
     368           0 : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     369           0 :   switch (VT.SimpleTy) {
     370           0 :   case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
     371           0 :   case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
     372           0 :   case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
     373             :   default: return 0;
     374             :   }
     375             : }
     376             : 
     377             : // FastEmit functions for ISD::BSWAP.
     378             : 
     379             : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     380           0 :   if (RetVT.SimpleTy != MVT::i32)
     381             :     return 0;
     382           0 :   return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     383             : }
     384             : 
     385             : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     386           0 :   if (RetVT.SimpleTy != MVT::i64)
     387             :     return 0;
     388           0 :   return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     389             : }
     390             : 
     391           0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     392           0 :   switch (VT.SimpleTy) {
     393           0 :   case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     394           0 :   case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     395             :   default: return 0;
     396             :   }
     397             : }
     398             : 
     399             : // FastEmit functions for ISD::CTLZ.
     400             : 
     401             : unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     402           0 :   if (RetVT.SimpleTy != MVT::i16)
     403             :     return 0;
     404           0 :   if ((Subtarget->hasLZCNT())) {
     405           0 :     return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     406             :   }
     407             :   return 0;
     408             : }
     409             : 
     410             : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     411           0 :   if (RetVT.SimpleTy != MVT::i32)
     412             :     return 0;
     413           0 :   if ((Subtarget->hasLZCNT())) {
     414           0 :     return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     415             :   }
     416             :   return 0;
     417             : }
     418             : 
     419             : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     420           0 :   if (RetVT.SimpleTy != MVT::i64)
     421             :     return 0;
     422           0 :   if ((Subtarget->hasLZCNT())) {
     423           0 :     return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     424             :   }
     425             :   return 0;
     426             : }
     427             : 
     428           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     429           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     430             :     return 0;
     431           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     432           0 :     return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     433             :   }
     434             :   return 0;
     435             : }
     436             : 
     437           0 : unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     438           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     439             :     return 0;
     440           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     441           0 :     return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     442             :   }
     443             :   return 0;
     444             : }
     445             : 
     446             : unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     447           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     448             :     return 0;
     449           0 :   if ((Subtarget->hasCDI())) {
     450           0 :     return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     451             :   }
     452             :   return 0;
     453             : }
     454             : 
     455           0 : unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     456           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     457             :     return 0;
     458           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     459           0 :     return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     460             :   }
     461             :   return 0;
     462             : }
     463             : 
     464           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     465           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     466             :     return 0;
     467           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     468           0 :     return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     469             :   }
     470             :   return 0;
     471             : }
     472             : 
     473             : unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     474           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     475             :     return 0;
     476           0 :   if ((Subtarget->hasCDI())) {
     477           0 :     return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     478             :   }
     479             :   return 0;
     480             : }
     481             : 
     482           0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     483           0 :   switch (VT.SimpleTy) {
     484           0 :   case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     485           0 :   case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     486           0 :   case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     487           0 :   case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     488           0 :   case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     489           0 :   case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     490           0 :   case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     491           0 :   case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     492           0 :   case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     493             :   default: return 0;
     494             :   }
     495             : }
     496             : 
     497             : // FastEmit functions for ISD::CTPOP.
     498             : 
     499             : unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     500           0 :   if (RetVT.SimpleTy != MVT::i16)
     501             :     return 0;
     502           0 :   if ((Subtarget->hasPOPCNT())) {
     503           0 :     return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     504             :   }
     505             :   return 0;
     506             : }
     507             : 
     508             : unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     509           0 :   if (RetVT.SimpleTy != MVT::i32)
     510             :     return 0;
     511           0 :   if ((Subtarget->hasPOPCNT())) {
     512           0 :     return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     513             :   }
     514             :   return 0;
     515             : }
     516             : 
     517             : unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     518           0 :   if (RetVT.SimpleTy != MVT::i64)
     519             :     return 0;
     520           0 :   if ((Subtarget->hasPOPCNT())) {
     521           0 :     return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     522             :   }
     523             :   return 0;
     524             : }
     525             : 
     526             : unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     527           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     528             :     return 0;
     529           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     530           0 :     return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     531             :   }
     532             :   return 0;
     533             : }
     534             : 
     535             : unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     536           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     537             :     return 0;
     538           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     539           0 :     return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     540             :   }
     541             :   return 0;
     542             : }
     543             : 
     544           0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     545           0 :   switch (VT.SimpleTy) {
     546           0 :   case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
     547           0 :   case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     548           0 :   case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     549           0 :   case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     550           0 :   case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     551             :   default: return 0;
     552             :   }
     553             : }
     554             : 
     555             : // FastEmit functions for ISD::CTTZ.
     556             : 
     557             : unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     558           0 :   if (RetVT.SimpleTy != MVT::i16)
     559             :     return 0;
     560           0 :   if ((Subtarget->hasBMI())) {
     561           0 :     return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     562             :   }
     563             :   return 0;
     564             : }
     565             : 
     566             : unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     567           0 :   if (RetVT.SimpleTy != MVT::i32)
     568             :     return 0;
     569           0 :   if ((Subtarget->hasBMI())) {
     570           0 :     return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     571             :   }
     572             :   return 0;
     573             : }
     574             : 
     575             : unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     576           0 :   if (RetVT.SimpleTy != MVT::i64)
     577             :     return 0;
     578           0 :   if ((Subtarget->hasBMI())) {
     579           0 :     return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     580             :   }
     581             :   return 0;
     582             : }
     583             : 
     584           0 : unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     585           0 :   switch (VT.SimpleTy) {
     586           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     587           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     588           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     589             :   default: return 0;
     590             :   }
     591             : }
     592             : 
     593             : // FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
     594             : 
     595             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     596           0 :   if (RetVT.SimpleTy != MVT::i16)
     597             :     return 0;
     598           0 :   return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     599             : }
     600             : 
     601             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     602           0 :   if (RetVT.SimpleTy != MVT::i32)
     603             :     return 0;
     604           0 :   return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     605             : }
     606             : 
     607             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     608           0 :   if (RetVT.SimpleTy != MVT::i64)
     609             :     return 0;
     610           0 :   return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     611             : }
     612             : 
     613           0 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     614           0 :   switch (VT.SimpleTy) {
     615           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
     616           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
     617           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
     618             :   default: return 0;
     619             :   }
     620             : }
     621             : 
     622             : // FastEmit functions for ISD::FABS.
     623             : 
     624             : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     625           0 :   if (RetVT.SimpleTy != MVT::f32)
     626             :     return 0;
     627           0 :   if ((!Subtarget->hasSSE1())) {
     628           0 :     return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     629             :   }
     630             :   return 0;
     631             : }
     632             : 
     633             : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     634           0 :   if (RetVT.SimpleTy != MVT::f64)
     635             :     return 0;
     636           0 :   if ((!Subtarget->hasSSE2())) {
     637           0 :     return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     638             :   }
     639             :   return 0;
     640             : }
     641             : 
     642             : unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     643           0 :   if (RetVT.SimpleTy != MVT::f80)
     644             :     return 0;
     645           0 :   return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     646             : }
     647             : 
     648           0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     649           0 :   switch (VT.SimpleTy) {
     650           0 :   case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     651           0 :   case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     652           0 :   case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     653             :   default: return 0;
     654             :   }
     655             : }
     656             : 
     657             : // FastEmit functions for ISD::FCOS.
     658             : 
     659             : unsigned fastEmit_ISD_FCOS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     660           0 :   if (RetVT.SimpleTy != MVT::f32)
     661             :     return 0;
     662           0 :   if ((!Subtarget->hasSSE1())) {
     663           0 :     return fastEmitInst_r(X86::COS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     664             :   }
     665             :   return 0;
     666             : }
     667             : 
     668             : unsigned fastEmit_ISD_FCOS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     669           0 :   if (RetVT.SimpleTy != MVT::f64)
     670             :     return 0;
     671           0 :   if ((!Subtarget->hasSSE2())) {
     672           0 :     return fastEmitInst_r(X86::COS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     673             :   }
     674             :   return 0;
     675             : }
     676             : 
     677             : unsigned fastEmit_ISD_FCOS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     678           0 :   if (RetVT.SimpleTy != MVT::f80)
     679             :     return 0;
     680           0 :   return fastEmitInst_r(X86::COS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     681             : }
     682             : 
     683           0 : unsigned fastEmit_ISD_FCOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     684           0 :   switch (VT.SimpleTy) {
     685           0 :   case MVT::f32: return fastEmit_ISD_FCOS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     686           0 :   case MVT::f64: return fastEmit_ISD_FCOS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     687           0 :   case MVT::f80: return fastEmit_ISD_FCOS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     688             :   default: return 0;
     689             :   }
     690             : }
     691             : 
     692             : // FastEmit functions for ISD::FNEG.
     693             : 
     694             : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     695           1 :   if (RetVT.SimpleTy != MVT::f32)
     696             :     return 0;
     697           2 :   if ((!Subtarget->hasSSE1())) {
     698           0 :     return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     699             :   }
     700             :   return 0;
     701             : }
     702             : 
     703             : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     704           2 :   if (RetVT.SimpleTy != MVT::f64)
     705             :     return 0;
     706           4 :   if ((!Subtarget->hasSSE2())) {
     707           0 :     return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     708             :   }
     709             :   return 0;
     710             : }
     711             : 
     712             : unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     713           0 :   if (RetVT.SimpleTy != MVT::f80)
     714             :     return 0;
     715           0 :   return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     716             : }
     717             : 
     718           3 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     719           3 :   switch (VT.SimpleTy) {
     720           1 :   case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
     721           2 :   case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
     722           0 :   case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill);
     723             :   default: return 0;
     724             :   }
     725             : }
     726             : 
     727             : // FastEmit functions for ISD::FP_EXTEND.
     728             : 
     729           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     730           0 :   if (RetVT.SimpleTy != MVT::f64)
     731             :     return 0;
     732           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     733           0 :     return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     734             :   }
     735             :   return 0;
     736             : }
     737             : 
     738           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     739           0 :   if (RetVT.SimpleTy != MVT::v4f64)
     740             :     return 0;
     741           0 :   if ((Subtarget->hasVLX())) {
     742           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     743             :   }
     744           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     745           0 :     return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     746             :   }
     747             :   return 0;
     748             : }
     749             : 
     750             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     751           0 :   if (RetVT.SimpleTy != MVT::v8f64)
     752             :     return 0;
     753           0 :   if ((Subtarget->hasAVX512())) {
     754           0 :     return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     755             :   }
     756             :   return 0;
     757             : }
     758             : 
     759           0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     760           0 :   switch (VT.SimpleTy) {
     761           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
     762           0 :   case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     763           0 :   case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
     764             :   default: return 0;
     765             :   }
     766             : }
     767             : 
     768             : // FastEmit functions for ISD::FP_ROUND.
     769             : 
     770           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     771           0 :   if (RetVT.SimpleTy != MVT::f32)
     772             :     return 0;
     773           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     774           0 :     return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     775             :   }
     776             :   return 0;
     777             : }
     778             : 
     779           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     780           0 :   if (RetVT.SimpleTy != MVT::v4f32)
     781             :     return 0;
     782           0 :   if ((Subtarget->hasVLX())) {
     783           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     784             :   }
     785           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     786           0 :     return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
     787             :   }
     788             :   return 0;
     789             : }
     790             : 
     791             : unsigned fastEmit_ISD_FP_ROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     792           0 :   if (RetVT.SimpleTy != MVT::v8f32)
     793             :     return 0;
     794           0 :   if ((Subtarget->hasAVX512())) {
     795           0 :     return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
     796             :   }
     797             :   return 0;
     798             : }
     799             : 
     800           0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     801           0 :   switch (VT.SimpleTy) {
     802           0 :   case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
     803           0 :   case MVT::v4f64: return fastEmit_ISD_FP_ROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
     804           0 :   case MVT::v8f64: return fastEmit_ISD_FP_ROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
     805             :   default: return 0;
     806             :   }
     807             : }
     808             : 
     809             : // FastEmit functions for ISD::FP_TO_SINT.
     810             : 
     811           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     812           0 :   if ((Subtarget->hasAVX512())) {
     813           0 :     return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     814             :   }
     815           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     816           0 :     return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     817             :   }
     818           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     819           0 :     return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     820             :   }
     821             :   return 0;
     822             : }
     823             : 
     824           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     825           0 :   if ((Subtarget->hasAVX512())) {
     826           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     827             :   }
     828           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     829           0 :     return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     830             :   }
     831           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     832           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     833             :   }
     834             :   return 0;
     835             : }
     836             : 
     837           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     838           0 : switch (RetVT.SimpleTy) {
     839           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
     840           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
     841             :   default: return 0;
     842             : }
     843             : }
     844             : 
     845           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     846           0 :   if ((Subtarget->hasAVX512())) {
     847           0 :     return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     848             :   }
     849           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     850           0 :     return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     851             :   }
     852           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     853           0 :     return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     854             :   }
     855             :   return 0;
     856             : }
     857             : 
     858           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     859           0 :   if ((Subtarget->hasAVX512())) {
     860           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     861             :   }
     862           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     863           0 :     return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     864             :   }
     865           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     866           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     867             :   }
     868             :   return 0;
     869             : }
     870             : 
     871           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     872           0 : switch (RetVT.SimpleTy) {
     873           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
     874           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     875             :   default: return 0;
     876             : }
     877             : }
     878             : 
     879           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     880           0 :   if ((Subtarget->hasVLX())) {
     881           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     882             :   }
     883           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     884           0 :     return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     885             :   }
     886           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     887           0 :     return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     888             :   }
     889             :   return 0;
     890             : }
     891             : 
     892           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
     893           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
     894           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     895             :   }
     896             :   return 0;
     897             : }
     898             : 
     899           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     900           0 : switch (RetVT.SimpleTy) {
     901           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
     902           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
     903             :   default: return 0;
     904             : }
     905             : }
     906             : 
     907           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
     908           0 :   if ((Subtarget->hasVLX())) {
     909           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     910             :   }
     911           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     912           0 :     return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     913             :   }
     914             :   return 0;
     915             : }
     916             : 
     917             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
     918           0 :   if ((Subtarget->hasDQI())) {
     919           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     920             :   }
     921             :   return 0;
     922             : }
     923             : 
     924           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     925           0 : switch (RetVT.SimpleTy) {
     926           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
     927           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
     928             :   default: return 0;
     929             : }
     930             : }
     931             : 
     932             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     933           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     934             :     return 0;
     935           0 :   if ((Subtarget->hasAVX512())) {
     936           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     937             :   }
     938             :   return 0;
     939             : }
     940             : 
     941           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     942           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     943             :     return 0;
     944           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
     945           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     946             :   }
     947             :   return 0;
     948             : }
     949             : 
     950           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     951           0 :   if ((Subtarget->hasVLX())) {
     952           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     953             :   }
     954           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     955           0 :     return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
     956             :   }
     957             :   return 0;
     958             : }
     959             : 
     960           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
     961           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
     962           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     963             :   }
     964             :   return 0;
     965             : }
     966             : 
     967           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     968           0 : switch (RetVT.SimpleTy) {
     969           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
     970           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
     971             :   default: return 0;
     972             : }
     973             : }
     974             : 
     975             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
     976           0 :   if ((Subtarget->hasAVX512())) {
     977           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
     978             :   }
     979             :   return 0;
     980             : }
     981             : 
     982             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
     983           0 :   if ((Subtarget->hasDQI())) {
     984           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     985             :   }
     986             :   return 0;
     987             : }
     988             : 
     989           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     990           0 : switch (RetVT.SimpleTy) {
     991           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
     992           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
     993             :   default: return 0;
     994             : }
     995             : }
     996             : 
     997           0 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     998           0 :   switch (VT.SimpleTy) {
     999           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1000           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1001           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1002           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1003           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1004           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1005           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1006           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1007             :   default: return 0;
    1008             :   }
    1009             : }
    1010             : 
    1011             : // FastEmit functions for ISD::FP_TO_UINT.
    1012             : 
    1013             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1014           0 :   if ((Subtarget->hasAVX512())) {
    1015           0 :     return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1016             :   }
    1017             :   return 0;
    1018             : }
    1019             : 
    1020             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1021           0 :   if ((Subtarget->hasAVX512())) {
    1022           0 :     return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1023             :   }
    1024             :   return 0;
    1025             : }
    1026             : 
    1027           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1028           0 : switch (RetVT.SimpleTy) {
    1029           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    1030           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    1031             :   default: return 0;
    1032             : }
    1033             : }
    1034             : 
    1035             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1036           0 :   if ((Subtarget->hasAVX512())) {
    1037           0 :     return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1038             :   }
    1039             :   return 0;
    1040             : }
    1041             : 
    1042             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1043           0 :   if ((Subtarget->hasAVX512())) {
    1044           0 :     return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1045             :   }
    1046             :   return 0;
    1047             : }
    1048             : 
    1049           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1050           0 : switch (RetVT.SimpleTy) {
    1051           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    1052           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    1053             :   default: return 0;
    1054             : }
    1055             : }
    1056             : 
    1057             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1058           0 :   if ((Subtarget->hasVLX())) {
    1059           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1060             :   }
    1061             :   return 0;
    1062             : }
    1063             : 
    1064           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1065           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1066           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1067             :   }
    1068             :   return 0;
    1069             : }
    1070             : 
    1071           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1072           0 : switch (RetVT.SimpleTy) {
    1073           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    1074           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    1075             :   default: return 0;
    1076             : }
    1077             : }
    1078             : 
    1079             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1080           0 :   if ((Subtarget->hasVLX())) {
    1081           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1082             :   }
    1083             :   return 0;
    1084             : }
    1085             : 
    1086             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1087           0 :   if ((Subtarget->hasDQI())) {
    1088           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1089             :   }
    1090             :   return 0;
    1091             : }
    1092             : 
    1093           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1094           0 : switch (RetVT.SimpleTy) {
    1095           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1096           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1097             :   default: return 0;
    1098             : }
    1099             : }
    1100             : 
    1101             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1102           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1103             :     return 0;
    1104           0 :   if ((Subtarget->hasAVX512())) {
    1105           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1106             :   }
    1107             :   return 0;
    1108             : }
    1109             : 
    1110           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1111           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1112             :     return 0;
    1113           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1114           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1115             :   }
    1116             :   return 0;
    1117             : }
    1118             : 
    1119             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1120           0 :   if ((Subtarget->hasVLX())) {
    1121           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1122             :   }
    1123             :   return 0;
    1124             : }
    1125             : 
    1126           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1127           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1128           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1129             :   }
    1130             :   return 0;
    1131             : }
    1132             : 
    1133           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1134           0 : switch (RetVT.SimpleTy) {
    1135           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1136           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1137             :   default: return 0;
    1138             : }
    1139             : }
    1140             : 
    1141             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1142           0 :   if ((Subtarget->hasAVX512())) {
    1143           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1144             :   }
    1145             :   return 0;
    1146             : }
    1147             : 
    1148             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1149           0 :   if ((Subtarget->hasDQI())) {
    1150           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1151             :   }
    1152             :   return 0;
    1153             : }
    1154             : 
    1155           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1156           0 : switch (RetVT.SimpleTy) {
    1157           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1158           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1159             :   default: return 0;
    1160             : }
    1161             : }
    1162             : 
    1163           0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1164           0 :   switch (VT.SimpleTy) {
    1165           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1166           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1167           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1168           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1169           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1170           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1171           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1172           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1173             :   default: return 0;
    1174             :   }
    1175             : }
    1176             : 
    1177             : // FastEmit functions for ISD::FSIN.
    1178             : 
    1179             : unsigned fastEmit_ISD_FSIN_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1180           0 :   if (RetVT.SimpleTy != MVT::f32)
    1181             :     return 0;
    1182           0 :   if ((!Subtarget->hasSSE1())) {
    1183           0 :     return fastEmitInst_r(X86::SIN_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1184             :   }
    1185             :   return 0;
    1186             : }
    1187             : 
    1188             : unsigned fastEmit_ISD_FSIN_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1189           0 :   if (RetVT.SimpleTy != MVT::f64)
    1190             :     return 0;
    1191           0 :   if ((!Subtarget->hasSSE2())) {
    1192           0 :     return fastEmitInst_r(X86::SIN_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1193             :   }
    1194             :   return 0;
    1195             : }
    1196             : 
    1197             : unsigned fastEmit_ISD_FSIN_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1198           0 :   if (RetVT.SimpleTy != MVT::f80)
    1199             :     return 0;
    1200           0 :   return fastEmitInst_r(X86::SIN_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1201             : }
    1202             : 
    1203           0 : unsigned fastEmit_ISD_FSIN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1204           0 :   switch (VT.SimpleTy) {
    1205           0 :   case MVT::f32: return fastEmit_ISD_FSIN_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1206           0 :   case MVT::f64: return fastEmit_ISD_FSIN_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1207           0 :   case MVT::f80: return fastEmit_ISD_FSIN_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1208             :   default: return 0;
    1209             :   }
    1210             : }
    1211             : 
    1212             : // FastEmit functions for ISD::FSQRT.
    1213             : 
    1214           0 : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1215           0 :   if (RetVT.SimpleTy != MVT::f32)
    1216             :     return 0;
    1217           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1218           0 :     return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    1219             :   }
    1220           0 :   if ((!Subtarget->hasSSE1())) {
    1221           0 :     return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1222             :   }
    1223             :   return 0;
    1224             : }
    1225             : 
    1226           0 : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1227           0 :   if (RetVT.SimpleTy != MVT::f64)
    1228             :     return 0;
    1229           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1230           0 :     return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
    1231             :   }
    1232           0 :   if ((!Subtarget->hasSSE2())) {
    1233           0 :     return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1234             :   }
    1235             :   return 0;
    1236             : }
    1237             : 
    1238             : unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1239           0 :   if (RetVT.SimpleTy != MVT::f80)
    1240             :     return 0;
    1241           0 :   return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1242             : }
    1243             : 
    1244           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1245           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1246             :     return 0;
    1247           0 :   if ((Subtarget->hasVLX())) {
    1248           0 :     return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1249             :   }
    1250           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1251           0 :     return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1252             :   }
    1253           0 :   if ((Subtarget->hasAVX())) {
    1254           0 :     return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1255             :   }
    1256             :   return 0;
    1257             : }
    1258             : 
    1259           0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1260           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    1261             :     return 0;
    1262           0 :   if ((Subtarget->hasVLX())) {
    1263           0 :     return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1264             :   }
    1265           0 :   if ((Subtarget->hasAVX())) {
    1266           0 :     return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1267             :   }
    1268             :   return 0;
    1269             : }
    1270             : 
    1271             : unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1272           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1273             :     return 0;
    1274           0 :   if ((Subtarget->hasAVX512())) {
    1275           0 :     return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1276             :   }
    1277             :   return 0;
    1278             : }
    1279             : 
    1280           0 : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1281           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1282             :     return 0;
    1283           0 :   if ((Subtarget->hasVLX())) {
    1284           0 :     return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1285             :   }
    1286           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1287           0 :     return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1288             :   }
    1289           0 :   if ((Subtarget->hasAVX())) {
    1290           0 :     return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1291             :   }
    1292             :   return 0;
    1293             : }
    1294             : 
    1295           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1296           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    1297             :     return 0;
    1298           0 :   if ((Subtarget->hasVLX())) {
    1299           0 :     return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1300             :   }
    1301           0 :   if ((Subtarget->hasAVX())) {
    1302           0 :     return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1303             :   }
    1304             :   return 0;
    1305             : }
    1306             : 
    1307             : unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1308           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    1309             :     return 0;
    1310           0 :   if ((Subtarget->hasAVX512())) {
    1311           0 :     return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1312             :   }
    1313             :   return 0;
    1314             : }
    1315             : 
    1316           0 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1317           0 :   switch (VT.SimpleTy) {
    1318           0 :   case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1319           0 :   case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1320           0 :   case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1321           0 :   case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1322           0 :   case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1323           0 :   case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1324           0 :   case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1325           0 :   case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1326           0 :   case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1327             :   default: return 0;
    1328             :   }
    1329             : }
    1330             : 
    1331             : // FastEmit functions for ISD::SCALAR_TO_VECTOR.
    1332             : 
    1333           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1334           2 :   if ((Subtarget->hasAVX512())) {
    1335           0 :     return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1336             :   }
    1337           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1338           0 :     return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1339             :   }
    1340           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1341           1 :     return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1342             :   }
    1343             :   return 0;
    1344             : }
    1345             : 
    1346             : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
    1347           0 :   if ((Subtarget->hasMMX())) {
    1348           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    1349             :   }
    1350             :   return 0;
    1351             : }
    1352             : 
    1353           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1354           1 : switch (RetVT.SimpleTy) {
    1355           1 :   case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    1356           0 :   case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill);
    1357             :   default: return 0;
    1358             : }
    1359             : }
    1360             : 
    1361           0 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1362           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1363             :     return 0;
    1364           0 :   if ((Subtarget->hasAVX512())) {
    1365           0 :     return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1366             :   }
    1367           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1368           0 :     return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1369             :   }
    1370           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1371           0 :     return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1372             :   }
    1373             :   return 0;
    1374             : }
    1375             : 
    1376           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1377           1 :   switch (VT.SimpleTy) {
    1378           1 :   case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1379           0 :   case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1380             :   default: return 0;
    1381             :   }
    1382             : }
    1383             : 
    1384             : // FastEmit functions for ISD::SIGN_EXTEND.
    1385             : 
    1386             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1387          12 :   return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    1388             : }
    1389             : 
    1390             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1391           3 :   return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
    1392             : }
    1393             : 
    1394          23 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1395          23 : switch (RetVT.SimpleTy) {
    1396          24 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill);
    1397           6 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill);
    1398             :   default: return 0;
    1399             : }
    1400             : }
    1401             : 
    1402             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1403           7 :   return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    1404             : }
    1405             : 
    1406             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1407           1 :   return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
    1408             : }
    1409             : 
    1410           8 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1411           8 : switch (RetVT.SimpleTy) {
    1412          14 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill);
    1413           2 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill);
    1414             :   default: return 0;
    1415             : }
    1416             : }
    1417             : 
    1418             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1419          35 :   if (RetVT.SimpleTy != MVT::i64)
    1420             :     return 0;
    1421          35 :   if ((Subtarget->is64Bit())) {
    1422          35 :     return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
    1423             :   }
    1424             :   return 0;
    1425             : }
    1426             : 
    1427          72 : unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1428          72 :   switch (VT.SimpleTy) {
    1429          23 :   case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    1430           8 :   case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    1431          35 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1432             :   default: return 0;
    1433             :   }
    1434             : }
    1435             : 
    1436             : // FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
    1437             : 
    1438           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1439           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1440           0 :     return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1441             :   }
    1442           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    1443           0 :     return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1444             :   }
    1445           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1446           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1447             :   }
    1448             :   return 0;
    1449             : }
    1450             : 
    1451           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1452           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1453           0 :     return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1454             :   }
    1455           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1456           0 :     return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1457             :   }
    1458           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1459           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1460             :   }
    1461             :   return 0;
    1462             : }
    1463             : 
    1464           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1465           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1466           0 :     return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1467             :   }
    1468           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1469           0 :     return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1470             :   }
    1471           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1472           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1473             :   }
    1474             :   return 0;
    1475             : }
    1476             : 
    1477           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1478           0 : switch (RetVT.SimpleTy) {
    1479           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1480           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1481           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1482             :   default: return 0;
    1483             : }
    1484             : }
    1485             : 
    1486           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1487           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1488           0 :     return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1489             :   }
    1490           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1491           0 :     return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1492             :   }
    1493           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1494           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1495             :   }
    1496             :   return 0;
    1497             : }
    1498             : 
    1499           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1500           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1501           0 :     return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1502             :   }
    1503           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1504           0 :     return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1505             :   }
    1506           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1507           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1508             :   }
    1509             :   return 0;
    1510             : }
    1511             : 
    1512           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1513           0 : switch (RetVT.SimpleTy) {
    1514           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    1515           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    1516             :   default: return 0;
    1517             : }
    1518             : }
    1519             : 
    1520           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1521           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1522             :     return 0;
    1523           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1524           0 :     return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1525             :   }
    1526           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1527           0 :     return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1528             :   }
    1529           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1530           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1531             :   }
    1532             :   return 0;
    1533             : }
    1534             : 
    1535           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1536           0 :   switch (VT.SimpleTy) {
    1537           0 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1538           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1539           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1540             :   default: return 0;
    1541             :   }
    1542             : }
    1543             : 
    1544             : // FastEmit functions for ISD::SINT_TO_FP.
    1545             : 
    1546           4 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1547           8 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1548           2 :     return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1549             :   }
    1550             :   return 0;
    1551             : }
    1552             : 
    1553           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1554          12 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1555           2 :     return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1556             :   }
    1557             :   return 0;
    1558             : }
    1559             : 
    1560          10 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1561          10 : switch (RetVT.SimpleTy) {
    1562           4 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    1563           6 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    1564             :   default: return 0;
    1565             : }
    1566             : }
    1567             : 
    1568           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1569           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1570           0 :     return fastEmitInst_r(X86::CVTSI2SS64rr, &X86::FR32RegClass, Op0, Op0IsKill);
    1571             :   }
    1572             :   return 0;
    1573             : }
    1574             : 
    1575           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1576           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1577           0 :     return fastEmitInst_r(X86::CVTSI2SD64rr, &X86::FR64RegClass, Op0, Op0IsKill);
    1578             :   }
    1579             :   return 0;
    1580             : }
    1581             : 
    1582           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1583           0 : switch (RetVT.SimpleTy) {
    1584           0 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    1585           0 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    1586             :   default: return 0;
    1587             : }
    1588             : }
    1589             : 
    1590           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1591           0 :   if ((Subtarget->hasVLX())) {
    1592           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1593             :   }
    1594           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1595           0 :     return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1596             :   }
    1597           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1598           0 :     return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1599             :   }
    1600             :   return 0;
    1601             : }
    1602             : 
    1603           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1604           2 :   if ((Subtarget->hasVLX())) {
    1605           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1606             :   }
    1607           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1608           2 :     return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1609             :   }
    1610             :   return 0;
    1611             : }
    1612             : 
    1613           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1614           2 : switch (RetVT.SimpleTy) {
    1615           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    1616           2 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    1617             :   default: return 0;
    1618             : }
    1619             : }
    1620             : 
    1621           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1622           0 :   if ((Subtarget->hasVLX())) {
    1623           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1624             :   }
    1625           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1626           0 :     return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1627             :   }
    1628             :   return 0;
    1629             : }
    1630             : 
    1631             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1632           0 :   if ((Subtarget->hasAVX512())) {
    1633           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1634             :   }
    1635             :   return 0;
    1636             : }
    1637             : 
    1638           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1639           0 : switch (RetVT.SimpleTy) {
    1640           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    1641           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    1642             :   default: return 0;
    1643             : }
    1644             : }
    1645             : 
    1646             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1647           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1648             :     return 0;
    1649           0 :   if ((Subtarget->hasAVX512())) {
    1650           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1651             :   }
    1652             :   return 0;
    1653             : }
    1654             : 
    1655           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1656           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1657             :     return 0;
    1658           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1659           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1660             :   }
    1661             :   return 0;
    1662             : }
    1663             : 
    1664           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1665           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1666           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1667             :   }
    1668             :   return 0;
    1669             : }
    1670             : 
    1671           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1672           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1673           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1674             :   }
    1675             :   return 0;
    1676             : }
    1677             : 
    1678           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1679           0 : switch (RetVT.SimpleTy) {
    1680           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    1681           0 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    1682             :   default: return 0;
    1683             : }
    1684             : }
    1685             : 
    1686             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1687           0 :   if ((Subtarget->hasDQI())) {
    1688           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1689             :   }
    1690             :   return 0;
    1691             : }
    1692             : 
    1693             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1694           0 :   if ((Subtarget->hasDQI())) {
    1695           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1696             :   }
    1697             :   return 0;
    1698             : }
    1699             : 
    1700           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1701           0 : switch (RetVT.SimpleTy) {
    1702           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    1703           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    1704             :   default: return 0;
    1705             : }
    1706             : }
    1707             : 
    1708          12 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1709          12 :   switch (VT.SimpleTy) {
    1710          10 :   case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1711           0 :   case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1712           2 :   case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1713           0 :   case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    1714           0 :   case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    1715           0 :   case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1716           0 :   case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    1717           0 :   case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    1718             :   default: return 0;
    1719             :   }
    1720             : }
    1721             : 
    1722             : // FastEmit functions for ISD::TRUNCATE.
    1723             : 
    1724             : unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1725           2 :   if (RetVT.SimpleTy != MVT::i8)
    1726             :     return 0;
    1727           2 :   if ((Subtarget->is64Bit())) {
    1728           1 :     return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
    1729             :   }
    1730             :   return 0;
    1731             : }
    1732             : 
    1733             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1734           8 :   if ((Subtarget->is64Bit())) {
    1735           8 :     return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1736             :   }
    1737             :   return 0;
    1738             : }
    1739             : 
    1740             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1741          18 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1742             : }
    1743             : 
    1744          17 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1745          17 : switch (RetVT.SimpleTy) {
    1746           8 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill);
    1747          18 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill);
    1748             :   default: return 0;
    1749             : }
    1750             : }
    1751             : 
    1752             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1753          14 :   return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1754             : }
    1755             : 
    1756             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1757           6 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1758             : }
    1759             : 
    1760             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1761          16 :   return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
    1762             : }
    1763             : 
    1764          18 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1765          18 : switch (RetVT.SimpleTy) {
    1766          14 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill);
    1767           6 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill);
    1768          16 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill);
    1769             :   default: return 0;
    1770             : }
    1771             : }
    1772             : 
    1773          44 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1774          44 :   switch (VT.SimpleTy) {
    1775           2 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill);
    1776          17 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1777          18 :   case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1778             :   default: return 0;
    1779             :   }
    1780             : }
    1781             : 
    1782             : // FastEmit functions for ISD::UINT_TO_FP.
    1783             : 
    1784             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1785           0 :   if ((Subtarget->hasVLX())) {
    1786           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1787             :   }
    1788             :   return 0;
    1789             : }
    1790             : 
    1791             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1792           0 :   if ((Subtarget->hasVLX())) {
    1793           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1794             :   }
    1795             :   return 0;
    1796             : }
    1797             : 
    1798           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1799           0 : switch (RetVT.SimpleTy) {
    1800           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    1801           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    1802             :   default: return 0;
    1803             : }
    1804             : }
    1805             : 
    1806             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1807           0 :   if ((Subtarget->hasVLX())) {
    1808           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1809             :   }
    1810             :   return 0;
    1811             : }
    1812             : 
    1813             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1814           0 :   if ((Subtarget->hasAVX512())) {
    1815           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1816             :   }
    1817             :   return 0;
    1818             : }
    1819             : 
    1820           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1821           0 : switch (RetVT.SimpleTy) {
    1822           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    1823           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    1824             :   default: return 0;
    1825             : }
    1826             : }
    1827             : 
    1828             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1829           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1830             :     return 0;
    1831           0 :   if ((Subtarget->hasAVX512())) {
    1832           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1833             :   }
    1834             :   return 0;
    1835             : }
    1836             : 
    1837           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1838           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1839             :     return 0;
    1840           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1841           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1842             :   }
    1843             :   return 0;
    1844             : }
    1845             : 
    1846           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1847           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1848           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1849             :   }
    1850             :   return 0;
    1851             : }
    1852             : 
    1853           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1854           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1855           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1856             :   }
    1857             :   return 0;
    1858             : }
    1859             : 
    1860           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1861           0 : switch (RetVT.SimpleTy) {
    1862           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    1863           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    1864             :   default: return 0;
    1865             : }
    1866             : }
    1867             : 
    1868             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1869           0 :   if ((Subtarget->hasDQI())) {
    1870           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1871             :   }
    1872             :   return 0;
    1873             : }
    1874             : 
    1875             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1876           0 :   if ((Subtarget->hasDQI())) {
    1877           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1878             :   }
    1879             :   return 0;
    1880             : }
    1881             : 
    1882           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1883           0 : switch (RetVT.SimpleTy) {
    1884           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    1885           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    1886             :   default: return 0;
    1887             : }
    1888             : }
    1889             : 
    1890           0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1891           0 :   switch (VT.SimpleTy) {
    1892           0 :   case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1893           0 :   case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    1894           0 :   case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    1895           0 :   case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1896           0 :   case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    1897           0 :   case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    1898             :   default: return 0;
    1899             :   }
    1900             : }
    1901             : 
    1902             : // FastEmit functions for ISD::ZERO_EXTEND.
    1903             : 
    1904             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1905         182 :   if (RetVT.SimpleTy != MVT::i32)
    1906             :     return 0;
    1907         171 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    1908             : }
    1909             : 
    1910             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1911          39 :   if (RetVT.SimpleTy != MVT::i32)
    1912             :     return 0;
    1913          37 :   return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    1914             : }
    1915             : 
    1916         232 : unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1917         232 :   switch (VT.SimpleTy) {
    1918         182 :   case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    1919          39 :   case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    1920             :   default: return 0;
    1921             :   }
    1922             : }
    1923             : 
    1924             : // FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
    1925             : 
    1926           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1927           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1928           0 :     return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1929             :   }
    1930           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    1931           0 :     return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1932             :   }
    1933           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1934           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1935             :   }
    1936             :   return 0;
    1937             : }
    1938             : 
    1939           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1940           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1941           0 :     return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1942             :   }
    1943           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1944           0 :     return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1945             :   }
    1946           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1947           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1948             :   }
    1949             :   return 0;
    1950             : }
    1951             : 
    1952           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1953           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1954           0 :     return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1955             :   }
    1956           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1957           0 :     return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1958             :   }
    1959           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1960           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1961             :   }
    1962             :   return 0;
    1963             : }
    1964             : 
    1965           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1966           0 : switch (RetVT.SimpleTy) {
    1967           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1968           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1969           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1970             :   default: return 0;
    1971             : }
    1972             : }
    1973             : 
    1974           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1975           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1976           0 :     return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1977             :   }
    1978           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1979           0 :     return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1980             :   }
    1981           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1982           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1983             :   }
    1984             :   return 0;
    1985             : }
    1986             : 
    1987           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1988           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1989           0 :     return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1990             :   }
    1991           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1992           0 :     return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1993             :   }
    1994           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1995           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1996             :   }
    1997             :   return 0;
    1998             : }
    1999             : 
    2000           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2001           0 : switch (RetVT.SimpleTy) {
    2002           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    2003           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    2004             :   default: return 0;
    2005             : }
    2006             : }
    2007             : 
    2008           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2009           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2010             :     return 0;
    2011           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2012           0 :     return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2013             :   }
    2014           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2015           0 :     return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2016             :   }
    2017           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2018           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2019             :   }
    2020             :   return 0;
    2021             : }
    2022             : 
    2023           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2024           0 :   switch (VT.SimpleTy) {
    2025           0 :   case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2026           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2027           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2028             :   default: return 0;
    2029             :   }
    2030             : }
    2031             : 
    2032             : // FastEmit functions for X86ISD::CALL.
    2033             : 
    2034             : unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2035           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2036             :     return 0;
    2037           0 :   if ((!Subtarget->is64Bit())) {
    2038           0 :     return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
    2039             :   }
    2040             :   return 0;
    2041             : }
    2042             : 
    2043             : unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2044           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2045             :     return 0;
    2046           0 :   if ((!Subtarget->is64Bit())) {
    2047           0 :     return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
    2048             :   }
    2049             :   return 0;
    2050             : }
    2051             : 
    2052             : unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2053           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2054             :     return 0;
    2055           0 :   if ((Subtarget->is64Bit())) {
    2056           0 :     return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
    2057             :   }
    2058             :   return 0;
    2059             : }
    2060             : 
    2061           0 : unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2062           0 :   switch (VT.SimpleTy) {
    2063           0 :   case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2064           0 :   case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2065           0 :   case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2066             :   default: return 0;
    2067             :   }
    2068             : }
    2069             : 
    2070             : // FastEmit functions for X86ISD::COMPRESS.
    2071             : 
    2072             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2073           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2074             :     return 0;
    2075           0 :   if ((Subtarget->hasVLX())) {
    2076           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2077             :   }
    2078             :   return 0;
    2079             : }
    2080             : 
    2081             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2082           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2083             :     return 0;
    2084           0 :   if ((Subtarget->hasVLX())) {
    2085           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2086             :   }
    2087             :   return 0;
    2088             : }
    2089             : 
    2090             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2091           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2092             :     return 0;
    2093           0 :   if ((Subtarget->hasAVX512())) {
    2094           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2095             :   }
    2096             :   return 0;
    2097             : }
    2098             : 
    2099             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2100           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2101             :     return 0;
    2102           0 :   if ((Subtarget->hasVLX())) {
    2103           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2104             :   }
    2105             :   return 0;
    2106             : }
    2107             : 
    2108             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2109           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2110             :     return 0;
    2111           0 :   if ((Subtarget->hasVLX())) {
    2112           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2113             :   }
    2114             :   return 0;
    2115             : }
    2116             : 
    2117             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2118           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2119             :     return 0;
    2120           0 :   if ((Subtarget->hasAVX512())) {
    2121           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2122             :   }
    2123             :   return 0;
    2124             : }
    2125             : 
    2126             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2127           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2128             :     return 0;
    2129           0 :   if ((Subtarget->hasVLX())) {
    2130           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2131             :   }
    2132             :   return 0;
    2133             : }
    2134             : 
    2135             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2136           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    2137             :     return 0;
    2138           0 :   if ((Subtarget->hasVLX())) {
    2139           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2140             :   }
    2141             :   return 0;
    2142             : }
    2143             : 
    2144             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2145           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2146             :     return 0;
    2147           0 :   if ((Subtarget->hasAVX512())) {
    2148           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2149             :   }
    2150             :   return 0;
    2151             : }
    2152             : 
    2153             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2154           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2155             :     return 0;
    2156           0 :   if ((Subtarget->hasVLX())) {
    2157           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2158             :   }
    2159             :   return 0;
    2160             : }
    2161             : 
    2162             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2163           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    2164             :     return 0;
    2165           0 :   if ((Subtarget->hasVLX())) {
    2166           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2167             :   }
    2168             :   return 0;
    2169             : }
    2170             : 
    2171             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2172           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    2173             :     return 0;
    2174           0 :   if ((Subtarget->hasAVX512())) {
    2175           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2176             :   }
    2177             :   return 0;
    2178             : }
    2179             : 
    2180           0 : unsigned fastEmit_X86ISD_COMPRESS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2181           0 :   switch (VT.SimpleTy) {
    2182           0 :   case MVT::v4i32: return fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2183           0 :   case MVT::v8i32: return fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2184           0 :   case MVT::v16i32: return fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2185           0 :   case MVT::v2i64: return fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2186           0 :   case MVT::v4i64: return fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2187           0 :   case MVT::v8i64: return fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2188           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2189           0 :   case MVT::v8f32: return fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2190           0 :   case MVT::v16f32: return fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2191           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2192           0 :   case MVT::v4f64: return fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2193           0 :   case MVT::v8f64: return fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2194             :   default: return 0;
    2195             :   }
    2196             : }
    2197             : 
    2198             : // FastEmit functions for X86ISD::CONFLICT.
    2199             : 
    2200           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2201           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2202             :     return 0;
    2203           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2204           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2205             :   }
    2206             :   return 0;
    2207             : }
    2208             : 
    2209           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2210           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2211             :     return 0;
    2212           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2213           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2214             :   }
    2215             :   return 0;
    2216             : }
    2217             : 
    2218             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2219           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2220             :     return 0;
    2221           0 :   if ((Subtarget->hasCDI())) {
    2222           0 :     return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2223             :   }
    2224             :   return 0;
    2225             : }
    2226             : 
    2227           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2228           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2229             :     return 0;
    2230           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2231           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2232             :   }
    2233             :   return 0;
    2234             : }
    2235             : 
    2236           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2237           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2238             :     return 0;
    2239           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2240           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2241             :   }
    2242             :   return 0;
    2243             : }
    2244             : 
    2245             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2246           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2247             :     return 0;
    2248           0 :   if ((Subtarget->hasCDI())) {
    2249           0 :     return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2250             :   }
    2251             :   return 0;
    2252             : }
    2253             : 
    2254           0 : unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2255           0 :   switch (VT.SimpleTy) {
    2256           0 :   case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2257           0 :   case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2258           0 :   case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2259           0 :   case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2260           0 :   case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2261           0 :   case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2262             :   default: return 0;
    2263             :   }
    2264             : }
    2265             : 
    2266             : // FastEmit functions for X86ISD::CVT2MASK.
    2267             : 
    2268           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2269           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    2270             :     return 0;
    2271           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2272           0 :     return fastEmitInst_r(X86::VPMOVB2MZ128rr, &X86::VK16RegClass, Op0, Op0IsKill);
    2273             :   }
    2274             :   return 0;
    2275             : }
    2276             : 
    2277           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2278           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    2279             :     return 0;
    2280           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2281           0 :     return fastEmitInst_r(X86::VPMOVB2MZ256rr, &X86::VK32RegClass, Op0, Op0IsKill);
    2282             :   }
    2283             :   return 0;
    2284             : }
    2285             : 
    2286             : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2287           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    2288             :     return 0;
    2289           0 :   if ((Subtarget->hasBWI())) {
    2290           0 :     return fastEmitInst_r(X86::VPMOVB2MZrr, &X86::VK64RegClass, Op0, Op0IsKill);
    2291             :   }
    2292             :   return 0;
    2293             : }
    2294             : 
    2295           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2296           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    2297             :     return 0;
    2298           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2299           0 :     return fastEmitInst_r(X86::VPMOVW2MZ128rr, &X86::VK8RegClass, Op0, Op0IsKill);
    2300             :   }
    2301             :   return 0;
    2302             : }
    2303             : 
    2304           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2305           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    2306             :     return 0;
    2307           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2308           0 :     return fastEmitInst_r(X86::VPMOVW2MZ256rr, &X86::VK16RegClass, Op0, Op0IsKill);
    2309             :   }
    2310             :   return 0;
    2311             : }
    2312             : 
    2313             : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2314           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    2315             :     return 0;
    2316           0 :   if ((Subtarget->hasBWI())) {
    2317           0 :     return fastEmitInst_r(X86::VPMOVW2MZrr, &X86::VK32RegClass, Op0, Op0IsKill);
    2318             :   }
    2319             :   return 0;
    2320             : }
    2321             : 
    2322           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2323           0 :   if (RetVT.SimpleTy != MVT::v4i1)
    2324             :     return 0;
    2325           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2326           0 :     return fastEmitInst_r(X86::VPMOVD2MZ128rr, &X86::VK4RegClass, Op0, Op0IsKill);
    2327             :   }
    2328             :   return 0;
    2329             : }
    2330             : 
    2331           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2332           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    2333             :     return 0;
    2334           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2335           0 :     return fastEmitInst_r(X86::VPMOVD2MZ256rr, &X86::VK8RegClass, Op0, Op0IsKill);
    2336             :   }
    2337             :   return 0;
    2338             : }
    2339             : 
    2340             : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2341           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    2342             :     return 0;
    2343           0 :   if ((Subtarget->hasDQI())) {
    2344           0 :     return fastEmitInst_r(X86::VPMOVD2MZrr, &X86::VK16RegClass, Op0, Op0IsKill);
    2345             :   }
    2346             :   return 0;
    2347             : }
    2348             : 
    2349           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2350           0 :   if (RetVT.SimpleTy != MVT::v2i1)
    2351             :     return 0;
    2352           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2353           0 :     return fastEmitInst_r(X86::VPMOVQ2MZ128rr, &X86::VK2RegClass, Op0, Op0IsKill);
    2354             :   }
    2355             :   return 0;
    2356             : }
    2357             : 
    2358           0 : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2359           0 :   if (RetVT.SimpleTy != MVT::v4i1)
    2360             :     return 0;
    2361           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2362           0 :     return fastEmitInst_r(X86::VPMOVQ2MZ256rr, &X86::VK4RegClass, Op0, Op0IsKill);
    2363             :   }
    2364             :   return 0;
    2365             : }
    2366             : 
    2367             : unsigned fastEmit_X86ISD_CVT2MASK_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2368           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    2369             :     return 0;
    2370           0 :   if ((Subtarget->hasDQI())) {
    2371           0 :     return fastEmitInst_r(X86::VPMOVQ2MZrr, &X86::VK8RegClass, Op0, Op0IsKill);
    2372             :   }
    2373             :   return 0;
    2374             : }
    2375             : 
    2376           0 : unsigned fastEmit_X86ISD_CVT2MASK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2377           0 :   switch (VT.SimpleTy) {
    2378           0 :   case MVT::v16i8: return fastEmit_X86ISD_CVT2MASK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2379           0 :   case MVT::v32i8: return fastEmit_X86ISD_CVT2MASK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    2380           0 :   case MVT::v64i8: return fastEmit_X86ISD_CVT2MASK_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    2381           0 :   case MVT::v8i16: return fastEmit_X86ISD_CVT2MASK_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2382           0 :   case MVT::v16i16: return fastEmit_X86ISD_CVT2MASK_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2383           0 :   case MVT::v32i16: return fastEmit_X86ISD_CVT2MASK_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2384           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVT2MASK_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2385           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVT2MASK_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2386           0 :   case MVT::v16i32: return fastEmit_X86ISD_CVT2MASK_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2387           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVT2MASK_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2388           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVT2MASK_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2389           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVT2MASK_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2390             :   default: return 0;
    2391             :   }
    2392             : }
    2393             : 
    2394             : // FastEmit functions for X86ISD::CVTP2SI.
    2395             : 
    2396             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2397           0 :   if ((Subtarget->hasVLX())) {
    2398           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2399             :   }
    2400             :   return 0;
    2401             : }
    2402             : 
    2403           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2404           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2405           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2406             :   }
    2407             :   return 0;
    2408             : }
    2409             : 
    2410           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2411           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2412           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2413             :   }
    2414             :   return 0;
    2415             : }
    2416             : 
    2417           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2418           0 : switch (RetVT.SimpleTy) {
    2419           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2420           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2421           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2422             :   default: return 0;
    2423             : }
    2424             : }
    2425             : 
    2426             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2427           0 :   if ((Subtarget->hasVLX())) {
    2428           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2429             :   }
    2430             :   return 0;
    2431             : }
    2432             : 
    2433             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2434           0 :   if ((Subtarget->hasDQI())) {
    2435           0 :     return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2436             :   }
    2437             :   return 0;
    2438             : }
    2439             : 
    2440           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2441           0 : switch (RetVT.SimpleTy) {
    2442           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2443           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2444             :   default: return 0;
    2445             : }
    2446             : }
    2447             : 
    2448             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2449           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2450             :     return 0;
    2451           0 :   if ((Subtarget->hasAVX512())) {
    2452           0 :     return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2453             :   }
    2454             :   return 0;
    2455             : }
    2456             : 
    2457           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2458           0 :   if ((Subtarget->hasVLX())) {
    2459           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2460             :   }
    2461           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2462           0 :     return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2463             :   }
    2464           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2465           0 :     return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2466             :   }
    2467             :   return 0;
    2468             : }
    2469             : 
    2470           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2471           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2472           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2473             :   }
    2474             :   return 0;
    2475             : }
    2476             : 
    2477           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2478           0 : switch (RetVT.SimpleTy) {
    2479           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2480           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2481             :   default: return 0;
    2482             : }
    2483             : }
    2484             : 
    2485           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2486           0 :   if ((Subtarget->hasVLX())) {
    2487           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2488             :   }
    2489           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2490           0 :     return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2491             :   }
    2492             :   return 0;
    2493             : }
    2494             : 
    2495           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2496           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2497           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2498             :   }
    2499             :   return 0;
    2500             : }
    2501             : 
    2502           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2503           0 : switch (RetVT.SimpleTy) {
    2504           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2505           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2506             :   default: return 0;
    2507             : }
    2508             : }
    2509             : 
    2510             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2511           0 :   if ((Subtarget->hasAVX512())) {
    2512           0 :     return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2513             :   }
    2514             :   return 0;
    2515             : }
    2516             : 
    2517             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2518           0 :   if ((Subtarget->hasDQI())) {
    2519           0 :     return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2520             :   }
    2521             :   return 0;
    2522             : }
    2523             : 
    2524           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2525           0 : switch (RetVT.SimpleTy) {
    2526           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2527           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2528             :   default: return 0;
    2529             : }
    2530             : }
    2531             : 
    2532           0 : unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2533           0 :   switch (VT.SimpleTy) {
    2534           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2535           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2536           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2537           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2538           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2539           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2540             :   default: return 0;
    2541             :   }
    2542             : }
    2543             : 
    2544             : // FastEmit functions for X86ISD::CVTP2UI.
    2545             : 
    2546             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2547           0 :   if ((Subtarget->hasVLX())) {
    2548           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2549             :   }
    2550             :   return 0;
    2551             : }
    2552             : 
    2553           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2554           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2555           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2556             :   }
    2557             :   return 0;
    2558             : }
    2559             : 
    2560           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2561           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2562           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2563             :   }
    2564             :   return 0;
    2565             : }
    2566             : 
    2567           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2568           0 : switch (RetVT.SimpleTy) {
    2569           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2570           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2571           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2572             :   default: return 0;
    2573             : }
    2574             : }
    2575             : 
    2576             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2577           0 :   if ((Subtarget->hasVLX())) {
    2578           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2579             :   }
    2580             :   return 0;
    2581             : }
    2582             : 
    2583             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2584           0 :   if ((Subtarget->hasDQI())) {
    2585           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2586             :   }
    2587             :   return 0;
    2588             : }
    2589             : 
    2590           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2591           0 : switch (RetVT.SimpleTy) {
    2592           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2593           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2594             :   default: return 0;
    2595             : }
    2596             : }
    2597             : 
    2598             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2599           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2600             :     return 0;
    2601           0 :   if ((Subtarget->hasAVX512())) {
    2602           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2603             :   }
    2604             :   return 0;
    2605             : }
    2606             : 
    2607             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2608           0 :   if ((Subtarget->hasVLX())) {
    2609           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2610             :   }
    2611             :   return 0;
    2612             : }
    2613             : 
    2614           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2615           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2616           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2617             :   }
    2618             :   return 0;
    2619             : }
    2620             : 
    2621           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2622           0 : switch (RetVT.SimpleTy) {
    2623           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2624           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2625             :   default: return 0;
    2626             : }
    2627             : }
    2628             : 
    2629             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2630           0 :   if ((Subtarget->hasVLX())) {
    2631           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2632             :   }
    2633             :   return 0;
    2634             : }
    2635             : 
    2636           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2637           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2638           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2639             :   }
    2640             :   return 0;
    2641             : }
    2642             : 
    2643           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2644           0 : switch (RetVT.SimpleTy) {
    2645           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2646           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2647             :   default: return 0;
    2648             : }
    2649             : }
    2650             : 
    2651             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2652           0 :   if ((Subtarget->hasAVX512())) {
    2653           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2654             :   }
    2655             :   return 0;
    2656             : }
    2657             : 
    2658             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2659           0 :   if ((Subtarget->hasDQI())) {
    2660           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2661             :   }
    2662             :   return 0;
    2663             : }
    2664             : 
    2665           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2666           0 : switch (RetVT.SimpleTy) {
    2667           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2668           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2669             :   default: return 0;
    2670             : }
    2671             : }
    2672             : 
    2673           0 : unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2674           0 :   switch (VT.SimpleTy) {
    2675           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2676           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2677           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2678           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2679           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2680           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2681             :   default: return 0;
    2682             :   }
    2683             : }
    2684             : 
    2685             : // FastEmit functions for X86ISD::CVTSI2P.
    2686             : 
    2687           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2688           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2689             :     return 0;
    2690           0 :   if ((Subtarget->hasVLX())) {
    2691           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2692             :   }
    2693           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2694           0 :     return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2695             :   }
    2696           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2697           0 :     return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2698             :   }
    2699             :   return 0;
    2700             : }
    2701             : 
    2702           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2703           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2704             :     return 0;
    2705           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2706           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2707             :   }
    2708             :   return 0;
    2709             : }
    2710             : 
    2711           0 : unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2712           0 :   switch (VT.SimpleTy) {
    2713           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2714           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2715             :   default: return 0;
    2716             :   }
    2717             : }
    2718             : 
    2719             : // FastEmit functions for X86ISD::CVTTP2SI.
    2720             : 
    2721           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2722           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2723             :     return 0;
    2724           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2725           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2726             :   }
    2727             :   return 0;
    2728             : }
    2729             : 
    2730           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2731           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2732             :     return 0;
    2733           0 :   if ((Subtarget->hasVLX())) {
    2734           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2735             :   }
    2736           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2737           0 :     return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2738             :   }
    2739           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2740           0 :     return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2741             :   }
    2742             :   return 0;
    2743             : }
    2744             : 
    2745           0 : unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2746           0 :   switch (VT.SimpleTy) {
    2747           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2748           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2749             :   default: return 0;
    2750             :   }
    2751             : }
    2752             : 
    2753             : // FastEmit functions for X86ISD::CVTTP2UI.
    2754             : 
    2755           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2756           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2757             :     return 0;
    2758           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2759           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2760             :   }
    2761             :   return 0;
    2762             : }
    2763             : 
    2764             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2765           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2766             :     return 0;
    2767           0 :   if ((Subtarget->hasVLX())) {
    2768           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2769             :   }
    2770             :   return 0;
    2771             : }
    2772             : 
    2773           0 : unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2774           0 :   switch (VT.SimpleTy) {
    2775           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2776           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2777             :   default: return 0;
    2778             :   }
    2779             : }
    2780             : 
    2781             : // FastEmit functions for X86ISD::CVTUI2P.
    2782             : 
    2783             : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2784           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2785             :     return 0;
    2786           0 :   if ((Subtarget->hasVLX())) {
    2787           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2788             :   }
    2789             :   return 0;
    2790             : }
    2791             : 
    2792           0 : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2793           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2794             :     return 0;
    2795           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2796           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2797             :   }
    2798             :   return 0;
    2799             : }
    2800             : 
    2801           0 : unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2802           0 :   switch (VT.SimpleTy) {
    2803           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2804           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2805             :   default: return 0;
    2806             :   }
    2807             : }
    2808             : 
    2809             : // FastEmit functions for X86ISD::EH_RETURN.
    2810             : 
    2811             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2812           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2813             :     return 0;
    2814           0 :   return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
    2815             : }
    2816             : 
    2817             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2818           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2819             :     return 0;
    2820           0 :   return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
    2821             : }
    2822             : 
    2823           0 : unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2824           0 :   switch (VT.SimpleTy) {
    2825           0 :   case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2826           0 :   case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2827             :   default: return 0;
    2828             :   }
    2829             : }
    2830             : 
    2831             : // FastEmit functions for X86ISD::EXPAND.
    2832             : 
    2833             : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2834           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2835             :     return 0;
    2836           0 :   if ((Subtarget->hasVLX())) {
    2837           0 :     return fastEmitInst_r(X86::VPEXPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2838             :   }
    2839             :   return 0;
    2840             : }
    2841             : 
    2842             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2843           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2844             :     return 0;
    2845           0 :   if ((Subtarget->hasVLX())) {
    2846           0 :     return fastEmitInst_r(X86::VPEXPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2847             :   }
    2848             :   return 0;
    2849             : }
    2850             : 
    2851             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2852           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2853             :     return 0;
    2854           0 :   if ((Subtarget->hasAVX512())) {
    2855           0 :     return fastEmitInst_r(X86::VPEXPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2856             :   }
    2857             :   return 0;
    2858             : }
    2859             : 
    2860             : unsigned fastEmit_X86ISD_EXPAND_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2861           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2862             :     return 0;
    2863           0 :   if ((Subtarget->hasVLX())) {
    2864           0 :     return fastEmitInst_r(X86::VPEXPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2865             :   }
    2866             :   return 0;
    2867             : }
    2868             : 
    2869             : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2870           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2871             :     return 0;
    2872           0 :   if ((Subtarget->hasVLX())) {
    2873           0 :     return fastEmitInst_r(X86::VPEXPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2874             :   }
    2875             :   return 0;
    2876             : }
    2877             : 
    2878             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2879           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2880             :     return 0;
    2881           0 :   if ((Subtarget->hasAVX512())) {
    2882           0 :     return fastEmitInst_r(X86::VPEXPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2883             :   }
    2884             :   return 0;
    2885             : }
    2886             : 
    2887             : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2888           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2889             :     return 0;
    2890           0 :   if ((Subtarget->hasVLX())) {
    2891           0 :     return fastEmitInst_r(X86::VEXPANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2892             :   }
    2893             :   return 0;
    2894             : }
    2895             : 
    2896             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2897           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    2898             :     return 0;
    2899           0 :   if ((Subtarget->hasVLX())) {
    2900           0 :     return fastEmitInst_r(X86::VEXPANDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2901             :   }
    2902             :   return 0;
    2903             : }
    2904             : 
    2905             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2906           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2907             :     return 0;
    2908           0 :   if ((Subtarget->hasAVX512())) {
    2909           0 :     return fastEmitInst_r(X86::VEXPANDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2910             :   }
    2911             :   return 0;
    2912             : }
    2913             : 
    2914             : unsigned fastEmit_X86ISD_EXPAND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2915           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2916             :     return 0;
    2917           0 :   if ((Subtarget->hasVLX())) {
    2918           0 :     return fastEmitInst_r(X86::VEXPANDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2919             :   }
    2920             :   return 0;
    2921             : }
    2922             : 
    2923             : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2924           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    2925             :     return 0;
    2926           0 :   if ((Subtarget->hasVLX())) {
    2927           0 :     return fastEmitInst_r(X86::VEXPANDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2928             :   }
    2929             :   return 0;
    2930             : }
    2931             : 
    2932             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2933           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    2934             :     return 0;
    2935           0 :   if ((Subtarget->hasAVX512())) {
    2936           0 :     return fastEmitInst_r(X86::VEXPANDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2937             :   }
    2938             :   return 0;
    2939             : }
    2940             : 
    2941           0 : unsigned fastEmit_X86ISD_EXPAND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2942           0 :   switch (VT.SimpleTy) {
    2943           0 :   case MVT::v4i32: return fastEmit_X86ISD_EXPAND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2944           0 :   case MVT::v8i32: return fastEmit_X86ISD_EXPAND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2945           0 :   case MVT::v16i32: return fastEmit_X86ISD_EXPAND_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2946           0 :   case MVT::v2i64: return fastEmit_X86ISD_EXPAND_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2947           0 :   case MVT::v4i64: return fastEmit_X86ISD_EXPAND_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2948           0 :   case MVT::v8i64: return fastEmit_X86ISD_EXPAND_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2949           0 :   case MVT::v4f32: return fastEmit_X86ISD_EXPAND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2950           0 :   case MVT::v8f32: return fastEmit_X86ISD_EXPAND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2951           0 :   case MVT::v16f32: return fastEmit_X86ISD_EXPAND_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2952           0 :   case MVT::v2f64: return fastEmit_X86ISD_EXPAND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2953           0 :   case MVT::v4f64: return fastEmit_X86ISD_EXPAND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2954           0 :   case MVT::v8f64: return fastEmit_X86ISD_EXPAND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2955             :   default: return 0;
    2956             :   }
    2957             : }
    2958             : 
    2959             : // FastEmit functions for X86ISD::FRCP.
    2960             : 
    2961           0 : unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2962           0 :   if (RetVT.SimpleTy != MVT::f32)
    2963             :     return 0;
    2964           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    2965           0 :     return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    2966             :   }
    2967             :   return 0;
    2968             : }
    2969             : 
    2970           0 : unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2971           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2972             :     return 0;
    2973           0 :   if ((Subtarget->hasVLX())) {
    2974           0 :     return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    2975             :   }
    2976           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    2977           0 :     return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    2978             :   }
    2979           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2980           0 :     return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    2981             :   }
    2982             :   return 0;
    2983             : }
    2984             : 
    2985           0 : unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2986           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    2987             :     return 0;
    2988           0 :   if ((Subtarget->hasVLX())) {
    2989           0 :     return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    2990             :   }
    2991           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2992           0 :     return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    2993             :   }
    2994             :   return 0;
    2995             : }
    2996             : 
    2997             : unsigned fastEmit_X86ISD_FRCP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2998           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2999             :     return 0;
    3000           0 :   if ((Subtarget->hasAVX512())) {
    3001           0 :     return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3002             :   }
    3003             :   return 0;
    3004             : }
    3005             : 
    3006             : unsigned fastEmit_X86ISD_FRCP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3007           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3008             :     return 0;
    3009           0 :   if ((Subtarget->hasVLX())) {
    3010           0 :     return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3011             :   }
    3012             :   return 0;
    3013             : }
    3014             : 
    3015             : unsigned fastEmit_X86ISD_FRCP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3016           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3017             :     return 0;
    3018           0 :   if ((Subtarget->hasVLX())) {
    3019           0 :     return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3020             :   }
    3021             :   return 0;
    3022             : }
    3023             : 
    3024             : unsigned fastEmit_X86ISD_FRCP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3025           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3026             :     return 0;
    3027           0 :   if ((Subtarget->hasAVX512())) {
    3028           0 :     return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3029             :   }
    3030             :   return 0;
    3031             : }
    3032             : 
    3033           0 : unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3034           0 :   switch (VT.SimpleTy) {
    3035           0 :   case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3036           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3037           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3038           0 :   case MVT::v16f32: return fastEmit_X86ISD_FRCP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3039           0 :   case MVT::v2f64: return fastEmit_X86ISD_FRCP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3040           0 :   case MVT::v4f64: return fastEmit_X86ISD_FRCP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3041           0 :   case MVT::v8f64: return fastEmit_X86ISD_FRCP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3042             :   default: return 0;
    3043             :   }
    3044             : }
    3045             : 
    3046             : // FastEmit functions for X86ISD::FRSQRT.
    3047             : 
    3048           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3049           0 :   if (RetVT.SimpleTy != MVT::f32)
    3050             :     return 0;
    3051           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3052           0 :     return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3053             :   }
    3054             :   return 0;
    3055             : }
    3056             : 
    3057           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3058           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3059             :     return 0;
    3060           0 :   if ((Subtarget->hasVLX())) {
    3061           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3062             :   }
    3063           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3064           0 :     return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3065             :   }
    3066           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3067           0 :     return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3068             :   }
    3069             :   return 0;
    3070             : }
    3071             : 
    3072           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3073           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3074             :     return 0;
    3075           0 :   if ((Subtarget->hasVLX())) {
    3076           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3077             :   }
    3078           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3079           0 :     return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3080             :   }
    3081             :   return 0;
    3082             : }
    3083             : 
    3084             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3085           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3086             :     return 0;
    3087           0 :   if ((Subtarget->hasAVX512())) {
    3088           0 :     return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3089             :   }
    3090             :   return 0;
    3091             : }
    3092             : 
    3093             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3094           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3095             :     return 0;
    3096           0 :   if ((Subtarget->hasVLX())) {
    3097           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3098             :   }
    3099             :   return 0;
    3100             : }
    3101             : 
    3102             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3103           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3104             :     return 0;
    3105           0 :   if ((Subtarget->hasVLX())) {
    3106           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3107             :   }
    3108             :   return 0;
    3109             : }
    3110             : 
    3111             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3112           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3113             :     return 0;
    3114           0 :   if ((Subtarget->hasAVX512())) {
    3115           0 :     return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3116             :   }
    3117             :   return 0;
    3118             : }
    3119             : 
    3120           0 : unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3121           0 :   switch (VT.SimpleTy) {
    3122           0 :   case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3123           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3124           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3125           0 :   case MVT::v16f32: return fastEmit_X86ISD_FRSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3126           0 :   case MVT::v2f64: return fastEmit_X86ISD_FRSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3127           0 :   case MVT::v4f64: return fastEmit_X86ISD_FRSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3128           0 :   case MVT::v8f64: return fastEmit_X86ISD_FRSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3129             :   default: return 0;
    3130             :   }
    3131             : }
    3132             : 
    3133             : // FastEmit functions for X86ISD::MMX_MOVD2W.
    3134             : 
    3135             : unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3136           0 :   if (RetVT.SimpleTy != MVT::i32)
    3137             :     return 0;
    3138           0 :   if ((Subtarget->hasMMX())) {
    3139           0 :     return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
    3140             :   }
    3141             :   return 0;
    3142             : }
    3143             : 
    3144           0 : unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3145           0 :   switch (VT.SimpleTy) {
    3146           0 :   case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
    3147             :   default: return 0;
    3148             :   }
    3149             : }
    3150             : 
    3151             : // FastEmit functions for X86ISD::MMX_MOVW2D.
    3152             : 
    3153             : unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3154           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3155             :     return 0;
    3156           0 :   if ((Subtarget->hasMMX())) {
    3157           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    3158             :   }
    3159             :   return 0;
    3160             : }
    3161             : 
    3162           0 : unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3163           0 :   switch (VT.SimpleTy) {
    3164           0 :   case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3165             :   default: return 0;
    3166             :   }
    3167             : }
    3168             : 
    3169             : // FastEmit functions for X86ISD::MOVDDUP.
    3170             : 
    3171           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3172           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    3173             :     return 0;
    3174           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3175           0 :     return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3176             :   }
    3177             :   return 0;
    3178             : }
    3179             : 
    3180           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3181           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3182             :     return 0;
    3183           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3184           0 :     return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3185             :   }
    3186           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3187           0 :     return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3188             :   }
    3189           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3190           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3191             :   }
    3192             :   return 0;
    3193             : }
    3194             : 
    3195           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3196           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3197             :     return 0;
    3198           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3199           0 :     return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3200             :   }
    3201           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3202           0 :     return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3203             :   }
    3204             :   return 0;
    3205             : }
    3206             : 
    3207             : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3208           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3209             :     return 0;
    3210           0 :   if ((Subtarget->hasAVX512())) {
    3211           0 :     return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3212             :   }
    3213             :   return 0;
    3214             : }
    3215             : 
    3216           0 : unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3217           0 :   switch (VT.SimpleTy) {
    3218           0 :   case MVT::v4i64: return fastEmit_X86ISD_MOVDDUP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    3219           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3220           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3221           0 :   case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3222             :   default: return 0;
    3223             :   }
    3224             : }
    3225             : 
    3226             : // FastEmit functions for X86ISD::MOVDQ2Q.
    3227             : 
    3228             : unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3229           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3230             :     return 0;
    3231           0 :   return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
    3232             : }
    3233             : 
    3234             : unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3235           0 :   switch (VT.SimpleTy) {
    3236           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3237             :   default: return 0;
    3238             :   }
    3239             : }
    3240             : 
    3241             : // FastEmit functions for X86ISD::MOVMSK.
    3242             : 
    3243           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3244           0 :   if (RetVT.SimpleTy != MVT::i32)
    3245             :     return 0;
    3246           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3247           0 :     return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3248             :   }
    3249           0 :   if ((Subtarget->hasAVX())) {
    3250           0 :     return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3251             :   }
    3252             :   return 0;
    3253             : }
    3254             : 
    3255             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3256           0 :   if (RetVT.SimpleTy != MVT::i32)
    3257             :     return 0;
    3258           0 :   if ((Subtarget->hasAVX2())) {
    3259           0 :     return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3260             :   }
    3261             :   return 0;
    3262             : }
    3263             : 
    3264           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3265           0 :   if (RetVT.SimpleTy != MVT::i32)
    3266             :     return 0;
    3267           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3268           0 :     return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3269             :   }
    3270           0 :   if ((Subtarget->hasAVX())) {
    3271           0 :     return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3272             :   }
    3273             :   return 0;
    3274             : }
    3275             : 
    3276             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3277           0 :   if (RetVT.SimpleTy != MVT::i32)
    3278             :     return 0;
    3279           0 :   if ((Subtarget->hasAVX())) {
    3280           0 :     return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3281             :   }
    3282             :   return 0;
    3283             : }
    3284             : 
    3285           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3286           0 :   if (RetVT.SimpleTy != MVT::i32)
    3287             :     return 0;
    3288           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3289           0 :     return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3290             :   }
    3291           0 :   if ((Subtarget->hasAVX())) {
    3292           0 :     return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3293             :   }
    3294             :   return 0;
    3295             : }
    3296             : 
    3297             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3298           0 :   if (RetVT.SimpleTy != MVT::i32)
    3299             :     return 0;
    3300           0 :   if ((Subtarget->hasAVX())) {
    3301           0 :     return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3302             :   }
    3303             :   return 0;
    3304             : }
    3305             : 
    3306           0 : unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3307           0 :   switch (VT.SimpleTy) {
    3308           0 :   case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3309           0 :   case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3310           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3311           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3312           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3313           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3314             :   default: return 0;
    3315             :   }
    3316             : }
    3317             : 
    3318             : // FastEmit functions for X86ISD::MOVSHDUP.
    3319             : 
    3320           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3321           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3322             :     return 0;
    3323           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3324           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3325             :   }
    3326           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3327           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3328             :   }
    3329             :   return 0;
    3330             : }
    3331             : 
    3332           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3333           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3334             :     return 0;
    3335           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3336           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3337             :   }
    3338             :   return 0;
    3339             : }
    3340             : 
    3341           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3342           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3343             :     return 0;
    3344           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3345           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3346             :   }
    3347           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3348           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3349             :   }
    3350           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3351           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3352             :   }
    3353             :   return 0;
    3354             : }
    3355             : 
    3356           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3357           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3358             :     return 0;
    3359           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3360           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3361             :   }
    3362           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3363           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3364             :   }
    3365             :   return 0;
    3366             : }
    3367             : 
    3368             : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3369           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3370             :     return 0;
    3371           0 :   if ((Subtarget->hasAVX512())) {
    3372           0 :     return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3373             :   }
    3374             :   return 0;
    3375             : }
    3376             : 
    3377           0 : unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3378           0 :   switch (VT.SimpleTy) {
    3379           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3380           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3381           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3382           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3383           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3384             :   default: return 0;
    3385             :   }
    3386             : }
    3387             : 
    3388             : // FastEmit functions for X86ISD::MOVSLDUP.
    3389             : 
    3390           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3391           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3392             :     return 0;
    3393           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3394           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3395             :   }
    3396           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3397           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3398             :   }
    3399             :   return 0;
    3400             : }
    3401             : 
    3402           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3403           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3404             :     return 0;
    3405           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3406           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3407             :   }
    3408             :   return 0;
    3409             : }
    3410             : 
    3411           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3412           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3413             :     return 0;
    3414           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3415           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3416             :   }
    3417           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3418           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3419             :   }
    3420           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3421           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3422             :   }
    3423             :   return 0;
    3424             : }
    3425             : 
    3426           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3427           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3428             :     return 0;
    3429           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3430           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3431             :   }
    3432           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3433           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3434             :   }
    3435             :   return 0;
    3436             : }
    3437             : 
    3438             : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3439           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3440             :     return 0;
    3441           0 :   if ((Subtarget->hasAVX512())) {
    3442           0 :     return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3443             :   }
    3444             :   return 0;
    3445             : }
    3446             : 
    3447           0 : unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3448           0 :   switch (VT.SimpleTy) {
    3449           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3450           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3451           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3452           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3453           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3454             :   default: return 0;
    3455             :   }
    3456             : }
    3457             : 
    3458             : // FastEmit functions for X86ISD::SEG_ALLOCA.
    3459             : 
    3460           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3461           0 :   if (RetVT.SimpleTy != MVT::i32)
    3462             :     return 0;
    3463           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    3464           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    3465             :   }
    3466             :   return 0;
    3467             : }
    3468             : 
    3469             : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3470           0 :   if (RetVT.SimpleTy != MVT::i64)
    3471             :     return 0;
    3472           0 :   if ((Subtarget->is64Bit())) {
    3473           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    3474             :   }
    3475             :   return 0;
    3476             : }
    3477             : 
    3478           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3479           0 :   switch (VT.SimpleTy) {
    3480           0 :   case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3481           0 :   case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3482             :   default: return 0;
    3483             :   }
    3484             : }
    3485             : 
    3486             : // FastEmit functions for X86ISD::VBROADCAST.
    3487             : 
    3488           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3489           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3490           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3491             :   }
    3492             :   return 0;
    3493             : }
    3494             : 
    3495           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3496           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3497           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3498             :   }
    3499             :   return 0;
    3500             : }
    3501             : 
    3502             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    3503           0 :   if ((Subtarget->hasAVX512())) {
    3504           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3505             :   }
    3506             :   return 0;
    3507             : }
    3508             : 
    3509           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3510           0 : switch (RetVT.SimpleTy) {
    3511           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    3512           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
    3513           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
    3514             :   default: return 0;
    3515             : }
    3516             : }
    3517             : 
    3518           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3519           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3520           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3521             :   }
    3522             :   return 0;
    3523             : }
    3524             : 
    3525           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3526           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3527           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3528             :   }
    3529             :   return 0;
    3530             : }
    3531             : 
    3532             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3533           0 :   if ((Subtarget->hasAVX512())) {
    3534           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3535             :   }
    3536             :   return 0;
    3537             : }
    3538             : 
    3539           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3540           0 : switch (RetVT.SimpleTy) {
    3541           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
    3542           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
    3543           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
    3544             :   default: return 0;
    3545             : }
    3546             : }
    3547             : 
    3548           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    3549           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3550           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3551             :   }
    3552           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    3553           0 :     return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3554             :   }
    3555             :   return 0;
    3556             : }
    3557             : 
    3558           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    3559           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3560           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3561             :   }
    3562           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    3563           0 :     return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3564             :   }
    3565             :   return 0;
    3566             : }
    3567             : 
    3568             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
    3569           0 :   if ((Subtarget->hasBWI())) {
    3570           0 :     return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3571             :   }
    3572             :   return 0;
    3573             : }
    3574             : 
    3575           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3576           0 : switch (RetVT.SimpleTy) {
    3577           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
    3578           0 :   case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
    3579           0 :   case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
    3580             :   default: return 0;
    3581             : }
    3582             : }
    3583             : 
    3584           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    3585           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3586           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3587             :   }
    3588           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    3589           0 :     return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3590             :   }
    3591             :   return 0;
    3592             : }
    3593             : 
    3594           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    3595           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3596           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3597             :   }
    3598           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    3599           0 :     return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3600             :   }
    3601             :   return 0;
    3602             : }
    3603             : 
    3604             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    3605           0 :   if ((Subtarget->hasBWI())) {
    3606           0 :     return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3607             :   }
    3608             :   return 0;
    3609             : }
    3610             : 
    3611           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3612           0 : switch (RetVT.SimpleTy) {
    3613           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
    3614           0 :   case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
    3615           0 :   case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
    3616             :   default: return 0;
    3617             : }
    3618             : }
    3619             : 
    3620           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3621           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3622           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3623             :   }
    3624           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    3625           0 :     return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3626             :   }
    3627             :   return 0;
    3628             : }
    3629             : 
    3630           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3631           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3632           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3633             :   }
    3634           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    3635           0 :     return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3636             :   }
    3637             :   return 0;
    3638             : }
    3639             : 
    3640             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    3641           0 :   if ((Subtarget->hasAVX512())) {
    3642           0 :     return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3643             :   }
    3644             :   return 0;
    3645             : }
    3646             : 
    3647           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3648           0 : switch (RetVT.SimpleTy) {
    3649           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
    3650           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
    3651           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
    3652             :   default: return 0;
    3653             : }
    3654             : }
    3655             : 
    3656           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3657           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3658           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3659             :   }
    3660           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    3661           0 :     return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3662             :   }
    3663             :   return 0;
    3664             : }
    3665             : 
    3666           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3667           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3668           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3669             :   }
    3670           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    3671           0 :     return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3672             :   }
    3673             :   return 0;
    3674             : }
    3675             : 
    3676             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3677           0 :   if ((Subtarget->hasAVX512())) {
    3678           0 :     return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3679             :   }
    3680             :   return 0;
    3681             : }
    3682             : 
    3683           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3684           0 : switch (RetVT.SimpleTy) {
    3685           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
    3686           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
    3687           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
    3688             :   default: return 0;
    3689             : }
    3690             : }
    3691             : 
    3692           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    3693           0 :   if ((Subtarget->hasVLX())) {
    3694           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3695             :   }
    3696           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    3697           0 :     return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3698             :   }
    3699             :   return 0;
    3700             : }
    3701             : 
    3702           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    3703           0 :   if ((Subtarget->hasVLX())) {
    3704           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3705             :   }
    3706           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    3707           0 :     return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3708             :   }
    3709             :   return 0;
    3710             : }
    3711             : 
    3712             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
    3713           0 :   if ((Subtarget->hasAVX512())) {
    3714           0 :     return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3715             :   }
    3716             :   return 0;
    3717             : }
    3718             : 
    3719           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3720           0 : switch (RetVT.SimpleTy) {
    3721           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
    3722           0 :   case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
    3723           0 :   case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
    3724             :   default: return 0;
    3725             : }
    3726             : }
    3727             : 
    3728           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    3729           0 :   if ((Subtarget->hasVLX())) {
    3730           0 :     return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3731             :   }
    3732           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    3733           0 :     return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3734             :   }
    3735             :   return 0;
    3736             : }
    3737             : 
    3738             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    3739           0 :   if ((Subtarget->hasAVX512())) {
    3740           0 :     return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3741             :   }
    3742             :   return 0;
    3743             : }
    3744             : 
    3745           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3746           0 : switch (RetVT.SimpleTy) {
    3747           0 :   case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
    3748           0 :   case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
    3749             :   default: return 0;
    3750             : }
    3751             : }
    3752             : 
    3753           0 : unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3754           0 :   switch (VT.SimpleTy) {
    3755           0 :   case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3756           0 :   case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3757           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3758           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3759           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3760           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3761           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3762           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3763             :   default: return 0;
    3764             :   }
    3765             : }
    3766             : 
    3767             : // FastEmit functions for X86ISD::VBROADCASTM.
    3768             : 
    3769           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3770           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    3771           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3772             :   }
    3773             :   return 0;
    3774             : }
    3775             : 
    3776           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3777           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    3778           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3779             :   }
    3780             :   return 0;
    3781             : }
    3782             : 
    3783             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3784           0 :   if ((Subtarget->hasCDI())) {
    3785           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3786             :   }
    3787             :   return 0;
    3788             : }
    3789             : 
    3790           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3791           0 : switch (RetVT.SimpleTy) {
    3792           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
    3793           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
    3794           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    3795             :   default: return 0;
    3796             : }
    3797             : }
    3798             : 
    3799           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3800           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    3801           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3802             :   }
    3803             :   return 0;
    3804             : }
    3805             : 
    3806           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3807           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    3808           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3809             :   }
    3810             :   return 0;
    3811             : }
    3812             : 
    3813             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    3814           0 :   if ((Subtarget->hasCDI())) {
    3815           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3816             :   }
    3817             :   return 0;
    3818             : }
    3819             : 
    3820           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3821           0 : switch (RetVT.SimpleTy) {
    3822           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
    3823           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
    3824           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    3825             :   default: return 0;
    3826             : }
    3827             : }
    3828             : 
    3829           0 : unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3830           0 :   switch (VT.SimpleTy) {
    3831           0 :   case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    3832           0 :   case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    3833             :   default: return 0;
    3834             :   }
    3835             : }
    3836             : 
    3837             : // FastEmit functions for X86ISD::VFPEXT.
    3838             : 
    3839           0 : unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3840           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3841             :     return 0;
    3842           0 :   if ((Subtarget->hasVLX())) {
    3843           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3844             :   }
    3845           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3846           0 :     return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3847             :   }
    3848           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3849           0 :     return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3850             :   }
    3851             :   return 0;
    3852             : }
    3853             : 
    3854             : unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3855           0 :   switch (VT.SimpleTy) {
    3856           0 :   case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3857             :   default: return 0;
    3858             :   }
    3859             : }
    3860             : 
    3861             : // FastEmit functions for X86ISD::VFPROUND.
    3862             : 
    3863           0 : unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3864           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3865             :     return 0;
    3866           0 :   if ((Subtarget->hasVLX())) {
    3867           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3868             :   }
    3869           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3870           0 :     return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3871             :   }
    3872           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3873           0 :     return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3874             :   }
    3875             :   return 0;
    3876             : }
    3877             : 
    3878             : unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3879           0 :   switch (VT.SimpleTy) {
    3880           0 :   case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3881             :   default: return 0;
    3882             :   }
    3883             : }
    3884             : 
    3885             : // FastEmit functions for X86ISD::VSEXT.
    3886             : 
    3887           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3888           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3889             :     return 0;
    3890           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3891           0 :     return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3892             :   }
    3893             :   return 0;
    3894             : }
    3895             : 
    3896           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3897           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3898           0 :     return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3899             :   }
    3900             :   return 0;
    3901             : }
    3902             : 
    3903           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3904           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3905           0 :     return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3906             :   }
    3907             :   return 0;
    3908             : }
    3909             : 
    3910           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3911           0 : switch (RetVT.SimpleTy) {
    3912           0 :   case MVT::v4i32: return fastEmit_X86ISD_VSEXT_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
    3913           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill);
    3914             :   default: return 0;
    3915             : }
    3916             : }
    3917             : 
    3918           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    3919           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3920           0 :     return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3921             :   }
    3922             :   return 0;
    3923             : }
    3924             : 
    3925           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3926           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3927           0 :     return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3928             :   }
    3929             :   return 0;
    3930             : }
    3931             : 
    3932             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3933           0 :   if ((Subtarget->hasDQI())) {
    3934           0 :     return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3935             :   }
    3936             :   return 0;
    3937             : }
    3938             : 
    3939           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3940           0 : switch (RetVT.SimpleTy) {
    3941           0 :   case MVT::v8i16: return fastEmit_X86ISD_VSEXT_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
    3942           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill);
    3943           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    3944             :   default: return 0;
    3945             : }
    3946             : }
    3947             : 
    3948           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    3949           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3950           0 :     return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3951             :   }
    3952             :   return 0;
    3953             : }
    3954             : 
    3955           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    3956           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3957           0 :     return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3958             :   }
    3959             :   return 0;
    3960             : }
    3961             : 
    3962             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    3963           0 :   if ((Subtarget->hasDQI())) {
    3964           0 :     return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3965             :   }
    3966             :   return 0;
    3967             : }
    3968             : 
    3969           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3970           0 : switch (RetVT.SimpleTy) {
    3971           0 :   case MVT::v16i8: return fastEmit_X86ISD_VSEXT_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
    3972           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill);
    3973           0 :   case MVT::v16i32: return fastEmit_X86ISD_VSEXT_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    3974             :   default: return 0;
    3975             : }
    3976             : }
    3977             : 
    3978           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    3979           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3980           0 :     return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3981             :   }
    3982             :   return 0;
    3983             : }
    3984             : 
    3985             : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    3986           0 :   if ((Subtarget->hasBWI())) {
    3987           0 :     return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3988             :   }
    3989             :   return 0;
    3990             : }
    3991             : 
    3992           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3993           0 : switch (RetVT.SimpleTy) {
    3994           0 :   case MVT::v32i8: return fastEmit_X86ISD_VSEXT_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
    3995           0 :   case MVT::v32i16: return fastEmit_X86ISD_VSEXT_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
    3996             :   default: return 0;
    3997             : }
    3998             : }
    3999             : 
    4000             : unsigned fastEmit_X86ISD_VSEXT_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4001           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    4002             :     return 0;
    4003           0 :   if ((Subtarget->hasBWI())) {
    4004           0 :     return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4005             :   }
    4006             :   return 0;
    4007             : }
    4008             : 
    4009           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4010           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4011           0 :     return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4012             :   }
    4013           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4014           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4015             :   }
    4016             :   return 0;
    4017             : }
    4018             : 
    4019           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4020           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4021           0 :     return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4022             :   }
    4023           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4024           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4025             :   }
    4026             :   return 0;
    4027             : }
    4028             : 
    4029             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4030           0 :   if ((Subtarget->hasAVX512())) {
    4031           0 :     return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4032             :   }
    4033             :   return 0;
    4034             : }
    4035             : 
    4036           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4037           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4038           0 :     return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4039             :   }
    4040           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4041           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4042             :   }
    4043             :   return 0;
    4044             : }
    4045             : 
    4046             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4047           0 :   if ((Subtarget->hasAVX512())) {
    4048           0 :     return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4049             :   }
    4050             :   return 0;
    4051             : }
    4052             : 
    4053           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4054           0 : switch (RetVT.SimpleTy) {
    4055           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    4056           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    4057           0 :   case MVT::v16i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    4058           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    4059           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    4060             :   default: return 0;
    4061             : }
    4062             : }
    4063             : 
    4064             : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4065           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    4066             :     return 0;
    4067           0 :   if ((Subtarget->hasBWI())) {
    4068           0 :     return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4069             :   }
    4070             :   return 0;
    4071             : }
    4072             : 
    4073           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4074           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4075           0 :     return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4076             :   }
    4077           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4078           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4079             :   }
    4080             :   return 0;
    4081             : }
    4082             : 
    4083           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4084           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4085           0 :     return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4086             :   }
    4087           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4088           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4089             :   }
    4090             :   return 0;
    4091             : }
    4092             : 
    4093             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4094           0 :   if ((Subtarget->hasAVX512())) {
    4095           0 :     return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4096             :   }
    4097             :   return 0;
    4098             : }
    4099             : 
    4100           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4101           0 : switch (RetVT.SimpleTy) {
    4102           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    4103           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    4104           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    4105             :   default: return 0;
    4106             : }
    4107             : }
    4108             : 
    4109             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4110           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    4111             :     return 0;
    4112           0 :   if ((Subtarget->hasAVX512())) {
    4113           0 :     return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4114             :   }
    4115             :   return 0;
    4116             : }
    4117             : 
    4118           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4119           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    4120             :     return 0;
    4121           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4122           0 :     return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4123             :   }
    4124           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4125           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4126             :   }
    4127             :   return 0;
    4128             : }
    4129             : 
    4130             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4131           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    4132             :     return 0;
    4133           0 :   if ((Subtarget->hasAVX512())) {
    4134           0 :     return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4135             :   }
    4136             :   return 0;
    4137             : }
    4138             : 
    4139           0 : unsigned fastEmit_X86ISD_VSEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4140           0 :   switch (VT.SimpleTy) {
    4141           0 :   case MVT::v2i1: return fastEmit_X86ISD_VSEXT_MVT_v2i1_r(RetVT, Op0, Op0IsKill);
    4142           0 :   case MVT::v4i1: return fastEmit_X86ISD_VSEXT_MVT_v4i1_r(RetVT, Op0, Op0IsKill);
    4143           0 :   case MVT::v8i1: return fastEmit_X86ISD_VSEXT_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    4144           0 :   case MVT::v16i1: return fastEmit_X86ISD_VSEXT_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    4145           0 :   case MVT::v32i1: return fastEmit_X86ISD_VSEXT_MVT_v32i1_r(RetVT, Op0, Op0IsKill);
    4146           0 :   case MVT::v64i1: return fastEmit_X86ISD_VSEXT_MVT_v64i1_r(RetVT, Op0, Op0IsKill);
    4147           0 :   case MVT::v16i8: return fastEmit_X86ISD_VSEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4148           0 :   case MVT::v32i8: return fastEmit_X86ISD_VSEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    4149           0 :   case MVT::v8i16: return fastEmit_X86ISD_VSEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4150           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4151           0 :   case MVT::v4i32: return fastEmit_X86ISD_VSEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4152           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4153             :   default: return 0;
    4154             :   }
    4155             : }
    4156             : 
    4157             : // FastEmit functions for X86ISD::VTRUNC.
    4158             : 
    4159           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4160           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4161             :     return 0;
    4162           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4163           0 :     return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4164             :   }
    4165             :   return 0;
    4166             : }
    4167             : 
    4168           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4169           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4170             :     return 0;
    4171           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4172           0 :     return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4173             :   }
    4174             :   return 0;
    4175             : }
    4176             : 
    4177             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4178           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4179             :     return 0;
    4180           0 :   if ((Subtarget->hasBWI())) {
    4181           0 :     return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4182             :   }
    4183             :   return 0;
    4184             : }
    4185             : 
    4186           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4187           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4188           0 :     return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4189             :   }
    4190             :   return 0;
    4191             : }
    4192             : 
    4193           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4194           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4195           0 :     return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4196             :   }
    4197             :   return 0;
    4198             : }
    4199             : 
    4200           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4201           0 : switch (RetVT.SimpleTy) {
    4202           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4203           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4204             :   default: return 0;
    4205             : }
    4206             : }
    4207             : 
    4208           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4209           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4210           0 :     return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4211             :   }
    4212             :   return 0;
    4213             : }
    4214             : 
    4215           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4216           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4217           0 :     return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4218             :   }
    4219             :   return 0;
    4220             : }
    4221             : 
    4222           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4223           0 : switch (RetVT.SimpleTy) {
    4224           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4225           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4226             :   default: return 0;
    4227             : }
    4228             : }
    4229             : 
    4230             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4231           0 :   if ((Subtarget->hasAVX512())) {
    4232           0 :     return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4233             :   }
    4234             :   return 0;
    4235             : }
    4236             : 
    4237             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4238           0 :   if ((Subtarget->hasAVX512())) {
    4239           0 :     return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4240             :   }
    4241             :   return 0;
    4242             : }
    4243             : 
    4244           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4245           0 : switch (RetVT.SimpleTy) {
    4246           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4247           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNC_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4248             :   default: return 0;
    4249             : }
    4250             : }
    4251             : 
    4252           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4253           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4254           0 :     return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4255             :   }
    4256             :   return 0;
    4257             : }
    4258             : 
    4259           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4260           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4261           0 :     return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4262             :   }
    4263             :   return 0;
    4264             : }
    4265             : 
    4266           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4267           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4268           0 :     return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4269             :   }
    4270             :   return 0;
    4271             : }
    4272             : 
    4273           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4274           0 : switch (RetVT.SimpleTy) {
    4275           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4276           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4277           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4278             :   default: return 0;
    4279             : }
    4280             : }
    4281             : 
    4282           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4283           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4284           0 :     return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4285             :   }
    4286             :   return 0;
    4287             : }
    4288             : 
    4289           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4290           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4291           0 :     return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4292             :   }
    4293             :   return 0;
    4294             : }
    4295             : 
    4296           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4297           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4298           0 :     return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4299             :   }
    4300             :   return 0;
    4301             : }
    4302             : 
    4303           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4304           0 : switch (RetVT.SimpleTy) {
    4305           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4306           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4307           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    4308             :   default: return 0;
    4309             : }
    4310             : }
    4311             : 
    4312             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4313           0 :   if ((Subtarget->hasAVX512())) {
    4314           0 :     return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4315             :   }
    4316             :   return 0;
    4317             : }
    4318             : 
    4319             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4320           0 :   if ((Subtarget->hasAVX512())) {
    4321           0 :     return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4322             :   }
    4323             :   return 0;
    4324             : }
    4325             : 
    4326             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4327           0 :   if ((Subtarget->hasAVX512())) {
    4328           0 :     return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4329             :   }
    4330             :   return 0;
    4331             : }
    4332             : 
    4333           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4334           0 : switch (RetVT.SimpleTy) {
    4335           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    4336           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    4337           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    4338             :   default: return 0;
    4339             : }
    4340             : }
    4341             : 
    4342           0 : unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4343           0 :   switch (VT.SimpleTy) {
    4344           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4345           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNC_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4346           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNC_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    4347           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4348           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4349           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNC_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    4350           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4351           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4352           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4353             :   default: return 0;
    4354             :   }
    4355             : }
    4356             : 
    4357             : // FastEmit functions for X86ISD::VTRUNCS.
    4358             : 
    4359           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4360           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4361             :     return 0;
    4362           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4363           0 :     return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4364             :   }
    4365             :   return 0;
    4366             : }
    4367             : 
    4368           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4369           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4370             :     return 0;
    4371           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4372           0 :     return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4373             :   }
    4374             :   return 0;
    4375             : }
    4376             : 
    4377             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4378           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4379             :     return 0;
    4380           0 :   if ((Subtarget->hasBWI())) {
    4381           0 :     return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4382             :   }
    4383             :   return 0;
    4384             : }
    4385             : 
    4386           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4387           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4388           0 :     return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4389             :   }
    4390             :   return 0;
    4391             : }
    4392             : 
    4393           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4394           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4395           0 :     return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4396             :   }
    4397             :   return 0;
    4398             : }
    4399             : 
    4400           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4401           0 : switch (RetVT.SimpleTy) {
    4402           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4403           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4404             :   default: return 0;
    4405             : }
    4406             : }
    4407             : 
    4408           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4409           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4410           0 :     return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4411             :   }
    4412             :   return 0;
    4413             : }
    4414             : 
    4415           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4416           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4417           0 :     return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4418             :   }
    4419             :   return 0;
    4420             : }
    4421             : 
    4422           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4423           0 : switch (RetVT.SimpleTy) {
    4424           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4425           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4426             :   default: return 0;
    4427             : }
    4428             : }
    4429             : 
    4430             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4431           0 :   if ((Subtarget->hasAVX512())) {
    4432           0 :     return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4433             :   }
    4434             :   return 0;
    4435             : }
    4436             : 
    4437             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4438           0 :   if ((Subtarget->hasAVX512())) {
    4439           0 :     return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4440             :   }
    4441             :   return 0;
    4442             : }
    4443             : 
    4444           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4445           0 : switch (RetVT.SimpleTy) {
    4446           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4447           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4448             :   default: return 0;
    4449             : }
    4450             : }
    4451             : 
    4452           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4453           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4454           0 :     return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4455             :   }
    4456             :   return 0;
    4457             : }
    4458             : 
    4459           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4460           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4461           0 :     return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4462             :   }
    4463             :   return 0;
    4464             : }
    4465             : 
    4466           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4467           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4468           0 :     return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4469             :   }
    4470             :   return 0;
    4471             : }
    4472             : 
    4473           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4474           0 : switch (RetVT.SimpleTy) {
    4475           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4476           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4477           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4478             :   default: return 0;
    4479             : }
    4480             : }
    4481             : 
    4482           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4483           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4484           0 :     return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4485             :   }
    4486             :   return 0;
    4487             : }
    4488             : 
    4489           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4490           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4491           0 :     return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4492             :   }
    4493             :   return 0;
    4494             : }
    4495             : 
    4496           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4497           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4498           0 :     return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4499             :   }
    4500             :   return 0;
    4501             : }
    4502             : 
    4503           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4504           0 : switch (RetVT.SimpleTy) {
    4505           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4506           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4507           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    4508             :   default: return 0;
    4509             : }
    4510             : }
    4511             : 
    4512             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4513           0 :   if ((Subtarget->hasAVX512())) {
    4514           0 :     return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4515             :   }
    4516             :   return 0;
    4517             : }
    4518             : 
    4519             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4520           0 :   if ((Subtarget->hasAVX512())) {
    4521           0 :     return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4522             :   }
    4523             :   return 0;
    4524             : }
    4525             : 
    4526             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4527           0 :   if ((Subtarget->hasAVX512())) {
    4528           0 :     return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4529             :   }
    4530             :   return 0;
    4531             : }
    4532             : 
    4533           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4534           0 : switch (RetVT.SimpleTy) {
    4535           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    4536           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    4537           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    4538             :   default: return 0;
    4539             : }
    4540             : }
    4541             : 
    4542           0 : unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4543           0 :   switch (VT.SimpleTy) {
    4544           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4545           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4546           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    4547           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4548           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4549           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    4550           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4551           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4552           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4553             :   default: return 0;
    4554             :   }
    4555             : }
    4556             : 
    4557             : // FastEmit functions for X86ISD::VTRUNCUS.
    4558             : 
    4559           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4560           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4561             :     return 0;
    4562           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4563           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4564             :   }
    4565             :   return 0;
    4566             : }
    4567             : 
    4568           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4569           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4570             :     return 0;
    4571           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4572           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4573             :   }
    4574             :   return 0;
    4575             : }
    4576             : 
    4577             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4578           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4579             :     return 0;
    4580           0 :   if ((Subtarget->hasBWI())) {
    4581           0 :     return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4582             :   }
    4583             :   return 0;
    4584             : }
    4585             : 
    4586           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4587           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4588           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4589             :   }
    4590             :   return 0;
    4591             : }
    4592             : 
    4593           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4594           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4595           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4596             :   }
    4597             :   return 0;
    4598             : }
    4599             : 
    4600           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4601           0 : switch (RetVT.SimpleTy) {
    4602           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4603           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4604             :   default: return 0;
    4605             : }
    4606             : }
    4607             : 
    4608           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4609           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4610           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4611             :   }
    4612             :   return 0;
    4613             : }
    4614             : 
    4615           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4616           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4617           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4618             :   }
    4619             :   return 0;
    4620             : }
    4621             : 
    4622           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4623           0 : switch (RetVT.SimpleTy) {
    4624           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4625           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4626             :   default: return 0;
    4627             : }
    4628             : }
    4629             : 
    4630             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4631           0 :   if ((Subtarget->hasAVX512())) {
    4632           0 :     return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4633             :   }
    4634             :   return 0;
    4635             : }
    4636             : 
    4637             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4638           0 :   if ((Subtarget->hasAVX512())) {
    4639           0 :     return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4640             :   }
    4641             :   return 0;
    4642             : }
    4643             : 
    4644           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4645           0 : switch (RetVT.SimpleTy) {
    4646           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4647           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4648             :   default: return 0;
    4649             : }
    4650             : }
    4651             : 
    4652           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4653           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4654           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4655             :   }
    4656             :   return 0;
    4657             : }
    4658             : 
    4659           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4660           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4661           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4662             :   }
    4663             :   return 0;
    4664             : }
    4665             : 
    4666           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4667           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4668           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4669             :   }
    4670             :   return 0;
    4671             : }
    4672             : 
    4673           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4674           0 : switch (RetVT.SimpleTy) {
    4675           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4676           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4677           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4678             :   default: return 0;
    4679             : }
    4680             : }
    4681             : 
    4682           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4683           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4684           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4685             :   }
    4686             :   return 0;
    4687             : }
    4688             : 
    4689           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4690           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4691           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4692             :   }
    4693             :   return 0;
    4694             : }
    4695             : 
    4696           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4697           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4698           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4699             :   }
    4700             :   return 0;
    4701             : }
    4702             : 
    4703           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4704           0 : switch (RetVT.SimpleTy) {
    4705           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4706           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4707           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    4708             :   default: return 0;
    4709             : }
    4710             : }
    4711             : 
    4712             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4713           0 :   if ((Subtarget->hasAVX512())) {
    4714           0 :     return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4715             :   }
    4716             :   return 0;
    4717             : }
    4718             : 
    4719             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4720           0 :   if ((Subtarget->hasAVX512())) {
    4721           0 :     return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4722             :   }
    4723             :   return 0;
    4724             : }
    4725             : 
    4726             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4727           0 :   if ((Subtarget->hasAVX512())) {
    4728           0 :     return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4729             :   }
    4730             :   return 0;
    4731             : }
    4732             : 
    4733           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4734           0 : switch (RetVT.SimpleTy) {
    4735           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    4736           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    4737           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    4738             :   default: return 0;
    4739             : }
    4740             : }
    4741             : 
    4742           0 : unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4743           0 :   switch (VT.SimpleTy) {
    4744           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4745           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4746           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    4747           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4748           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4749           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    4750           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4751           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4752           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4753             :   default: return 0;
    4754             :   }
    4755             : }
    4756             : 
    4757             : // FastEmit functions for X86ISD::VZEXT.
    4758             : 
    4759           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4760           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4761           0 :     return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4762             :   }
    4763           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4764           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4765             :   }
    4766             :   return 0;
    4767             : }
    4768             : 
    4769           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4770           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4771           0 :     return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4772             :   }
    4773           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4774           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4775             :   }
    4776             :   return 0;
    4777             : }
    4778             : 
    4779             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4780           0 :   if ((Subtarget->hasAVX512())) {
    4781           0 :     return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4782             :   }
    4783             :   return 0;
    4784             : }
    4785             : 
    4786           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4787           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4788           0 :     return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4789             :   }
    4790           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4791           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4792             :   }
    4793             :   return 0;
    4794             : }
    4795             : 
    4796             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4797           0 :   if ((Subtarget->hasAVX512())) {
    4798           0 :     return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4799             :   }
    4800             :   return 0;
    4801             : }
    4802             : 
    4803           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4804           0 : switch (RetVT.SimpleTy) {
    4805           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    4806           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    4807           0 :   case MVT::v16i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    4808           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    4809           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    4810             :   default: return 0;
    4811             : }
    4812             : }
    4813             : 
    4814             : unsigned fastEmit_X86ISD_VZEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4815           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    4816             :     return 0;
    4817           0 :   if ((Subtarget->hasBWI())) {
    4818           0 :     return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4819             :   }
    4820             :   return 0;
    4821             : }
    4822             : 
    4823           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4824           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4825           0 :     return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4826             :   }
    4827           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4828           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4829             :   }
    4830             :   return 0;
    4831             : }
    4832             : 
    4833           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4834           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4835           0 :     return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4836             :   }
    4837           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4838           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4839             :   }
    4840             :   return 0;
    4841             : }
    4842             : 
    4843             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4844           0 :   if ((Subtarget->hasAVX512())) {
    4845           0 :     return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4846             :   }
    4847             :   return 0;
    4848             : }
    4849             : 
    4850           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4851           0 : switch (RetVT.SimpleTy) {
    4852           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    4853           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    4854           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    4855             :   default: return 0;
    4856             : }
    4857             : }
    4858             : 
    4859             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4860           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    4861             :     return 0;
    4862           0 :   if ((Subtarget->hasAVX512())) {
    4863           0 :     return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4864             :   }
    4865             :   return 0;
    4866             : }
    4867             : 
    4868           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4869           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    4870             :     return 0;
    4871           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4872           0 :     return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4873             :   }
    4874           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4875           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4876             :   }
    4877             :   return 0;
    4878             : }
    4879             : 
    4880             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4881           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    4882             :     return 0;
    4883           0 :   if ((Subtarget->hasAVX512())) {
    4884           0 :     return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4885             :   }
    4886             :   return 0;
    4887             : }
    4888             : 
    4889           0 : unsigned fastEmit_X86ISD_VZEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4890           0 :   switch (VT.SimpleTy) {
    4891           0 :   case MVT::v16i8: return fastEmit_X86ISD_VZEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4892           0 :   case MVT::v32i8: return fastEmit_X86ISD_VZEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    4893           0 :   case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4894           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4895           0 :   case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4896           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4897             :   default: return 0;
    4898             :   }
    4899             : }
    4900             : 
    4901             : // FastEmit functions for X86ISD::VZEXT_MOVL.
    4902             : 
    4903           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4904           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    4905             :     return 0;
    4906           0 :   if ((Subtarget->hasAVX512())) {
    4907           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4908             :   }
    4909           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4910           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4911             :   }
    4912           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    4913           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4914             :   }
    4915             :   return 0;
    4916             : }
    4917             : 
    4918           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4919           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4920             :     return 0;
    4921           0 :   if ((Subtarget->hasAVX512())) {
    4922           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4923             :   }
    4924           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4925           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4926             :   }
    4927           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    4928           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4929             :   }
    4930             :   return 0;
    4931             : }
    4932             : 
    4933           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4934           0 :   switch (VT.SimpleTy) {
    4935           0 :   case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4936           0 :   case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4937             :   default: return 0;
    4938             :   }
    4939             : }
    4940             : 
    4941             : // FastEmit functions for X86ISD::WIN_ALLOCA.
    4942             : 
    4943           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4944           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4945             :     return 0;
    4946           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    4947           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    4948             :   }
    4949             :   return 0;
    4950             : }
    4951             : 
    4952             : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4953           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    4954             :     return 0;
    4955           0 :   if ((Subtarget->is64Bit())) {
    4956           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    4957             :   }
    4958             :   return 0;
    4959             : }
    4960             : 
    4961           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4962           0 :   switch (VT.SimpleTy) {
    4963           0 :   case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4964           0 :   case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4965             :   default: return 0;
    4966             :   }
    4967             : }
    4968             : 
    4969             : // Top-level FastEmit function.
    4970             : 
    4971        2035 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
    4972        2035 :   switch (Opcode) {
    4973           0 :   case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
    4974           6 :   case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    4975        1665 :   case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
    4976           0 :   case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    4977           0 :   case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
    4978           0 :   case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
    4979           0 :   case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
    4980           0 :   case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill);
    4981           0 :   case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill);
    4982           0 :   case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
    4983           0 :   case ISD::FCOS: return fastEmit_ISD_FCOS_r(VT, RetVT, Op0, Op0IsKill);
    4984           3 :   case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
    4985           0 :   case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    4986           0 :   case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
    4987           0 :   case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
    4988           0 :   case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
    4989           0 :   case ISD::FSIN: return fastEmit_ISD_FSIN_r(VT, RetVT, Op0, Op0IsKill);
    4990           0 :   case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
    4991           1 :   case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill);
    4992          72 :   case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    4993           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    4994          12 :   case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    4995          44 :   case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
    4996           0 :   case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    4997         232 :   case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    4998           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    4999           0 :   case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5000           0 :   case X86ISD::COMPRESS: return fastEmit_X86ISD_COMPRESS_r(VT, RetVT, Op0, Op0IsKill);
    5001           0 :   case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill);
    5002           0 :   case X86ISD::CVT2MASK: return fastEmit_X86ISD_CVT2MASK_r(VT, RetVT, Op0, Op0IsKill);
    5003           0 :   case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5004           0 :   case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5005           0 :   case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
    5006           0 :   case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5007           0 :   case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5008           0 :   case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
    5009           0 :   case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill);
    5010           0 :   case X86ISD::EXPAND: return fastEmit_X86ISD_EXPAND_r(VT, RetVT, Op0, Op0IsKill);
    5011           0 :   case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill);
    5012           0 :   case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5013           0 :   case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill);
    5014           0 :   case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill);
    5015           0 :   case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill);
    5016           0 :   case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill);
    5017           0 :   case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill);
    5018           0 :   case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill);
    5019           0 :   case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill);
    5020           0 :   case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5021           0 :   case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill);
    5022           0 :   case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill);
    5023           0 :   case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
    5024           0 :   case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
    5025           0 :   case X86ISD::VSEXT: return fastEmit_X86ISD_VSEXT_r(VT, RetVT, Op0, Op0IsKill);
    5026           0 :   case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill);
    5027           0 :   case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill);
    5028           0 :   case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill);
    5029           0 :   case X86ISD::VZEXT: return fastEmit_X86ISD_VZEXT_r(VT, RetVT, Op0, Op0IsKill);
    5030           0 :   case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill);
    5031           0 :   case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5032             :   default: return 0;
    5033             :   }
    5034             : }
    5035             : 
    5036             : // FastEmit functions for ISD::ADD.
    5037             : 
    5038             : unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5039           1 :   if (RetVT.SimpleTy != MVT::i8)
    5040             :     return 0;
    5041           1 :   return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5042             : }
    5043             : 
    5044             : unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5045           1 :   if (RetVT.SimpleTy != MVT::i16)
    5046             :     return 0;
    5047           1 :   return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5048             : }
    5049             : 
    5050             : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5051          80 :   if (RetVT.SimpleTy != MVT::i32)
    5052             :     return 0;
    5053          80 :   return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5054             : }
    5055             : 
    5056             : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5057          30 :   if (RetVT.SimpleTy != MVT::i64)
    5058             :     return 0;
    5059          30 :   return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5060             : }
    5061             : 
    5062             : unsigned fastEmit_ISD_ADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5063           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    5064             :     return 0;
    5065           0 :   if ((Subtarget->hasDQI())) {
    5066           0 :     return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5067             :   }
    5068             :   return 0;
    5069             : }
    5070             : 
    5071             : unsigned fastEmit_ISD_ADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5072           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    5073             :     return 0;
    5074           0 :   if ((Subtarget->hasDQI())) {
    5075           0 :     return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5076             :   }
    5077             :   return 0;
    5078             : }
    5079             : 
    5080             : unsigned fastEmit_ISD_ADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5081           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    5082             :     return 0;
    5083           0 :   if ((Subtarget->hasBWI())) {
    5084           0 :     return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5085             :   }
    5086             :   return 0;
    5087             : }
    5088             : 
    5089             : unsigned fastEmit_ISD_ADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5090           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    5091             :     return 0;
    5092           0 :   if ((Subtarget->hasBWI())) {
    5093           0 :     return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5094             :   }
    5095             :   return 0;
    5096             : }
    5097             : 
    5098           2 : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5099           2 :   if (RetVT.SimpleTy != MVT::v16i8)
    5100             :     return 0;
    5101           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5102           0 :     return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5103             :   }
    5104           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5105           2 :     return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5106             :   }
    5107           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5108           0 :     return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5109             :   }
    5110             :   return 0;
    5111             : }
    5112             : 
    5113           2 : unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5114           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    5115             :     return 0;
    5116           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5117           0 :     return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5118             :   }
    5119           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5120           2 :     return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5121             :   }
    5122             :   return 0;
    5123             : }
    5124             : 
    5125             : unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5126           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    5127             :     return 0;
    5128           0 :   if ((Subtarget->hasBWI())) {
    5129           0 :     return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5130             :   }
    5131             :   return 0;
    5132             : }
    5133             : 
    5134           2 : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5135           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    5136             :     return 0;
    5137           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5138           0 :     return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5139             :   }
    5140           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5141           2 :     return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5142             :   }
    5143           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5144           0 :     return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5145             :   }
    5146             :   return 0;
    5147             : }
    5148             : 
    5149           2 : unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5150           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    5151             :     return 0;
    5152           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5153           0 :     return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5154             :   }
    5155           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5156           2 :     return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5157             :   }
    5158             :   return 0;
    5159             : }
    5160             : 
    5161             : unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5162           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5163             :     return 0;
    5164           0 :   if ((Subtarget->hasBWI())) {
    5165           0 :     return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5166             :   }
    5167             :   return 0;
    5168             : }
    5169             : 
    5170          18 : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5171          18 :   if (RetVT.SimpleTy != MVT::v4i32)
    5172             :     return 0;
    5173          36 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5174           2 :     return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5175             :   }
    5176          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5177           6 :     return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5178             :   }
    5179          20 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5180          10 :     return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5181             :   }
    5182             :   return 0;
    5183             : }
    5184             : 
    5185           3 : unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5186           3 :   if (RetVT.SimpleTy != MVT::v8i32)
    5187             :     return 0;
    5188           6 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5189           1 :     return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5190             :   }
    5191           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5192           2 :     return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5193             :   }
    5194             :   return 0;
    5195             : }
    5196             : 
    5197             : unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5198           2 :   if (RetVT.SimpleTy != MVT::v16i32)
    5199             :     return 0;
    5200           4 :   if ((Subtarget->hasAVX512())) {
    5201           2 :     return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5202             :   }
    5203             :   return 0;
    5204             : }
    5205             : 
    5206           2 : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5207           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    5208             :     return 0;
    5209           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5210           0 :     return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5211             :   }
    5212           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5213           2 :     return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5214             :   }
    5215           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5216           0 :     return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5217             :   }
    5218             :   return 0;
    5219             : }
    5220             : 
    5221           4 : unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5222           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    5223             :     return 0;
    5224           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5225           2 :     return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5226             :   }
    5227           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5228           2 :     return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5229             :   }
    5230             :   return 0;
    5231             : }
    5232             : 
    5233             : unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5234           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5235             :     return 0;
    5236           0 :   if ((Subtarget->hasAVX512())) {
    5237           0 :     return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5238             :   }
    5239             :   return 0;
    5240             : }
    5241             : 
    5242         149 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5243         149 :   switch (VT.SimpleTy) {
    5244           1 :   case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5245           1 :   case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5246          80 :   case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5247          30 :   case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5248           0 :   case MVT::v8i1: return fastEmit_ISD_ADD_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5249           0 :   case MVT::v16i1: return fastEmit_ISD_ADD_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5250           0 :   case MVT::v32i1: return fastEmit_ISD_ADD_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5251           0 :   case MVT::v64i1: return fastEmit_ISD_ADD_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5252           2 :   case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5253           2 :   case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5254           0 :   case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5255           2 :   case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5256           2 :   case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5257           0 :   case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5258          18 :   case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5259           3 :   case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5260           2 :   case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5261           2 :   case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5262           4 :   case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5263           0 :   case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5264             :   default: return 0;
    5265             :   }
    5266             : }
    5267             : 
    5268             : // FastEmit functions for ISD::AND.
    5269             : 
    5270             : unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5271           0 :   if (RetVT.SimpleTy != MVT::i8)
    5272             :     return 0;
    5273           0 :   return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5274             : }
    5275             : 
    5276             : unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5277           0 :   if (RetVT.SimpleTy != MVT::i16)
    5278             :     return 0;
    5279           0 :   return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5280             : }
    5281             : 
    5282             : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5283          20 :   if (RetVT.SimpleTy != MVT::i32)
    5284             :     return 0;
    5285          20 :   return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5286             : }
    5287             : 
    5288             : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5289           9 :   if (RetVT.SimpleTy != MVT::i64)
    5290             :     return 0;
    5291           9 :   return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5292             : }
    5293             : 
    5294             : unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5295           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    5296             :     return 0;
    5297           0 :   if ((Subtarget->hasDQI())) {
    5298           0 :     return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5299             :   }
    5300             :   return 0;
    5301             : }
    5302             : 
    5303             : unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5304           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    5305             :     return 0;
    5306           0 :   if ((Subtarget->hasAVX512())) {
    5307           0 :     return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5308             :   }
    5309             :   return 0;
    5310             : }
    5311             : 
    5312             : unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5313           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    5314             :     return 0;
    5315           0 :   if ((Subtarget->hasBWI())) {
    5316           0 :     return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5317             :   }
    5318             :   return 0;
    5319             : }
    5320             : 
    5321             : unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5322           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    5323             :     return 0;
    5324           0 :   if ((Subtarget->hasBWI())) {
    5325           0 :     return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5326             :   }
    5327             :   return 0;
    5328             : }
    5329             : 
    5330           8 : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5331           8 :   if (RetVT.SimpleTy != MVT::v2i64)
    5332             :     return 0;
    5333          16 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5334           0 :     return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5335             :   }
    5336           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5337           4 :     return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5338             :   }
    5339           8 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5340           4 :     return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5341             :   }
    5342             :   return 0;
    5343             : }
    5344             : 
    5345          12 : unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5346          12 :   if (RetVT.SimpleTy != MVT::v4i64)
    5347             :     return 0;
    5348          24 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    5349           8 :     return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5350             :   }
    5351           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5352           0 :     return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5353             :   }
    5354           8 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5355           4 :     return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5356             :   }
    5357             :   return 0;
    5358             : }
    5359             : 
    5360             : unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5361           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5362             :     return 0;
    5363           0 :   if ((Subtarget->hasAVX512())) {
    5364           0 :     return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5365             :   }
    5366             :   return 0;
    5367             : }
    5368             : 
    5369          57 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5370          57 :   switch (VT.SimpleTy) {
    5371           0 :   case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5372           0 :   case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5373          20 :   case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5374           9 :   case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5375           0 :   case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5376           0 :   case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5377           0 :   case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5378           0 :   case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5379           8 :   case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5380          12 :   case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5381           0 :   case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5382             :   default: return 0;
    5383             :   }
    5384             : }
    5385             : 
    5386             : // FastEmit functions for ISD::FADD.
    5387             : 
    5388          13 : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5389          13 :   if (RetVT.SimpleTy != MVT::f32)
    5390             :     return 0;
    5391          26 :   if ((Subtarget->hasAVX512())) {
    5392           0 :     return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5393             :   }
    5394          13 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5395          13 :     return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5396             :   }
    5397           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5398           0 :     return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5399             :   }
    5400           0 :   if ((!Subtarget->hasSSE1())) {
    5401           0 :     return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5402             :   }
    5403             :   return 0;
    5404             : }
    5405             : 
    5406          13 : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5407          13 :   if (RetVT.SimpleTy != MVT::f64)
    5408             :     return 0;
    5409          26 :   if ((Subtarget->hasAVX512())) {
    5410           0 :     return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5411             :   }
    5412          13 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5413          13 :     return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5414             :   }
    5415           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5416           0 :     return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5417             :   }
    5418           0 :   if ((!Subtarget->hasSSE2())) {
    5419           0 :     return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5420             :   }
    5421             :   return 0;
    5422             : }
    5423             : 
    5424             : unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5425           0 :   if (RetVT.SimpleTy != MVT::f80)
    5426             :     return 0;
    5427           0 :   return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5428             : }
    5429             : 
    5430           2 : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5431           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5432             :     return 0;
    5433           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5434           0 :     return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5435             :   }
    5436           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5437           2 :     return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5438             :   }
    5439           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5440           0 :     return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5441             :   }
    5442             :   return 0;
    5443             : }
    5444             : 
    5445           2 : unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5446           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5447             :     return 0;
    5448           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5449           0 :     return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5450             :   }
    5451           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5452           2 :     return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5453             :   }
    5454             :   return 0;
    5455             : }
    5456             : 
    5457             : unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5458           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5459             :     return 0;
    5460           0 :   if ((Subtarget->hasAVX512())) {
    5461           0 :     return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5462             :   }
    5463             :   return 0;
    5464             : }
    5465             : 
    5466          18 : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5467          18 :   if (RetVT.SimpleTy != MVT::v2f64)
    5468             :     return 0;
    5469          36 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5470           2 :     return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5471             :   }
    5472          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5473           6 :     return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5474             :   }
    5475          20 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5476          10 :     return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5477             :   }
    5478             :   return 0;
    5479             : }
    5480             : 
    5481          14 : unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5482          14 :   if (RetVT.SimpleTy != MVT::v4f64)
    5483             :     return 0;
    5484          28 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5485           2 :     return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5486             :   }
    5487          12 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5488          12 :     return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5489             :   }
    5490             :   return 0;
    5491             : }
    5492             : 
    5493             : unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5494           8 :   if (RetVT.SimpleTy != MVT::v8f64)
    5495             :     return 0;
    5496          16 :   if ((Subtarget->hasAVX512())) {
    5497           8 :     return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5498             :   }
    5499             :   return 0;
    5500             : }
    5501             : 
    5502          70 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5503          70 :   switch (VT.SimpleTy) {
    5504          13 :   case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5505          13 :   case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5506           0 :   case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5507           2 :   case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5508           2 :   case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5509           0 :   case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5510          18 :   case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5511          14 :   case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5512           8 :   case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5513             :   default: return 0;
    5514             :   }
    5515             : }
    5516             : 
    5517             : // FastEmit functions for ISD::FDIV.
    5518             : 
    5519           0 : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5520           0 :   if (RetVT.SimpleTy != MVT::f32)
    5521             :     return 0;
    5522           0 :   if ((Subtarget->hasAVX512())) {
    5523           0 :     return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5524             :   }
    5525           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5526           0 :     return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5527             :   }
    5528           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5529           0 :     return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5530             :   }
    5531           0 :   if ((!Subtarget->hasSSE1())) {
    5532           0 :     return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5533             :   }
    5534             :   return 0;
    5535             : }
    5536             : 
    5537           0 : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5538           0 :   if (RetVT.SimpleTy != MVT::f64)
    5539             :     return 0;
    5540           0 :   if ((Subtarget->hasAVX512())) {
    5541           0 :     return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5542             :   }
    5543           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5544           0 :     return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5545             :   }
    5546           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5547           0 :     return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5548             :   }
    5549           0 :   if ((!Subtarget->hasSSE2())) {
    5550           0 :     return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5551             :   }
    5552             :   return 0;
    5553             : }
    5554             : 
    5555             : unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5556           0 :   if (RetVT.SimpleTy != MVT::f80)
    5557             :     return 0;
    5558           0 :   return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5559             : }
    5560             : 
    5561           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5562           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5563             :     return 0;
    5564           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5565           0 :     return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5566             :   }
    5567           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5568           2 :     return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5569             :   }
    5570           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5571           0 :     return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5572             :   }
    5573             :   return 0;
    5574             : }
    5575             : 
    5576           2 : unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5577           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5578             :     return 0;
    5579           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5580           0 :     return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5581             :   }
    5582           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5583           2 :     return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5584             :   }
    5585             :   return 0;
    5586             : }
    5587             : 
    5588             : unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5589           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5590             :     return 0;
    5591           0 :   if ((Subtarget->hasAVX512())) {
    5592           0 :     return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5593             :   }
    5594             :   return 0;
    5595             : }
    5596             : 
    5597           2 : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5598           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    5599             :     return 0;
    5600           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5601           0 :     return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5602             :   }
    5603           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5604           2 :     return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5605             :   }
    5606           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5607           0 :     return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5608             :   }
    5609             :   return 0;
    5610             : }
    5611             : 
    5612           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5613           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    5614             :     return 0;
    5615           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5616           0 :     return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5617             :   }
    5618           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5619           2 :     return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5620             :   }
    5621             :   return 0;
    5622             : }
    5623             : 
    5624             : unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5625           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    5626             :     return 0;
    5627           0 :   if ((Subtarget->hasAVX512())) {
    5628           0 :     return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5629             :   }
    5630             :   return 0;
    5631             : }
    5632             : 
    5633           8 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5634           8 :   switch (VT.SimpleTy) {
    5635           0 :   case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5636           0 :   case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5637           0 :   case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5638           2 :   case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5639           2 :   case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5640           0 :   case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5641           2 :   case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5642           2 :   case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5643           0 :   case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5644             :   default: return 0;
    5645             :   }
    5646             : }
    5647             : 
    5648             : // FastEmit functions for ISD::FMUL.
    5649             : 
    5650           0 : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5651           0 :   if (RetVT.SimpleTy != MVT::f32)
    5652             :     return 0;
    5653           0 :   if ((Subtarget->hasAVX512())) {
    5654           0 :     return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5655             :   }
    5656           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5657           0 :     return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5658             :   }
    5659           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5660           0 :     return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5661             :   }
    5662           0 :   if ((!Subtarget->hasSSE1())) {
    5663           0 :     return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5664             :   }
    5665             :   return 0;
    5666             : }
    5667             : 
    5668           2 : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5669           2 :   if (RetVT.SimpleTy != MVT::f64)
    5670             :     return 0;
    5671           4 :   if ((Subtarget->hasAVX512())) {
    5672           0 :     return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5673             :   }
    5674           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5675           2 :     return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5676             :   }
    5677           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5678           0 :     return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5679             :   }
    5680           0 :   if ((!Subtarget->hasSSE2())) {
    5681           0 :     return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5682             :   }
    5683             :   return 0;
    5684             : }
    5685             : 
    5686             : unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5687           0 :   if (RetVT.SimpleTy != MVT::f80)
    5688             :     return 0;
    5689           0 :   return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5690             : }
    5691             : 
    5692           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5693           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5694             :     return 0;
    5695           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5696           0 :     return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5697             :   }
    5698           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5699           2 :     return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5700             :   }
    5701           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5702           0 :     return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5703             :   }
    5704             :   return 0;
    5705             : }
    5706             : 
    5707           2 : unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5708           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5709             :     return 0;
    5710           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5711           0 :     return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5712             :   }
    5713           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5714           2 :     return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5715             :   }
    5716             :   return 0;
    5717             : }
    5718             : 
    5719             : unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5720           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5721             :     return 0;
    5722           0 :   if ((Subtarget->hasAVX512())) {
    5723           0 :     return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5724             :   }
    5725             :   return 0;
    5726             : }
    5727             : 
    5728           2 : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5729           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    5730             :     return 0;
    5731           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5732           0 :     return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5733             :   }
    5734           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5735           2 :     return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5736             :   }
    5737           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5738           0 :     return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5739             :   }
    5740             :   return 0;
    5741             : }
    5742             : 
    5743           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5744           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    5745             :     return 0;
    5746           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5747           0 :     return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5748             :   }
    5749           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5750           2 :     return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5751             :   }
    5752             :   return 0;
    5753             : }
    5754             : 
    5755             : unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5756           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    5757             :     return 0;
    5758           0 :   if ((Subtarget->hasAVX512())) {
    5759           0 :     return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5760             :   }
    5761             :   return 0;
    5762             : }
    5763             : 
    5764          10 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5765          10 :   switch (VT.SimpleTy) {
    5766           0 :   case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5767           2 :   case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5768           0 :   case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5769           2 :   case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5770           2 :   case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5771           0 :   case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5772           2 :   case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5773           2 :   case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5774           0 :   case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5775             :   default: return 0;
    5776             :   }
    5777             : }
    5778             : 
    5779             : // FastEmit functions for ISD::FSUB.
    5780             : 
    5781           0 : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5782           0 :   if (RetVT.SimpleTy != MVT::f32)
    5783             :     return 0;
    5784           0 :   if ((Subtarget->hasAVX512())) {
    5785           0 :     return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5786             :   }
    5787           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5788           0 :     return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5789             :   }
    5790           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5791           0 :     return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5792             :   }
    5793           0 :   if ((!Subtarget->hasSSE1())) {
    5794           0 :     return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5795             :   }
    5796             :   return 0;
    5797             : }
    5798             : 
    5799           2 : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5800           2 :   if (RetVT.SimpleTy != MVT::f64)
    5801             :     return 0;
    5802           4 :   if ((Subtarget->hasAVX512())) {
    5803           0 :     return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5804             :   }
    5805           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5806           2 :     return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5807             :   }
    5808           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5809           0 :     return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5810             :   }
    5811           0 :   if ((!Subtarget->hasSSE2())) {
    5812           0 :     return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5813             :   }
    5814             :   return 0;
    5815             : }
    5816             : 
    5817             : unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5818           0 :   if (RetVT.SimpleTy != MVT::f80)
    5819             :     return 0;
    5820           0 :   return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5821             : }
    5822             : 
    5823           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5824           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5825             :     return 0;
    5826           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5827           0 :     return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5828             :   }
    5829           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5830           2 :     return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5831             :   }
    5832           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5833           0 :     return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5834             :   }
    5835             :   return 0;
    5836             : }
    5837             : 
    5838           2 : unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5839           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5840             :     return 0;
    5841           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5842           0 :     return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5843             :   }
    5844           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5845           2 :     return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5846             :   }
    5847             :   return 0;
    5848             : }
    5849             : 
    5850             : unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5851           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5852             :     return 0;
    5853           0 :   if ((Subtarget->hasAVX512())) {
    5854           0 :     return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5855             :   }
    5856             :   return 0;
    5857             : }
    5858             : 
    5859           2 : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5860           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    5861             :     return 0;
    5862           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5863           0 :     return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5864             :   }
    5865           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5866           2 :     return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5867             :   }
    5868           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5869           0 :     return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5870             :   }
    5871             :   return 0;
    5872             : }
    5873             : 
    5874           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5875           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    5876             :     return 0;
    5877           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5878           0 :     return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5879             :   }
    5880           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5881           2 :     return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5882             :   }
    5883             :   return 0;
    5884             : }
    5885             : 
    5886             : unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5887           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    5888             :     return 0;
    5889           0 :   if ((Subtarget->hasAVX512())) {
    5890           0 :     return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5891             :   }
    5892             :   return 0;
    5893             : }
    5894             : 
    5895          10 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5896          10 :   switch (VT.SimpleTy) {
    5897           0 :   case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5898           2 :   case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5899           0 :   case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5900           2 :   case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5901           2 :   case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5902           0 :   case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5903           2 :   case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5904           2 :   case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5905           0 :   case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5906             :   default: return 0;
    5907             :   }
    5908             : }
    5909             : 
    5910             : // FastEmit functions for ISD::MUL.
    5911             : 
    5912           2 : unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5913           2 :   if (RetVT.SimpleTy != MVT::i8)
    5914             :     return 0;
    5915           4 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
    5916           2 :   return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill);
    5917             : }
    5918             : 
    5919             : unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5920           0 :   if (RetVT.SimpleTy != MVT::i16)
    5921             :     return 0;
    5922           0 :   return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5923             : }
    5924             : 
    5925             : unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5926           6 :   if (RetVT.SimpleTy != MVT::i32)
    5927             :     return 0;
    5928           6 :   return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5929             : }
    5930             : 
    5931             : unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5932           0 :   if (RetVT.SimpleTy != MVT::i64)
    5933             :     return 0;
    5934           0 :   return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5935             : }
    5936             : 
    5937           2 : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5938           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    5939             :     return 0;
    5940           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5941           0 :     return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5942             :   }
    5943           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5944           2 :     return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5945             :   }
    5946           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5947           0 :     return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5948             :   }
    5949             :   return 0;
    5950             : }
    5951             : 
    5952           2 : unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5953           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    5954             :     return 0;
    5955           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5956           0 :     return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5957             :   }
    5958           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5959           2 :     return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5960             :   }
    5961             :   return 0;
    5962             : }
    5963             : 
    5964             : unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5965           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5966             :     return 0;
    5967           0 :   if ((Subtarget->hasBWI())) {
    5968           0 :     return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5969             :   }
    5970             :   return 0;
    5971             : }
    5972             : 
    5973           2 : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5974           2 :   if (RetVT.SimpleTy != MVT::v4i32)
    5975             :     return 0;
    5976           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5977           0 :     return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5978             :   }
    5979           2 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    5980           2 :     return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5981             :   }
    5982           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5983           0 :     return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5984             :   }
    5985             :   return 0;
    5986             : }
    5987             : 
    5988           2 : unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5989           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    5990             :     return 0;
    5991           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5992           0 :     return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5993             :   }
    5994           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5995           2 :     return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5996             :   }
    5997             :   return 0;
    5998             : }
    5999             : 
    6000             : unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6001           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6002             :     return 0;
    6003           0 :   if ((Subtarget->hasAVX512())) {
    6004           0 :     return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6005             :   }
    6006             :   return 0;
    6007             : }
    6008             : 
    6009           0 : unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6010           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6011             :     return 0;
    6012           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6013           0 :     return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6014             :   }
    6015             :   return 0;
    6016             : }
    6017             : 
    6018           0 : unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6019           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6020             :     return 0;
    6021           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6022           0 :     return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6023             :   }
    6024             :   return 0;
    6025             : }
    6026             : 
    6027             : unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6028           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6029             :     return 0;
    6030           0 :   if ((Subtarget->hasDQI())) {
    6031           0 :     return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6032             :   }
    6033             :   return 0;
    6034             : }
    6035             : 
    6036          16 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6037          16 :   switch (VT.SimpleTy) {
    6038           2 :   case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6039           0 :   case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6040           6 :   case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6041           0 :   case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6042           2 :   case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6043           2 :   case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6044           0 :   case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6045           2 :   case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6046           2 :   case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6047           0 :   case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6048           0 :   case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6049           0 :   case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6050           0 :   case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6051             :   default: return 0;
    6052             :   }
    6053             : }
    6054             : 
    6055             : // FastEmit functions for ISD::MULHS.
    6056             : 
    6057           0 : unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6058           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6059             :     return 0;
    6060           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6061           0 :     return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6062             :   }
    6063           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6064           0 :     return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6065             :   }
    6066           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6067           0 :     return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6068             :   }
    6069             :   return 0;
    6070             : }
    6071             : 
    6072           0 : unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6073           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6074             :     return 0;
    6075           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6076           0 :     return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6077             :   }
    6078           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6079           0 :     return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6080             :   }
    6081             :   return 0;
    6082             : }
    6083             : 
    6084             : unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6085           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6086             :     return 0;
    6087           0 :   if ((Subtarget->hasBWI())) {
    6088           0 :     return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6089             :   }
    6090             :   return 0;
    6091             : }
    6092             : 
    6093           0 : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6094           0 :   switch (VT.SimpleTy) {
    6095           0 :   case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6096           0 :   case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6097           0 :   case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6098             :   default: return 0;
    6099             :   }
    6100             : }
    6101             : 
    6102             : // FastEmit functions for ISD::MULHU.
    6103             : 
    6104           0 : unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6105           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6106             :     return 0;
    6107           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6108           0 :     return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6109             :   }
    6110           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6111           0 :     return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6112             :   }
    6113           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6114           0 :     return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6115             :   }
    6116             :   return 0;
    6117             : }
    6118             : 
    6119           0 : unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6120           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6121             :     return 0;
    6122           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6123           0 :     return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6124             :   }
    6125           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6126           0 :     return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6127             :   }
    6128             :   return 0;
    6129             : }
    6130             : 
    6131             : unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6132           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6133             :     return 0;
    6134           0 :   if ((Subtarget->hasBWI())) {
    6135           0 :     return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6136             :   }
    6137             :   return 0;
    6138             : }
    6139             : 
    6140           0 : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6141           0 :   switch (VT.SimpleTy) {
    6142           0 :   case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6143           0 :   case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6144           0 :   case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6145             :   default: return 0;
    6146             :   }
    6147             : }
    6148             : 
    6149             : // FastEmit functions for ISD::OR.
    6150             : 
    6151             : unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6152           0 :   if (RetVT.SimpleTy != MVT::i8)
    6153             :     return 0;
    6154           0 :   return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6155             : }
    6156             : 
    6157             : unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6158           2 :   if (RetVT.SimpleTy != MVT::i16)
    6159             :     return 0;
    6160           2 :   return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6161             : }
    6162             : 
    6163             : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6164          26 :   if (RetVT.SimpleTy != MVT::i32)
    6165             :     return 0;
    6166          26 :   return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6167             : }
    6168             : 
    6169             : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6170           7 :   if (RetVT.SimpleTy != MVT::i64)
    6171             :     return 0;
    6172           7 :   return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6173             : }
    6174             : 
    6175             : unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6176           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    6177             :     return 0;
    6178           0 :   if ((Subtarget->hasDQI())) {
    6179           0 :     return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6180             :   }
    6181             :   return 0;
    6182             : }
    6183             : 
    6184             : unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6185           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    6186             :     return 0;
    6187           0 :   if ((Subtarget->hasAVX512())) {
    6188           0 :     return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6189             :   }
    6190             :   return 0;
    6191             : }
    6192             : 
    6193             : unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6194           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    6195             :     return 0;
    6196           0 :   if ((Subtarget->hasBWI())) {
    6197           0 :     return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6198             :   }
    6199             :   return 0;
    6200             : }
    6201             : 
    6202             : unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6203           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    6204             :     return 0;
    6205           0 :   if ((Subtarget->hasBWI())) {
    6206           0 :     return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6207             :   }
    6208             :   return 0;
    6209             : }
    6210             : 
    6211           4 : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6212           4 :   if (RetVT.SimpleTy != MVT::v2i64)
    6213             :     return 0;
    6214           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6215           0 :     return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6216             :   }
    6217           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6218           2 :     return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6219             :   }
    6220           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6221           2 :     return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6222             :   }
    6223             :   return 0;
    6224             : }
    6225             : 
    6226           6 : unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6227           6 :   if (RetVT.SimpleTy != MVT::v4i64)
    6228             :     return 0;
    6229          12 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    6230           4 :     return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6231             :   }
    6232           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6233           0 :     return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6234             :   }
    6235           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6236           2 :     return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6237             :   }
    6238             :   return 0;
    6239             : }
    6240             : 
    6241             : unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6242           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6243             :     return 0;
    6244           0 :   if ((Subtarget->hasAVX512())) {
    6245           0 :     return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6246             :   }
    6247             :   return 0;
    6248             : }
    6249             : 
    6250          49 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6251          49 :   switch (VT.SimpleTy) {
    6252           0 :   case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6253           2 :   case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6254          26 :   case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6255           7 :   case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6256           0 :   case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6257           0 :   case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6258           0 :   case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6259           0 :   case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6260           4 :   case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6261           6 :   case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6262           0 :   case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6263             :   default: return 0;
    6264             :   }
    6265             : }
    6266             : 
    6267             : // FastEmit functions for ISD::ROTL.
    6268             : 
    6269           0 : unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6270           0 :   if (RetVT.SimpleTy != MVT::i8)
    6271             :     return 0;
    6272           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6273           0 :   return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6274             : }
    6275             : 
    6276           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6277           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6278             :     return 0;
    6279           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6280           0 :     return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6281             :   }
    6282             :   return 0;
    6283             : }
    6284             : 
    6285           0 : unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6286           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6287             :     return 0;
    6288           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6289           0 :     return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6290             :   }
    6291             :   return 0;
    6292             : }
    6293             : 
    6294             : unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6295           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6296             :     return 0;
    6297           0 :   if ((Subtarget->hasAVX512())) {
    6298           0 :     return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6299             :   }
    6300             :   return 0;
    6301             : }
    6302             : 
    6303           0 : unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6304           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6305             :     return 0;
    6306           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6307           0 :     return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6308             :   }
    6309             :   return 0;
    6310             : }
    6311             : 
    6312           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6313           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6314             :     return 0;
    6315           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6316           0 :     return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6317             :   }
    6318             :   return 0;
    6319             : }
    6320             : 
    6321             : unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6322           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6323             :     return 0;
    6324           0 :   if ((Subtarget->hasAVX512())) {
    6325           0 :     return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6326             :   }
    6327             :   return 0;
    6328             : }
    6329             : 
    6330           0 : unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6331           0 :   switch (VT.SimpleTy) {
    6332           0 :   case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6333           0 :   case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6334           0 :   case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6335           0 :   case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6336           0 :   case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6337           0 :   case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6338           0 :   case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6339             :   default: return 0;
    6340             :   }
    6341             : }
    6342             : 
    6343             : // FastEmit functions for ISD::ROTR.
    6344             : 
    6345           0 : unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6346           0 :   if (RetVT.SimpleTy != MVT::i8)
    6347             :     return 0;
    6348           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6349           0 :   return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6350             : }
    6351             : 
    6352           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6353           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6354             :     return 0;
    6355           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6356           0 :     return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6357             :   }
    6358             :   return 0;
    6359             : }
    6360             : 
    6361           0 : unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6362           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6363             :     return 0;
    6364           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6365           0 :     return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6366             :   }
    6367             :   return 0;
    6368             : }
    6369             : 
    6370             : unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6371           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6372             :     return 0;
    6373           0 :   if ((Subtarget->hasAVX512())) {
    6374           0 :     return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6375             :   }
    6376             :   return 0;
    6377             : }
    6378             : 
    6379           0 : unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6380           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6381             :     return 0;
    6382           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6383           0 :     return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6384             :   }
    6385             :   return 0;
    6386             : }
    6387             : 
    6388           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6389           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6390             :     return 0;
    6391           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6392           0 :     return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6393             :   }
    6394             :   return 0;
    6395             : }
    6396             : 
    6397             : unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6398           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6399             :     return 0;
    6400           0 :   if ((Subtarget->hasAVX512())) {
    6401           0 :     return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6402             :   }
    6403             :   return 0;
    6404             : }
    6405             : 
    6406           0 : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6407           0 :   switch (VT.SimpleTy) {
    6408           0 :   case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6409           0 :   case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6410           0 :   case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6411           0 :   case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6412           0 :   case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6413           0 :   case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6414           0 :   case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6415             :   default: return 0;
    6416             :   }
    6417             : }
    6418             : 
    6419             : // FastEmit functions for ISD::SHL.
    6420             : 
    6421           2 : unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6422           2 :   if (RetVT.SimpleTy != MVT::i8)
    6423             :     return 0;
    6424           4 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6425           2 :   return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6426             : }
    6427             : 
    6428           0 : unsigned fastEmit_ISD_SHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6429           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6430             :     return 0;
    6431           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6432           0 :     return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6433             :   }
    6434             :   return 0;
    6435             : }
    6436             : 
    6437           0 : unsigned fastEmit_ISD_SHL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6438           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6439             :     return 0;
    6440           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6441           0 :     return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6442             :   }
    6443             :   return 0;
    6444             : }
    6445             : 
    6446             : unsigned fastEmit_ISD_SHL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6447           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6448             :     return 0;
    6449           0 :   if ((Subtarget->hasBWI())) {
    6450           0 :     return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6451             :   }
    6452             :   return 0;
    6453             : }
    6454             : 
    6455           0 : unsigned fastEmit_ISD_SHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6456           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6457             :     return 0;
    6458           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6459           0 :     return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6460             :   }
    6461           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6462           0 :     return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6463             :   }
    6464             :   return 0;
    6465             : }
    6466             : 
    6467           0 : unsigned fastEmit_ISD_SHL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6468           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6469             :     return 0;
    6470           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6471           0 :     return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6472             :   }
    6473           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6474           0 :     return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6475             :   }
    6476             :   return 0;
    6477             : }
    6478             : 
    6479             : unsigned fastEmit_ISD_SHL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6480           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6481             :     return 0;
    6482           0 :   if ((Subtarget->hasAVX512())) {
    6483           0 :     return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6484             :   }
    6485             :   return 0;
    6486             : }
    6487             : 
    6488           0 : unsigned fastEmit_ISD_SHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6489           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6490             :     return 0;
    6491           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6492           0 :     return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6493             :   }
    6494           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6495           0 :     return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6496             :   }
    6497             :   return 0;
    6498             : }
    6499             : 
    6500           0 : unsigned fastEmit_ISD_SHL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6501           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6502             :     return 0;
    6503           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6504           0 :     return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6505             :   }
    6506           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6507           0 :     return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6508             :   }
    6509             :   return 0;
    6510             : }
    6511             : 
    6512             : unsigned fastEmit_ISD_SHL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6513           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6514             :     return 0;
    6515           0 :   if ((Subtarget->hasAVX512())) {
    6516           0 :     return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6517             :   }
    6518             :   return 0;
    6519             : }
    6520             : 
    6521           4 : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6522           4 :   switch (VT.SimpleTy) {
    6523           2 :   case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6524           0 :   case MVT::v8i16: return fastEmit_ISD_SHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6525           0 :   case MVT::v16i16: return fastEmit_ISD_SHL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6526           0 :   case MVT::v32i16: return fastEmit_ISD_SHL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6527           0 :   case MVT::v4i32: return fastEmit_ISD_SHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6528           0 :   case MVT::v8i32: return fastEmit_ISD_SHL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6529           0 :   case MVT::v16i32: return fastEmit_ISD_SHL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6530           0 :   case MVT::v2i64: return fastEmit_ISD_SHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6531           0 :   case MVT::v4i64: return fastEmit_ISD_SHL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6532           0 :   case MVT::v8i64: return fastEmit_ISD_SHL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6533             :   default: return 0;
    6534             :   }
    6535             : }
    6536             : 
    6537             : // FastEmit functions for ISD::SMAX.
    6538             : 
    6539           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6540           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6541             :     return 0;
    6542           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6543           0 :     return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6544             :   }
    6545           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6546           0 :     return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6547             :   }
    6548           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6549           0 :     return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6550             :   }
    6551             :   return 0;
    6552             : }
    6553             : 
    6554           0 : unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6555           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    6556             :     return 0;
    6557           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6558           0 :     return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6559             :   }
    6560           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6561           0 :     return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6562             :   }
    6563             :   return 0;
    6564             : }
    6565             : 
    6566             : unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6567           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    6568             :     return 0;
    6569           0 :   if ((Subtarget->hasBWI())) {
    6570           0 :     return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6571             :   }
    6572             :   return 0;
    6573             : }
    6574             : 
    6575           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6576           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6577             :     return 0;
    6578           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6579           0 :     return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6580             :   }
    6581           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6582           0 :     return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6583             :   }
    6584           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6585           0 :     return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6586             :   }
    6587             :   return 0;
    6588             : }
    6589             : 
    6590           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6591           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6592             :     return 0;
    6593           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6594           0 :     return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6595             :   }
    6596           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6597           0 :     return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6598             :   }
    6599             :   return 0;
    6600             : }
    6601             : 
    6602             : unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6603           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6604             :     return 0;
    6605           0 :   if ((Subtarget->hasBWI())) {
    6606           0 :     return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6607             :   }
    6608             :   return 0;
    6609             : }
    6610             : 
    6611           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6612           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6613             :     return 0;
    6614           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6615           0 :     return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6616             :   }
    6617           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6618           0 :     return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6619             :   }
    6620           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6621           0 :     return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6622             :   }
    6623             :   return 0;
    6624             : }
    6625             : 
    6626           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6627           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6628             :     return 0;
    6629           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6630           0 :     return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6631             :   }
    6632           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6633           0 :     return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6634             :   }
    6635             :   return 0;
    6636             : }
    6637             : 
    6638             : unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6639           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6640             :     return 0;
    6641           0 :   if ((Subtarget->hasAVX512())) {
    6642           0 :     return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6643             :   }
    6644             :   return 0;
    6645             : }
    6646             : 
    6647           0 : unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6648           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6649             :     return 0;
    6650           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6651           0 :     return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6652             :   }
    6653             :   return 0;
    6654             : }
    6655             : 
    6656           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6657           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6658             :     return 0;
    6659           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6660           0 :     return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6661             :   }
    6662             :   return 0;
    6663             : }
    6664             : 
    6665             : unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6666           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6667             :     return 0;
    6668           0 :   if ((Subtarget->hasAVX512())) {
    6669           0 :     return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6670             :   }
    6671             :   return 0;
    6672             : }
    6673             : 
    6674           0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6675           0 :   switch (VT.SimpleTy) {
    6676           0 :   case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6677           0 :   case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6678           0 :   case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6679           0 :   case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6680           0 :   case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6681           0 :   case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6682           0 :   case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6683           0 :   case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6684           0 :   case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6685           0 :   case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6686           0 :   case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6687           0 :   case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6688             :   default: return 0;
    6689             :   }
    6690             : }
    6691             : 
    6692             : // FastEmit functions for ISD::SMIN.
    6693             : 
    6694           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6695           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6696             :     return 0;
    6697           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6698           0 :     return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6699             :   }
    6700           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6701           0 :     return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6702             :   }
    6703           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6704           0 :     return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6705             :   }
    6706             :   return 0;
    6707             : }
    6708             : 
    6709           0 : unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6710           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    6711             :     return 0;
    6712           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6713           0 :     return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6714             :   }
    6715           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6716           0 :     return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6717             :   }
    6718             :   return 0;
    6719             : }
    6720             : 
    6721             : unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6722           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    6723             :     return 0;
    6724           0 :   if ((Subtarget->hasBWI())) {
    6725           0 :     return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6726             :   }
    6727             :   return 0;
    6728             : }
    6729             : 
    6730           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6731           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6732             :     return 0;
    6733           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6734           0 :     return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6735             :   }
    6736           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6737           0 :     return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6738             :   }
    6739           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6740           0 :     return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6741             :   }
    6742             :   return 0;
    6743             : }
    6744             : 
    6745           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6746           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6747             :     return 0;
    6748           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6749           0 :     return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6750             :   }
    6751           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6752           0 :     return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6753             :   }
    6754             :   return 0;
    6755             : }
    6756             : 
    6757             : unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6758           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6759             :     return 0;
    6760           0 :   if ((Subtarget->hasBWI())) {
    6761           0 :     return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6762             :   }
    6763             :   return 0;
    6764             : }
    6765             : 
    6766           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6767           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6768             :     return 0;
    6769           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6770           0 :     return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6771             :   }
    6772           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6773           0 :     return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6774             :   }
    6775           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6776           0 :     return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6777             :   }
    6778             :   return 0;
    6779             : }
    6780             : 
    6781           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6782           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6783             :     return 0;
    6784           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6785           0 :     return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6786             :   }
    6787           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6788           0 :     return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6789             :   }
    6790             :   return 0;
    6791             : }
    6792             : 
    6793             : unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6794           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6795             :     return 0;
    6796           0 :   if ((Subtarget->hasAVX512())) {
    6797           0 :     return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6798             :   }
    6799             :   return 0;
    6800             : }
    6801             : 
    6802           0 : unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6803           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6804             :     return 0;
    6805           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6806           0 :     return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6807             :   }
    6808             :   return 0;
    6809             : }
    6810             : 
    6811           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6812           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6813             :     return 0;
    6814           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6815           0 :     return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6816             :   }
    6817             :   return 0;
    6818             : }
    6819             : 
    6820             : unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6821           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6822             :     return 0;
    6823           0 :   if ((Subtarget->hasAVX512())) {
    6824           0 :     return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6825             :   }
    6826             :   return 0;
    6827             : }
    6828             : 
    6829           0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6830           0 :   switch (VT.SimpleTy) {
    6831           0 :   case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6832           0 :   case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6833           0 :   case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6834           0 :   case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6835           0 :   case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6836           0 :   case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6837           0 :   case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6838           0 :   case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6839           0 :   case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6840           0 :   case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6841           0 :   case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6842           0 :   case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6843             :   default: return 0;
    6844             :   }
    6845             : }
    6846             : 
    6847             : // FastEmit functions for ISD::SRA.
    6848             : 
    6849           0 : unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6850           0 :   if (RetVT.SimpleTy != MVT::i8)
    6851             :     return 0;
    6852           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6853           0 :   return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6854             : }
    6855             : 
    6856           0 : unsigned fastEmit_ISD_SRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6857           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6858             :     return 0;
    6859           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6860           0 :     return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6861             :   }
    6862             :   return 0;
    6863             : }
    6864             : 
    6865           0 : unsigned fastEmit_ISD_SRA_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6866           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6867             :     return 0;
    6868           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6869           0 :     return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6870             :   }
    6871             :   return 0;
    6872             : }
    6873             : 
    6874             : unsigned fastEmit_ISD_SRA_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6875           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6876             :     return 0;
    6877           0 :   if ((Subtarget->hasBWI())) {
    6878           0 :     return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6879             :   }
    6880             :   return 0;
    6881             : }
    6882             : 
    6883           0 : unsigned fastEmit_ISD_SRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6884           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6885             :     return 0;
    6886           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6887           0 :     return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6888             :   }
    6889           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6890           0 :     return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6891             :   }
    6892             :   return 0;
    6893             : }
    6894             : 
    6895           0 : unsigned fastEmit_ISD_SRA_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6896           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6897             :     return 0;
    6898           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6899           0 :     return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6900             :   }
    6901           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6902           0 :     return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6903             :   }
    6904             :   return 0;
    6905             : }
    6906             : 
    6907             : unsigned fastEmit_ISD_SRA_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6908           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6909             :     return 0;
    6910           0 :   if ((Subtarget->hasAVX512())) {
    6911           0 :     return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6912             :   }
    6913             :   return 0;
    6914             : }
    6915             : 
    6916           0 : unsigned fastEmit_ISD_SRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6917           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6918             :     return 0;
    6919           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6920           0 :     return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6921             :   }
    6922             :   return 0;
    6923             : }
    6924             : 
    6925           0 : unsigned fastEmit_ISD_SRA_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6926           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6927             :     return 0;
    6928           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6929           0 :     return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6930             :   }
    6931             :   return 0;
    6932             : }
    6933             : 
    6934             : unsigned fastEmit_ISD_SRA_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6935           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6936             :     return 0;
    6937           0 :   if ((Subtarget->hasAVX512())) {
    6938           0 :     return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6939             :   }
    6940             :   return 0;
    6941             : }
    6942             : 
    6943           1 : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6944           1 :   switch (VT.SimpleTy) {
    6945           0 :   case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6946           0 :   case MVT::v8i16: return fastEmit_ISD_SRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6947           0 :   case MVT::v16i16: return fastEmit_ISD_SRA_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6948           0 :   case MVT::v32i16: return fastEmit_ISD_SRA_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6949           0 :   case MVT::v4i32: return fastEmit_ISD_SRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6950           0 :   case MVT::v8i32: return fastEmit_ISD_SRA_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6951           0 :   case MVT::v16i32: return fastEmit_ISD_SRA_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6952           0 :   case MVT::v2i64: return fastEmit_ISD_SRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6953           0 :   case MVT::v4i64: return fastEmit_ISD_SRA_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6954           0 :   case MVT::v8i64: return fastEmit_ISD_SRA_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6955             :   default: return 0;
    6956             :   }
    6957             : }
    6958             : 
    6959             : // FastEmit functions for ISD::SRL.
    6960             : 
    6961           0 : unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6962           0 :   if (RetVT.SimpleTy != MVT::i8)
    6963             :     return 0;
    6964           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6965           0 :   return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6966             : }
    6967             : 
    6968           0 : unsigned fastEmit_ISD_SRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6969           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6970             :     return 0;
    6971           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6972           0 :     return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6973             :   }
    6974             :   return 0;
    6975             : }
    6976             : 
    6977           0 : unsigned fastEmit_ISD_SRL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6978           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6979             :     return 0;
    6980           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6981           0 :     return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6982             :   }
    6983             :   return 0;
    6984             : }
    6985             : 
    6986             : unsigned fastEmit_ISD_SRL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6987           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6988             :     return 0;
    6989           0 :   if ((Subtarget->hasBWI())) {
    6990           0 :     return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6991             :   }
    6992             :   return 0;
    6993             : }
    6994             : 
    6995           0 : unsigned fastEmit_ISD_SRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6996           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6997             :     return 0;
    6998           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6999           0 :     return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7000             :   }
    7001           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7002           0 :     return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7003             :   }
    7004             :   return 0;
    7005             : }
    7006             : 
    7007           0 : unsigned fastEmit_ISD_SRL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7008           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7009             :     return 0;
    7010           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7011           0 :     return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7012             :   }
    7013           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7014           0 :     return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7015             :   }
    7016             :   return 0;
    7017             : }
    7018             : 
    7019             : unsigned fastEmit_ISD_SRL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7020           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7021             :     return 0;
    7022           0 :   if ((Subtarget->hasAVX512())) {
    7023           0 :     return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7024             :   }
    7025             :   return 0;
    7026             : }
    7027             : 
    7028           0 : unsigned fastEmit_ISD_SRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7029           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7030             :     return 0;
    7031           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7032           0 :     return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7033             :   }
    7034           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7035           0 :     return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7036             :   }
    7037             :   return 0;
    7038             : }
    7039             : 
    7040           0 : unsigned fastEmit_ISD_SRL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7041           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7042             :     return 0;
    7043           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7044           0 :     return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7045             :   }
    7046           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7047           0 :     return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7048             :   }
    7049             :   return 0;
    7050             : }
    7051             : 
    7052             : unsigned fastEmit_ISD_SRL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7053           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7054             :     return 0;
    7055           0 :   if ((Subtarget->hasAVX512())) {
    7056           0 :     return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7057             :   }
    7058             :   return 0;
    7059             : }
    7060             : 
    7061           0 : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7062           0 :   switch (VT.SimpleTy) {
    7063           0 :   case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7064           0 :   case MVT::v8i16: return fastEmit_ISD_SRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7065           0 :   case MVT::v16i16: return fastEmit_ISD_SRL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7066           0 :   case MVT::v32i16: return fastEmit_ISD_SRL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7067           0 :   case MVT::v4i32: return fastEmit_ISD_SRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7068           0 :   case MVT::v8i32: return fastEmit_ISD_SRL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7069           0 :   case MVT::v16i32: return fastEmit_ISD_SRL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7070           0 :   case MVT::v2i64: return fastEmit_ISD_SRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7071           0 :   case MVT::v4i64: return fastEmit_ISD_SRL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7072           0 :   case MVT::v8i64: return fastEmit_ISD_SRL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7073             :   default: return 0;
    7074             :   }
    7075             : }
    7076             : 
    7077             : // FastEmit functions for ISD::SUB.
    7078             : 
    7079             : unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7080           0 :   if (RetVT.SimpleTy != MVT::i8)
    7081             :     return 0;
    7082           0 :   return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7083             : }
    7084             : 
    7085             : unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7086           0 :   if (RetVT.SimpleTy != MVT::i16)
    7087             :     return 0;
    7088           0 :   return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7089             : }
    7090             : 
    7091             : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7092          12 :   if (RetVT.SimpleTy != MVT::i32)
    7093             :     return 0;
    7094          12 :   return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7095             : }
    7096             : 
    7097             : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7098          14 :   if (RetVT.SimpleTy != MVT::i64)
    7099             :     return 0;
    7100          14 :   return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7101             : }
    7102             : 
    7103           2 : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7104           2 :   if (RetVT.SimpleTy != MVT::v16i8)
    7105             :     return 0;
    7106           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7107           0 :     return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7108             :   }
    7109           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7110           2 :     return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7111             :   }
    7112           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7113           0 :     return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7114             :   }
    7115             :   return 0;
    7116             : }
    7117             : 
    7118           2 : unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7119           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    7120             :     return 0;
    7121           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7122           0 :     return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7123             :   }
    7124           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7125           2 :     return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7126             :   }
    7127             :   return 0;
    7128             : }
    7129             : 
    7130             : unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7131           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7132             :     return 0;
    7133           0 :   if ((Subtarget->hasBWI())) {
    7134           0 :     return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7135             :   }
    7136             :   return 0;
    7137             : }
    7138             : 
    7139           2 : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7140           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    7141             :     return 0;
    7142           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7143           0 :     return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7144             :   }
    7145           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7146           2 :     return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7147             :   }
    7148           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7149           0 :     return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7150             :   }
    7151             :   return 0;
    7152             : }
    7153             : 
    7154           2 : unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7155           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    7156             :     return 0;
    7157           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7158           0 :     return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7159             :   }
    7160           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7161           2 :     return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7162             :   }
    7163             :   return 0;
    7164             : }
    7165             : 
    7166             : unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7167           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7168             :     return 0;
    7169           0 :   if ((Subtarget->hasBWI())) {
    7170           0 :     return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7171             :   }
    7172             :   return 0;
    7173             : }
    7174             : 
    7175           2 : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7176           2 :   if (RetVT.SimpleTy != MVT::v4i32)
    7177             :     return 0;
    7178           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7179           0 :     return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7180             :   }
    7181           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7182           2 :     return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7183             :   }
    7184           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7185           0 :     return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7186             :   }
    7187             :   return 0;
    7188             : }
    7189             : 
    7190           2 : unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7191           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    7192             :     return 0;
    7193           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7194           0 :     return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7195             :   }
    7196           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7197           2 :     return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7198             :   }
    7199             :   return 0;
    7200             : }
    7201             : 
    7202             : unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7203           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7204             :     return 0;
    7205           0 :   if ((Subtarget->hasAVX512())) {
    7206           0 :     return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7207             :   }
    7208             :   return 0;
    7209             : }
    7210             : 
    7211           2 : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7212           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    7213             :     return 0;
    7214           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7215           0 :     return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7216             :   }
    7217           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7218           2 :     return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7219             :   }
    7220           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7221           0 :     return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7222             :   }
    7223             :   return 0;
    7224             : }
    7225             : 
    7226           2 : unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7227           2 :   if (RetVT.SimpleTy != MVT::v4i64)
    7228             :     return 0;
    7229           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7230           0 :     return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7231             :   }
    7232           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7233           2 :     return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7234             :   }
    7235             :   return 0;
    7236             : }
    7237             : 
    7238             : unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7239           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7240             :     return 0;
    7241           0 :   if ((Subtarget->hasAVX512())) {
    7242           0 :     return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7243             :   }
    7244             :   return 0;
    7245             : }
    7246             : 
    7247          42 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7248          42 :   switch (VT.SimpleTy) {
    7249           0 :   case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7250           0 :   case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7251          12 :   case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7252          14 :   case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7253           2 :   case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7254           2 :   case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7255           0 :   case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7256           2 :   case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7257           2 :   case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7258           0 :   case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7259           2 :   case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7260           2 :   case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7261           0 :   case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7262           2 :   case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7263           2 :   case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7264           0 :   case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7265             :   default: return 0;
    7266             :   }
    7267             : }
    7268             : 
    7269             : // FastEmit functions for ISD::UMAX.
    7270             : 
    7271           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7272           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7273             :     return 0;
    7274           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7275           0 :     return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7276             :   }
    7277           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7278           0 :     return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7279             :   }
    7280           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7281           0 :     return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7282             :   }
    7283             :   return 0;
    7284             : }
    7285             : 
    7286           0 : unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7287           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7288             :     return 0;
    7289           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7290           0 :     return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7291             :   }
    7292           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7293           0 :     return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7294             :   }
    7295             :   return 0;
    7296             : }
    7297             : 
    7298             : unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7299           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7300             :     return 0;
    7301           0 :   if ((Subtarget->hasBWI())) {
    7302           0 :     return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7303             :   }
    7304             :   return 0;
    7305             : }
    7306             : 
    7307           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7308           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7309             :     return 0;
    7310           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7311           0 :     return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7312             :   }
    7313           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7314           0 :     return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7315             :   }
    7316           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7317           0 :     return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7318             :   }
    7319             :   return 0;
    7320             : }
    7321             : 
    7322           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7323           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7324             :     return 0;
    7325           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7326           0 :     return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7327             :   }
    7328           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7329           0 :     return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7330             :   }
    7331             :   return 0;
    7332             : }
    7333             : 
    7334             : unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7335           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7336             :     return 0;
    7337           0 :   if ((Subtarget->hasBWI())) {
    7338           0 :     return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7339             :   }
    7340             :   return 0;
    7341             : }
    7342             : 
    7343           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7344           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7345             :     return 0;
    7346           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7347           0 :     return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7348             :   }
    7349           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7350           0 :     return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7351             :   }
    7352           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7353           0 :     return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7354             :   }
    7355             :   return 0;
    7356             : }
    7357             : 
    7358           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7359           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7360             :     return 0;
    7361           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7362           0 :     return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7363             :   }
    7364           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7365           0 :     return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7366             :   }
    7367             :   return 0;
    7368             : }
    7369             : 
    7370             : unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7371           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7372             :     return 0;
    7373           0 :   if ((Subtarget->hasAVX512())) {
    7374           0 :     return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7375             :   }
    7376             :   return 0;
    7377             : }
    7378             : 
    7379           0 : unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7380           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7381             :     return 0;
    7382           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7383           0 :     return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7384             :   }
    7385             :   return 0;
    7386             : }
    7387             : 
    7388           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7389           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7390             :     return 0;
    7391           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7392           0 :     return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7393             :   }
    7394             :   return 0;
    7395             : }
    7396             : 
    7397             : unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7398           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7399             :     return 0;
    7400           0 :   if ((Subtarget->hasAVX512())) {
    7401           0 :     return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7402             :   }
    7403             :   return 0;
    7404             : }
    7405             : 
    7406           0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7407           0 :   switch (VT.SimpleTy) {
    7408           0 :   case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7409           0 :   case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7410           0 :   case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7411           0 :   case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7412           0 :   case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7413           0 :   case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7414           0 :   case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7415           0 :   case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7416           0 :   case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7417           0 :   case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7418           0 :   case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7419           0 :   case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7420             :   default: return 0;
    7421             :   }
    7422             : }
    7423             : 
    7424             : // FastEmit functions for ISD::UMIN.
    7425             : 
    7426           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7427           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7428             :     return 0;
    7429           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7430           0 :     return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7431             :   }
    7432           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7433           0 :     return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7434             :   }
    7435           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7436           0 :     return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7437             :   }
    7438             :   return 0;
    7439             : }
    7440             : 
    7441           0 : unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7442           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7443             :     return 0;
    7444           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7445           0 :     return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7446             :   }
    7447           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7448           0 :     return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7449             :   }
    7450             :   return 0;
    7451             : }
    7452             : 
    7453             : unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7454           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7455             :     return 0;
    7456           0 :   if ((Subtarget->hasBWI())) {
    7457           0 :     return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7458             :   }
    7459             :   return 0;
    7460             : }
    7461             : 
    7462           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7463           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7464             :     return 0;
    7465           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7466           0 :     return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7467             :   }
    7468           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7469           0 :     return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7470             :   }
    7471           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7472           0 :     return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7473             :   }
    7474             :   return 0;
    7475             : }
    7476             : 
    7477           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7478           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7479             :     return 0;
    7480           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7481           0 :     return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7482             :   }
    7483           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7484           0 :     return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7485             :   }
    7486             :   return 0;
    7487             : }
    7488             : 
    7489             : unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7490           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7491             :     return 0;
    7492           0 :   if ((Subtarget->hasBWI())) {
    7493           0 :     return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7494             :   }
    7495             :   return 0;
    7496             : }
    7497             : 
    7498           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7499           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7500             :     return 0;
    7501           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7502           0 :     return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7503             :   }
    7504           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7505           0 :     return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7506             :   }
    7507           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7508           0 :     return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7509             :   }
    7510             :   return 0;
    7511             : }
    7512             : 
    7513           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7514           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7515             :     return 0;
    7516           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7517           0 :     return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7518             :   }
    7519           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7520           0 :     return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7521             :   }
    7522             :   return 0;
    7523             : }
    7524             : 
    7525             : unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7526           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7527             :     return 0;
    7528           0 :   if ((Subtarget->hasAVX512())) {
    7529           0 :     return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7530             :   }
    7531             :   return 0;
    7532             : }
    7533             : 
    7534           0 : unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7535           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7536             :     return 0;
    7537           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7538           0 :     return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7539             :   }
    7540             :   return 0;
    7541             : }
    7542             : 
    7543           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7544           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7545             :     return 0;
    7546           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7547           0 :     return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7548             :   }
    7549             :   return 0;
    7550             : }
    7551             : 
    7552             : unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7553           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7554             :     return 0;
    7555           0 :   if ((Subtarget->hasAVX512())) {
    7556           0 :     return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7557             :   }
    7558             :   return 0;
    7559             : }
    7560             : 
    7561           0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7562           0 :   switch (VT.SimpleTy) {
    7563           0 :   case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7564           0 :   case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7565           0 :   case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7566           0 :   case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7567           0 :   case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7568           0 :   case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7569           0 :   case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7570           0 :   case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7571           0 :   case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7572           0 :   case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7573           0 :   case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7574           0 :   case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7575             :   default: return 0;
    7576             :   }
    7577             : }
    7578             : 
    7579             : // FastEmit functions for ISD::XOR.
    7580             : 
    7581             : unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7582           0 :   if (RetVT.SimpleTy != MVT::i8)
    7583             :     return 0;
    7584           0 :   return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7585             : }
    7586             : 
    7587             : unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7588           0 :   if (RetVT.SimpleTy != MVT::i16)
    7589             :     return 0;
    7590           0 :   return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7591             : }
    7592             : 
    7593             : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7594          14 :   if (RetVT.SimpleTy != MVT::i32)
    7595             :     return 0;
    7596          14 :   return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7597             : }
    7598             : 
    7599             : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7600           8 :   if (RetVT.SimpleTy != MVT::i64)
    7601             :     return 0;
    7602           8 :   return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7603             : }
    7604             : 
    7605             : unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7606           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    7607             :     return 0;
    7608           0 :   if ((Subtarget->hasDQI())) {
    7609           0 :     return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7610             :   }
    7611             :   return 0;
    7612             : }
    7613             : 
    7614             : unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7615           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    7616             :     return 0;
    7617           0 :   if ((Subtarget->hasAVX512())) {
    7618           0 :     return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7619             :   }
    7620             :   return 0;
    7621             : }
    7622             : 
    7623             : unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7624           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    7625             :     return 0;
    7626           0 :   if ((Subtarget->hasBWI())) {
    7627           0 :     return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7628             :   }
    7629             :   return 0;
    7630             : }
    7631             : 
    7632             : unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7633           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    7634             :     return 0;
    7635           0 :   if ((Subtarget->hasBWI())) {
    7636           0 :     return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7637             :   }
    7638             :   return 0;
    7639             : }
    7640             : 
    7641           2 : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7642           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    7643             :     return 0;
    7644           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7645           0 :     return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7646             :   }
    7647           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7648           2 :     return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7649             :   }
    7650           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7651           0 :     return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7652             :   }
    7653             :   return 0;
    7654             : }
    7655             : 
    7656           4 : unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7657           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    7658             :     return 0;
    7659           8 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    7660           2 :     return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7661             :   }
    7662           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7663           0 :     return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7664             :   }
    7665           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7666           2 :     return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7667             :   }
    7668             :   return 0;
    7669             : }
    7670             : 
    7671             : unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7672           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7673             :     return 0;
    7674           0 :   if ((Subtarget->hasAVX512())) {
    7675           0 :     return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7676             :   }
    7677             :   return 0;
    7678             : }
    7679             : 
    7680          32 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7681          32 :   switch (VT.SimpleTy) {
    7682           0 :   case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7683           0 :   case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7684          14 :   case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7685           8 :   case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7686           0 :   case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7687           0 :   case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7688           0 :   case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7689           0 :   case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7690           2 :   case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7691           4 :   case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7692           0 :   case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7693             :   default: return 0;
    7694             :   }
    7695             : }
    7696             : 
    7697             : // FastEmit functions for X86ISD::ADDS.
    7698             : 
    7699           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7700           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7701             :     return 0;
    7702           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7703           0 :     return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7704             :   }
    7705           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7706           0 :     return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7707             :   }
    7708           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7709           0 :     return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7710             :   }
    7711             :   return 0;
    7712             : }
    7713             : 
    7714           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7715           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7716             :     return 0;
    7717           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7718           0 :     return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7719             :   }
    7720           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7721           0 :     return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7722             :   }
    7723             :   return 0;
    7724             : }
    7725             : 
    7726             : unsigned fastEmit_X86ISD_ADDS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7727           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7728             :     return 0;
    7729           0 :   if ((Subtarget->hasBWI())) {
    7730           0 :     return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7731             :   }
    7732             :   return 0;
    7733             : }
    7734             : 
    7735           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7736           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7737             :     return 0;
    7738           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7739           0 :     return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7740             :   }
    7741           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7742           0 :     return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7743             :   }
    7744           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7745           0 :     return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7746             :   }
    7747             :   return 0;
    7748             : }
    7749             : 
    7750           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7751           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7752             :     return 0;
    7753           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7754           0 :     return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7755             :   }
    7756           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7757           0 :     return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7758             :   }
    7759             :   return 0;
    7760             : }
    7761             : 
    7762             : unsigned fastEmit_X86ISD_ADDS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7763           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7764             :     return 0;
    7765           0 :   if ((Subtarget->hasBWI())) {
    7766           0 :     return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7767             :   }
    7768             :   return 0;
    7769             : }
    7770             : 
    7771           0 : unsigned fastEmit_X86ISD_ADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7772           0 :   switch (VT.SimpleTy) {
    7773           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7774           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7775           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7776           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7777           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7778           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7779             :   default: return 0;
    7780             :   }
    7781             : }
    7782             : 
    7783             : // FastEmit functions for X86ISD::ADDSUB.
    7784             : 
    7785           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7786           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    7787             :     return 0;
    7788           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    7789           0 :     return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7790             :   }
    7791           0 :   if ((Subtarget->hasAVX())) {
    7792           0 :     return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7793             :   }
    7794             :   return 0;
    7795             : }
    7796             : 
    7797             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7798           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    7799             :     return 0;
    7800           0 :   if ((Subtarget->hasAVX())) {
    7801           0 :     return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7802             :   }
    7803             :   return 0;
    7804             : }
    7805             : 
    7806           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7807           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    7808             :     return 0;
    7809           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    7810           0 :     return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7811             :   }
    7812           0 :   if ((Subtarget->hasAVX())) {
    7813           0 :     return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7814             :   }
    7815             :   return 0;
    7816             : }
    7817             : 
    7818             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7819           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    7820             :     return 0;
    7821           0 :   if ((Subtarget->hasAVX())) {
    7822           0 :     return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7823             :   }
    7824             :   return 0;
    7825             : }
    7826             : 
    7827           0 : unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7828           0 :   switch (VT.SimpleTy) {
    7829           0 :   case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7830           0 :   case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7831           0 :   case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7832           0 :   case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7833             :   default: return 0;
    7834             :   }
    7835             : }
    7836             : 
    7837             : // FastEmit functions for X86ISD::ADDUS.
    7838             : 
    7839           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7840           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7841             :     return 0;
    7842           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7843           0 :     return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7844             :   }
    7845           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7846           0 :     return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7847             :   }
    7848           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7849           0 :     return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7850             :   }
    7851             :   return 0;
    7852             : }
    7853             : 
    7854           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7855           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7856             :     return 0;
    7857           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7858           0 :     return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7859             :   }
    7860           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7861           0 :     return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7862             :   }
    7863             :   return 0;
    7864             : }
    7865             : 
    7866             : unsigned fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7867           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7868             :     return 0;
    7869           0 :   if ((Subtarget->hasBWI())) {
    7870           0 :     return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7871             :   }
    7872             :   return 0;
    7873             : }
    7874             : 
    7875           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7876           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7877             :     return 0;
    7878           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7879           0 :     return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7880             :   }
    7881           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7882           0 :     return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7883             :   }
    7884           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7885           0 :     return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7886             :   }
    7887             :   return 0;
    7888             : }
    7889             : 
    7890           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7891           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7892             :     return 0;
    7893           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7894           0 :     return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7895             :   }
    7896           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7897           0 :     return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7898             :   }
    7899             :   return 0;
    7900             : }
    7901             : 
    7902             : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7903           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7904             :     return 0;
    7905           0 :   if ((Subtarget->hasBWI())) {
    7906           0 :     return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7907             :   }
    7908             :   return 0;
    7909             : }
    7910             : 
    7911           0 : unsigned fastEmit_X86ISD_ADDUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7912           0 :   switch (VT.SimpleTy) {
    7913           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7914           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7915           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7916           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7917           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7918           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7919             :   default: return 0;
    7920             :   }
    7921             : }
    7922             : 
    7923             : // FastEmit functions for X86ISD::ANDNP.
    7924             : 
    7925           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7926           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7927             :     return 0;
    7928           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7929           0 :     return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7930             :   }
    7931           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7932           0 :     return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7933             :   }
    7934           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7935           0 :     return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7936             :   }
    7937             :   return 0;
    7938             : }
    7939             : 
    7940           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7941           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7942             :     return 0;
    7943           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    7944           0 :     return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7945             :   }
    7946           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7947           0 :     return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7948             :   }
    7949           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7950           0 :     return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7951             :   }
    7952             :   return 0;
    7953             : }
    7954             : 
    7955             : unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7956           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7957             :     return 0;
    7958           0 :   if ((Subtarget->hasAVX512())) {
    7959           0 :     return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7960             :   }
    7961             :   return 0;
    7962             : }
    7963             : 
    7964           0 : unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7965           0 :   switch (VT.SimpleTy) {
    7966           0 :   case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7967           0 :   case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7968           0 :   case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7969             :   default: return 0;
    7970             :   }
    7971             : }
    7972             : 
    7973             : // FastEmit functions for X86ISD::AVG.
    7974             : 
    7975           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7976           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7977             :     return 0;
    7978           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7979           0 :     return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7980             :   }
    7981           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7982           0 :     return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7983             :   }
    7984           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7985           0 :     return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7986             :   }
    7987             :   return 0;
    7988             : }
    7989             : 
    7990           0 : unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7991           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7992             :     return 0;
    7993           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7994           0 :     return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7995             :   }
    7996           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7997           0 :     return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7998             :   }
    7999             :   return 0;
    8000             : }
    8001             : 
    8002             : unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8003           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8004             :     return 0;
    8005           0 :   if ((Subtarget->hasBWI())) {
    8006           0 :     return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8007             :   }
    8008             :   return 0;
    8009             : }
    8010             : 
    8011           0 : unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8012           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8013             :     return 0;
    8014           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8015           0 :     return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8016             :   }
    8017           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8018           0 :     return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8019             :   }
    8020           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8021           0 :     return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8022             :   }
    8023             :   return 0;
    8024             : }
    8025             : 
    8026           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8027           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8028             :     return 0;
    8029           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8030           0 :     return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8031             :   }
    8032           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8033           0 :     return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8034             :   }
    8035             :   return 0;
    8036             : }
    8037             : 
    8038             : unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8039           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8040             :     return 0;
    8041           0 :   if ((Subtarget->hasBWI())) {
    8042           0 :     return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8043             :   }
    8044             :   return 0;
    8045             : }
    8046             : 
    8047           0 : unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8048           0 :   switch (VT.SimpleTy) {
    8049           0 :   case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8050           0 :   case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8051           0 :   case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8052           0 :   case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8053           0 :   case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8054           0 :   case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8055             :   default: return 0;
    8056             :   }
    8057             : }
    8058             : 
    8059             : // FastEmit functions for X86ISD::BT.
    8060             : 
    8061             : unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8062           0 :   if (RetVT.SimpleTy != MVT::i32)
    8063             :     return 0;
    8064           0 :   return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8065             : }
    8066             : 
    8067             : unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8068           0 :   if (RetVT.SimpleTy != MVT::i32)
    8069             :     return 0;
    8070           0 :   return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8071             : }
    8072             : 
    8073             : unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8074           0 :   if (RetVT.SimpleTy != MVT::i32)
    8075             :     return 0;
    8076           0 :   return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8077             : }
    8078             : 
    8079           0 : unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8080           0 :   switch (VT.SimpleTy) {
    8081           0 :   case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8082           0 :   case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8083           0 :   case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8084             :   default: return 0;
    8085             :   }
    8086             : }
    8087             : 
    8088             : // FastEmit functions for X86ISD::CMP.
    8089             : 
    8090             : unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8091           0 :   if (RetVT.SimpleTy != MVT::i32)
    8092             :     return 0;
    8093           0 :   return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8094             : }
    8095             : 
    8096             : unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8097           0 :   if (RetVT.SimpleTy != MVT::i32)
    8098             :     return 0;
    8099           0 :   return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8100             : }
    8101             : 
    8102             : unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8103           0 :   if (RetVT.SimpleTy != MVT::i32)
    8104             :     return 0;
    8105           0 :   return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8106             : }
    8107             : 
    8108             : unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8109           0 :   if (RetVT.SimpleTy != MVT::i32)
    8110             :     return 0;
    8111           0 :   return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8112             : }
    8113             : 
    8114           0 : unsigned fastEmit_X86ISD_CMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8115           0 :   if (RetVT.SimpleTy != MVT::i32)
    8116             :     return 0;
    8117           0 :   if ((Subtarget->hasAVX512())) {
    8118           0 :     return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8119             :   }
    8120           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8121           0 :     return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8122             :   }
    8123           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8124           0 :     return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8125             :   }
    8126           0 :   if ((!Subtarget->hasSSE1())) {
    8127           0 :     return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8128             :   }
    8129             :   return 0;
    8130             : }
    8131             : 
    8132           0 : unsigned fastEmit_X86ISD_CMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8133           0 :   if (RetVT.SimpleTy != MVT::i32)
    8134             :     return 0;
    8135           0 :   if ((Subtarget->hasAVX512())) {
    8136           0 :     return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8137             :   }
    8138           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8139           0 :     return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8140             :   }
    8141           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8142           0 :     return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8143             :   }
    8144           0 :   if ((!Subtarget->hasSSE2())) {
    8145           0 :     return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8146             :   }
    8147             :   return 0;
    8148             : }
    8149             : 
    8150             : unsigned fastEmit_X86ISD_CMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8151           0 :   if (RetVT.SimpleTy != MVT::i32)
    8152             :     return 0;
    8153           0 :   return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8154             : }
    8155             : 
    8156           0 : unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8157           0 :   switch (VT.SimpleTy) {
    8158           0 :   case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8159           0 :   case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8160           0 :   case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8161           0 :   case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8162           0 :   case MVT::f32: return fastEmit_X86ISD_CMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8163           0 :   case MVT::f64: return fastEmit_X86ISD_CMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8164           0 :   case MVT::f80: return fastEmit_X86ISD_CMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8165             :   default: return 0;
    8166             :   }
    8167             : }
    8168             : 
    8169             : // FastEmit functions for X86ISD::COMI.
    8170             : 
    8171           0 : unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8172           0 :   if (RetVT.SimpleTy != MVT::i32)
    8173             :     return 0;
    8174           0 :   if ((Subtarget->hasAVX512())) {
    8175           0 :     return fastEmitInst_rr(X86::Int_VCOMISSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8176             :   }
    8177           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8178           0 :     return fastEmitInst_rr(X86::Int_COMISSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8179             :   }
    8180           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8181           0 :     return fastEmitInst_rr(X86::Int_VCOMISSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8182             :   }
    8183             :   return 0;
    8184             : }
    8185             : 
    8186           0 : unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8187           0 :   if (RetVT.SimpleTy != MVT::i32)
    8188             :     return 0;
    8189           0 :   if ((Subtarget->hasAVX512())) {
    8190           0 :     return fastEmitInst_rr(X86::Int_VCOMISDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8191             :   }
    8192           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8193           0 :     return fastEmitInst_rr(X86::Int_COMISDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8194             :   }
    8195           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8196           0 :     return fastEmitInst_rr(X86::Int_VCOMISDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8197             :   }
    8198             :   return 0;
    8199             : }
    8200             : 
    8201           0 : unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8202           0 :   switch (VT.SimpleTy) {
    8203           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8204           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8205             :   default: return 0;
    8206             :   }
    8207             : }
    8208             : 
    8209             : // FastEmit functions for X86ISD::FAND.
    8210             : 
    8211             : unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8212           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8213             :     return 0;
    8214           0 :   return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8215             : }
    8216             : 
    8217             : unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8218           0 :   switch (VT.SimpleTy) {
    8219           0 :   case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8220             :   default: return 0;
    8221             :   }
    8222             : }
    8223             : 
    8224             : // FastEmit functions for X86ISD::FANDN.
    8225             : 
    8226             : unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8227           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8228             :     return 0;
    8229           0 :   return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8230             : }
    8231             : 
    8232             : unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8233           0 :   switch (VT.SimpleTy) {
    8234           0 :   case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8235             :   default: return 0;
    8236             :   }
    8237             : }
    8238             : 
    8239             : // FastEmit functions for X86ISD::FHADD.
    8240             : 
    8241           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8242           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8243             :     return 0;
    8244           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8245           0 :     return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8246             :   }
    8247           0 :   if ((Subtarget->hasAVX())) {
    8248           0 :     return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8249             :   }
    8250             :   return 0;
    8251             : }
    8252             : 
    8253             : unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8254           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8255             :     return 0;
    8256           0 :   if ((Subtarget->hasAVX())) {
    8257           0 :     return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8258             :   }
    8259             :   return 0;
    8260             : }
    8261             : 
    8262           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8263           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8264             :     return 0;
    8265           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8266           0 :     return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8267             :   }
    8268           0 :   if ((Subtarget->hasAVX())) {
    8269           0 :     return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8270             :   }
    8271             :   return 0;
    8272             : }
    8273             : 
    8274             : unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8275           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8276             :     return 0;
    8277           0 :   if ((Subtarget->hasAVX())) {
    8278           0 :     return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8279             :   }
    8280             :   return 0;
    8281             : }
    8282             : 
    8283           0 : unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8284           0 :   switch (VT.SimpleTy) {
    8285           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8286           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8287           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8288           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8289             :   default: return 0;
    8290             :   }
    8291             : }
    8292             : 
    8293             : // FastEmit functions for X86ISD::FHSUB.
    8294             : 
    8295           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8296           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8297             :     return 0;
    8298           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8299           0 :     return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8300             :   }
    8301           0 :   if ((Subtarget->hasAVX())) {
    8302           0 :     return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8303             :   }
    8304             :   return 0;
    8305             : }
    8306             : 
    8307             : unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8308           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8309             :     return 0;
    8310           0 :   if ((Subtarget->hasAVX())) {
    8311           0 :     return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8312             :   }
    8313             :   return 0;
    8314             : }
    8315             : 
    8316           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8317           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8318             :     return 0;
    8319           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8320           0 :     return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8321             :   }
    8322           0 :   if ((Subtarget->hasAVX())) {
    8323           0 :     return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8324             :   }
    8325             :   return 0;
    8326             : }
    8327             : 
    8328             : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8329           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8330             :     return 0;
    8331           0 :   if ((Subtarget->hasAVX())) {
    8332           0 :     return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8333             :   }
    8334             :   return 0;
    8335             : }
    8336             : 
    8337           0 : unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8338           0 :   switch (VT.SimpleTy) {
    8339           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8340           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8341           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8342           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8343             :   default: return 0;
    8344             :   }
    8345             : }
    8346             : 
    8347             : // FastEmit functions for X86ISD::FMAX.
    8348             : 
    8349           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8350           0 :   if (RetVT.SimpleTy != MVT::f32)
    8351             :     return 0;
    8352           0 :   if ((Subtarget->hasAVX512())) {
    8353           0 :     return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8354             :   }
    8355           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8356           0 :     return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8357             :   }
    8358           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8359           0 :     return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8360             :   }
    8361             :   return 0;
    8362             : }
    8363             : 
    8364           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8365           0 :   if (RetVT.SimpleTy != MVT::f64)
    8366             :     return 0;
    8367           0 :   if ((Subtarget->hasAVX512())) {
    8368           0 :     return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8369             :   }
    8370           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8371           0 :     return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8372             :   }
    8373           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8374           0 :     return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8375             :   }
    8376             :   return 0;
    8377             : }
    8378             : 
    8379           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8380           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8381             :     return 0;
    8382           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8383           0 :     return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8384             :   }
    8385           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8386           0 :     return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8387             :   }
    8388           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8389           0 :     return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8390             :   }
    8391             :   return 0;
    8392             : }
    8393             : 
    8394           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8395           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8396             :     return 0;
    8397           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8398           0 :     return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8399             :   }
    8400           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8401           0 :     return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8402             :   }
    8403             :   return 0;
    8404             : }
    8405             : 
    8406             : unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8407           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8408             :     return 0;
    8409           0 :   if ((Subtarget->hasAVX512())) {
    8410           0 :     return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8411             :   }
    8412             :   return 0;
    8413             : }
    8414             : 
    8415           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8416           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8417             :     return 0;
    8418           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8419           0 :     return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8420             :   }
    8421           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8422           0 :     return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8423             :   }
    8424           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8425           0 :     return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8426             :   }
    8427             :   return 0;
    8428             : }
    8429             : 
    8430           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8431           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8432             :     return 0;
    8433           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8434           0 :     return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8435             :   }
    8436           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8437           0 :     return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8438             :   }
    8439             :   return 0;
    8440             : }
    8441             : 
    8442             : unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8443           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8444             :     return 0;
    8445           0 :   if ((Subtarget->hasAVX512())) {
    8446           0 :     return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8447             :   }
    8448             :   return 0;
    8449             : }
    8450             : 
    8451           0 : unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8452           0 :   switch (VT.SimpleTy) {
    8453           0 :   case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8454           0 :   case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8455           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8456           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8457           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8458           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8459           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8460           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8461             :   default: return 0;
    8462             :   }
    8463             : }
    8464             : 
    8465             : // FastEmit functions for X86ISD::FMAXC.
    8466             : 
    8467           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8468           0 :   if (RetVT.SimpleTy != MVT::f32)
    8469             :     return 0;
    8470           0 :   if ((Subtarget->hasAVX512())) {
    8471           0 :     return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8472             :   }
    8473           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8474           0 :     return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8475             :   }
    8476           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8477           0 :     return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8478             :   }
    8479             :   return 0;
    8480             : }
    8481             : 
    8482           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8483           0 :   if (RetVT.SimpleTy != MVT::f64)
    8484             :     return 0;
    8485           0 :   if ((Subtarget->hasAVX512())) {
    8486           0 :     return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8487             :   }
    8488           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8489           0 :     return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8490             :   }
    8491           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8492           0 :     return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8493             :   }
    8494             :   return 0;
    8495             : }
    8496             : 
    8497           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8498           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8499             :     return 0;
    8500           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8501           0 :     return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8502             :   }
    8503           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8504           0 :     return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8505             :   }
    8506           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8507           0 :     return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8508             :   }
    8509             :   return 0;
    8510             : }
    8511             : 
    8512           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8513           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8514             :     return 0;
    8515           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8516           0 :     return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8517             :   }
    8518           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8519           0 :     return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8520             :   }
    8521             :   return 0;
    8522             : }
    8523             : 
    8524             : unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8525           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8526             :     return 0;
    8527           0 :   if ((Subtarget->hasAVX512())) {
    8528           0 :     return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8529             :   }
    8530             :   return 0;
    8531             : }
    8532             : 
    8533           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8534           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8535             :     return 0;
    8536           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8537           0 :     return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8538             :   }
    8539           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8540           0 :     return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8541             :   }
    8542           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8543           0 :     return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8544             :   }
    8545             :   return 0;
    8546             : }
    8547             : 
    8548           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8549           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8550             :     return 0;
    8551           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8552           0 :     return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8553             :   }
    8554           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8555           0 :     return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8556             :   }
    8557             :   return 0;
    8558             : }
    8559             : 
    8560             : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8561           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8562             :     return 0;
    8563           0 :   if ((Subtarget->hasAVX512())) {
    8564           0 :     return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8565             :   }
    8566             :   return 0;
    8567             : }
    8568             : 
    8569           0 : unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8570           0 :   switch (VT.SimpleTy) {
    8571           0 :   case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8572           0 :   case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8573           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8574           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8575           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8576           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8577           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8578           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8579             :   default: return 0;
    8580             :   }
    8581             : }
    8582             : 
    8583             : // FastEmit functions for X86ISD::FMAXS.
    8584             : 
    8585           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8586           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8587             :     return 0;
    8588           0 :   if ((Subtarget->hasAVX512())) {
    8589           0 :     return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8590             :   }
    8591           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8592           0 :     return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8593             :   }
    8594           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8595           0 :     return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8596             :   }
    8597             :   return 0;
    8598             : }
    8599             : 
    8600           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8601           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8602             :     return 0;
    8603           0 :   if ((Subtarget->hasAVX512())) {
    8604           0 :     return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8605             :   }
    8606           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8607           0 :     return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8608             :   }
    8609           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8610           0 :     return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8611             :   }
    8612             :   return 0;
    8613             : }
    8614             : 
    8615           0 : unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8616           0 :   switch (VT.SimpleTy) {
    8617           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8618           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8619             :   default: return 0;
    8620             :   }
    8621             : }
    8622             : 
    8623             : // FastEmit functions for X86ISD::FMIN.
    8624             : 
    8625           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8626           0 :   if (RetVT.SimpleTy != MVT::f32)
    8627             :     return 0;
    8628           0 :   if ((Subtarget->hasAVX512())) {
    8629           0 :     return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8630             :   }
    8631           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8632           0 :     return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8633             :   }
    8634           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8635           0 :     return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8636             :   }
    8637             :   return 0;
    8638             : }
    8639             : 
    8640           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8641           0 :   if (RetVT.SimpleTy != MVT::f64)
    8642             :     return 0;
    8643           0 :   if ((Subtarget->hasAVX512())) {
    8644           0 :     return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8645             :   }
    8646           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8647           0 :     return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8648             :   }
    8649           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8650           0 :     return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8651             :   }
    8652             :   return 0;
    8653             : }
    8654             : 
    8655           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8656           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8657             :     return 0;
    8658           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8659           0 :     return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8660             :   }
    8661           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8662           0 :     return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8663             :   }
    8664           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8665           0 :     return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8666             :   }
    8667             :   return 0;
    8668             : }
    8669             : 
    8670           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8671           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8672             :     return 0;
    8673           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8674           0 :     return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8675             :   }
    8676           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8677           0 :     return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8678             :   }
    8679             :   return 0;
    8680             : }
    8681             : 
    8682             : unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8683           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8684             :     return 0;
    8685           0 :   if ((Subtarget->hasAVX512())) {
    8686           0 :     return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8687             :   }
    8688             :   return 0;
    8689             : }
    8690             : 
    8691           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8692           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8693             :     return 0;
    8694           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8695           0 :     return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8696             :   }
    8697           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8698           0 :     return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8699             :   }
    8700           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8701           0 :     return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8702             :   }
    8703             :   return 0;
    8704             : }
    8705             : 
    8706           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8707           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8708             :     return 0;
    8709           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8710           0 :     return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8711             :   }
    8712           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8713           0 :     return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8714             :   }
    8715             :   return 0;
    8716             : }
    8717             : 
    8718             : unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8719           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8720             :     return 0;
    8721           0 :   if ((Subtarget->hasAVX512())) {
    8722           0 :     return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8723             :   }
    8724             :   return 0;
    8725             : }
    8726             : 
    8727           0 : unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8728           0 :   switch (VT.SimpleTy) {
    8729           0 :   case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8730           0 :   case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8731           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8732           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8733           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8734           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8735           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8736           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8737             :   default: return 0;
    8738             :   }
    8739             : }
    8740             : 
    8741             : // FastEmit functions for X86ISD::FMINC.
    8742             : 
    8743           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8744           0 :   if (RetVT.SimpleTy != MVT::f32)
    8745             :     return 0;
    8746           0 :   if ((Subtarget->hasAVX512())) {
    8747           0 :     return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8748             :   }
    8749           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8750           0 :     return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8751             :   }
    8752           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8753           0 :     return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8754             :   }
    8755             :   return 0;
    8756             : }
    8757             : 
    8758           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8759           0 :   if (RetVT.SimpleTy != MVT::f64)
    8760             :     return 0;
    8761           0 :   if ((Subtarget->hasAVX512())) {
    8762           0 :     return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8763             :   }
    8764           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8765           0 :     return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8766             :   }
    8767           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8768           0 :     return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8769             :   }
    8770             :   return 0;
    8771             : }
    8772             : 
    8773           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8774           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8775             :     return 0;
    8776           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8777           0 :     return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8778             :   }
    8779           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8780           0 :     return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8781             :   }
    8782           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8783           0 :     return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8784             :   }
    8785             :   return 0;
    8786             : }
    8787             : 
    8788           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8789           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8790             :     return 0;
    8791           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8792           0 :     return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8793             :   }
    8794           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8795           0 :     return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8796             :   }
    8797             :   return 0;
    8798             : }
    8799             : 
    8800             : unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8801           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8802             :     return 0;
    8803           0 :   if ((Subtarget->hasAVX512())) {
    8804           0 :     return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8805             :   }
    8806             :   return 0;
    8807             : }
    8808             : 
    8809           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8810           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8811             :     return 0;
    8812           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8813           0 :     return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8814             :   }
    8815           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8816           0 :     return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8817             :   }
    8818           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8819           0 :     return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8820             :   }
    8821             :   return 0;
    8822             : }
    8823             : 
    8824           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8825           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8826             :     return 0;
    8827           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8828           0 :     return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8829             :   }
    8830           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8831           0 :     return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8832             :   }
    8833             :   return 0;
    8834             : }
    8835             : 
    8836             : unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8837           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8838             :     return 0;
    8839           0 :   if ((Subtarget->hasAVX512())) {
    8840           0 :     return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8841             :   }
    8842             :   return 0;
    8843             : }
    8844             : 
    8845           0 : unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8846           0 :   switch (VT.SimpleTy) {
    8847           0 :   case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8848           0 :   case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8849           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8850           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8851           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8852           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8853           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8854           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8855             :   default: return 0;
    8856             :   }
    8857             : }
    8858             : 
    8859             : // FastEmit functions for X86ISD::FMINS.
    8860             : 
    8861           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8862           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8863             :     return 0;
    8864           0 :   if ((Subtarget->hasAVX512())) {
    8865           0 :     return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8866             :   }
    8867           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8868           0 :     return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8869             :   }
    8870           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8871           0 :     return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8872             :   }
    8873             :   return 0;
    8874             : }
    8875             : 
    8876           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8877           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8878             :     return 0;
    8879           0 :   if ((Subtarget->hasAVX512())) {
    8880           0 :     return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8881             :   }
    8882           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8883           0 :     return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8884             :   }
    8885           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8886           0 :     return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8887             :   }
    8888             :   return 0;
    8889             : }
    8890             : 
    8891           0 : unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8892           0 :   switch (VT.SimpleTy) {
    8893           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8894           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8895             :   default: return 0;
    8896             :   }
    8897             : }
    8898             : 
    8899             : // FastEmit functions for X86ISD::FOR.
    8900             : 
    8901             : unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8902           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8903             :     return 0;
    8904           0 :   return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8905             : }
    8906             : 
    8907             : unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8908           0 :   switch (VT.SimpleTy) {
    8909           0 :   case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8910             :   default: return 0;
    8911             :   }
    8912             : }
    8913             : 
    8914             : // FastEmit functions for X86ISD::FRCPS.
    8915             : 
    8916             : unsigned fastEmit_X86ISD_FRCPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8917           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8918             :     return 0;
    8919           0 :   if ((Subtarget->hasAVX512())) {
    8920           0 :     return fastEmitInst_rr(X86::VRCP14SSrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8921             :   }
    8922             :   return 0;
    8923             : }
    8924             : 
    8925             : unsigned fastEmit_X86ISD_FRCPS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8926           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8927             :     return 0;
    8928           0 :   if ((Subtarget->hasAVX512())) {
    8929           0 :     return fastEmitInst_rr(X86::VRCP14SDrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8930             :   }
    8931             :   return 0;
    8932             : }
    8933             : 
    8934           0 : unsigned fastEmit_X86ISD_FRCPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8935           0 :   switch (VT.SimpleTy) {
    8936           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRCPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8937           0 :   case MVT::v2f64: return fastEmit_X86ISD_FRCPS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8938             :   default: return 0;
    8939             :   }
    8940             : }
    8941             : 
    8942             : // FastEmit functions for X86ISD::FRSQRTS.
    8943             : 
    8944             : unsigned fastEmit_X86ISD_FRSQRTS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8945           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8946             :     return 0;
    8947           0 :   if ((Subtarget->hasAVX512())) {
    8948           0 :     return fastEmitInst_rr(X86::VRSQRT14SSrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8949             :   }
    8950             :   return 0;
    8951             : }
    8952             : 
    8953             : unsigned fastEmit_X86ISD_FRSQRTS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8954           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8955             :     return 0;
    8956           0 :   if ((Subtarget->hasAVX512())) {
    8957           0 :     return fastEmitInst_rr(X86::VRSQRT14SDrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8958             :   }
    8959             :   return 0;
    8960             : }
    8961             : 
    8962           0 : unsigned fastEmit_X86ISD_FRSQRTS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8963           0 :   switch (VT.SimpleTy) {
    8964           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRSQRTS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8965           0 :   case MVT::v2f64: return fastEmit_X86ISD_FRSQRTS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8966             :   default: return 0;
    8967             :   }
    8968             : }
    8969             : 
    8970             : // FastEmit functions for X86ISD::FXOR.
    8971             : 
    8972             : unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8973           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8974             :     return 0;
    8975           0 :   return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8976             : }
    8977             : 
    8978             : unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8979           0 :   switch (VT.SimpleTy) {
    8980           0 :   case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8981             :   default: return 0;
    8982             :   }
    8983             : }
    8984             : 
    8985             : // FastEmit functions for X86ISD::HADD.
    8986             : 
    8987           0 : unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8988           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8989             :     return 0;
    8990           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    8991           0 :     return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8992             :   }
    8993           0 :   if ((Subtarget->hasAVX())) {
    8994           0 :     return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8995             :   }
    8996             :   return 0;
    8997             : }
    8998             : 
    8999             : unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9000           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9001             :     return 0;
    9002           0 :   if ((Subtarget->hasAVX2())) {
    9003           0 :     return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9004             :   }
    9005             :   return 0;
    9006             : }
    9007             : 
    9008           0 : unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9009           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9010             :     return 0;
    9011           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9012           0 :     return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9013             :   }
    9014           0 :   if ((Subtarget->hasAVX())) {
    9015           0 :     return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9016             :   }
    9017             :   return 0;
    9018             : }
    9019             : 
    9020             : unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9021           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    9022             :     return 0;
    9023           0 :   if ((Subtarget->hasAVX2())) {
    9024           0 :     return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9025             :   }
    9026             :   return 0;
    9027             : }
    9028             : 
    9029           0 : unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9030           0 :   switch (VT.SimpleTy) {
    9031           0 :   case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9032           0 :   case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9033           0 :   case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9034           0 :   case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9035             :   default: return 0;
    9036             :   }
    9037             : }
    9038             : 
    9039             : // FastEmit functions for X86ISD::HSUB.
    9040             : 
    9041           0 : unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9042           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9043             :     return 0;
    9044           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9045           0 :     return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9046             :   }
    9047           0 :   if ((Subtarget->hasAVX())) {
    9048           0 :     return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9049             :   }
    9050             :   return 0;
    9051             : }
    9052             : 
    9053             : unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9054           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9055             :     return 0;
    9056           0 :   if ((Subtarget->hasAVX2())) {
    9057           0 :     return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9058             :   }
    9059             :   return 0;
    9060             : }
    9061             : 
    9062           0 : unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9063           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9064             :     return 0;
    9065           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9066           0 :     return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9067             :   }
    9068           0 :   if ((Subtarget->hasAVX())) {
    9069           0 :     return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9070             :   }
    9071             :   return 0;
    9072             : }
    9073             : 
    9074             : unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9075           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    9076             :     return 0;
    9077           0 :   if ((Subtarget->hasAVX2())) {
    9078           0 :     return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9079             :   }
    9080             :   return 0;
    9081             : }
    9082             : 
    9083           0 : unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9084           0 :   switch (VT.SimpleTy) {
    9085           0 :   case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9086           0 :   case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9087           0 :   case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9088           0 :   case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9089             :   default: return 0;
    9090             :   }
    9091             : }
    9092             : 
    9093             : // FastEmit functions for X86ISD::KORTEST.
    9094             : 
    9095             : unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9096           0 :   if (RetVT.SimpleTy != MVT::i32)
    9097             :     return 0;
    9098           0 :   if ((Subtarget->hasDQI())) {
    9099           0 :     return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9100             :   }
    9101             :   return 0;
    9102             : }
    9103             : 
    9104             : unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9105           0 :   if (RetVT.SimpleTy != MVT::i32)
    9106             :     return 0;
    9107           0 :   if ((Subtarget->hasAVX512())) {
    9108           0 :     return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9109             :   }
    9110             :   return 0;
    9111             : }
    9112             : 
    9113             : unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9114           0 :   if (RetVT.SimpleTy != MVT::i32)
    9115             :     return 0;
    9116           0 :   if ((Subtarget->hasBWI())) {
    9117           0 :     return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9118             :   }
    9119             :   return 0;
    9120             : }
    9121             : 
    9122             : unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9123           0 :   if (RetVT.SimpleTy != MVT::i32)
    9124             :     return 0;
    9125           0 :   if ((Subtarget->hasBWI())) {
    9126           0 :     return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9127             :   }
    9128             :   return 0;
    9129             : }
    9130             : 
    9131           0 : unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9132           0 :   switch (VT.SimpleTy) {
    9133           0 :   case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9134           0 :   case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9135           0 :   case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9136           0 :   case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9137             :   default: return 0;
    9138             :   }
    9139             : }
    9140             : 
    9141             : // FastEmit functions for X86ISD::KTEST.
    9142             : 
    9143             : unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9144           0 :   if (RetVT.SimpleTy != MVT::i32)
    9145             :     return 0;
    9146           0 :   if ((Subtarget->hasDQI())) {
    9147           0 :     return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9148             :   }
    9149             :   return 0;
    9150             : }
    9151             : 
    9152             : unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9153           0 :   if (RetVT.SimpleTy != MVT::i32)
    9154             :     return 0;
    9155           0 :   if ((Subtarget->hasDQI())) {
    9156           0 :     return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9157             :   }
    9158             :   return 0;
    9159             : }
    9160             : 
    9161             : unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9162           0 :   if (RetVT.SimpleTy != MVT::i32)
    9163             :     return 0;
    9164           0 :   if ((Subtarget->hasBWI())) {
    9165           0 :     return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9166             :   }
    9167             :   return 0;
    9168             : }
    9169             : 
    9170             : unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9171           0 :   if (RetVT.SimpleTy != MVT::i32)
    9172             :     return 0;
    9173           0 :   if ((Subtarget->hasBWI())) {
    9174           0 :     return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9175             :   }
    9176             :   return 0;
    9177             : }
    9178             : 
    9179           0 : unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9180           0 :   switch (VT.SimpleTy) {
    9181           0 :   case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9182           0 :   case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9183           0 :   case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9184           0 :   case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9185             :   default: return 0;
    9186             :   }
    9187             : }
    9188             : 
    9189             : // FastEmit functions for X86ISD::MOVHLPS.
    9190             : 
    9191           0 : unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9192           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9193             :     return 0;
    9194           0 :   if ((Subtarget->hasAVX512())) {
    9195           0 :     return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9196             :   }
    9197           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9198           0 :     return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9199             :   }
    9200           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9201           0 :     return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9202             :   }
    9203             :   return 0;
    9204             : }
    9205             : 
    9206           0 : unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9207           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9208             :     return 0;
    9209           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9210           0 :     return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9211             :   }
    9212           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9213           0 :     return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9214             :   }
    9215           0 :   if ((Subtarget->hasAVX512())) {
    9216           0 :     return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9217             :   }
    9218             :   return 0;
    9219             : }
    9220             : 
    9221           0 : unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9222           0 :   switch (VT.SimpleTy) {
    9223           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVHLPS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9224           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9225             :   default: return 0;
    9226             :   }
    9227             : }
    9228             : 
    9229             : // FastEmit functions for X86ISD::MOVLHPS.
    9230             : 
    9231           0 : unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9232           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9233             :     return 0;
    9234           0 :   if ((Subtarget->hasAVX512())) {
    9235           0 :     return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9236             :   }
    9237           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9238           0 :     return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9239             :   }
    9240           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9241           0 :     return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9242             :   }
    9243             :   return 0;
    9244             : }
    9245             : 
    9246           0 : unsigned fastEmit_X86ISD_MOVLHPS_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9247           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    9248             :     return 0;
    9249           0 :   if ((Subtarget->hasAVX512())) {
    9250           0 :     return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9251             :   }
    9252           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9253           0 :     return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9254             :   }
    9255           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9256           0 :     return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9257             :   }
    9258             :   return 0;
    9259             : }
    9260             : 
    9261           0 : unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9262           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9263             :     return 0;
    9264           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9265           0 :     return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9266             :   }
    9267           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9268           0 :     return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9269             :   }
    9270           0 :   if ((Subtarget->hasAVX512())) {
    9271           0 :     return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9272             :   }
    9273             :   return 0;
    9274             : }
    9275             : 
    9276           0 : unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9277           0 :   switch (VT.SimpleTy) {
    9278           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVLHPS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9279           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVLHPS_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9280           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9281             :   default: return 0;
    9282             :   }
    9283             : }
    9284             : 
    9285             : // FastEmit functions for X86ISD::MULHRS.
    9286             : 
    9287           0 : unsigned fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9288           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9289             :     return 0;
    9290           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9291           0 :     return fastEmitInst_rr(X86::VPMULHRSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9292             :   }
    9293           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9294           0 :     return fastEmitInst_rr(X86::PMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9295             :   }
    9296           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9297           0 :     return fastEmitInst_rr(X86::VPMULHRSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9298             :   }
    9299             :   return 0;
    9300             : }
    9301             : 
    9302           0 : unsigned fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9303           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9304             :     return 0;
    9305           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9306           0 :     return fastEmitInst_rr(X86::VPMULHRSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9307             :   }
    9308           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9309           0 :     return fastEmitInst_rr(X86::VPMULHRSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9310             :   }
    9311             :   return 0;
    9312             : }
    9313             : 
    9314             : unsigned fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9315           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    9316             :     return 0;
    9317           0 :   if ((Subtarget->hasBWI())) {
    9318           0 :     return fastEmitInst_rr(X86::VPMULHRSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9319             :   }
    9320             :   return 0;
    9321             : }
    9322             : 
    9323           0 : unsigned fastEmit_X86ISD_MULHRS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9324           0 :   switch (VT.SimpleTy) {
    9325           0 :   case MVT::v8i16: return fastEmit_X86ISD_MULHRS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9326           0 :   case MVT::v16i16: return fastEmit_X86ISD_MULHRS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9327           0 :   case MVT::v32i16: return fastEmit_X86ISD_MULHRS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9328             :   default: return 0;
    9329             :   }
    9330             : }
    9331             : 
    9332             : // FastEmit functions for X86ISD::MULTISHIFT.
    9333             : 
    9334           0 : unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9335           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    9336             :     return 0;
    9337           0 :   if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
    9338           0 :     return fastEmitInst_rr(X86::VPMULTISHIFTQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9339             :   }
    9340             :   return 0;
    9341             : }
    9342             : 
    9343           0 : unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9344           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    9345             :     return 0;
    9346           0 :   if ((Subtarget->hasVBMI()) && (Subtarget->hasVLX())) {
    9347           0 :     return fastEmitInst_rr(X86::VPMULTISHIFTQBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9348             :   }
    9349             :   return 0;
    9350             : }
    9351             : 
    9352             : unsigned fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9353           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    9354             :     return 0;
    9355           0 :   if ((Subtarget->hasVBMI())) {
    9356           0 :     return fastEmitInst_rr(X86::VPMULTISHIFTQBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9357             :   }
    9358             :   return 0;
    9359             : }
    9360             : 
    9361           0 : unsigned fastEmit_X86ISD_MULTISHIFT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9362           0 :   switch (VT.SimpleTy) {
    9363           0 :   case MVT::v16i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9364           0 :   case MVT::v32i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9365           0 :   case MVT::v64i8: return fastEmit_X86ISD_MULTISHIFT_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9366             :   default: return 0;
    9367             :   }
    9368             : }
    9369             : 
    9370             : // FastEmit functions for X86ISD::PACKSS.
    9371             : 
    9372           0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9373           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    9374             :     return 0;
    9375           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9376           0 :     return fastEmitInst_rr(X86::VPACKSSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9377             :   }
    9378           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9379           0 :     return fastEmitInst_rr(X86::PACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9380             :   }
    9381           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9382           0 :     return fastEmitInst_rr(X86::VPACKSSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9383             :   }
    9384             :   return 0;
    9385             : }
    9386             : 
    9387           0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9388           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    9389             :     return 0;
    9390           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9391           0 :     return fastEmitInst_rr(X86::VPACKSSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9392             :   }
    9393           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9394           0 :     return fastEmitInst_rr(X86::VPACKSSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9395             :   }
    9396             :   return 0;
    9397             : }
    9398             : 
    9399             : unsigned fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9400           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    9401             :     return 0;
    9402           0 :   if ((Subtarget->hasBWI())) {
    9403           0 :     return fastEmitInst_rr(X86::VPACKSSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9404             :   }
    9405             :   return 0;
    9406             : }
    9407             : 
    9408           0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9409           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9410             :     return 0;
    9411           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9412           0 :     return fastEmitInst_rr(X86::VPACKSSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9413             :   }
    9414           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9415           0 :     return fastEmitInst_rr(X86::PACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9416             :   }
    9417           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9418           0 :     return fastEmitInst_rr(X86::VPACKSSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9419             :   }
    9420             :   return 0;
    9421             : }
    9422             : 
    9423           0 : unsigned fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9424           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9425             :     return 0;
    9426           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9427           0 :     return fastEmitInst_rr(X86::VPACKSSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9428             :   }
    9429           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9430           0 :     return fastEmitInst_rr(X86::VPACKSSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9431             :   }
    9432             :   return 0;
    9433             : }
    9434             : 
    9435             : unsigned fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9436           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    9437             :     return 0;
    9438           0 :   if ((Subtarget->hasBWI())) {
    9439           0 :     return fastEmitInst_rr(X86::VPACKSSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9440             :   }
    9441             :   return 0;
    9442             : }
    9443             : 
    9444           0 : unsigned fastEmit_X86ISD_PACKSS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9445           0 :   switch (VT.SimpleTy) {
    9446           0 :   case MVT::v8i16: return fastEmit_X86ISD_PACKSS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9447           0 :   case MVT::v16i16: return fastEmit_X86ISD_PACKSS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9448           0 :   case MVT::v32i16: return fastEmit_X86ISD_PACKSS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9449           0 :   case MVT::v4i32: return fastEmit_X86ISD_PACKSS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9450           0 :   case MVT::v8i32: return fastEmit_X86ISD_PACKSS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9451           0 :   case MVT::v16i32: return fastEmit_X86ISD_PACKSS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9452             :   default: return 0;
    9453             :   }
    9454             : }
    9455             : 
    9456             : // FastEmit functions for X86ISD::PACKUS.
    9457             : 
    9458           0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9459           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    9460             :     return 0;
    9461           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9462           0 :     return fastEmitInst_rr(X86::VPACKUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9463             :   }
    9464           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9465           0 :     return fastEmitInst_rr(X86::PACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9466             :   }
    9467           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9468           0 :     return fastEmitInst_rr(X86::VPACKUSWBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9469             :   }
    9470             :   return 0;
    9471             : }
    9472             : 
    9473           0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9474           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    9475             :     return 0;
    9476           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9477           0 :     return fastEmitInst_rr(X86::VPACKUSWBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9478             :   }
    9479           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9480           0 :     return fastEmitInst_rr(X86::VPACKUSWBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9481             :   }
    9482             :   return 0;
    9483             : }
    9484             : 
    9485             : unsigned fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9486           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    9487             :     return 0;
    9488           0 :   if ((Subtarget->hasBWI())) {
    9489           0 :     return fastEmitInst_rr(X86::VPACKUSWBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9490             :   }
    9491             :   return 0;
    9492             : }
    9493             : 
    9494           0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9495           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9496             :     return 0;
    9497           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9498           0 :     return fastEmitInst_rr(X86::VPACKUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9499             :   }
    9500           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    9501           0 :     return fastEmitInst_rr(X86::PACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9502             :   }
    9503           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9504           0 :     return fastEmitInst_rr(X86::VPACKUSDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9505             :   }
    9506             :   return 0;
    9507             : }
    9508             : 
    9509           0 : unsigned fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9510           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9511             :     return 0;
    9512           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    9513           0 :     return fastEmitInst_rr(X86::VPACKUSDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9514             :   }
    9515           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9516           0 :     return fastEmitInst_rr(X86::VPACKUSDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9517             :   }
    9518             :   return 0;
    9519             : }
    9520             : 
    9521             : unsigned fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9522           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    9523             :     return 0;
    9524           0 :   if ((Subtarget->hasBWI())) {
    9525           0 :     return fastEmitInst_rr(X86::VPACKUSDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9526             :   }
    9527             :   return 0;
    9528             : }
    9529             : 
    9530           0 : unsigned fastEmit_X86ISD_PACKUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9531           0 :   switch (VT.SimpleTy) {
    9532           0 :   case MVT::v8i16: return fastEmit_X86ISD_PACKUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9533           0 :   case MVT::v16i16: return fastEmit_X86ISD_PACKUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9534           0 :   case MVT::v32i16: return fastEmit_X86ISD_PACKUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9535           0 :   case MVT::v4i32: return fastEmit_X86ISD_PACKUS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9536           0 :   case MVT::v8i32: return fastEmit_X86ISD_PACKUS_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9537           0 :   case MVT::v16i32: return fastEmit_X86ISD_PACKUS_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9538             :   default: return 0;
    9539             :   }
    9540             : }
    9541             : 
    9542             : // FastEmit functions for X86ISD::PCMPEQ.
    9543             : 
    9544           0 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9545           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    9546             :     return 0;
    9547           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9548           0 :     return fastEmitInst_rr(X86::PCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9549             :   }
    9550           0 :   if ((Subtarget->hasAVX()) && (true)) {
    9551           0 :     return fastEmitInst_rr(X86::VPCMPEQBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9552             :   }
    9553             :   return 0;
    9554             : }
    9555             : 
    9556             : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9557           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    9558             :     return 0;
    9559           0 :   if ((Subtarget->hasAVX2()) && (true)) {
    9560           0 :     return fastEmitInst_rr(X86::VPCMPEQBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9561             :   }
    9562             :   return 0;
    9563             : }
    9564             : 
    9565           0 : unsigned fastEmit_X86ISD_PCMPEQ_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9566           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9567             :     return 0;
    9568           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9569           0 :     return fastEmitInst_rr(X86::PCMPEQWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9570             :   }
    9571           0 :   if ((Subtarget->hasAVX()) && (true)) {
    9572           0 :     return fastEmitInst_rr(X86::VPCMPEQWrr, &X86::VR128Reg