LCOV - code coverage report
Current view: top level - build-llvm/lib/Target/X86 - X86GenFastISel.inc (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 797 6732 11.8 %
Date: 2018-02-19 03:08:00 Functions: 112 847 13.2 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
       2             : |*                                                                            *|
       3             : |* "Fast" Instruction Selector for the X86 target                             *|
       4             : |*                                                                            *|
       5             : |* Automatically generated file, do not edit!                                 *|
       6             : |*                                                                            *|
       7             : \*===----------------------------------------------------------------------===*/
       8             : 
       9             : 
      10             : // FastEmit Immediate Predicate functions.
      11             : static bool Predicate_i16immSExt8(int64_t Imm) {
      12             :  return isInt<8>(Imm); 
      13             : }
      14             : static bool Predicate_i32immSExt8(int64_t Imm) {
      15             :  return isInt<8>(Imm); 
      16             : }
      17             : static bool Predicate_i64immSExt8(int64_t Imm) {
      18             :  return isInt<8>(Imm); 
      19             : }
      20             : static bool Predicate_FROUND_NO_EXC(int64_t Imm) {
      21             :  return Imm == 8; 
      22             : }
      23             : static bool Predicate_FROUND_CURRENT(int64_t Imm) {
      24             : 
      25             :   return Imm == X86::STATIC_ROUNDING::CUR_DIRECTION;
      26             : 
      27             : }
      28             : static bool Predicate_BTRMask64(int64_t Imm) {
      29             : 
      30             :   return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
      31             : 
      32             : }
      33             : static bool Predicate_BTCBTSMask64(int64_t Imm) {
      34             : 
      35             :   return !isInt<32>(Imm) && isPowerOf2_64(Imm);
      36             : 
      37             : }
      38             : static bool Predicate_i64immSExt32(int64_t Imm) {
      39             :  return isInt<32>(Imm); 
      40             : }
      41             : static bool Predicate_AndMask64(int64_t Imm) {
      42             : 
      43             :   return isMask_64(Imm) && !isUInt<32>(Imm);
      44             : 
      45             : }
      46             : 
      47             : 
      48             : // FastEmit functions for ISD::ABS.
      49             : 
      50           0 : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      51           0 :   if (RetVT.SimpleTy != MVT::v16i8)
      52             :     return 0;
      53           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      54           0 :     return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      55             :   }
      56           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      57           0 :     return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      58             :   }
      59           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      60           0 :     return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      61             :   }
      62             :   return 0;
      63             : }
      64             : 
      65           0 : unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      66           0 :   if (RetVT.SimpleTy != MVT::v32i8)
      67             :     return 0;
      68           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      69           0 :     return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
      70             :   }
      71           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      72           0 :     return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
      73             :   }
      74             :   return 0;
      75             : }
      76             : 
      77             : unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      78           0 :   if (RetVT.SimpleTy != MVT::v64i8)
      79             :     return 0;
      80           0 :   if ((Subtarget->hasBWI())) {
      81           0 :     return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
      82             :   }
      83             :   return 0;
      84             : }
      85             : 
      86           0 : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      87           0 :   if (RetVT.SimpleTy != MVT::v8i16)
      88             :     return 0;
      89           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      90           0 :     return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      91             :   }
      92           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      93           0 :     return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      94             :   }
      95           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      96           0 :     return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      97             :   }
      98             :   return 0;
      99             : }
     100             : 
     101           0 : unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     102           0 :   if (RetVT.SimpleTy != MVT::v16i16)
     103             :     return 0;
     104           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
     105           0 :     return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     106             :   }
     107           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
     108           0 :     return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     109             :   }
     110             :   return 0;
     111             : }
     112             : 
     113             : unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     114           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     115             :     return 0;
     116           0 :   if ((Subtarget->hasBWI())) {
     117           0 :     return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     118             :   }
     119             :   return 0;
     120             : }
     121             : 
     122           0 : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     123           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     124             :     return 0;
     125           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     126           0 :     return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     127             :   }
     128           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
     129           0 :     return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     130             :   }
     131           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     132           0 :     return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     133             :   }
     134             :   return 0;
     135             : }
     136             : 
     137           0 : unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     138           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     139             :     return 0;
     140           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     141           0 :     return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     142             :   }
     143           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
     144           0 :     return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     145             :   }
     146             :   return 0;
     147             : }
     148             : 
     149             : unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     150           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     151             :     return 0;
     152           0 :   if ((Subtarget->hasAVX512())) {
     153           0 :     return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     154             :   }
     155             :   return 0;
     156             : }
     157             : 
     158           0 : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     159           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     160             :     return 0;
     161           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     162           0 :     return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     163             :   }
     164             :   return 0;
     165             : }
     166             : 
     167           0 : unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     168           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     169             :     return 0;
     170           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     171           0 :     return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     172             :   }
     173             :   return 0;
     174             : }
     175             : 
     176             : unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     177           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     178             :     return 0;
     179           0 :   if ((Subtarget->hasAVX512())) {
     180           0 :     return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     181             :   }
     182             :   return 0;
     183             : }
     184             : 
     185           0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     186           0 :   switch (VT.SimpleTy) {
     187           0 :   case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     188           0 :   case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     189           0 :   case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     190           0 :   case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     191           0 :   case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     192           0 :   case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     193           0 :   case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     194           0 :   case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     195           0 :   case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     196           0 :   case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     197           0 :   case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     198           0 :   case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     199             :   default: return 0;
     200             :   }
     201             : }
     202             : 
     203             : // FastEmit functions for ISD::ANY_EXTEND.
     204             : 
     205             : unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     206          15 :   if (RetVT.SimpleTy != MVT::i32)
     207             :     return 0;
     208          15 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
     209             : }
     210             : 
     211             : unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     212          18 :   switch (VT.SimpleTy) {
     213          15 :   case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
     214             :   default: return 0;
     215             :   }
     216             : }
     217             : 
     218             : // FastEmit functions for ISD::BITCAST.
     219             : 
     220           1 : unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     221           1 :   if (RetVT.SimpleTy != MVT::f32)
     222             :     return 0;
     223           2 :   if ((Subtarget->hasAVX512())) {
     224           0 :     return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
     225             :   }
     226           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     227           1 :     return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     228             :   }
     229           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     230           0 :     return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     231             :   }
     232             :   return 0;
     233             : }
     234             : 
     235           2 : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     236           4 :   if ((Subtarget->hasAVX512())) {
     237           0 :     return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
     238             :   }
     239           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     240           2 :     return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     241             :   }
     242           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     243           0 :     return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     244             :   }
     245             :   return 0;
     246             : }
     247             : 
     248             : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     249           0 :   if ((Subtarget->hasMMX())) {
     250           0 :     return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
     251             :   }
     252             :   return 0;
     253             : }
     254             : 
     255           2 : unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     256           2 : switch (RetVT.SimpleTy) {
     257           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
     258           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill);
     259             :   default: return 0;
     260             : }
     261             : }
     262             : 
     263           3 : unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     264           3 :   if (RetVT.SimpleTy != MVT::i32)
     265             :     return 0;
     266           6 :   if ((Subtarget->hasAVX512())) {
     267           0 :     return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     268             :   }
     269           3 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     270           3 :     return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     271             :   }
     272           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     273           0 :     return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     274             :   }
     275             :   return 0;
     276             : }
     277             : 
     278           3 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     279           6 :   if ((Subtarget->hasAVX512())) {
     280           0 :     return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     281             :   }
     282           3 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     283           2 :     return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     284             :   }
     285           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     286           1 :     return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     287             :   }
     288             :   return 0;
     289             : }
     290             : 
     291             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     292           0 :   if ((Subtarget->hasSSE2())) {
     293           0 :     return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
     294             :   }
     295             :   return 0;
     296             : }
     297             : 
     298           3 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     299           3 : switch (RetVT.SimpleTy) {
     300           3 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     301           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill);
     302             :   default: return 0;
     303             : }
     304             : }
     305             : 
     306             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     307           0 :   if ((Subtarget->hasMMX())) {
     308           0 :     return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     309             :   }
     310             :   return 0;
     311             : }
     312             : 
     313             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     314           0 :   if ((Subtarget->hasSSE2())) {
     315           0 :     return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
     316             :   }
     317             :   return 0;
     318             : }
     319             : 
     320           0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     321           0 : switch (RetVT.SimpleTy) {
     322           0 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
     323           0 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
     324             :   default: return 0;
     325             : }
     326             : }
     327             : 
     328        1809 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     329        1809 :   switch (VT.SimpleTy) {
     330           1 :   case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
     331           2 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
     332           3 :   case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
     333           3 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
     334           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
     335             :   default: return 0;
     336             :   }
     337             : }
     338             : 
     339             : // FastEmit functions for ISD::BRIND.
     340             : 
     341             : unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     342           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     343             :     return 0;
     344           0 :   if ((!Subtarget->is64Bit())) {
     345           0 :     return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
     346             :   }
     347             :   return 0;
     348             : }
     349             : 
     350             : unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     351           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     352             :     return 0;
     353           0 :   if ((!Subtarget->is64Bit())) {
     354           0 :     return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     355             :   }
     356             :   return 0;
     357             : }
     358             : 
     359             : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     360           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     361             :     return 0;
     362           0 :   if ((Subtarget->is64Bit())) {
     363           0 :     return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     364             :   }
     365             :   return 0;
     366             : }
     367             : 
     368           0 : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     369           0 :   switch (VT.SimpleTy) {
     370           0 :   case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
     371           0 :   case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
     372           0 :   case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
     373             :   default: return 0;
     374             :   }
     375             : }
     376             : 
     377             : // FastEmit functions for ISD::BSWAP.
     378             : 
     379             : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     380           0 :   if (RetVT.SimpleTy != MVT::i32)
     381             :     return 0;
     382           0 :   return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     383             : }
     384             : 
     385             : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     386           0 :   if (RetVT.SimpleTy != MVT::i64)
     387             :     return 0;
     388           0 :   return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     389             : }
     390             : 
     391           0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     392           0 :   switch (VT.SimpleTy) {
     393           0 :   case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     394           0 :   case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     395             :   default: return 0;
     396             :   }
     397             : }
     398             : 
     399             : // FastEmit functions for ISD::CTLZ.
     400             : 
     401             : unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     402           0 :   if (RetVT.SimpleTy != MVT::i16)
     403             :     return 0;
     404           0 :   if ((Subtarget->hasLZCNT())) {
     405           0 :     return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     406             :   }
     407             :   return 0;
     408             : }
     409             : 
     410             : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     411           0 :   if (RetVT.SimpleTy != MVT::i32)
     412             :     return 0;
     413           0 :   if ((Subtarget->hasLZCNT())) {
     414           0 :     return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     415             :   }
     416             :   return 0;
     417             : }
     418             : 
     419             : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     420           0 :   if (RetVT.SimpleTy != MVT::i64)
     421             :     return 0;
     422           0 :   if ((Subtarget->hasLZCNT())) {
     423           0 :     return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     424             :   }
     425             :   return 0;
     426             : }
     427             : 
     428           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     429           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     430             :     return 0;
     431           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     432           0 :     return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     433             :   }
     434             :   return 0;
     435             : }
     436             : 
     437           0 : unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     438           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     439             :     return 0;
     440           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     441           0 :     return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     442             :   }
     443             :   return 0;
     444             : }
     445             : 
     446             : unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     447           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     448             :     return 0;
     449           0 :   if ((Subtarget->hasCDI())) {
     450           0 :     return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     451             :   }
     452             :   return 0;
     453             : }
     454             : 
     455           0 : unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     456           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     457             :     return 0;
     458           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     459           0 :     return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     460             :   }
     461             :   return 0;
     462             : }
     463             : 
     464           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     465           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     466             :     return 0;
     467           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     468           0 :     return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     469             :   }
     470             :   return 0;
     471             : }
     472             : 
     473             : unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     474           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     475             :     return 0;
     476           0 :   if ((Subtarget->hasCDI())) {
     477           0 :     return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     478             :   }
     479             :   return 0;
     480             : }
     481             : 
     482           0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     483           0 :   switch (VT.SimpleTy) {
     484           0 :   case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     485           0 :   case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     486           0 :   case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     487           0 :   case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     488           0 :   case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     489           0 :   case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     490           0 :   case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     491           0 :   case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     492           0 :   case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     493             :   default: return 0;
     494             :   }
     495             : }
     496             : 
     497             : // FastEmit functions for ISD::CTPOP.
     498             : 
     499             : unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     500           0 :   if (RetVT.SimpleTy != MVT::i16)
     501             :     return 0;
     502           0 :   if ((Subtarget->hasPOPCNT())) {
     503           0 :     return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     504             :   }
     505             :   return 0;
     506             : }
     507             : 
     508             : unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     509           0 :   if (RetVT.SimpleTy != MVT::i32)
     510             :     return 0;
     511           0 :   if ((Subtarget->hasPOPCNT())) {
     512           0 :     return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     513             :   }
     514             :   return 0;
     515             : }
     516             : 
     517             : unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     518           0 :   if (RetVT.SimpleTy != MVT::i64)
     519             :     return 0;
     520           0 :   if ((Subtarget->hasPOPCNT())) {
     521           0 :     return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     522             :   }
     523             :   return 0;
     524             : }
     525             : 
     526           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     527           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     528             :     return 0;
     529           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     530           0 :     return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     531             :   }
     532             :   return 0;
     533             : }
     534             : 
     535           0 : unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     536           0 :   if (RetVT.SimpleTy != MVT::v32i8)
     537             :     return 0;
     538           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     539           0 :     return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     540             :   }
     541             :   return 0;
     542             : }
     543             : 
     544             : unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     545           0 :   if (RetVT.SimpleTy != MVT::v64i8)
     546             :     return 0;
     547           0 :   if ((Subtarget->hasBITALG())) {
     548           0 :     return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     549             :   }
     550             :   return 0;
     551             : }
     552             : 
     553           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     554           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     555             :     return 0;
     556           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     557           0 :     return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     558             :   }
     559             :   return 0;
     560             : }
     561             : 
     562           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     563           0 :   if (RetVT.SimpleTy != MVT::v16i16)
     564             :     return 0;
     565           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     566           0 :     return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     567             :   }
     568             :   return 0;
     569             : }
     570             : 
     571             : unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     572           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     573             :     return 0;
     574           0 :   if ((Subtarget->hasBITALG())) {
     575           0 :     return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     576             :   }
     577             :   return 0;
     578             : }
     579             : 
     580           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     581           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     582             :     return 0;
     583           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     584           0 :     return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     585             :   }
     586             :   return 0;
     587             : }
     588             : 
     589           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     590           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     591             :     return 0;
     592           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     593           0 :     return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     594             :   }
     595             :   return 0;
     596             : }
     597             : 
     598             : unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     599           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     600             :     return 0;
     601           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     602           0 :     return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     603             :   }
     604             :   return 0;
     605             : }
     606             : 
     607           0 : unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     608           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     609             :     return 0;
     610           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     611           0 :     return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     612             :   }
     613             :   return 0;
     614             : }
     615             : 
     616           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     617           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     618             :     return 0;
     619           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     620           0 :     return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     621             :   }
     622             :   return 0;
     623             : }
     624             : 
     625             : unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     626           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     627             :     return 0;
     628           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     629           0 :     return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     630             :   }
     631             :   return 0;
     632             : }
     633             : 
     634           0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     635           0 :   switch (VT.SimpleTy) {
     636           0 :   case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
     637           0 :   case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     638           0 :   case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     639           0 :   case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     640           0 :   case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     641           0 :   case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     642           0 :   case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     643           0 :   case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     644           0 :   case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     645           0 :   case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     646           0 :   case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     647           0 :   case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     648           0 :   case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     649           0 :   case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     650           0 :   case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     651             :   default: return 0;
     652             :   }
     653             : }
     654             : 
     655             : // FastEmit functions for ISD::CTTZ.
     656             : 
     657             : unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     658           0 :   if (RetVT.SimpleTy != MVT::i16)
     659             :     return 0;
     660           0 :   if ((Subtarget->hasBMI())) {
     661           0 :     return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     662             :   }
     663             :   return 0;
     664             : }
     665             : 
     666             : unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     667           0 :   if (RetVT.SimpleTy != MVT::i32)
     668             :     return 0;
     669           0 :   if ((Subtarget->hasBMI())) {
     670           0 :     return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     671             :   }
     672             :   return 0;
     673             : }
     674             : 
     675             : unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     676           0 :   if (RetVT.SimpleTy != MVT::i64)
     677             :     return 0;
     678           0 :   if ((Subtarget->hasBMI())) {
     679           0 :     return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     680             :   }
     681             :   return 0;
     682             : }
     683             : 
     684           0 : unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     685           0 :   switch (VT.SimpleTy) {
     686           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     687           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     688           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     689             :   default: return 0;
     690             :   }
     691             : }
     692             : 
     693             : // FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
     694             : 
     695             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     696           0 :   if (RetVT.SimpleTy != MVT::i16)
     697             :     return 0;
     698           0 :   return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     699             : }
     700             : 
     701             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     702           0 :   if (RetVT.SimpleTy != MVT::i32)
     703             :     return 0;
     704           0 :   return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     705             : }
     706             : 
     707             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     708           0 :   if (RetVT.SimpleTy != MVT::i64)
     709             :     return 0;
     710           0 :   return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     711             : }
     712             : 
     713           0 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     714           0 :   switch (VT.SimpleTy) {
     715           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
     716           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
     717           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
     718             :   default: return 0;
     719             :   }
     720             : }
     721             : 
     722             : // FastEmit functions for ISD::FABS.
     723             : 
     724             : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     725           0 :   if (RetVT.SimpleTy != MVT::f32)
     726             :     return 0;
     727           0 :   if ((!Subtarget->hasSSE1())) {
     728           0 :     return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     729             :   }
     730             :   return 0;
     731             : }
     732             : 
     733             : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     734           0 :   if (RetVT.SimpleTy != MVT::f64)
     735             :     return 0;
     736           0 :   if ((!Subtarget->hasSSE2())) {
     737           0 :     return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     738             :   }
     739             :   return 0;
     740             : }
     741             : 
     742             : unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     743           0 :   if (RetVT.SimpleTy != MVT::f80)
     744             :     return 0;
     745           0 :   return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     746             : }
     747             : 
     748           0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     749           0 :   switch (VT.SimpleTy) {
     750           0 :   case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     751           0 :   case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     752           0 :   case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     753             :   default: return 0;
     754             :   }
     755             : }
     756             : 
     757             : // FastEmit functions for ISD::FCOS.
     758             : 
     759             : unsigned fastEmit_ISD_FCOS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     760           0 :   if (RetVT.SimpleTy != MVT::f32)
     761             :     return 0;
     762           0 :   if ((!Subtarget->hasSSE1())) {
     763           0 :     return fastEmitInst_r(X86::COS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     764             :   }
     765             :   return 0;
     766             : }
     767             : 
     768             : unsigned fastEmit_ISD_FCOS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     769           0 :   if (RetVT.SimpleTy != MVT::f64)
     770             :     return 0;
     771           0 :   if ((!Subtarget->hasSSE2())) {
     772           0 :     return fastEmitInst_r(X86::COS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     773             :   }
     774             :   return 0;
     775             : }
     776             : 
     777             : unsigned fastEmit_ISD_FCOS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     778           0 :   if (RetVT.SimpleTy != MVT::f80)
     779             :     return 0;
     780           0 :   return fastEmitInst_r(X86::COS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     781             : }
     782             : 
     783           0 : unsigned fastEmit_ISD_FCOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     784           0 :   switch (VT.SimpleTy) {
     785           0 :   case MVT::f32: return fastEmit_ISD_FCOS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     786           0 :   case MVT::f64: return fastEmit_ISD_FCOS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     787           0 :   case MVT::f80: return fastEmit_ISD_FCOS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     788             :   default: return 0;
     789             :   }
     790             : }
     791             : 
     792             : // FastEmit functions for ISD::FNEG.
     793             : 
     794             : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     795           1 :   if (RetVT.SimpleTy != MVT::f32)
     796             :     return 0;
     797           2 :   if ((!Subtarget->hasSSE1())) {
     798           0 :     return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     799             :   }
     800             :   return 0;
     801             : }
     802             : 
     803             : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     804           2 :   if (RetVT.SimpleTy != MVT::f64)
     805             :     return 0;
     806           4 :   if ((!Subtarget->hasSSE2())) {
     807           0 :     return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     808             :   }
     809             :   return 0;
     810             : }
     811             : 
     812             : unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     813           0 :   if (RetVT.SimpleTy != MVT::f80)
     814             :     return 0;
     815           0 :   return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     816             : }
     817             : 
     818           3 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     819           3 :   switch (VT.SimpleTy) {
     820           1 :   case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
     821           2 :   case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
     822           0 :   case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill);
     823             :   default: return 0;
     824             :   }
     825             : }
     826             : 
     827             : // FastEmit functions for ISD::FP_EXTEND.
     828             : 
     829           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     830           0 :   if (RetVT.SimpleTy != MVT::f64)
     831             :     return 0;
     832           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     833           0 :     return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     834             :   }
     835             :   return 0;
     836             : }
     837             : 
     838           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     839           0 :   if (RetVT.SimpleTy != MVT::v4f64)
     840             :     return 0;
     841           0 :   if ((Subtarget->hasVLX())) {
     842           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     843             :   }
     844           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     845           0 :     return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     846             :   }
     847             :   return 0;
     848             : }
     849             : 
     850             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     851           0 :   if (RetVT.SimpleTy != MVT::v8f64)
     852             :     return 0;
     853           0 :   if ((Subtarget->hasAVX512())) {
     854           0 :     return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     855             :   }
     856             :   return 0;
     857             : }
     858             : 
     859           0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     860           0 :   switch (VT.SimpleTy) {
     861           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
     862           0 :   case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     863           0 :   case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
     864             :   default: return 0;
     865             :   }
     866             : }
     867             : 
     868             : // FastEmit functions for ISD::FP_ROUND.
     869             : 
     870           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     871           0 :   if (RetVT.SimpleTy != MVT::f32)
     872             :     return 0;
     873           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     874           0 :     return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     875             :   }
     876             :   return 0;
     877             : }
     878             : 
     879           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     880           0 :   if (RetVT.SimpleTy != MVT::v4f32)
     881             :     return 0;
     882           0 :   if ((Subtarget->hasVLX())) {
     883           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     884             :   }
     885           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     886           0 :     return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
     887             :   }
     888             :   return 0;
     889             : }
     890             : 
     891             : unsigned fastEmit_ISD_FP_ROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     892           0 :   if (RetVT.SimpleTy != MVT::v8f32)
     893             :     return 0;
     894           0 :   if ((Subtarget->hasAVX512())) {
     895           0 :     return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
     896             :   }
     897             :   return 0;
     898             : }
     899             : 
     900           0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     901           0 :   switch (VT.SimpleTy) {
     902           0 :   case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
     903           0 :   case MVT::v4f64: return fastEmit_ISD_FP_ROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
     904           0 :   case MVT::v8f64: return fastEmit_ISD_FP_ROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
     905             :   default: return 0;
     906             :   }
     907             : }
     908             : 
     909             : // FastEmit functions for ISD::FP_TO_SINT.
     910             : 
     911           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     912           0 :   if ((Subtarget->hasAVX512())) {
     913           0 :     return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     914             :   }
     915           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     916           0 :     return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     917             :   }
     918           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     919           0 :     return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     920             :   }
     921             :   return 0;
     922             : }
     923             : 
     924           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     925           0 :   if ((Subtarget->hasAVX512())) {
     926           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     927             :   }
     928           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     929           0 :     return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     930             :   }
     931           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     932           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     933             :   }
     934             :   return 0;
     935             : }
     936             : 
     937           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     938           0 : switch (RetVT.SimpleTy) {
     939           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
     940           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
     941             :   default: return 0;
     942             : }
     943             : }
     944             : 
     945           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     946           0 :   if ((Subtarget->hasAVX512())) {
     947           0 :     return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     948             :   }
     949           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     950           0 :     return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     951             :   }
     952           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     953           0 :     return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     954             :   }
     955             :   return 0;
     956             : }
     957             : 
     958           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     959           0 :   if ((Subtarget->hasAVX512())) {
     960           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     961             :   }
     962           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     963           0 :     return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     964             :   }
     965           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     966           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     967             :   }
     968             :   return 0;
     969             : }
     970             : 
     971           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     972           0 : switch (RetVT.SimpleTy) {
     973           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
     974           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     975             :   default: return 0;
     976             : }
     977             : }
     978             : 
     979           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     980           0 :   if ((Subtarget->hasVLX())) {
     981           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     982             :   }
     983           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     984           0 :     return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     985             :   }
     986           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     987           0 :     return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     988             :   }
     989             :   return 0;
     990             : }
     991             : 
     992           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
     993           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
     994           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     995             :   }
     996             :   return 0;
     997             : }
     998             : 
     999           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1000           0 : switch (RetVT.SimpleTy) {
    1001           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    1002           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    1003             :   default: return 0;
    1004             : }
    1005             : }
    1006             : 
    1007           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1008           0 :   if ((Subtarget->hasVLX())) {
    1009           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1010             :   }
    1011           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1012           0 :     return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1013             :   }
    1014             :   return 0;
    1015             : }
    1016             : 
    1017             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1018           0 :   if ((Subtarget->hasDQI())) {
    1019           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1020             :   }
    1021             :   return 0;
    1022             : }
    1023             : 
    1024           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1025           0 : switch (RetVT.SimpleTy) {
    1026           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1027           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1028             :   default: return 0;
    1029             : }
    1030             : }
    1031             : 
    1032             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1033           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1034             :     return 0;
    1035           0 :   if ((Subtarget->hasAVX512())) {
    1036           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1037             :   }
    1038             :   return 0;
    1039             : }
    1040             : 
    1041           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1042           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1043             :     return 0;
    1044           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1045           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1046             :   }
    1047             :   return 0;
    1048             : }
    1049             : 
    1050           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1051           0 :   if ((Subtarget->hasVLX())) {
    1052           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1053             :   }
    1054           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1055           0 :     return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1056             :   }
    1057             :   return 0;
    1058             : }
    1059             : 
    1060           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1061           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1062           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1063             :   }
    1064             :   return 0;
    1065             : }
    1066             : 
    1067           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1068           0 : switch (RetVT.SimpleTy) {
    1069           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1070           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1071             :   default: return 0;
    1072             : }
    1073             : }
    1074             : 
    1075             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1076           0 :   if ((Subtarget->hasAVX512())) {
    1077           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1078             :   }
    1079             :   return 0;
    1080             : }
    1081             : 
    1082             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1083           0 :   if ((Subtarget->hasDQI())) {
    1084           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1085             :   }
    1086             :   return 0;
    1087             : }
    1088             : 
    1089           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1090           0 : switch (RetVT.SimpleTy) {
    1091           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1092           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1093             :   default: return 0;
    1094             : }
    1095             : }
    1096             : 
    1097           0 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1098           0 :   switch (VT.SimpleTy) {
    1099           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1100           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1101           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1102           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1103           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1104           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1105           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1106           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1107             :   default: return 0;
    1108             :   }
    1109             : }
    1110             : 
    1111             : // FastEmit functions for ISD::FP_TO_UINT.
    1112             : 
    1113             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1114           0 :   if ((Subtarget->hasAVX512())) {
    1115           0 :     return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1116             :   }
    1117             :   return 0;
    1118             : }
    1119             : 
    1120             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1121           0 :   if ((Subtarget->hasAVX512())) {
    1122           0 :     return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1123             :   }
    1124             :   return 0;
    1125             : }
    1126             : 
    1127           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1128           0 : switch (RetVT.SimpleTy) {
    1129           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    1130           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    1131             :   default: return 0;
    1132             : }
    1133             : }
    1134             : 
    1135             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1136           0 :   if ((Subtarget->hasAVX512())) {
    1137           0 :     return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1138             :   }
    1139             :   return 0;
    1140             : }
    1141             : 
    1142             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1143           0 :   if ((Subtarget->hasAVX512())) {
    1144           0 :     return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1145             :   }
    1146             :   return 0;
    1147             : }
    1148             : 
    1149           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1150           0 : switch (RetVT.SimpleTy) {
    1151           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    1152           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    1153             :   default: return 0;
    1154             : }
    1155             : }
    1156             : 
    1157             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1158           0 :   if ((Subtarget->hasVLX())) {
    1159           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1160             :   }
    1161             :   return 0;
    1162             : }
    1163             : 
    1164           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1165           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1166           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1167             :   }
    1168             :   return 0;
    1169             : }
    1170             : 
    1171           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1172           0 : switch (RetVT.SimpleTy) {
    1173           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    1174           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    1175             :   default: return 0;
    1176             : }
    1177             : }
    1178             : 
    1179             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1180           0 :   if ((Subtarget->hasVLX())) {
    1181           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1182             :   }
    1183             :   return 0;
    1184             : }
    1185             : 
    1186             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1187           0 :   if ((Subtarget->hasDQI())) {
    1188           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1189             :   }
    1190             :   return 0;
    1191             : }
    1192             : 
    1193           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1194           0 : switch (RetVT.SimpleTy) {
    1195           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1196           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1197             :   default: return 0;
    1198             : }
    1199             : }
    1200             : 
    1201             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1202           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1203             :     return 0;
    1204           0 :   if ((Subtarget->hasAVX512())) {
    1205           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1206             :   }
    1207             :   return 0;
    1208             : }
    1209             : 
    1210           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1211           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1212             :     return 0;
    1213           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1214           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1215             :   }
    1216             :   return 0;
    1217             : }
    1218             : 
    1219             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1220           0 :   if ((Subtarget->hasVLX())) {
    1221           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1222             :   }
    1223             :   return 0;
    1224             : }
    1225             : 
    1226           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1227           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1228           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1229             :   }
    1230             :   return 0;
    1231             : }
    1232             : 
    1233           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1234           0 : switch (RetVT.SimpleTy) {
    1235           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1236           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1237             :   default: return 0;
    1238             : }
    1239             : }
    1240             : 
    1241             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1242           0 :   if ((Subtarget->hasAVX512())) {
    1243           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1244             :   }
    1245             :   return 0;
    1246             : }
    1247             : 
    1248             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1249           0 :   if ((Subtarget->hasDQI())) {
    1250           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1251             :   }
    1252             :   return 0;
    1253             : }
    1254             : 
    1255           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1256           0 : switch (RetVT.SimpleTy) {
    1257           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1258           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1259             :   default: return 0;
    1260             : }
    1261             : }
    1262             : 
    1263           0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1264           0 :   switch (VT.SimpleTy) {
    1265           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1266           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1267           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1268           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1269           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1270           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1271           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1272           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1273             :   default: return 0;
    1274             :   }
    1275             : }
    1276             : 
    1277             : // FastEmit functions for ISD::FSIN.
    1278             : 
    1279             : unsigned fastEmit_ISD_FSIN_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1280           0 :   if (RetVT.SimpleTy != MVT::f32)
    1281             :     return 0;
    1282           0 :   if ((!Subtarget->hasSSE1())) {
    1283           0 :     return fastEmitInst_r(X86::SIN_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1284             :   }
    1285             :   return 0;
    1286             : }
    1287             : 
    1288             : unsigned fastEmit_ISD_FSIN_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1289           0 :   if (RetVT.SimpleTy != MVT::f64)
    1290             :     return 0;
    1291           0 :   if ((!Subtarget->hasSSE2())) {
    1292           0 :     return fastEmitInst_r(X86::SIN_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1293             :   }
    1294             :   return 0;
    1295             : }
    1296             : 
    1297             : unsigned fastEmit_ISD_FSIN_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1298           0 :   if (RetVT.SimpleTy != MVT::f80)
    1299             :     return 0;
    1300           0 :   return fastEmitInst_r(X86::SIN_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1301             : }
    1302             : 
    1303           0 : unsigned fastEmit_ISD_FSIN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1304           0 :   switch (VT.SimpleTy) {
    1305           0 :   case MVT::f32: return fastEmit_ISD_FSIN_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1306           0 :   case MVT::f64: return fastEmit_ISD_FSIN_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1307           0 :   case MVT::f80: return fastEmit_ISD_FSIN_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1308             :   default: return 0;
    1309             :   }
    1310             : }
    1311             : 
    1312             : // FastEmit functions for ISD::FSQRT.
    1313             : 
    1314           0 : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1315           0 :   if (RetVT.SimpleTy != MVT::f32)
    1316             :     return 0;
    1317           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1318           0 :     return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    1319             :   }
    1320           0 :   if ((!Subtarget->hasSSE1())) {
    1321           0 :     return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1322             :   }
    1323             :   return 0;
    1324             : }
    1325             : 
    1326           0 : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1327           0 :   if (RetVT.SimpleTy != MVT::f64)
    1328             :     return 0;
    1329           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1330           0 :     return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
    1331             :   }
    1332           0 :   if ((!Subtarget->hasSSE2())) {
    1333           0 :     return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1334             :   }
    1335             :   return 0;
    1336             : }
    1337             : 
    1338             : unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1339           0 :   if (RetVT.SimpleTy != MVT::f80)
    1340             :     return 0;
    1341           0 :   return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1342             : }
    1343             : 
    1344           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1345           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1346             :     return 0;
    1347           0 :   if ((Subtarget->hasVLX())) {
    1348           0 :     return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1349             :   }
    1350           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1351           0 :     return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1352             :   }
    1353           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1354           0 :     return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1355             :   }
    1356             :   return 0;
    1357             : }
    1358             : 
    1359           0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1360           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    1361             :     return 0;
    1362           0 :   if ((Subtarget->hasVLX())) {
    1363           0 :     return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1364             :   }
    1365           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1366           0 :     return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1367             :   }
    1368             :   return 0;
    1369             : }
    1370             : 
    1371             : unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1372           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1373             :     return 0;
    1374           0 :   if ((Subtarget->hasAVX512())) {
    1375           0 :     return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1376             :   }
    1377             :   return 0;
    1378             : }
    1379             : 
    1380           0 : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1381           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1382             :     return 0;
    1383           0 :   if ((Subtarget->hasVLX())) {
    1384           0 :     return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1385             :   }
    1386           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1387           0 :     return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1388             :   }
    1389           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1390           0 :     return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1391             :   }
    1392             :   return 0;
    1393             : }
    1394             : 
    1395           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1396           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    1397             :     return 0;
    1398           0 :   if ((Subtarget->hasVLX())) {
    1399           0 :     return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1400             :   }
    1401           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1402           0 :     return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1403             :   }
    1404             :   return 0;
    1405             : }
    1406             : 
    1407             : unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1408           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    1409             :     return 0;
    1410           0 :   if ((Subtarget->hasAVX512())) {
    1411           0 :     return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1412             :   }
    1413             :   return 0;
    1414             : }
    1415             : 
    1416           0 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1417           0 :   switch (VT.SimpleTy) {
    1418           0 :   case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1419           0 :   case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1420           0 :   case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1421           0 :   case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1422           0 :   case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1423           0 :   case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1424           0 :   case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1425           0 :   case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1426           0 :   case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1427             :   default: return 0;
    1428             :   }
    1429             : }
    1430             : 
    1431             : // FastEmit functions for ISD::SCALAR_TO_VECTOR.
    1432             : 
    1433           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1434           2 :   if ((Subtarget->hasAVX512())) {
    1435           0 :     return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1436             :   }
    1437           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1438           0 :     return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1439             :   }
    1440           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1441           1 :     return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1442             :   }
    1443             :   return 0;
    1444             : }
    1445             : 
    1446             : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
    1447           0 :   if ((Subtarget->hasMMX())) {
    1448           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    1449             :   }
    1450             :   return 0;
    1451             : }
    1452             : 
    1453           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1454           1 : switch (RetVT.SimpleTy) {
    1455           1 :   case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    1456           0 :   case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill);
    1457             :   default: return 0;
    1458             : }
    1459             : }
    1460             : 
    1461           0 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1462           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1463             :     return 0;
    1464           0 :   if ((Subtarget->hasAVX512())) {
    1465           0 :     return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1466             :   }
    1467           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1468           0 :     return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1469             :   }
    1470           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1471           0 :     return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1472             :   }
    1473             :   return 0;
    1474             : }
    1475             : 
    1476           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1477           1 :   switch (VT.SimpleTy) {
    1478           1 :   case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1479           0 :   case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1480             :   default: return 0;
    1481             :   }
    1482             : }
    1483             : 
    1484             : // FastEmit functions for ISD::SIGN_EXTEND.
    1485             : 
    1486             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1487         417 :   return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    1488             : }
    1489             : 
    1490             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1491           6 :   return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
    1492             : }
    1493             : 
    1494         431 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1495         431 : switch (RetVT.SimpleTy) {
    1496         834 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill);
    1497          12 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill);
    1498             :   default: return 0;
    1499             : }
    1500             : }
    1501             : 
    1502             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1503          67 :   return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    1504             : }
    1505             : 
    1506             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1507           4 :   return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
    1508             : }
    1509             : 
    1510          71 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1511          71 : switch (RetVT.SimpleTy) {
    1512         134 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill);
    1513           8 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill);
    1514             :   default: return 0;
    1515             : }
    1516             : }
    1517             : 
    1518             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1519         867 :   if (RetVT.SimpleTy != MVT::i64)
    1520             :     return 0;
    1521         867 :   if ((Subtarget->is64Bit())) {
    1522         867 :     return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
    1523             :   }
    1524             :   return 0;
    1525             : }
    1526             : 
    1527           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1528           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1529             :     return 0;
    1530           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1531           0 :     return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1532             :   }
    1533             :   return 0;
    1534             : }
    1535             : 
    1536           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1537           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1538           0 :     return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1539             :   }
    1540             :   return 0;
    1541             : }
    1542             : 
    1543           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1544           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1545           0 :     return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1546             :   }
    1547             :   return 0;
    1548             : }
    1549             : 
    1550           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1551           0 : switch (RetVT.SimpleTy) {
    1552           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
    1553           0 :   case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill);
    1554             :   default: return 0;
    1555             : }
    1556             : }
    1557             : 
    1558           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1559           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1560           0 :     return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1561             :   }
    1562             :   return 0;
    1563             : }
    1564             : 
    1565           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1566           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1567           0 :     return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1568             :   }
    1569             :   return 0;
    1570             : }
    1571             : 
    1572             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1573           0 :   if ((Subtarget->hasDQI())) {
    1574           0 :     return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1575             :   }
    1576             :   return 0;
    1577             : }
    1578             : 
    1579           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1580           0 : switch (RetVT.SimpleTy) {
    1581           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
    1582           0 :   case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill);
    1583           0 :   case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    1584             :   default: return 0;
    1585             : }
    1586             : }
    1587             : 
    1588           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1589           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1590           0 :     return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1591             :   }
    1592             :   return 0;
    1593             : }
    1594             : 
    1595           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    1596           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1597           0 :     return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1598             :   }
    1599             :   return 0;
    1600             : }
    1601             : 
    1602             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    1603           0 :   if ((Subtarget->hasDQI())) {
    1604           0 :     return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1605             :   }
    1606             :   return 0;
    1607             : }
    1608             : 
    1609           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1610           0 : switch (RetVT.SimpleTy) {
    1611           0 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
    1612           0 :   case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill);
    1613           0 :   case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    1614             :   default: return 0;
    1615             : }
    1616             : }
    1617             : 
    1618           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    1619           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1620           0 :     return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1621             :   }
    1622             :   return 0;
    1623             : }
    1624             : 
    1625             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    1626           0 :   if ((Subtarget->hasBWI())) {
    1627           0 :     return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1628             :   }
    1629             :   return 0;
    1630             : }
    1631             : 
    1632           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1633           0 : switch (RetVT.SimpleTy) {
    1634           0 :   case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
    1635           0 :   case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
    1636             :   default: return 0;
    1637             : }
    1638             : }
    1639             : 
    1640             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1641           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    1642             :     return 0;
    1643           0 :   if ((Subtarget->hasBWI())) {
    1644           0 :     return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1645             :   }
    1646             :   return 0;
    1647             : }
    1648             : 
    1649        1375 : unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1650        1375 :   switch (VT.SimpleTy) {
    1651         431 :   case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    1652          71 :   case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    1653         867 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1654           0 :   case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill);
    1655           0 :   case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill);
    1656           0 :   case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    1657           0 :   case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    1658           0 :   case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill);
    1659           0 :   case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill);
    1660             :   default: return 0;
    1661             :   }
    1662             : }
    1663             : 
    1664             : // FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
    1665             : 
    1666           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1667           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1668           0 :     return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1669             :   }
    1670           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    1671           0 :     return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1672             :   }
    1673           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1674           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1675             :   }
    1676             :   return 0;
    1677             : }
    1678             : 
    1679           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1680           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1681           0 :     return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1682             :   }
    1683           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1684           0 :     return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1685             :   }
    1686           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1687           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1688             :   }
    1689             :   return 0;
    1690             : }
    1691             : 
    1692           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1693           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1694           0 :     return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1695             :   }
    1696           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1697           0 :     return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1698             :   }
    1699           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1700           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1701             :   }
    1702             :   return 0;
    1703             : }
    1704             : 
    1705           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1706           0 : switch (RetVT.SimpleTy) {
    1707           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1708           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1709           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1710             :   default: return 0;
    1711             : }
    1712             : }
    1713             : 
    1714           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1715           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1716           0 :     return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1717             :   }
    1718           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1719           0 :     return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1720             :   }
    1721           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1722           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1723             :   }
    1724             :   return 0;
    1725             : }
    1726             : 
    1727           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1728           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1729           0 :     return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1730             :   }
    1731           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1732           0 :     return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1733             :   }
    1734           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1735           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1736             :   }
    1737             :   return 0;
    1738             : }
    1739             : 
    1740           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1741           0 : switch (RetVT.SimpleTy) {
    1742           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    1743           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    1744             :   default: return 0;
    1745             : }
    1746             : }
    1747             : 
    1748           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1749           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1750             :     return 0;
    1751           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1752           0 :     return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1753             :   }
    1754           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1755           0 :     return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1756             :   }
    1757           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1758           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1759             :   }
    1760             :   return 0;
    1761             : }
    1762             : 
    1763           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1764           0 :   switch (VT.SimpleTy) {
    1765           0 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1766           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1767           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1768             :   default: return 0;
    1769             :   }
    1770             : }
    1771             : 
    1772             : // FastEmit functions for ISD::SINT_TO_FP.
    1773             : 
    1774          12 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1775          24 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1776           9 :     return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1777             :   }
    1778             :   return 0;
    1779             : }
    1780             : 
    1781          14 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1782          28 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1783           9 :     return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1784             :   }
    1785             :   return 0;
    1786             : }
    1787             : 
    1788          26 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1789          26 : switch (RetVT.SimpleTy) {
    1790          12 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    1791          14 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    1792             :   default: return 0;
    1793             : }
    1794             : }
    1795             : 
    1796           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1797          12 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1798           3 :     return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1799             :   }
    1800             :   return 0;
    1801             : }
    1802             : 
    1803           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1804          12 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1805           3 :     return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1806             :   }
    1807             :   return 0;
    1808             : }
    1809             : 
    1810          12 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1811          12 : switch (RetVT.SimpleTy) {
    1812           6 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    1813           6 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    1814             :   default: return 0;
    1815             : }
    1816             : }
    1817             : 
    1818           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1819           0 :   if ((Subtarget->hasVLX())) {
    1820           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1821             :   }
    1822           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1823           0 :     return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1824             :   }
    1825           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1826           0 :     return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1827             :   }
    1828             :   return 0;
    1829             : }
    1830             : 
    1831           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1832           2 :   if ((Subtarget->hasVLX())) {
    1833           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1834             :   }
    1835           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1836           2 :     return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1837             :   }
    1838             :   return 0;
    1839             : }
    1840             : 
    1841           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1842           2 : switch (RetVT.SimpleTy) {
    1843           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    1844           2 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    1845             :   default: return 0;
    1846             : }
    1847             : }
    1848             : 
    1849           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1850           0 :   if ((Subtarget->hasVLX())) {
    1851           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1852             :   }
    1853           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1854           0 :     return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1855             :   }
    1856             :   return 0;
    1857             : }
    1858             : 
    1859             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1860           0 :   if ((Subtarget->hasAVX512())) {
    1861           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1862             :   }
    1863             :   return 0;
    1864             : }
    1865             : 
    1866           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1867           0 : switch (RetVT.SimpleTy) {
    1868           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    1869           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    1870             :   default: return 0;
    1871             : }
    1872             : }
    1873             : 
    1874             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1875           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1876             :     return 0;
    1877           0 :   if ((Subtarget->hasAVX512())) {
    1878           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1879             :   }
    1880             :   return 0;
    1881             : }
    1882             : 
    1883           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1884           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1885             :     return 0;
    1886           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1887           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1888             :   }
    1889             :   return 0;
    1890             : }
    1891             : 
    1892           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1893           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1894           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1895             :   }
    1896             :   return 0;
    1897             : }
    1898             : 
    1899           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1900           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1901           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1902             :   }
    1903             :   return 0;
    1904             : }
    1905             : 
    1906           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1907           0 : switch (RetVT.SimpleTy) {
    1908           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    1909           0 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    1910             :   default: return 0;
    1911             : }
    1912             : }
    1913             : 
    1914             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1915           0 :   if ((Subtarget->hasDQI())) {
    1916           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1917             :   }
    1918             :   return 0;
    1919             : }
    1920             : 
    1921             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1922           0 :   if ((Subtarget->hasDQI())) {
    1923           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1924             :   }
    1925             :   return 0;
    1926             : }
    1927             : 
    1928           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1929           0 : switch (RetVT.SimpleTy) {
    1930           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    1931           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    1932             :   default: return 0;
    1933             : }
    1934             : }
    1935             : 
    1936          40 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1937          40 :   switch (VT.SimpleTy) {
    1938          26 :   case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1939          12 :   case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1940           2 :   case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1941           0 :   case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    1942           0 :   case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    1943           0 :   case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1944           0 :   case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    1945           0 :   case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    1946             :   default: return 0;
    1947             :   }
    1948             : }
    1949             : 
    1950             : // FastEmit functions for ISD::TRUNCATE.
    1951             : 
    1952             : unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1953           2 :   if (RetVT.SimpleTy != MVT::i8)
    1954             :     return 0;
    1955           2 :   if ((Subtarget->is64Bit())) {
    1956           1 :     return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
    1957             :   }
    1958             :   return 0;
    1959             : }
    1960             : 
    1961             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1962          59 :   if ((Subtarget->is64Bit())) {
    1963         110 :     return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1964             :   }
    1965             :   return 0;
    1966             : }
    1967             : 
    1968             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1969          62 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1970             : }
    1971             : 
    1972          90 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1973          90 : switch (RetVT.SimpleTy) {
    1974          59 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill);
    1975          62 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill);
    1976             :   default: return 0;
    1977             : }
    1978             : }
    1979             : 
    1980             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1981          18 :   return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1982             : }
    1983             : 
    1984             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1985           4 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1986             : }
    1987             : 
    1988             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1989         302 :   return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
    1990             : }
    1991             : 
    1992         162 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1993         162 : switch (RetVT.SimpleTy) {
    1994          18 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill);
    1995           4 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill);
    1996         302 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill);
    1997             :   default: return 0;
    1998             : }
    1999             : }
    2000             : 
    2001           0 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2002           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2003             :     return 0;
    2004           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2005           0 :     return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2006             :   }
    2007             :   return 0;
    2008             : }
    2009             : 
    2010             : unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2011           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    2012             :     return 0;
    2013           0 :   if ((Subtarget->hasBWI())) {
    2014           0 :     return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2015             :   }
    2016             :   return 0;
    2017             : }
    2018             : 
    2019           0 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2020           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    2021             :     return 0;
    2022           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2023           0 :     return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2024             :   }
    2025             :   return 0;
    2026             : }
    2027             : 
    2028             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2029           0 :   if ((Subtarget->hasAVX512())) {
    2030           0 :     return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2031             :   }
    2032             :   return 0;
    2033             : }
    2034             : 
    2035             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    2036           0 :   if ((Subtarget->hasAVX512())) {
    2037           0 :     return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2038             :   }
    2039             :   return 0;
    2040             : }
    2041             : 
    2042           0 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2043           0 : switch (RetVT.SimpleTy) {
    2044           0 :   case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    2045           0 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    2046             :   default: return 0;
    2047             : }
    2048             : }
    2049             : 
    2050           2 : unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2051           2 :   if (RetVT.SimpleTy != MVT::v4i32)
    2052             :     return 0;
    2053           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2054           0 :     return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2055             :   }
    2056             :   return 0;
    2057             : }
    2058             : 
    2059             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2060           0 :   if ((Subtarget->hasAVX512())) {
    2061           0 :     return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2062             :   }
    2063             :   return 0;
    2064             : }
    2065             : 
    2066             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2067           0 :   if ((Subtarget->hasAVX512())) {
    2068           0 :     return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2069             :   }
    2070             :   return 0;
    2071             : }
    2072             : 
    2073           4 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2074           4 : switch (RetVT.SimpleTy) {
    2075           0 :   case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    2076           0 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    2077             :   default: return 0;
    2078             : }
    2079             : }
    2080             : 
    2081         263 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2082         263 :   switch (VT.SimpleTy) {
    2083           2 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2084          90 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2085         162 :   case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2086           0 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2087           0 :   case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2088           0 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2089           0 :   case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2090           2 :   case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2091           4 :   case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2092             :   default: return 0;
    2093             :   }
    2094             : }
    2095             : 
    2096             : // FastEmit functions for ISD::UINT_TO_FP.
    2097             : 
    2098             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2099           0 :   if ((Subtarget->hasVLX())) {
    2100           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2101             :   }
    2102             :   return 0;
    2103             : }
    2104             : 
    2105             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2106           0 :   if ((Subtarget->hasVLX())) {
    2107           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2108             :   }
    2109             :   return 0;
    2110             : }
    2111             : 
    2112           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2113           0 : switch (RetVT.SimpleTy) {
    2114           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    2115           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    2116             :   default: return 0;
    2117             : }
    2118             : }
    2119             : 
    2120             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2121           0 :   if ((Subtarget->hasVLX())) {
    2122           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2123             :   }
    2124             :   return 0;
    2125             : }
    2126             : 
    2127             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2128           0 :   if ((Subtarget->hasAVX512())) {
    2129           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2130             :   }
    2131             :   return 0;
    2132             : }
    2133             : 
    2134           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2135           0 : switch (RetVT.SimpleTy) {
    2136           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    2137           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    2138             :   default: return 0;
    2139             : }
    2140             : }
    2141             : 
    2142             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2143           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2144             :     return 0;
    2145           0 :   if ((Subtarget->hasAVX512())) {
    2146           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2147             :   }
    2148             :   return 0;
    2149             : }
    2150             : 
    2151           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2152           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2153             :     return 0;
    2154           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2155           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2156             :   }
    2157             :   return 0;
    2158             : }
    2159             : 
    2160           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2161           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2162           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2163             :   }
    2164             :   return 0;
    2165             : }
    2166             : 
    2167           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2168           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2169           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2170             :   }
    2171             :   return 0;
    2172             : }
    2173             : 
    2174           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2175           0 : switch (RetVT.SimpleTy) {
    2176           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    2177           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    2178             :   default: return 0;
    2179             : }
    2180             : }
    2181             : 
    2182             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2183           0 :   if ((Subtarget->hasDQI())) {
    2184           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2185             :   }
    2186             :   return 0;
    2187             : }
    2188             : 
    2189             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2190           0 :   if ((Subtarget->hasDQI())) {
    2191           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2192             :   }
    2193             :   return 0;
    2194             : }
    2195             : 
    2196           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2197           0 : switch (RetVT.SimpleTy) {
    2198           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    2199           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    2200             :   default: return 0;
    2201             : }
    2202             : }
    2203             : 
    2204           0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2205           0 :   switch (VT.SimpleTy) {
    2206           0 :   case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2207           0 :   case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2208           0 :   case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2209           0 :   case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2210           0 :   case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2211           0 :   case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2212             :   default: return 0;
    2213             :   }
    2214             : }
    2215             : 
    2216             : // FastEmit functions for ISD::ZERO_EXTEND.
    2217             : 
    2218             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2219        1368 :   if (RetVT.SimpleTy != MVT::i32)
    2220             :     return 0;
    2221        1354 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    2222             : }
    2223             : 
    2224             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2225          76 :   if (RetVT.SimpleTy != MVT::i32)
    2226             :     return 0;
    2227          74 :   return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    2228             : }
    2229             : 
    2230        1500 : unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2231        1500 :   switch (VT.SimpleTy) {
    2232        1368 :   case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    2233          76 :   case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2234             :   default: return 0;
    2235             :   }
    2236             : }
    2237             : 
    2238             : // FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
    2239             : 
    2240           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2241           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2242           0 :     return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2243             :   }
    2244           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    2245           0 :     return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2246             :   }
    2247           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2248           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2249             :   }
    2250             :   return 0;
    2251             : }
    2252             : 
    2253           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2254           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2255           0 :     return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2256             :   }
    2257           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2258           0 :     return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2259             :   }
    2260           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2261           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2262             :   }
    2263             :   return 0;
    2264             : }
    2265             : 
    2266           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2267           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2268           0 :     return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2269             :   }
    2270           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2271           0 :     return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2272             :   }
    2273           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2274           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2275             :   }
    2276             :   return 0;
    2277             : }
    2278             : 
    2279           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2280           0 : switch (RetVT.SimpleTy) {
    2281           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    2282           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    2283           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    2284             :   default: return 0;
    2285             : }
    2286             : }
    2287             : 
    2288           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2289           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2290           0 :     return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2291             :   }
    2292           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2293           0 :     return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2294             :   }
    2295           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2296           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2297             :   }
    2298             :   return 0;
    2299             : }
    2300             : 
    2301           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2302           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2303           0 :     return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2304             :   }
    2305           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2306           0 :     return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2307             :   }
    2308           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2309           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2310             :   }
    2311             :   return 0;
    2312             : }
    2313             : 
    2314           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2315           0 : switch (RetVT.SimpleTy) {
    2316           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    2317           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    2318             :   default: return 0;
    2319             : }
    2320             : }
    2321             : 
    2322           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2323           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2324             :     return 0;
    2325           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2326           0 :     return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2327             :   }
    2328           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2329           0 :     return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2330             :   }
    2331           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2332           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2333             :   }
    2334             :   return 0;
    2335             : }
    2336             : 
    2337           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2338           0 :   switch (VT.SimpleTy) {
    2339           0 :   case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2340           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2341           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2342             :   default: return 0;
    2343             :   }
    2344             : }
    2345             : 
    2346             : // FastEmit functions for X86ISD::CALL.
    2347             : 
    2348             : unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2349           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2350             :     return 0;
    2351           0 :   if ((!Subtarget->is64Bit())) {
    2352           0 :     return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
    2353             :   }
    2354             :   return 0;
    2355             : }
    2356             : 
    2357           0 : unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2358           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2359             :     return 0;
    2360           0 :   if ((!Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
    2361           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL32, &X86::GR32RegClass, Op0, Op0IsKill);
    2362             :   }
    2363           0 :   if ((!Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
    2364           0 :     return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
    2365             :   }
    2366             :   return 0;
    2367             : }
    2368             : 
    2369           0 : unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2370           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2371             :     return 0;
    2372           0 :   if ((Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
    2373           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL64, &X86::GR64RegClass, Op0, Op0IsKill);
    2374             :   }
    2375           0 :   if ((Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
    2376           0 :     return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
    2377             :   }
    2378             :   return 0;
    2379             : }
    2380             : 
    2381           0 : unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2382           0 :   switch (VT.SimpleTy) {
    2383           0 :   case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2384           0 :   case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2385           0 :   case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2386             :   default: return 0;
    2387             :   }
    2388             : }
    2389             : 
    2390             : // FastEmit functions for X86ISD::COMPRESS.
    2391             : 
    2392           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2393           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2394             :     return 0;
    2395           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2396           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2397             :   }
    2398             :   return 0;
    2399             : }
    2400             : 
    2401           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2402           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    2403             :     return 0;
    2404           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2405           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2406             :   }
    2407             :   return 0;
    2408             : }
    2409             : 
    2410             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2411           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    2412             :     return 0;
    2413           0 :   if ((Subtarget->hasVBMI2())) {
    2414           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2415             :   }
    2416             :   return 0;
    2417             : }
    2418             : 
    2419           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2420           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    2421             :     return 0;
    2422           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2423           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2424             :   }
    2425             :   return 0;
    2426             : }
    2427             : 
    2428           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2429           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    2430             :     return 0;
    2431           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2432           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2433             :   }
    2434             :   return 0;
    2435             : }
    2436             : 
    2437             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2438           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    2439             :     return 0;
    2440           0 :   if ((Subtarget->hasVBMI2())) {
    2441           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2442             :   }
    2443             :   return 0;
    2444             : }
    2445             : 
    2446           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2447           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2448             :     return 0;
    2449           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2450           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2451             :   }
    2452             :   return 0;
    2453             : }
    2454             : 
    2455           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2456           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2457             :     return 0;
    2458           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2459           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2460             :   }
    2461             :   return 0;
    2462             : }
    2463             : 
    2464             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2465           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2466             :     return 0;
    2467           0 :   if ((Subtarget->hasAVX512())) {
    2468           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2469             :   }
    2470             :   return 0;
    2471             : }
    2472             : 
    2473           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2474           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2475             :     return 0;
    2476           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2477           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2478             :   }
    2479             :   return 0;
    2480             : }
    2481             : 
    2482           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2483           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2484             :     return 0;
    2485           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2486           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2487             :   }
    2488             :   return 0;
    2489             : }
    2490             : 
    2491             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2492           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2493             :     return 0;
    2494           0 :   if ((Subtarget->hasAVX512())) {
    2495           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2496             :   }
    2497             :   return 0;
    2498             : }
    2499             : 
    2500           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2501           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2502             :     return 0;
    2503           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2504           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2505             :   }
    2506             :   return 0;
    2507             : }
    2508             : 
    2509           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2510           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    2511             :     return 0;
    2512           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2513           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2514             :   }
    2515             :   return 0;
    2516             : }
    2517             : 
    2518             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2519           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2520             :     return 0;
    2521           0 :   if ((Subtarget->hasAVX512())) {
    2522           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2523             :   }
    2524             :   return 0;
    2525             : }
    2526             : 
    2527           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2528           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2529             :     return 0;
    2530           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2531           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2532             :   }
    2533             :   return 0;
    2534             : }
    2535             : 
    2536           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2537           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    2538             :     return 0;
    2539           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2540           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2541             :   }
    2542             :   return 0;
    2543             : }
    2544             : 
    2545             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2546           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    2547             :     return 0;
    2548           0 :   if ((Subtarget->hasAVX512())) {
    2549           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2550             :   }
    2551             :   return 0;
    2552             : }
    2553             : 
    2554           0 : unsigned fastEmit_X86ISD_COMPRESS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2555           0 :   switch (VT.SimpleTy) {
    2556           0 :   case MVT::v16i8: return fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2557           0 :   case MVT::v32i8: return fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    2558           0 :   case MVT::v64i8: return fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    2559           0 :   case MVT::v8i16: return fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2560           0 :   case MVT::v16i16: return fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2561           0 :   case MVT::v32i16: return fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2562           0 :   case MVT::v4i32: return fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2563           0 :   case MVT::v8i32: return fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2564           0 :   case MVT::v16i32: return fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2565           0 :   case MVT::v2i64: return fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2566           0 :   case MVT::v4i64: return fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2567           0 :   case MVT::v8i64: return fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2568           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2569           0 :   case MVT::v8f32: return fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2570           0 :   case MVT::v16f32: return fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2571           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2572           0 :   case MVT::v4f64: return fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2573           0 :   case MVT::v8f64: return fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2574             :   default: return 0;
    2575             :   }
    2576             : }
    2577             : 
    2578             : // FastEmit functions for X86ISD::CONFLICT.
    2579             : 
    2580           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2581           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2582             :     return 0;
    2583           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2584           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2585             :   }
    2586             :   return 0;
    2587             : }
    2588             : 
    2589           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2590           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2591             :     return 0;
    2592           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2593           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2594             :   }
    2595             :   return 0;
    2596             : }
    2597             : 
    2598             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2599           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2600             :     return 0;
    2601           0 :   if ((Subtarget->hasCDI())) {
    2602           0 :     return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2603             :   }
    2604             :   return 0;
    2605             : }
    2606             : 
    2607           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2608           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2609             :     return 0;
    2610           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2611           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2612             :   }
    2613             :   return 0;
    2614             : }
    2615             : 
    2616           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2617           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2618             :     return 0;
    2619           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2620           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2621             :   }
    2622             :   return 0;
    2623             : }
    2624             : 
    2625             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2626           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2627             :     return 0;
    2628           0 :   if ((Subtarget->hasCDI())) {
    2629           0 :     return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2630             :   }
    2631             :   return 0;
    2632             : }
    2633             : 
    2634           0 : unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2635           0 :   switch (VT.SimpleTy) {
    2636           0 :   case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2637           0 :   case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2638           0 :   case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2639           0 :   case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2640           0 :   case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2641           0 :   case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2642             :   default: return 0;
    2643             :   }
    2644             : }
    2645             : 
    2646             : // FastEmit functions for X86ISD::CVTP2SI.
    2647             : 
    2648             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2649           0 :   if ((Subtarget->hasVLX())) {
    2650           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2651             :   }
    2652             :   return 0;
    2653             : }
    2654             : 
    2655           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2656           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2657           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2658             :   }
    2659             :   return 0;
    2660             : }
    2661             : 
    2662           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2663           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2664           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2665             :   }
    2666             :   return 0;
    2667             : }
    2668             : 
    2669           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2670           0 : switch (RetVT.SimpleTy) {
    2671           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2672           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2673           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2674             :   default: return 0;
    2675             : }
    2676             : }
    2677             : 
    2678             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2679           0 :   if ((Subtarget->hasVLX())) {
    2680           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2681             :   }
    2682             :   return 0;
    2683             : }
    2684             : 
    2685             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2686           0 :   if ((Subtarget->hasDQI())) {
    2687           0 :     return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2688             :   }
    2689             :   return 0;
    2690             : }
    2691             : 
    2692           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2693           0 : switch (RetVT.SimpleTy) {
    2694           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2695           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2696             :   default: return 0;
    2697             : }
    2698             : }
    2699             : 
    2700             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2701           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2702             :     return 0;
    2703           0 :   if ((Subtarget->hasAVX512())) {
    2704           0 :     return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2705             :   }
    2706             :   return 0;
    2707             : }
    2708             : 
    2709           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2710           0 :   if ((Subtarget->hasVLX())) {
    2711           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2712             :   }
    2713           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2714           0 :     return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2715             :   }
    2716           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2717           0 :     return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2718             :   }
    2719             :   return 0;
    2720             : }
    2721             : 
    2722           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2723           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2724           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2725             :   }
    2726             :   return 0;
    2727             : }
    2728             : 
    2729           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2730           0 : switch (RetVT.SimpleTy) {
    2731           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2732           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2733             :   default: return 0;
    2734             : }
    2735             : }
    2736             : 
    2737           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2738           0 :   if ((Subtarget->hasVLX())) {
    2739           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2740             :   }
    2741           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2742           0 :     return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2743             :   }
    2744             :   return 0;
    2745             : }
    2746             : 
    2747           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2748           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2749           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2750             :   }
    2751             :   return 0;
    2752             : }
    2753             : 
    2754           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2755           0 : switch (RetVT.SimpleTy) {
    2756           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2757           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2758             :   default: return 0;
    2759             : }
    2760             : }
    2761             : 
    2762             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2763           0 :   if ((Subtarget->hasAVX512())) {
    2764           0 :     return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2765             :   }
    2766             :   return 0;
    2767             : }
    2768             : 
    2769             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2770           0 :   if ((Subtarget->hasDQI())) {
    2771           0 :     return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2772             :   }
    2773             :   return 0;
    2774             : }
    2775             : 
    2776           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2777           0 : switch (RetVT.SimpleTy) {
    2778           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2779           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2780             :   default: return 0;
    2781             : }
    2782             : }
    2783             : 
    2784           0 : unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2785           0 :   switch (VT.SimpleTy) {
    2786           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2787           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2788           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2789           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2790           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2791           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2792             :   default: return 0;
    2793             :   }
    2794             : }
    2795             : 
    2796             : // FastEmit functions for X86ISD::CVTP2UI.
    2797             : 
    2798             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2799           0 :   if ((Subtarget->hasVLX())) {
    2800           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2801             :   }
    2802             :   return 0;
    2803             : }
    2804             : 
    2805           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2806           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2807           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2808             :   }
    2809             :   return 0;
    2810             : }
    2811             : 
    2812           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2813           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2814           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2815             :   }
    2816             :   return 0;
    2817             : }
    2818             : 
    2819           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2820           0 : switch (RetVT.SimpleTy) {
    2821           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2822           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2823           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2824             :   default: return 0;
    2825             : }
    2826             : }
    2827             : 
    2828             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2829           0 :   if ((Subtarget->hasVLX())) {
    2830           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2831             :   }
    2832             :   return 0;
    2833             : }
    2834             : 
    2835             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2836           0 :   if ((Subtarget->hasDQI())) {
    2837           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2838             :   }
    2839             :   return 0;
    2840             : }
    2841             : 
    2842           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2843           0 : switch (RetVT.SimpleTy) {
    2844           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2845           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2846             :   default: return 0;
    2847             : }
    2848             : }
    2849             : 
    2850             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2851           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2852             :     return 0;
    2853           0 :   if ((Subtarget->hasAVX512())) {
    2854           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2855             :   }
    2856             :   return 0;
    2857             : }
    2858             : 
    2859             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2860           0 :   if ((Subtarget->hasVLX())) {
    2861           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2862             :   }
    2863             :   return 0;
    2864             : }
    2865             : 
    2866           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2867           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2868           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2869             :   }
    2870             :   return 0;
    2871             : }
    2872             : 
    2873           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2874           0 : switch (RetVT.SimpleTy) {
    2875           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2876           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2877             :   default: return 0;
    2878             : }
    2879             : }
    2880             : 
    2881             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2882           0 :   if ((Subtarget->hasVLX())) {
    2883           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2884             :   }
    2885             :   return 0;
    2886             : }
    2887             : 
    2888           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2889           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2890           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2891             :   }
    2892             :   return 0;
    2893             : }
    2894             : 
    2895           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2896           0 : switch (RetVT.SimpleTy) {
    2897           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2898           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2899             :   default: return 0;
    2900             : }
    2901             : }
    2902             : 
    2903             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2904           0 :   if ((Subtarget->hasAVX512())) {
    2905           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2906             :   }
    2907             :   return 0;
    2908             : }
    2909             : 
    2910             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2911           0 :   if ((Subtarget->hasDQI())) {
    2912           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2913             :   }
    2914             :   return 0;
    2915             : }
    2916             : 
    2917           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2918           0 : switch (RetVT.SimpleTy) {
    2919           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2920           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2921             :   default: return 0;
    2922             : }
    2923             : }
    2924             : 
    2925           0 : unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2926           0 :   switch (VT.SimpleTy) {
    2927           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2928           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2929           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2930           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2931           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2932           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2933             :   default: return 0;
    2934             :   }
    2935             : }
    2936             : 
    2937             : // FastEmit functions for X86ISD::CVTPH2PS.
    2938             : 
    2939           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2940           0 :   if ((Subtarget->hasVLX())) {
    2941           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2942             :   }
    2943           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2944           0 :     return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2945             :   }
    2946             :   return 0;
    2947             : }
    2948             : 
    2949           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2950           0 :   if ((Subtarget->hasVLX())) {
    2951           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2952             :   }
    2953           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2954           0 :     return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    2955             :   }
    2956             :   return 0;
    2957             : }
    2958             : 
    2959           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2960           0 : switch (RetVT.SimpleTy) {
    2961           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
    2962           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
    2963             :   default: return 0;
    2964             : }
    2965             : }
    2966             : 
    2967             : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2968           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2969             :     return 0;
    2970           0 :   if ((Subtarget->hasAVX512())) {
    2971           0 :     return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2972             :   }
    2973             :   return 0;
    2974             : }
    2975             : 
    2976           0 : unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2977           0 :   switch (VT.SimpleTy) {
    2978           0 :   case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2979           0 :   case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2980             :   default: return 0;
    2981             :   }
    2982             : }
    2983             : 
    2984             : // FastEmit functions for X86ISD::CVTSI2P.
    2985             : 
    2986           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2987           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2988             :     return 0;
    2989           0 :   if ((Subtarget->hasVLX())) {
    2990           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2991             :   }
    2992           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2993           0 :     return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2994             :   }
    2995           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2996           0 :     return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2997             :   }
    2998             :   return 0;
    2999             : }
    3000             : 
    3001           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3002           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3003             :     return 0;
    3004           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3005           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3006             :   }
    3007             :   return 0;
    3008             : }
    3009             : 
    3010           0 : unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3011           0 :   switch (VT.SimpleTy) {
    3012           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3013           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3014             :   default: return 0;
    3015             :   }
    3016             : }
    3017             : 
    3018             : // FastEmit functions for X86ISD::CVTTP2SI.
    3019             : 
    3020           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3021           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3022             :     return 0;
    3023           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3024           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3025             :   }
    3026             :   return 0;
    3027             : }
    3028             : 
    3029           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3030           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3031             :     return 0;
    3032           0 :   if ((Subtarget->hasVLX())) {
    3033           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3034             :   }
    3035           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3036           0 :     return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3037             :   }
    3038           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3039           0 :     return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3040             :   }
    3041             :   return 0;
    3042             : }
    3043             : 
    3044           0 : unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3045           0 :   switch (VT.SimpleTy) {
    3046           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3047           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3048             :   default: return 0;
    3049             :   }
    3050             : }
    3051             : 
    3052             : // FastEmit functions for X86ISD::CVTTP2UI.
    3053             : 
    3054           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3055           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3056             :     return 0;
    3057           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3058           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3059             :   }
    3060             :   return 0;
    3061             : }
    3062             : 
    3063             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3064           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3065             :     return 0;
    3066           0 :   if ((Subtarget->hasVLX())) {
    3067           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3068             :   }
    3069             :   return 0;
    3070             : }
    3071             : 
    3072           0 : unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3073           0 :   switch (VT.SimpleTy) {
    3074           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3075           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3076             :   default: return 0;
    3077             :   }
    3078             : }
    3079             : 
    3080             : // FastEmit functions for X86ISD::CVTUI2P.
    3081             : 
    3082             : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3083           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3084             :     return 0;
    3085           0 :   if ((Subtarget->hasVLX())) {
    3086           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3087             :   }
    3088             :   return 0;
    3089             : }
    3090             : 
    3091           0 : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3092           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3093             :     return 0;
    3094           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3095           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3096             :   }
    3097             :   return 0;
    3098             : }
    3099             : 
    3100           0 : unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3101           0 :   switch (VT.SimpleTy) {
    3102           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3103           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3104             :   default: return 0;
    3105             :   }
    3106             : }
    3107             : 
    3108             : // FastEmit functions for X86ISD::EH_RETURN.
    3109             : 
    3110             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3111           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3112             :     return 0;
    3113           0 :   return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
    3114             : }
    3115             : 
    3116             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3117           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3118             :     return 0;
    3119           0 :   return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
    3120             : }
    3121             : 
    3122           0 : unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3123           0 :   switch (VT.SimpleTy) {
    3124           0 :   case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3125           0 :   case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3126             :   default: return 0;
    3127             :   }
    3128             : }
    3129             : 
    3130             : // FastEmit functions for X86ISD::EXPAND.
    3131             : 
    3132           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3133           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    3134             :     return 0;
    3135           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3136           0 :     return fastEmitInst_r(X86::VPEXPANDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3137             :   }
    3138             :   return 0;
    3139             : }
    3140             : 
    3141           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3142           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    3143             :     return 0;
    3144           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3145           0 :     return fastEmitInst_r(X86::VPEXPANDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3146             :   }
    3147             :   return 0;
    3148             : }
    3149             : 
    3150             : unsigned fastEmit_X86ISD_EXPAND_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3151           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    3152             :     return 0;
    3153           0 :   if ((Subtarget->hasVBMI2())) {
    3154           0 :     return fastEmitInst_r(X86::VPEXPANDBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3155             :   }
    3156             :   return 0;
    3157             : }
    3158             : 
    3159           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3160           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3161             :     return 0;
    3162           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3163           0 :     return fastEmitInst_r(X86::VPEXPANDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3164             :   }
    3165             :   return 0;
    3166             : }
    3167             : 
    3168           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3169           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    3170             :     return 0;
    3171           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3172           0 :     return fastEmitInst_r(X86::VPEXPANDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3173             :   }
    3174             :   return 0;
    3175             : }
    3176             : 
    3177             : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3178           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    3179             :     return 0;
    3180           0 :   if ((Subtarget->hasVBMI2())) {
    3181           0 :     return fastEmitInst_r(X86::VPEXPANDWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3182             :   }
    3183             :   return 0;
    3184             : }
    3185             : 
    3186           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3187           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3188             :     return 0;
    3189           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3190           0 :     return fastEmitInst_r(X86::VPEXPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3191             :   }
    3192             :   return 0;
    3193             : }
    3194             : 
    3195           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3196           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3197             :     return 0;
    3198           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3199           0 :     return fastEmitInst_r(X86::VPEXPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3200             :   }
    3201             :   return 0;
    3202             : }
    3203             : 
    3204             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3205           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3206             :     return 0;
    3207           0 :   if ((Subtarget->hasAVX512())) {
    3208           0 :     return fastEmitInst_r(X86::VPEXPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3209             :   }
    3210             :   return 0;
    3211             : }
    3212             : 
    3213           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3214           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3215             :     return 0;
    3216           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3217           0 :     return fastEmitInst_r(X86::VPEXPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3218             :   }
    3219             :   return 0;
    3220             : }
    3221             : 
    3222           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3223           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    3224             :     return 0;
    3225           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3226           0 :     return fastEmitInst_r(X86::VPEXPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3227             :   }
    3228             :   return 0;
    3229             : }
    3230             : 
    3231             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3232           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    3233             :     return 0;
    3234           0 :   if ((Subtarget->hasAVX512())) {
    3235           0 :     return fastEmitInst_r(X86::VPEXPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3236             :   }
    3237             :   return 0;
    3238             : }
    3239             : 
    3240           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3241           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3242             :     return 0;
    3243           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3244           0 :     return fastEmitInst_r(X86::VEXPANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3245             :   }
    3246             :   return 0;
    3247             : }
    3248             : 
    3249           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3250           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3251             :     return 0;
    3252           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3253           0 :     return fastEmitInst_r(X86::VEXPANDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3254             :   }
    3255             :   return 0;
    3256             : }
    3257             : 
    3258             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3259           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3260             :     return 0;
    3261           0 :   if ((Subtarget->hasAVX512())) {
    3262           0 :     return fastEmitInst_r(X86::VEXPANDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3263             :   }
    3264             :   return 0;
    3265             : }
    3266             : 
    3267           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3268           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3269             :     return 0;
    3270           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3271           0 :     return fastEmitInst_r(X86::VEXPANDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3272             :   }
    3273             :   return 0;
    3274             : }
    3275             : 
    3276           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3277           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3278             :     return 0;
    3279           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3280           0 :     return fastEmitInst_r(X86::VEXPANDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3281             :   }
    3282             :   return 0;
    3283             : }
    3284             : 
    3285             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3286           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3287             :     return 0;
    3288           0 :   if ((Subtarget->hasAVX512())) {
    3289           0 :     return fastEmitInst_r(X86::VEXPANDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3290             :   }
    3291             :   return 0;
    3292             : }
    3293             : 
    3294           0 : unsigned fastEmit_X86ISD_EXPAND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3295           0 :   switch (VT.SimpleTy) {
    3296           0 :   case MVT::v16i8: return fastEmit_X86ISD_EXPAND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3297           0 :   case MVT::v32i8: return fastEmit_X86ISD_EXPAND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3298           0 :   case MVT::v64i8: return fastEmit_X86ISD_EXPAND_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    3299           0 :   case MVT::v8i16: return fastEmit_X86ISD_EXPAND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3300           0 :   case MVT::v16i16: return fastEmit_X86ISD_EXPAND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    3301           0 :   case MVT::v32i16: return fastEmit_X86ISD_EXPAND_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    3302           0 :   case MVT::v4i32: return fastEmit_X86ISD_EXPAND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3303           0 :   case MVT::v8i32: return fastEmit_X86ISD_EXPAND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3304           0 :   case MVT::v16i32: return fastEmit_X86ISD_EXPAND_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    3305           0 :   case MVT::v2i64: return fastEmit_X86ISD_EXPAND_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3306           0 :   case MVT::v4i64: return fastEmit_X86ISD_EXPAND_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    3307           0 :   case MVT::v8i64: return fastEmit_X86ISD_EXPAND_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    3308           0 :   case MVT::v4f32: return fastEmit_X86ISD_EXPAND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3309           0 :   case MVT::v8f32: return fastEmit_X86ISD_EXPAND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3310           0 :   case MVT::v16f32: return fastEmit_X86ISD_EXPAND_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3311           0 :   case MVT::v2f64: return fastEmit_X86ISD_EXPAND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3312           0 :   case MVT::v4f64: return fastEmit_X86ISD_EXPAND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3313           0 :   case MVT::v8f64: return fastEmit_X86ISD_EXPAND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3314             :   default: return 0;
    3315             :   }
    3316             : }
    3317             : 
    3318             : // FastEmit functions for X86ISD::FRCP.
    3319             : 
    3320           0 : unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3321           0 :   if (RetVT.SimpleTy != MVT::f32)
    3322             :     return 0;
    3323           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3324           0 :     return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3325             :   }
    3326             :   return 0;
    3327             : }
    3328             : 
    3329           0 : unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3330           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3331             :     return 0;
    3332           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3333           0 :     return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3334             :   }
    3335           0 :   if ((Subtarget->hasAVX())) {
    3336           0 :     return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3337             :   }
    3338             :   return 0;
    3339             : }
    3340             : 
    3341             : unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3342           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3343             :     return 0;
    3344           0 :   if ((Subtarget->hasAVX())) {
    3345           0 :     return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3346             :   }
    3347             :   return 0;
    3348             : }
    3349             : 
    3350           0 : unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3351           0 :   switch (VT.SimpleTy) {
    3352           0 :   case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3353           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3354           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3355             :   default: return 0;
    3356             :   }
    3357             : }
    3358             : 
    3359             : // FastEmit functions for X86ISD::FRSQRT.
    3360             : 
    3361           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3362           0 :   if (RetVT.SimpleTy != MVT::f32)
    3363             :     return 0;
    3364           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3365           0 :     return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3366             :   }
    3367             :   return 0;
    3368             : }
    3369             : 
    3370           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3371           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3372             :     return 0;
    3373           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3374           0 :     return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3375             :   }
    3376           0 :   if ((Subtarget->hasAVX())) {
    3377           0 :     return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3378             :   }
    3379             :   return 0;
    3380             : }
    3381             : 
    3382             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3383           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3384             :     return 0;
    3385           0 :   if ((Subtarget->hasAVX())) {
    3386           0 :     return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3387             :   }
    3388             :   return 0;
    3389             : }
    3390             : 
    3391           0 : unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3392           0 :   switch (VT.SimpleTy) {
    3393           0 :   case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3394           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3395           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3396             :   default: return 0;
    3397             :   }
    3398             : }
    3399             : 
    3400             : // FastEmit functions for X86ISD::MMX_MOVD2W.
    3401             : 
    3402             : unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3403           0 :   if (RetVT.SimpleTy != MVT::i32)
    3404             :     return 0;
    3405           0 :   if ((Subtarget->hasMMX())) {
    3406           0 :     return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
    3407             :   }
    3408             :   return 0;
    3409             : }
    3410             : 
    3411           0 : unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3412           0 :   switch (VT.SimpleTy) {
    3413           0 :   case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
    3414             :   default: return 0;
    3415             :   }
    3416             : }
    3417             : 
    3418             : // FastEmit functions for X86ISD::MMX_MOVW2D.
    3419             : 
    3420             : unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3421           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3422             :     return 0;
    3423           0 :   if ((Subtarget->hasMMX())) {
    3424           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    3425             :   }
    3426             :   return 0;
    3427             : }
    3428             : 
    3429           0 : unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3430           0 :   switch (VT.SimpleTy) {
    3431           0 :   case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3432             :   default: return 0;
    3433             :   }
    3434             : }
    3435             : 
    3436             : // FastEmit functions for X86ISD::MOVDDUP.
    3437             : 
    3438           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3439           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3440             :     return 0;
    3441           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3442           0 :     return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3443             :   }
    3444           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3445           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3446             :   }
    3447             :   return 0;
    3448             : }
    3449             : 
    3450           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3451           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3452             :     return 0;
    3453           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3454           0 :     return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3455             :   }
    3456           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3457           0 :     return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3458             :   }
    3459             :   return 0;
    3460             : }
    3461             : 
    3462             : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3463           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3464             :     return 0;
    3465           0 :   if ((Subtarget->hasAVX512())) {
    3466           0 :     return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3467             :   }
    3468             :   return 0;
    3469             : }
    3470             : 
    3471           0 : unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3472           0 :   switch (VT.SimpleTy) {
    3473           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3474           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3475           0 :   case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3476             :   default: return 0;
    3477             :   }
    3478             : }
    3479             : 
    3480             : // FastEmit functions for X86ISD::MOVDQ2Q.
    3481             : 
    3482             : unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3483           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3484             :     return 0;
    3485           0 :   return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
    3486             : }
    3487             : 
    3488             : unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3489           0 :   switch (VT.SimpleTy) {
    3490           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3491             :   default: return 0;
    3492             :   }
    3493             : }
    3494             : 
    3495             : // FastEmit functions for X86ISD::MOVMSK.
    3496             : 
    3497           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3498           0 :   if (RetVT.SimpleTy != MVT::i32)
    3499             :     return 0;
    3500           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3501           0 :     return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3502             :   }
    3503           0 :   if ((Subtarget->hasAVX())) {
    3504           0 :     return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3505             :   }
    3506             :   return 0;
    3507             : }
    3508             : 
    3509             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3510           0 :   if (RetVT.SimpleTy != MVT::i32)
    3511             :     return 0;
    3512           0 :   if ((Subtarget->hasAVX2())) {
    3513           0 :     return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3514             :   }
    3515             :   return 0;
    3516             : }
    3517             : 
    3518           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3519           0 :   if (RetVT.SimpleTy != MVT::i32)
    3520             :     return 0;
    3521           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3522           0 :     return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3523             :   }
    3524           0 :   if ((Subtarget->hasAVX())) {
    3525           0 :     return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3526             :   }
    3527             :   return 0;
    3528             : }
    3529             : 
    3530             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3531           0 :   if (RetVT.SimpleTy != MVT::i32)
    3532             :     return 0;
    3533           0 :   if ((Subtarget->hasAVX())) {
    3534           0 :     return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3535             :   }
    3536             :   return 0;
    3537             : }
    3538             : 
    3539           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3540           0 :   if (RetVT.SimpleTy != MVT::i32)
    3541             :     return 0;
    3542           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3543           0 :     return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3544             :   }
    3545           0 :   if ((Subtarget->hasAVX())) {
    3546           0 :     return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3547             :   }
    3548             :   return 0;
    3549             : }
    3550             : 
    3551             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3552           0 :   if (RetVT.SimpleTy != MVT::i32)
    3553             :     return 0;
    3554           0 :   if ((Subtarget->hasAVX())) {
    3555           0 :     return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3556             :   }
    3557             :   return 0;
    3558             : }
    3559             : 
    3560           0 : unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3561           0 :   switch (VT.SimpleTy) {
    3562           0 :   case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3563           0 :   case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3564           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3565           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3566           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3567           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3568             :   default: return 0;
    3569             :   }
    3570             : }
    3571             : 
    3572             : // FastEmit functions for X86ISD::MOVSHDUP.
    3573             : 
    3574           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3575           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3576             :     return 0;
    3577           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3578           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3579             :   }
    3580           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3581           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3582             :   }
    3583             :   return 0;
    3584             : }
    3585             : 
    3586           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3587           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3588             :     return 0;
    3589           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3590           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3591             :   }
    3592             :   return 0;
    3593             : }
    3594             : 
    3595           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3596           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3597             :     return 0;
    3598           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3599           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3600             :   }
    3601           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3602           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3603             :   }
    3604           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3605           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3606             :   }
    3607             :   return 0;
    3608             : }
    3609             : 
    3610           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3611           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3612             :     return 0;
    3613           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3614           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3615             :   }
    3616           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3617           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3618             :   }
    3619             :   return 0;
    3620             : }
    3621             : 
    3622             : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3623           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3624             :     return 0;
    3625           0 :   if ((Subtarget->hasAVX512())) {
    3626           0 :     return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3627             :   }
    3628             :   return 0;
    3629             : }
    3630             : 
    3631           0 : unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3632           0 :   switch (VT.SimpleTy) {
    3633           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3634           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3635           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3636           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3637           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3638             :   default: return 0;
    3639             :   }
    3640             : }
    3641             : 
    3642             : // FastEmit functions for X86ISD::MOVSLDUP.
    3643             : 
    3644           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3645           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3646             :     return 0;
    3647           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3648           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3649             :   }
    3650           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3651           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3652             :   }
    3653             :   return 0;
    3654             : }
    3655             : 
    3656           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3657           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3658             :     return 0;
    3659           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3660           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3661             :   }
    3662             :   return 0;
    3663             : }
    3664             : 
    3665           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3666           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3667             :     return 0;
    3668           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3669           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3670             :   }
    3671           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3672           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3673             :   }
    3674           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3675           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3676             :   }
    3677             :   return 0;
    3678             : }
    3679             : 
    3680           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3681           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3682             :     return 0;
    3683           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3684           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3685             :   }
    3686           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3687           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3688             :   }
    3689             :   return 0;
    3690             : }
    3691             : 
    3692             : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3693           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3694             :     return 0;
    3695           0 :   if ((Subtarget->hasAVX512())) {
    3696           0 :     return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3697             :   }
    3698             :   return 0;
    3699             : }
    3700             : 
    3701           0 : unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3702           0 :   switch (VT.SimpleTy) {
    3703           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3704           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3705           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3706           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3707           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3708             :   default: return 0;
    3709             :   }
    3710             : }
    3711             : 
    3712             : // FastEmit functions for X86ISD::PHMINPOS.
    3713             : 
    3714           0 : unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3715           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3716             :     return 0;
    3717           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    3718           0 :     return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3719             :   }
    3720           0 :   if ((Subtarget->hasAVX())) {
    3721           0 :     return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3722             :   }
    3723             :   return 0;
    3724             : }
    3725             : 
    3726             : unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3727           0 :   switch (VT.SimpleTy) {
    3728           0 :   case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3729             :   default: return 0;
    3730             :   }
    3731             : }
    3732             : 
    3733             : // FastEmit functions for X86ISD::RCP14.
    3734             : 
    3735             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3736           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3737             :     return 0;
    3738           0 :   if ((Subtarget->hasVLX())) {
    3739           0 :     return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3740             :   }
    3741             :   return 0;
    3742             : }
    3743             : 
    3744             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3745           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3746             :     return 0;
    3747           0 :   if ((Subtarget->hasVLX())) {
    3748           0 :     return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3749             :   }
    3750             :   return 0;
    3751             : }
    3752             : 
    3753             : unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3754           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3755             :     return 0;
    3756           0 :   if ((Subtarget->hasAVX512())) {
    3757           0 :     return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3758             :   }
    3759             :   return 0;
    3760             : }
    3761             : 
    3762             : unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3763           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3764             :     return 0;
    3765           0 :   if ((Subtarget->hasVLX())) {
    3766           0 :     return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3767             :   }
    3768             :   return 0;
    3769             : }
    3770             : 
    3771             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3772           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3773             :     return 0;
    3774           0 :   if ((Subtarget->hasVLX())) {
    3775           0 :     return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3776             :   }
    3777             :   return 0;
    3778             : }
    3779             : 
    3780             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3781           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3782             :     return 0;
    3783           0 :   if ((Subtarget->hasAVX512())) {
    3784           0 :     return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3785             :   }
    3786             :   return 0;
    3787             : }
    3788             : 
    3789           0 : unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3790           0 :   switch (VT.SimpleTy) {
    3791           0 :   case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3792           0 :   case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3793           0 :   case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3794           0 :   case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3795           0 :   case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3796           0 :   case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3797             :   default: return 0;
    3798             :   }
    3799             : }
    3800             : 
    3801             : // FastEmit functions for X86ISD::RSQRT14.
    3802             : 
    3803             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3804           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3805             :     return 0;
    3806           0 :   if ((Subtarget->hasVLX())) {
    3807           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3808             :   }
    3809             :   return 0;
    3810             : }
    3811             : 
    3812             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3813           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3814             :     return 0;
    3815           0 :   if ((Subtarget->hasVLX())) {
    3816           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3817             :   }
    3818             :   return 0;
    3819             : }
    3820             : 
    3821             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3822           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3823             :     return 0;
    3824           0 :   if ((Subtarget->hasAVX512())) {
    3825           0 :     return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3826             :   }
    3827             :   return 0;
    3828             : }
    3829             : 
    3830             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3831           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3832             :     return 0;
    3833           0 :   if ((Subtarget->hasVLX())) {
    3834           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3835             :   }
    3836             :   return 0;
    3837             : }
    3838             : 
    3839             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3840           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3841             :     return 0;
    3842           0 :   if ((Subtarget->hasVLX())) {
    3843           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3844             :   }
    3845             :   return 0;
    3846             : }
    3847             : 
    3848             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3849           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3850             :     return 0;
    3851           0 :   if ((Subtarget->hasAVX512())) {
    3852           0 :     return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3853             :   }
    3854             :   return 0;
    3855             : }
    3856             : 
    3857           0 : unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3858           0 :   switch (VT.SimpleTy) {
    3859           0 :   case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3860           0 :   case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3861           0 :   case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3862           0 :   case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3863           0 :   case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3864           0 :   case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3865             :   default: return 0;
    3866             :   }
    3867             : }
    3868             : 
    3869             : // FastEmit functions for X86ISD::SEG_ALLOCA.
    3870             : 
    3871           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3872           0 :   if (RetVT.SimpleTy != MVT::i32)
    3873             :     return 0;
    3874           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    3875           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    3876             :   }
    3877             :   return 0;
    3878             : }
    3879             : 
    3880             : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3881           0 :   if (RetVT.SimpleTy != MVT::i64)
    3882             :     return 0;
    3883           0 :   if ((Subtarget->is64Bit())) {
    3884           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    3885             :   }
    3886             :   return 0;
    3887             : }
    3888             : 
    3889           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3890           0 :   switch (VT.SimpleTy) {
    3891           0 :   case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3892           0 :   case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3893             :   default: return 0;
    3894             :   }
    3895             : }
    3896             : 
    3897             : // FastEmit functions for X86ISD::VBROADCAST.
    3898             : 
    3899           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3900           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3901           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3902             :   }
    3903             :   return 0;
    3904             : }
    3905             : 
    3906           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3907           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3908           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3909             :   }
    3910             :   return 0;
    3911             : }
    3912             : 
    3913             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    3914           0 :   if ((Subtarget->hasAVX512())) {
    3915           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3916             :   }
    3917             :   return 0;
    3918             : }
    3919             : 
    3920           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3921           0 : switch (RetVT.SimpleTy) {
    3922           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    3923           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
    3924           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
    3925             :   default: return 0;
    3926             : }
    3927             : }
    3928             : 
    3929           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    3930           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3931           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3932             :   }
    3933             :   return 0;
    3934             : }
    3935             : 
    3936           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    3937           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3938           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3939             :   }
    3940             :   return 0;
    3941             : }
    3942             : 
    3943             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    3944           0 :   if ((Subtarget->hasAVX512())) {
    3945           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3946             :   }
    3947             :   return 0;
    3948             : }
    3949             : 
    3950           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3951           0 : switch (RetVT.SimpleTy) {
    3952           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
    3953           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
    3954           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
    3955             :   default: return 0;
    3956             : }
    3957             : }
    3958             : 
    3959           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    3960           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3961           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3962             :   }
    3963           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    3964           0 :     return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3965             :   }
    3966             :   return 0;
    3967             : }
    3968             : 
    3969           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    3970           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3971           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3972             :   }
    3973           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    3974           0 :     return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3975             :   }
    3976             :   return 0;
    3977             : }
    3978             : 
    3979             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
    3980           0 :   if ((Subtarget->hasBWI())) {
    3981           0 :     return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3982             :   }
    3983             :   return 0;
    3984             : }
    3985             : 
    3986           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3987           0 : switch (RetVT.SimpleTy) {
    3988           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
    3989           0 :   case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
    3990           0 :   case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
    3991             :   default: return 0;
    3992             : }
    3993             : }
    3994             : 
    3995           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    3996           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    3997           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3998             :   }
    3999           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4000           0 :     return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4001             :   }
    4002             :   return 0;
    4003             : }
    4004             : 
    4005           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4006           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4007           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4008             :   }
    4009           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4010           0 :     return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4011             :   }
    4012             :   return 0;
    4013             : }
    4014             : 
    4015             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    4016           0 :   if ((Subtarget->hasBWI())) {
    4017           0 :     return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4018             :   }
    4019             :   return 0;
    4020             : }
    4021             : 
    4022           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4023           0 : switch (RetVT.SimpleTy) {
    4024           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
    4025           0 :   case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
    4026           0 :   case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
    4027             :   default: return 0;
    4028             : }
    4029             : }
    4030             : 
    4031           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4032           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4033           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4034             :   }
    4035           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4036           0 :     return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4037             :   }
    4038             :   return 0;
    4039             : }
    4040             : 
    4041           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4042           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4043           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4044             :   }
    4045           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4046           0 :     return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4047             :   }
    4048             :   return 0;
    4049             : }
    4050             : 
    4051             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4052           0 :   if ((Subtarget->hasAVX512())) {
    4053           0 :     return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4054             :   }
    4055             :   return 0;
    4056             : }
    4057             : 
    4058           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4059           0 : switch (RetVT.SimpleTy) {
    4060           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
    4061           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
    4062           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
    4063             :   default: return 0;
    4064             : }
    4065             : }
    4066             : 
    4067           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4068           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4069           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4070             :   }
    4071           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4072           0 :     return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4073             :   }
    4074             :   return 0;
    4075             : }
    4076             : 
    4077           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4078           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4079           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4080             :   }
    4081           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4082           0 :     return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4083             :   }
    4084             :   return 0;
    4085             : }
    4086             : 
    4087             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4088           0 :   if ((Subtarget->hasAVX512())) {
    4089           0 :     return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4090             :   }
    4091             :   return 0;
    4092             : }
    4093             : 
    4094           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4095           0 : switch (RetVT.SimpleTy) {
    4096           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
    4097           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
    4098           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
    4099             :   default: return 0;
    4100             : }
    4101             : }
    4102             : 
    4103           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    4104           0 :   if ((Subtarget->hasVLX())) {
    4105           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4106             :   }
    4107           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4108           0 :     return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4109             :   }
    4110             :   return 0;
    4111             : }
    4112             : 
    4113           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    4114           0 :   if ((Subtarget->hasVLX())) {
    4115           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4116             :   }
    4117           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4118           0 :     return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4119             :   }
    4120             :   return 0;
    4121             : }
    4122             : 
    4123             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
    4124           0 :   if ((Subtarget->hasAVX512())) {
    4125           0 :     return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4126             :   }
    4127             :   return 0;
    4128             : }
    4129             : 
    4130           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4131           0 : switch (RetVT.SimpleTy) {
    4132           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
    4133           0 :   case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
    4134           0 :   case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
    4135             :   default: return 0;
    4136             : }
    4137             : }
    4138             : 
    4139           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    4140           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4141           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4142             :   }
    4143           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4144           0 :     return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4145             :   }
    4146             :   return 0;
    4147             : }
    4148             : 
    4149           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    4150           0 :   if ((Subtarget->hasVLX())) {
    4151           0 :     return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4152             :   }
    4153           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4154           0 :     return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4155             :   }
    4156             :   return 0;
    4157             : }
    4158             : 
    4159             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    4160           0 :   if ((Subtarget->hasAVX512())) {
    4161           0 :     return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4162             :   }
    4163             :   return 0;
    4164             : }
    4165             : 
    4166           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4167           0 : switch (RetVT.SimpleTy) {
    4168           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill);
    4169           0 :   case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
    4170           0 :   case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
    4171             :   default: return 0;
    4172             : }
    4173             : }
    4174             : 
    4175           0 : unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4176           0 :   switch (VT.SimpleTy) {
    4177           0 :   case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4178           0 :   case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4179           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4180           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4181           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4182           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4183           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4184           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4185             :   default: return 0;
    4186             :   }
    4187             : }
    4188             : 
    4189             : // FastEmit functions for X86ISD::VBROADCASTM.
    4190             : 
    4191           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4192           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4193           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4194             :   }
    4195             :   return 0;
    4196             : }
    4197             : 
    4198           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4199           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4200           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4201             :   }
    4202             :   return 0;
    4203             : }
    4204             : 
    4205             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4206           0 :   if ((Subtarget->hasCDI())) {
    4207           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4208             :   }
    4209             :   return 0;
    4210             : }
    4211             : 
    4212           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4213           0 : switch (RetVT.SimpleTy) {
    4214           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
    4215           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
    4216           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    4217             :   default: return 0;
    4218             : }
    4219             : }
    4220             : 
    4221           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4222           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4223           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4224             :   }
    4225             :   return 0;
    4226             : }
    4227             : 
    4228           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4229           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4230           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4231             :   }
    4232             :   return 0;
    4233             : }
    4234             : 
    4235             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4236           0 :   if ((Subtarget->hasCDI())) {
    4237           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4238             :   }
    4239             :   return 0;
    4240             : }
    4241             : 
    4242           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4243           0 : switch (RetVT.SimpleTy) {
    4244           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
    4245           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
    4246           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    4247             :   default: return 0;
    4248             : }
    4249             : }
    4250             : 
    4251           0 : unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4252           0 :   switch (VT.SimpleTy) {
    4253           0 :   case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    4254           0 :   case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    4255             :   default: return 0;
    4256             :   }
    4257             : }
    4258             : 
    4259             : // FastEmit functions for X86ISD::VFPEXT.
    4260             : 
    4261           0 : unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4262           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4263             :     return 0;
    4264           0 :   if ((Subtarget->hasVLX())) {
    4265           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4266             :   }
    4267           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4268           0 :     return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4269             :   }
    4270           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4271           0 :     return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4272             :   }
    4273             :   return 0;
    4274             : }
    4275             : 
    4276             : unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4277           0 :   switch (VT.SimpleTy) {
    4278           0 :   case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4279             :   default: return 0;
    4280             :   }
    4281             : }
    4282             : 
    4283             : // FastEmit functions for X86ISD::VFPROUND.
    4284             : 
    4285           0 : unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4286           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4287             :     return 0;
    4288           0 :   if ((Subtarget->hasVLX())) {
    4289           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4290             :   }
    4291           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4292           0 :     return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4293             :   }
    4294           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4295           0 :     return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4296             :   }
    4297             :   return 0;
    4298             : }
    4299             : 
    4300             : unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4301           0 :   switch (VT.SimpleTy) {
    4302           0 :   case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4303             :   default: return 0;
    4304             :   }
    4305             : }
    4306             : 
    4307             : // FastEmit functions for X86ISD::VSEXT.
    4308             : 
    4309           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4310           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4311           0 :     return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4312             :   }
    4313           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4314           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4315             :   }
    4316             :   return 0;
    4317             : }
    4318             : 
    4319           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4320           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4321           0 :     return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4322             :   }
    4323           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4324           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4325             :   }
    4326             :   return 0;
    4327             : }
    4328             : 
    4329             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4330           0 :   if ((Subtarget->hasAVX512())) {
    4331           0 :     return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4332             :   }
    4333             :   return 0;
    4334             : }
    4335             : 
    4336           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4337           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4338           0 :     return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4339             :   }
    4340           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4341           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4342             :   }
    4343             :   return 0;
    4344             : }
    4345             : 
    4346             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4347           0 :   if ((Subtarget->hasAVX512())) {
    4348           0 :     return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4349             :   }
    4350             :   return 0;
    4351             : }
    4352             : 
    4353           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4354           0 : switch (RetVT.SimpleTy) {
    4355           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    4356           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    4357           0 :   case MVT::v16i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    4358           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    4359           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    4360             :   default: return 0;
    4361             : }
    4362             : }
    4363             : 
    4364             : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4365           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    4366             :     return 0;
    4367           0 :   if ((Subtarget->hasBWI())) {
    4368           0 :     return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4369             :   }
    4370             :   return 0;
    4371             : }
    4372             : 
    4373           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4374           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4375           0 :     return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4376             :   }
    4377           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4378           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4379             :   }
    4380             :   return 0;
    4381             : }
    4382             : 
    4383           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4384           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4385           0 :     return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4386             :   }
    4387           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4388           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4389             :   }
    4390             :   return 0;
    4391             : }
    4392             : 
    4393             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4394           0 :   if ((Subtarget->hasAVX512())) {
    4395           0 :     return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4396             :   }
    4397             :   return 0;
    4398             : }
    4399             : 
    4400           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4401           0 : switch (RetVT.SimpleTy) {
    4402           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    4403           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    4404           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    4405             :   default: return 0;
    4406             : }
    4407             : }
    4408             : 
    4409             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4410           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    4411             :     return 0;
    4412           0 :   if ((Subtarget->hasAVX512())) {
    4413           0 :     return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4414             :   }
    4415             :   return 0;
    4416             : }
    4417             : 
    4418           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4419           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    4420             :     return 0;
    4421           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4422           0 :     return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4423             :   }
    4424           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4425           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4426             :   }
    4427             :   return 0;
    4428             : }
    4429             : 
    4430             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4431           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    4432             :     return 0;
    4433           0 :   if ((Subtarget->hasAVX512())) {
    4434           0 :     return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4435             :   }
    4436             :   return 0;
    4437             : }
    4438             : 
    4439           0 : unsigned fastEmit_X86ISD_VSEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4440           0 :   switch (VT.SimpleTy) {
    4441           0 :   case MVT::v16i8: return fastEmit_X86ISD_VSEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4442           0 :   case MVT::v32i8: return fastEmit_X86ISD_VSEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    4443           0 :   case MVT::v8i16: return fastEmit_X86ISD_VSEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4444           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4445           0 :   case MVT::v4i32: return fastEmit_X86ISD_VSEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4446           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4447             :   default: return 0;
    4448             :   }
    4449             : }
    4450             : 
    4451             : // FastEmit functions for X86ISD::VTRUNC.
    4452             : 
    4453           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4454           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4455             :     return 0;
    4456           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4457           0 :     return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4458             :   }
    4459             :   return 0;
    4460             : }
    4461             : 
    4462           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4463           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4464           0 :     return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4465             :   }
    4466             :   return 0;
    4467             : }
    4468             : 
    4469           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4470           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4471           0 :     return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4472             :   }
    4473             :   return 0;
    4474             : }
    4475             : 
    4476           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4477           0 : switch (RetVT.SimpleTy) {
    4478           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4479           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4480             :   default: return 0;
    4481             : }
    4482             : }
    4483             : 
    4484           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4485           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4486             :     return 0;
    4487           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4488           0 :     return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4489             :   }
    4490             :   return 0;
    4491             : }
    4492             : 
    4493           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4494           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4495           0 :     return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4496             :   }
    4497             :   return 0;
    4498             : }
    4499             : 
    4500           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4501           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4502           0 :     return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4503             :   }
    4504             :   return 0;
    4505             : }
    4506             : 
    4507           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4508           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4509           0 :     return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4510             :   }
    4511             :   return 0;
    4512             : }
    4513             : 
    4514           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4515           0 : switch (RetVT.SimpleTy) {
    4516           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4517           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4518           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4519             :   default: return 0;
    4520             : }
    4521             : }
    4522             : 
    4523           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4524           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4525           0 :     return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4526             :   }
    4527             :   return 0;
    4528             : }
    4529             : 
    4530           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4531           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4532           0 :     return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4533             :   }
    4534             :   return 0;
    4535             : }
    4536             : 
    4537           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4538           0 : switch (RetVT.SimpleTy) {
    4539           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4540           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4541             :   default: return 0;
    4542             : }
    4543             : }
    4544             : 
    4545             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4546           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4547             :     return 0;
    4548           0 :   if ((Subtarget->hasAVX512())) {
    4549           0 :     return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4550             :   }
    4551             :   return 0;
    4552             : }
    4553             : 
    4554           0 : unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4555           0 :   switch (VT.SimpleTy) {
    4556           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4557           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4558           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4559           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4560           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4561           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4562             :   default: return 0;
    4563             :   }
    4564             : }
    4565             : 
    4566             : // FastEmit functions for X86ISD::VTRUNCS.
    4567             : 
    4568           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4569           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4570             :     return 0;
    4571           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4572           0 :     return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4573             :   }
    4574             :   return 0;
    4575             : }
    4576             : 
    4577           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4578           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4579             :     return 0;
    4580           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4581           0 :     return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4582             :   }
    4583             :   return 0;
    4584             : }
    4585             : 
    4586             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4587           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4588             :     return 0;
    4589           0 :   if ((Subtarget->hasBWI())) {
    4590           0 :     return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4591             :   }
    4592             :   return 0;
    4593             : }
    4594             : 
    4595           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4596           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4597           0 :     return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4598             :   }
    4599             :   return 0;
    4600             : }
    4601             : 
    4602           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4603           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4604           0 :     return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4605             :   }
    4606             :   return 0;
    4607             : }
    4608             : 
    4609           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4610           0 : switch (RetVT.SimpleTy) {
    4611           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4612           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4613             :   default: return 0;
    4614             : }
    4615             : }
    4616             : 
    4617           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4618           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4619           0 :     return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4620             :   }
    4621             :   return 0;
    4622             : }
    4623             : 
    4624           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4625           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4626           0 :     return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4627             :   }
    4628             :   return 0;
    4629             : }
    4630             : 
    4631           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4632           0 : switch (RetVT.SimpleTy) {
    4633           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4634           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4635             :   default: return 0;
    4636             : }
    4637             : }
    4638             : 
    4639             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4640           0 :   if ((Subtarget->hasAVX512())) {
    4641           0 :     return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4642             :   }
    4643             :   return 0;
    4644             : }
    4645             : 
    4646             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4647           0 :   if ((Subtarget->hasAVX512())) {
    4648           0 :     return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4649             :   }
    4650             :   return 0;
    4651             : }
    4652             : 
    4653           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4654           0 : switch (RetVT.SimpleTy) {
    4655           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4656           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4657             :   default: return 0;
    4658             : }
    4659             : }
    4660             : 
    4661           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4662           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4663           0 :     return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4664             :   }
    4665             :   return 0;
    4666             : }
    4667             : 
    4668           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4669           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4670           0 :     return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4671             :   }
    4672             :   return 0;
    4673             : }
    4674             : 
    4675           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4676           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4677           0 :     return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4678             :   }
    4679             :   return 0;
    4680             : }
    4681             : 
    4682           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4683           0 : switch (RetVT.SimpleTy) {
    4684           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4685           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4686           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4687             :   default: return 0;
    4688             : }
    4689             : }
    4690             : 
    4691           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4692           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4693           0 :     return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4694             :   }
    4695             :   return 0;
    4696             : }
    4697             : 
    4698           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4699           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4700           0 :     return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4701             :   }
    4702             :   return 0;
    4703             : }
    4704             : 
    4705           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4706           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4707           0 :     return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4708             :   }
    4709             :   return 0;
    4710             : }
    4711             : 
    4712           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4713           0 : switch (RetVT.SimpleTy) {
    4714           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4715           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4716           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    4717             :   default: return 0;
    4718             : }
    4719             : }
    4720             : 
    4721             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4722           0 :   if ((Subtarget->hasAVX512())) {
    4723           0 :     return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4724             :   }
    4725             :   return 0;
    4726             : }
    4727             : 
    4728             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4729           0 :   if ((Subtarget->hasAVX512())) {
    4730           0 :     return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4731             :   }
    4732             :   return 0;
    4733             : }
    4734             : 
    4735             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4736           0 :   if ((Subtarget->hasAVX512())) {
    4737           0 :     return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4738             :   }
    4739             :   return 0;
    4740             : }
    4741             : 
    4742           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4743           0 : switch (RetVT.SimpleTy) {
    4744           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    4745           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    4746           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    4747             :   default: return 0;
    4748             : }
    4749             : }
    4750             : 
    4751           0 : unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4752           0 :   switch (VT.SimpleTy) {
    4753           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4754           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4755           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    4756           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4757           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4758           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    4759           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4760           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4761           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4762             :   default: return 0;
    4763             :   }
    4764             : }
    4765             : 
    4766             : // FastEmit functions for X86ISD::VTRUNCUS.
    4767             : 
    4768           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4769           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4770             :     return 0;
    4771           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4772           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4773             :   }
    4774             :   return 0;
    4775             : }
    4776             : 
    4777           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4778           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4779             :     return 0;
    4780           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4781           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4782             :   }
    4783             :   return 0;
    4784             : }
    4785             : 
    4786             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4787           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4788             :     return 0;
    4789           0 :   if ((Subtarget->hasBWI())) {
    4790           0 :     return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4791             :   }
    4792             :   return 0;
    4793             : }
    4794             : 
    4795           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4796           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4797           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4798             :   }
    4799             :   return 0;
    4800             : }
    4801             : 
    4802           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4803           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4804           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4805             :   }
    4806             :   return 0;
    4807             : }
    4808             : 
    4809           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4810           0 : switch (RetVT.SimpleTy) {
    4811           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4812           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4813             :   default: return 0;
    4814             : }
    4815             : }
    4816             : 
    4817           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4818           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4819           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4820             :   }
    4821             :   return 0;
    4822             : }
    4823             : 
    4824           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4825           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4826           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4827             :   }
    4828             :   return 0;
    4829             : }
    4830             : 
    4831           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4832           0 : switch (RetVT.SimpleTy) {
    4833           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4834           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4835             :   default: return 0;
    4836             : }
    4837             : }
    4838             : 
    4839             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4840           0 :   if ((Subtarget->hasAVX512())) {
    4841           0 :     return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4842             :   }
    4843             :   return 0;
    4844             : }
    4845             : 
    4846             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4847           0 :   if ((Subtarget->hasAVX512())) {
    4848           0 :     return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4849             :   }
    4850             :   return 0;
    4851             : }
    4852             : 
    4853           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4854           0 : switch (RetVT.SimpleTy) {
    4855           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4856           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4857             :   default: return 0;
    4858             : }
    4859             : }
    4860             : 
    4861           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4862           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4863           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4864             :   }
    4865             :   return 0;
    4866             : }
    4867             : 
    4868           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4869           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4870           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4871             :   }
    4872             :   return 0;
    4873             : }
    4874             : 
    4875           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4876           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4877           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4878             :   }
    4879             :   return 0;
    4880             : }
    4881             : 
    4882           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4883           0 : switch (RetVT.SimpleTy) {
    4884           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4885           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4886           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4887             :   default: return 0;
    4888             : }
    4889             : }
    4890             : 
    4891           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4892           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4893           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4894             :   }
    4895             :   return 0;
    4896             : }
    4897             : 
    4898           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4899           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4900           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4901             :   }
    4902             :   return 0;
    4903             : }
    4904             : 
    4905           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4906           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4907           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4908             :   }
    4909             :   return 0;
    4910             : }
    4911             : 
    4912           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4913           0 : switch (RetVT.SimpleTy) {
    4914           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4915           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4916           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    4917             :   default: return 0;
    4918             : }
    4919             : }
    4920             : 
    4921             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4922           0 :   if ((Subtarget->hasAVX512())) {
    4923           0 :     return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4924             :   }
    4925             :   return 0;
    4926             : }
    4927             : 
    4928             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4929           0 :   if ((Subtarget->hasAVX512())) {
    4930           0 :     return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4931             :   }
    4932             :   return 0;
    4933             : }
    4934             : 
    4935             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4936           0 :   if ((Subtarget->hasAVX512())) {
    4937           0 :     return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4938             :   }
    4939             :   return 0;
    4940             : }
    4941             : 
    4942           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4943           0 : switch (RetVT.SimpleTy) {
    4944           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    4945           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    4946           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    4947             :   default: return 0;
    4948             : }
    4949             : }
    4950             : 
    4951           0 : unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4952           0 :   switch (VT.SimpleTy) {
    4953           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4954           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4955           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    4956           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4957           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4958           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    4959           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4960           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4961           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4962             :   default: return 0;
    4963             :   }
    4964             : }
    4965             : 
    4966             : // FastEmit functions for X86ISD::VZEXT.
    4967             : 
    4968           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4969           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4970           0 :     return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4971             :   }
    4972           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4973           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4974             :   }
    4975             :   return 0;
    4976             : }
    4977             : 
    4978           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4979           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4980           0 :     return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4981             :   }
    4982           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4983           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4984             :   }
    4985             :   return 0;
    4986             : }
    4987             : 
    4988             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4989           0 :   if ((Subtarget->hasAVX512())) {
    4990           0 :     return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4991             :   }
    4992             :   return 0;
    4993             : }
    4994             : 
    4995           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4996           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4997           0 :     return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4998             :   }
    4999           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5000           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5001             :   }
    5002             :   return 0;
    5003             : }
    5004             : 
    5005             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5006           0 :   if ((Subtarget->hasAVX512())) {
    5007           0 :     return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5008             :   }
    5009             :   return 0;
    5010             : }
    5011             : 
    5012           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5013           0 : switch (RetVT.SimpleTy) {
    5014           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    5015           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    5016           0 :   case MVT::v16i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    5017           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    5018           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    5019             :   default: return 0;
    5020             : }
    5021             : }
    5022             : 
    5023             : unsigned fastEmit_X86ISD_VZEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5024           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5025             :     return 0;
    5026           0 :   if ((Subtarget->hasBWI())) {
    5027           0 :     return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5028             :   }
    5029             :   return 0;
    5030             : }
    5031             : 
    5032           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5033           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5034           0 :     return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5035             :   }
    5036           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5037           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5038             :   }
    5039             :   return 0;
    5040             : }
    5041             : 
    5042           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5043           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5044           0 :     return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5045             :   }
    5046           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5047           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5048             :   }
    5049             :   return 0;
    5050             : }
    5051             : 
    5052             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5053           0 :   if ((Subtarget->hasAVX512())) {
    5054           0 :     return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5055             :   }
    5056             :   return 0;
    5057             : }
    5058             : 
    5059           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5060           0 : switch (RetVT.SimpleTy) {
    5061           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    5062           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    5063           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    5064             :   default: return 0;
    5065             : }
    5066             : }
    5067             : 
    5068             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5069           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    5070             :     return 0;
    5071           0 :   if ((Subtarget->hasAVX512())) {
    5072           0 :     return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5073             :   }
    5074             :   return 0;
    5075             : }
    5076             : 
    5077           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5078           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    5079             :     return 0;
    5080           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5081           0 :     return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5082             :   }
    5083           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5084           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5085             :   }
    5086             :   return 0;
    5087             : }
    5088             : 
    5089             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5090           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5091             :     return 0;
    5092           0 :   if ((Subtarget->hasAVX512())) {
    5093           0 :     return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5094             :   }
    5095             :   return 0;
    5096             : }
    5097             : 
    5098           0 : unsigned fastEmit_X86ISD_VZEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5099           0 :   switch (VT.SimpleTy) {
    5100           0 :   case MVT::v16i8: return fastEmit_X86ISD_VZEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    5101           0 :   case MVT::v32i8: return fastEmit_X86ISD_VZEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    5102           0 :   case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5103           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5104           0 :   case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5105           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5106             :   default: return 0;
    5107             :   }
    5108             : }
    5109             : 
    5110             : // FastEmit functions for X86ISD::VZEXT_MOVL.
    5111             : 
    5112           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5113           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5114             :     return 0;
    5115           0 :   if ((Subtarget->hasAVX512())) {
    5116           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5117             :   }
    5118           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5119           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5120             :   }
    5121           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5122           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5123             :   }
    5124             :   return 0;
    5125             : }
    5126             : 
    5127           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5128           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5129             :     return 0;
    5130           0 :   if ((Subtarget->hasAVX512())) {
    5131           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5132             :   }
    5133           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5134           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5135             :   }
    5136           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5137           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5138             :   }
    5139             :   return 0;
    5140             : }
    5141             : 
    5142           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5143           0 :   switch (VT.SimpleTy) {
    5144           0 :   case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5145           0 :   case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    5146             :   default: return 0;
    5147             :   }
    5148             : }
    5149             : 
    5150             : // FastEmit functions for X86ISD::WIN_ALLOCA.
    5151             : 
    5152           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5153           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5154             :     return 0;
    5155           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    5156           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    5157             :   }
    5158             :   return 0;
    5159             : }
    5160             : 
    5161             : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5162           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5163             :     return 0;
    5164           0 :   if ((Subtarget->is64Bit())) {
    5165           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    5166             :   }
    5167             :   return 0;
    5168             : }
    5169             : 
    5170           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5171           0 :   switch (VT.SimpleTy) {
    5172           0 :   case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    5173           0 :   case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    5174             :   default: return 0;
    5175             :   }
    5176             : }
    5177             : 
    5178             : // Top-level FastEmit function.
    5179             : 
    5180        5009 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
    5181        5009 :   switch (Opcode) {
    5182           0 :   case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
    5183          18 :   case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5184        1809 :   case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
    5185           0 :   case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    5186           0 :   case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
    5187           0 :   case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
    5188           0 :   case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
    5189           0 :   case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill);
    5190           0 :   case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill);
    5191           0 :   case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
    5192           0 :   case ISD::FCOS: return fastEmit_ISD_FCOS_r(VT, RetVT, Op0, Op0IsKill);
    5193           3 :   case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
    5194           0 :   case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5195           0 :   case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
    5196           0 :   case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
    5197           0 :   case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
    5198           0 :   case ISD::FSIN: return fastEmit_ISD_FSIN_r(VT, RetVT, Op0, Op0IsKill);
    5199           0 :   case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5200           1 :   case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill);
    5201        1375 :   case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5202           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5203          40 :   case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5204         263 :   case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
    5205           0 :   case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5206        1500 :   case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5207           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5208           0 :   case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5209           0 :   case X86ISD::COMPRESS: return fastEmit_X86ISD_COMPRESS_r(VT, RetVT, Op0, Op0IsKill);
    5210           0 :   case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill);
    5211           0 :   case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5212           0 :   case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5213           0 :   case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill);
    5214           0 :   case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
    5215           0 :   case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5216           0 :   case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5217           0 :   case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
    5218           0 :   case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill);
    5219           0 :   case X86ISD::EXPAND: return fastEmit_X86ISD_EXPAND_r(VT, RetVT, Op0, Op0IsKill);
    5220           0 :   case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill);
    5221           0 :   case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5222           0 :   case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill);
    5223           0 :   case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill);
    5224           0 :   case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill);
    5225           0 :   case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill);
    5226           0 :   case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill);
    5227           0 :   case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill);
    5228           0 :   case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill);
    5229           0 :   case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0, Op0IsKill);
    5230           0 :   case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0, Op0IsKill);
    5231           0 :   case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0, Op0IsKill);
    5232           0 :   case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5233           0 :   case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill);
    5234           0 :   case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill);
    5235           0 :   case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
    5236           0 :   case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
    5237           0 :   case X86ISD::VSEXT: return fastEmit_X86ISD_VSEXT_r(VT, RetVT, Op0, Op0IsKill);
    5238           0 :   case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill);
    5239           0 :   case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill);
    5240           0 :   case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill);
    5241           0 :   case X86ISD::VZEXT: return fastEmit_X86ISD_VZEXT_r(VT, RetVT, Op0, Op0IsKill);
    5242           0 :   case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill);
    5243           0 :   case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5244             :   default: return 0;
    5245             :   }
    5246             : }
    5247             : 
    5248             : // FastEmit functions for ISD::ADD.
    5249             : 
    5250             : unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5251           1 :   if (RetVT.SimpleTy != MVT::i8)
    5252             :     return 0;
    5253           1 :   return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5254             : }
    5255             : 
    5256             : unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5257           1 :   if (RetVT.SimpleTy != MVT::i16)
    5258             :     return 0;
    5259           1 :   return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5260             : }
    5261             : 
    5262             : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5263         944 :   if (RetVT.SimpleTy != MVT::i32)
    5264             :     return 0;
    5265         944 :   return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5266             : }
    5267             : 
    5268             : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5269        1114 :   if (RetVT.SimpleTy != MVT::i64)
    5270             :     return 0;
    5271        1114 :   return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5272             : }
    5273             : 
    5274           2 : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5275           2 :   if (RetVT.SimpleTy != MVT::v16i8)
    5276             :     return 0;
    5277           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5278           0 :     return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5279             :   }
    5280           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5281           2 :     return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5282             :   }
    5283           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5284           0 :     return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5285             :   }
    5286             :   return 0;
    5287             : }
    5288             : 
    5289           2 : unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5290           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    5291             :     return 0;
    5292           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5293           0 :     return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5294             :   }
    5295           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5296           2 :     return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5297             :   }
    5298             :   return 0;
    5299             : }
    5300             : 
    5301             : unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5302           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    5303             :     return 0;
    5304           0 :   if ((Subtarget->hasBWI())) {
    5305           0 :     return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5306             :   }
    5307             :   return 0;
    5308             : }
    5309             : 
    5310           2 : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5311           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    5312             :     return 0;
    5313           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5314           0 :     return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5315             :   }
    5316           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5317           2 :     return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5318             :   }
    5319           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5320           0 :     return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5321             :   }
    5322             :   return 0;
    5323             : }
    5324             : 
    5325           2 : unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5326           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    5327             :     return 0;
    5328           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5329           0 :     return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5330             :   }
    5331           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5332           2 :     return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5333             :   }
    5334             :   return 0;
    5335             : }
    5336             : 
    5337             : unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5338           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5339             :     return 0;
    5340           0 :   if ((Subtarget->hasBWI())) {
    5341           0 :     return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5342             :   }
    5343             :   return 0;
    5344             : }
    5345             : 
    5346          18 : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5347          18 :   if (RetVT.SimpleTy != MVT::v4i32)
    5348             :     return 0;
    5349          36 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5350           2 :     return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5351             :   }
    5352          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5353           6 :     return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5354             :   }
    5355          20 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5356          10 :     return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5357             :   }
    5358             :   return 0;
    5359             : }
    5360             : 
    5361           3 : unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5362           3 :   if (RetVT.SimpleTy != MVT::v8i32)
    5363             :     return 0;
    5364           6 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5365           1 :     return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5366             :   }
    5367           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5368           2 :     return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5369             :   }
    5370             :   return 0;
    5371             : }
    5372             : 
    5373             : unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5374           2 :   if (RetVT.SimpleTy != MVT::v16i32)
    5375             :     return 0;
    5376           4 :   if ((Subtarget->hasAVX512())) {
    5377           2 :     return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5378             :   }
    5379             :   return 0;
    5380             : }
    5381             : 
    5382           2 : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5383           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    5384             :     return 0;
    5385           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5386           0 :     return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5387             :   }
    5388           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5389           2 :     return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5390             :   }
    5391           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5392           0 :     return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5393             :   }
    5394             :   return 0;
    5395             : }
    5396             : 
    5397           4 : unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5398           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    5399             :     return 0;
    5400           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5401           2 :     return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5402             :   }
    5403           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5404           2 :     return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5405             :   }
    5406             :   return 0;
    5407             : }
    5408             : 
    5409             : unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5410           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5411             :     return 0;
    5412           0 :   if ((Subtarget->hasAVX512())) {
    5413           0 :     return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5414             :   }
    5415             :   return 0;
    5416             : }
    5417             : 
    5418        2097 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5419        2097 :   switch (VT.SimpleTy) {
    5420           1 :   case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5421           1 :   case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5422         944 :   case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5423        1114 :   case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5424           2 :   case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5425           2 :   case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5426           0 :   case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5427           2 :   case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5428           2 :   case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5429           0 :   case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5430          18 :   case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5431           3 :   case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5432           2 :   case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5433           2 :   case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5434           4 :   case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5435           0 :   case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5436             :   default: return 0;
    5437             :   }
    5438             : }
    5439             : 
    5440             : // FastEmit functions for ISD::AND.
    5441             : 
    5442             : unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5443           6 :   if (RetVT.SimpleTy != MVT::i8)
    5444             :     return 0;
    5445           6 :   return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5446             : }
    5447             : 
    5448             : unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5449           0 :   if (RetVT.SimpleTy != MVT::i16)
    5450             :     return 0;
    5451           0 :   return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5452             : }
    5453             : 
    5454             : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5455         142 :   if (RetVT.SimpleTy != MVT::i32)
    5456             :     return 0;
    5457         142 :   return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5458             : }
    5459             : 
    5460             : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5461          16 :   if (RetVT.SimpleTy != MVT::i64)
    5462             :     return 0;
    5463          16 :   return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5464             : }
    5465             : 
    5466             : unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5467           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    5468             :     return 0;
    5469           0 :   if ((Subtarget->hasDQI())) {
    5470           0 :     return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5471             :   }
    5472             :   return 0;
    5473             : }
    5474             : 
    5475             : unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5476           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    5477             :     return 0;
    5478           0 :   if ((Subtarget->hasAVX512())) {
    5479           0 :     return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5480             :   }
    5481             :   return 0;
    5482             : }
    5483             : 
    5484             : unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5485           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    5486             :     return 0;
    5487           0 :   if ((Subtarget->hasBWI())) {
    5488           0 :     return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5489             :   }
    5490             :   return 0;
    5491             : }
    5492             : 
    5493             : unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5494           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    5495             :     return 0;
    5496           0 :   if ((Subtarget->hasBWI())) {
    5497           0 :     return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5498             :   }
    5499             :   return 0;
    5500             : }
    5501             : 
    5502           8 : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5503           8 :   if (RetVT.SimpleTy != MVT::v2i64)
    5504             :     return 0;
    5505          16 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5506           0 :     return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5507             :   }
    5508           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5509           4 :     return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5510             :   }
    5511           8 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5512           4 :     return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5513             :   }
    5514             :   return 0;
    5515             : }
    5516             : 
    5517          12 : unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5518          12 :   if (RetVT.SimpleTy != MVT::v4i64)
    5519             :     return 0;
    5520          24 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    5521           8 :     return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5522             :   }
    5523           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5524           0 :     return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5525             :   }
    5526           8 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5527           4 :     return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5528             :   }
    5529             :   return 0;
    5530             : }
    5531             : 
    5532             : unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5533           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5534             :     return 0;
    5535           0 :   if ((Subtarget->hasAVX512())) {
    5536           0 :     return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5537             :   }
    5538             :   return 0;
    5539             : }
    5540             : 
    5541         192 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5542         192 :   switch (VT.SimpleTy) {
    5543           6 :   case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5544           0 :   case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5545         142 :   case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5546          16 :   case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5547           0 :   case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5548           0 :   case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5549           0 :   case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5550           0 :   case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5551           8 :   case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5552          12 :   case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5553           0 :   case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5554             :   default: return 0;
    5555             :   }
    5556             : }
    5557             : 
    5558             : // FastEmit functions for ISD::FADD.
    5559             : 
    5560          25 : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5561          25 :   if (RetVT.SimpleTy != MVT::f32)
    5562             :     return 0;
    5563          50 :   if ((Subtarget->hasAVX512())) {
    5564           2 :     return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5565             :   }
    5566          23 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5567          21 :     return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5568             :   }
    5569           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5570           2 :     return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5571             :   }
    5572           0 :   if ((!Subtarget->hasSSE1())) {
    5573           0 :     return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5574             :   }
    5575             :   return 0;
    5576             : }
    5577             : 
    5578          26 : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5579          26 :   if (RetVT.SimpleTy != MVT::f64)
    5580             :     return 0;
    5581          52 :   if ((Subtarget->hasAVX512())) {
    5582           2 :     return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5583             :   }
    5584          24 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5585          22 :     return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5586             :   }
    5587           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5588           2 :     return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5589             :   }
    5590           0 :   if ((!Subtarget->hasSSE2())) {
    5591           0 :     return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5592             :   }
    5593             :   return 0;
    5594             : }
    5595             : 
    5596             : unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5597           0 :   if (RetVT.SimpleTy != MVT::f80)
    5598             :     return 0;
    5599           0 :   return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5600             : }
    5601             : 
    5602           2 : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5603           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5604             :     return 0;
    5605           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5606           0 :     return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5607             :   }
    5608           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5609           2 :     return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5610             :   }
    5611           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5612           0 :     return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5613             :   }
    5614             :   return 0;
    5615             : }
    5616             : 
    5617           2 : unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5618           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5619             :     return 0;
    5620           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5621           0 :     return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5622             :   }
    5623           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5624           2 :     return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5625             :   }
    5626             :   return 0;
    5627             : }
    5628             : 
    5629             : unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5630           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5631             :     return 0;
    5632           0 :   if ((Subtarget->hasAVX512())) {
    5633           0 :     return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5634             :   }
    5635             :   return 0;
    5636             : }
    5637             : 
    5638          18 : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5639          18 :   if (RetVT.SimpleTy != MVT::v2f64)
    5640             :     return 0;
    5641          36 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5642           2 :     return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5643             :   }
    5644          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5645           6 :     return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5646             :   }
    5647          20 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5648          10 :     return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5649             :   }
    5650             :   return 0;
    5651             : }
    5652             : 
    5653          14 : unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5654          14 :   if (RetVT.SimpleTy != MVT::v4f64)
    5655             :     return 0;
    5656          28 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5657           2 :     return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5658             :   }
    5659          12 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5660          12 :     return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5661             :   }
    5662             :   return 0;
    5663             : }
    5664             : 
    5665             : unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5666           8 :   if (RetVT.SimpleTy != MVT::v8f64)
    5667             :     return 0;
    5668          16 :   if ((Subtarget->hasAVX512())) {
    5669           8 :     return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5670             :   }
    5671             :   return 0;
    5672             : }
    5673             : 
    5674          95 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5675          95 :   switch (VT.SimpleTy) {
    5676          25 :   case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5677          26 :   case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5678           0 :   case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5679           2 :   case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5680           2 :   case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5681           0 :   case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5682          18 :   case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5683          14 :   case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5684           8 :   case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5685             :   default: return 0;
    5686             :   }
    5687             : }
    5688             : 
    5689             : // FastEmit functions for ISD::FDIV.
    5690             : 
    5691           8 : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5692           8 :   if (RetVT.SimpleTy != MVT::f32)
    5693             :     return 0;
    5694          16 :   if ((Subtarget->hasAVX512())) {
    5695           0 :     return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5696             :   }
    5697           8 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5698           8 :     return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5699             :   }
    5700           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5701           0 :     return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5702             :   }
    5703           0 :   if ((!Subtarget->hasSSE1())) {
    5704           0 :     return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5705             :   }
    5706             :   return 0;
    5707             : }
    5708             : 
    5709           0 : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5710           0 :   if (RetVT.SimpleTy != MVT::f64)
    5711             :     return 0;
    5712           0 :   if ((Subtarget->hasAVX512())) {
    5713           0 :     return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5714             :   }
    5715           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5716           0 :     return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5717             :   }
    5718           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5719           0 :     return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5720             :   }
    5721           0 :   if ((!Subtarget->hasSSE2())) {
    5722           0 :     return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5723             :   }
    5724             :   return 0;
    5725             : }
    5726             : 
    5727             : unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5728           0 :   if (RetVT.SimpleTy != MVT::f80)
    5729             :     return 0;
    5730           0 :   return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5731             : }
    5732             : 
    5733           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5734           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5735             :     return 0;
    5736           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5737           0 :     return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5738             :   }
    5739           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5740           2 :     return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5741             :   }
    5742           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5743           0 :     return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5744             :   }
    5745             :   return 0;
    5746             : }
    5747             : 
    5748           2 : unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5749           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5750             :     return 0;
    5751           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5752           0 :     return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5753             :   }
    5754           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5755           2 :     return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5756             :   }
    5757             :   return 0;
    5758             : }
    5759             : 
    5760             : unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5761           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5762             :     return 0;
    5763           0 :   if ((Subtarget->hasAVX512())) {
    5764           0 :     return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5765             :   }
    5766             :   return 0;
    5767             : }
    5768             : 
    5769           2 : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5770           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    5771             :     return 0;
    5772           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5773           0 :     return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5774             :   }
    5775           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5776           2 :     return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5777             :   }
    5778           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5779           0 :     return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5780             :   }
    5781             :   return 0;
    5782             : }
    5783             : 
    5784           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5785           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    5786             :     return 0;
    5787           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5788           0 :     return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5789             :   }
    5790           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5791           2 :     return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5792             :   }
    5793             :   return 0;
    5794             : }
    5795             : 
    5796             : unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5797           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    5798             :     return 0;
    5799           0 :   if ((Subtarget->hasAVX512())) {
    5800           0 :     return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5801             :   }
    5802             :   return 0;
    5803             : }
    5804             : 
    5805          16 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5806          16 :   switch (VT.SimpleTy) {
    5807           8 :   case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5808           0 :   case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5809           0 :   case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5810           2 :   case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5811           2 :   case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5812           0 :   case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5813           2 :   case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5814           2 :   case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5815           0 :   case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5816             :   default: return 0;
    5817             :   }
    5818             : }
    5819             : 
    5820             : // FastEmit functions for ISD::FMUL.
    5821             : 
    5822           9 : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5823           9 :   if (RetVT.SimpleTy != MVT::f32)
    5824             :     return 0;
    5825          18 :   if ((Subtarget->hasAVX512())) {
    5826           0 :     return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5827             :   }
    5828           9 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5829           9 :     return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5830             :   }
    5831           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5832           0 :     return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5833             :   }
    5834           0 :   if ((!Subtarget->hasSSE1())) {
    5835           0 :     return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5836             :   }
    5837             :   return 0;
    5838             : }
    5839             : 
    5840           5 : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5841           5 :   if (RetVT.SimpleTy != MVT::f64)
    5842             :     return 0;
    5843          10 :   if ((Subtarget->hasAVX512())) {
    5844           0 :     return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5845             :   }
    5846           5 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5847           5 :     return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5848             :   }
    5849           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5850           0 :     return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5851             :   }
    5852           0 :   if ((!Subtarget->hasSSE2())) {
    5853           0 :     return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5854             :   }
    5855             :   return 0;
    5856             : }
    5857             : 
    5858             : unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5859           0 :   if (RetVT.SimpleTy != MVT::f80)
    5860             :     return 0;
    5861           0 :   return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5862             : }
    5863             : 
    5864           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5865           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5866             :     return 0;
    5867           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5868           0 :     return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5869             :   }
    5870           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5871           2 :     return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5872             :   }
    5873           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5874           0 :     return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5875             :   }
    5876             :   return 0;
    5877             : }
    5878             : 
    5879           2 : unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5880           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5881             :     return 0;
    5882           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5883           0 :     return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5884             :   }
    5885           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5886           2 :     return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5887             :   }
    5888             :   return 0;
    5889             : }
    5890             : 
    5891             : unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5892           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5893             :     return 0;
    5894           0 :   if ((Subtarget->hasAVX512())) {
    5895           0 :     return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5896             :   }
    5897             :   return 0;
    5898             : }
    5899             : 
    5900           2 : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5901           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    5902             :     return 0;
    5903           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5904           0 :     return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5905             :   }
    5906           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5907           2 :     return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5908             :   }
    5909           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5910           0 :     return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5911             :   }
    5912             :   return 0;
    5913             : }
    5914             : 
    5915           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5916           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    5917             :     return 0;
    5918           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5919           0 :     return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5920             :   }
    5921           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5922           2 :     return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5923             :   }
    5924             :   return 0;
    5925             : }
    5926             : 
    5927             : unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5928           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    5929             :     return 0;
    5930           0 :   if ((Subtarget->hasAVX512())) {
    5931           0 :     return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5932             :   }
    5933             :   return 0;
    5934             : }
    5935             : 
    5936          22 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5937          22 :   switch (VT.SimpleTy) {
    5938           9 :   case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5939           5 :   case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5940           0 :   case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5941           2 :   case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5942           2 :   case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5943           0 :   case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5944           2 :   case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5945           2 :   case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5946           0 :   case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5947             :   default: return 0;
    5948             :   }
    5949             : }
    5950             : 
    5951             : // FastEmit functions for ISD::FSUB.
    5952             : 
    5953           0 : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5954           0 :   if (RetVT.SimpleTy != MVT::f32)
    5955             :     return 0;
    5956           0 :   if ((Subtarget->hasAVX512())) {
    5957           0 :     return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5958             :   }
    5959           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5960           0 :     return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5961             :   }
    5962           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5963           0 :     return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5964             :   }
    5965           0 :   if ((!Subtarget->hasSSE1())) {
    5966           0 :     return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5967             :   }
    5968             :   return 0;
    5969             : }
    5970             : 
    5971          25 : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5972          25 :   if (RetVT.SimpleTy != MVT::f64)
    5973             :     return 0;
    5974          50 :   if ((Subtarget->hasAVX512())) {
    5975           0 :     return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5976             :   }
    5977          25 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5978          25 :     return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5979             :   }
    5980           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5981           0 :     return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5982             :   }
    5983           0 :   if ((!Subtarget->hasSSE2())) {
    5984           0 :     return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5985             :   }
    5986             :   return 0;
    5987             : }
    5988             : 
    5989             : unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5990           0 :   if (RetVT.SimpleTy != MVT::f80)
    5991             :     return 0;
    5992           0 :   return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5993             : }
    5994             : 
    5995           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5996           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5997             :     return 0;
    5998           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5999           0 :     return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6000             :   }
    6001           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6002           2 :     return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6003             :   }
    6004           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6005           0 :     return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6006             :   }
    6007             :   return 0;
    6008             : }
    6009             : 
    6010           2 : unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6011           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6012             :     return 0;
    6013           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6014           0 :     return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6015             :   }
    6016           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6017           2 :     return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6018             :   }
    6019             :   return 0;
    6020             : }
    6021             : 
    6022             : unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6023           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6024             :     return 0;
    6025           0 :   if ((Subtarget->hasAVX512())) {
    6026           0 :     return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6027             :   }
    6028             :   return 0;
    6029             : }
    6030             : 
    6031           2 : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6032           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    6033             :     return 0;
    6034           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6035           0 :     return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6036             :   }
    6037           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6038           2 :     return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6039             :   }
    6040           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6041           0 :     return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6042             :   }
    6043             :   return 0;
    6044             : }
    6045             : 
    6046           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6047           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6048             :     return 0;
    6049           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6050           0 :     return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6051             :   }
    6052           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6053           2 :     return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6054             :   }
    6055             :   return 0;
    6056             : }
    6057             : 
    6058             : unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6059           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6060             :     return 0;
    6061           0 :   if ((Subtarget->hasAVX512())) {
    6062           0 :     return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6063             :   }
    6064             :   return 0;
    6065             : }
    6066             : 
    6067          33 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6068          33 :   switch (VT.SimpleTy) {
    6069           0 :   case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6070          25 :   case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6071           0 :   case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6072           2 :   case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6073           2 :   case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6074           0 :   case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6075           2 :   case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6076           2 :   case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6077           0 :   case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6078             :   default: return 0;
    6079             :   }
    6080             : }
    6081             : 
    6082             : // FastEmit functions for ISD::MUL.
    6083             : 
    6084           2 : unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6085           2 :   if (RetVT.SimpleTy != MVT::i8)
    6086             :     return 0;
    6087           4 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
    6088           2 :   return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill);
    6089             : }
    6090             : 
    6091             : unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6092           0 :   if (RetVT.SimpleTy != MVT::i16)
    6093             :     return 0;
    6094           0 :   return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6095             : }
    6096             : 
    6097             : unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6098          22 :   if (RetVT.SimpleTy != MVT::i32)
    6099             :     return 0;
    6100          22 :   return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6101             : }
    6102             : 
    6103             : unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6104         102 :   if (RetVT.SimpleTy != MVT::i64)
    6105             :     return 0;
    6106         102 :   return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6107             : }
    6108             : 
    6109           2 : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6110           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    6111             :     return 0;
    6112           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6113           0 :     return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6114             :   }
    6115           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6116           2 :     return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6117             :   }
    6118           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6119           0 :     return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6120             :   }
    6121             :   return 0;
    6122             : }
    6123             : 
    6124           2 : unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6125           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    6126             :     return 0;
    6127           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6128           0 :     return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6129             :   }
    6130           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6131           2 :     return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6132             :   }
    6133             :   return 0;
    6134             : }
    6135             : 
    6136             : unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6137           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6138             :     return 0;
    6139           0 :   if ((Subtarget->hasBWI())) {
    6140           0 :     return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6141             :   }
    6142             :   return 0;
    6143             : }
    6144             : 
    6145           2 : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6146           2 :   if (RetVT.SimpleTy != MVT::v4i32)
    6147             :     return 0;
    6148           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6149           0 :     return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6150             :   }
    6151           2 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6152           2 :     return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6153             :   }
    6154           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6155           0 :     return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6156             :   }
    6157             :   return 0;
    6158             : }
    6159             : 
    6160           2 : unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6161           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    6162             :     return 0;
    6163           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6164           0 :     return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6165             :   }
    6166           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6167           2 :     return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6168             :   }
    6169             :   return 0;
    6170             : }
    6171             : 
    6172             : unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6173           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6174             :     return 0;
    6175           0 :   if ((Subtarget->hasAVX512())) {
    6176           0 :     return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6177             :   }
    6178             :   return 0;
    6179             : }
    6180             : 
    6181           0 : unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6182           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6183             :     return 0;
    6184           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6185           0 :     return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6186             :   }
    6187             :   return 0;
    6188             : }
    6189             : 
    6190           0 : unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6191           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6192             :     return 0;
    6193           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6194           0 :     return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6195             :   }
    6196             :   return 0;
    6197             : }
    6198             : 
    6199             : unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6200           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6201             :     return 0;
    6202           0 :   if ((Subtarget->hasDQI())) {
    6203           0 :     return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6204             :   }
    6205             :   return 0;
    6206             : }
    6207             : 
    6208         134 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6209         134 :   switch (VT.SimpleTy) {
    6210           2 :   case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6211           0 :   case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6212          22 :   case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6213         102 :   case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6214           2 :   case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6215           2 :   case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6216           0 :   case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6217           2 :   case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6218           2 :   case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6219           0 :   case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6220           0 :   case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6221           0 :   case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6222           0 :   case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6223             :   default: return 0;
    6224             :   }
    6225             : }
    6226             : 
    6227             : // FastEmit functions for ISD::MULHS.
    6228             : 
    6229           0 : unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6230           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6231             :     return 0;
    6232           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6233           0 :     return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6234             :   }
    6235           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6236           0 :     return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6237             :   }
    6238           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6239           0 :     return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6240             :   }
    6241             :   return 0;
    6242             : }
    6243             : 
    6244           0 : unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6245           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6246             :     return 0;
    6247           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6248           0 :     return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6249             :   }
    6250           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6251           0 :     return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6252             :   }
    6253             :   return 0;
    6254             : }
    6255             : 
    6256             : unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6257           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6258             :     return 0;
    6259           0 :   if ((Subtarget->hasBWI())) {
    6260           0 :     return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6261             :   }
    6262             :   return 0;
    6263             : }
    6264             : 
    6265           0 : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6266           0 :   switch (VT.SimpleTy) {
    6267           0 :   case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6268           0 :   case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6269           0 :   case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6270             :   default: return 0;
    6271             :   }
    6272             : }
    6273             : 
    6274             : // FastEmit functions for ISD::MULHU.
    6275             : 
    6276           0 : unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6277           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6278             :     return 0;
    6279           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6280           0 :     return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6281             :   }
    6282           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6283           0 :     return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6284             :   }
    6285           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6286           0 :     return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6287             :   }
    6288             :   return 0;
    6289             : }
    6290             : 
    6291           0 : unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6292           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6293             :     return 0;
    6294           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6295           0 :     return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6296             :   }
    6297           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6298           0 :     return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6299             :   }
    6300             :   return 0;
    6301             : }
    6302             : 
    6303             : unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6304           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6305             :     return 0;
    6306           0 :   if ((Subtarget->hasBWI())) {
    6307           0 :     return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6308             :   }
    6309             :   return 0;
    6310             : }
    6311             : 
    6312           0 : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6313           0 :   switch (VT.SimpleTy) {
    6314           0 :   case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6315           0 :   case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6316           0 :   case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6317             :   default: return 0;
    6318             :   }
    6319             : }
    6320             : 
    6321             : // FastEmit functions for ISD::OR.
    6322             : 
    6323             : unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6324           6 :   if (RetVT.SimpleTy != MVT::i8)
    6325             :     return 0;
    6326           6 :   return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6327             : }
    6328             : 
    6329             : unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6330          20 :   if (RetVT.SimpleTy != MVT::i16)
    6331             :     return 0;
    6332          20 :   return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6333             : }
    6334             : 
    6335             : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6336         143 :   if (RetVT.SimpleTy != MVT::i32)
    6337             :     return 0;
    6338         143 :   return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6339             : }
    6340             : 
    6341             : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6342          13 :   if (RetVT.SimpleTy != MVT::i64)
    6343             :     return 0;
    6344          13 :   return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6345             : }
    6346             : 
    6347             : unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6348           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    6349             :     return 0;
    6350           0 :   if ((Subtarget->hasDQI())) {
    6351           0 :     return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6352             :   }
    6353             :   return 0;
    6354             : }
    6355             : 
    6356             : unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6357           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    6358             :     return 0;
    6359           0 :   if ((Subtarget->hasAVX512())) {
    6360           0 :     return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6361             :   }
    6362             :   return 0;
    6363             : }
    6364             : 
    6365             : unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6366           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    6367             :     return 0;
    6368           0 :   if ((Subtarget->hasBWI())) {
    6369           0 :     return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6370             :   }
    6371             :   return 0;
    6372             : }
    6373             : 
    6374             : unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6375           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    6376             :     return 0;
    6377           0 :   if ((Subtarget->hasBWI())) {
    6378           0 :     return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6379             :   }
    6380             :   return 0;
    6381             : }
    6382             : 
    6383           4 : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6384           4 :   if (RetVT.SimpleTy != MVT::v2i64)
    6385             :     return 0;
    6386           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6387           0 :     return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6388             :   }
    6389           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6390           2 :     return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6391             :   }
    6392           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6393           2 :     return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6394             :   }
    6395             :   return 0;
    6396             : }
    6397             : 
    6398           6 : unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6399           6 :   if (RetVT.SimpleTy != MVT::v4i64)
    6400             :     return 0;
    6401          12 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    6402           4 :     return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6403             :   }
    6404           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6405           0 :     return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6406             :   }
    6407           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6408           2 :     return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6409             :   }
    6410             :   return 0;
    6411             : }
    6412             : 
    6413             : unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6414           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6415             :     return 0;
    6416           0 :   if ((Subtarget->hasAVX512())) {
    6417           0 :     return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6418             :   }
    6419             :   return 0;
    6420             : }
    6421             : 
    6422         196 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6423         196 :   switch (VT.SimpleTy) {
    6424           6 :   case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6425          20 :   case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6426         143 :   case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6427          13 :   case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6428           0 :   case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6429           0 :   case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6430           0 :   case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6431           0 :   case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6432           4 :   case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6433           6 :   case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6434           0 :   case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6435             :   default: return 0;
    6436             :   }
    6437             : }
    6438             : 
    6439             : // FastEmit functions for ISD::ROTL.
    6440             : 
    6441           0 : unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6442           0 :   if (RetVT.SimpleTy != MVT::i8)
    6443             :     return 0;
    6444           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6445           0 :   return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6446             : }
    6447             : 
    6448             : unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6449           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6450             :     return 0;
    6451           0 :   if ((Subtarget->hasXOP())) {
    6452           0 :     return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6453             :   }
    6454             :   return 0;
    6455             : }
    6456             : 
    6457             : unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6458           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6459             :     return 0;
    6460           0 :   if ((Subtarget->hasXOP())) {
    6461           0 :     return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6462             :   }
    6463             :   return 0;
    6464             : }
    6465             : 
    6466           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6467           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6468             :     return 0;
    6469           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6470           0 :     return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6471             :   }
    6472           0 :   if ((Subtarget->hasXOP())) {
    6473           0 :     return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6474             :   }
    6475             :   return 0;
    6476             : }
    6477             : 
    6478           0 : unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6479           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6480             :     return 0;
    6481           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6482           0 :     return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6483             :   }
    6484             :   return 0;
    6485             : }
    6486             : 
    6487             : unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6488           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6489             :     return 0;
    6490           0 :   if ((Subtarget->hasAVX512())) {
    6491           0 :     return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6492             :   }
    6493             :   return 0;
    6494             : }
    6495             : 
    6496           0 : unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6497           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6498             :     return 0;
    6499           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6500           0 :     return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6501             :   }
    6502           0 :   if ((Subtarget->hasXOP())) {
    6503           0 :     return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6504             :   }
    6505             :   return 0;
    6506             : }
    6507             : 
    6508           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6509           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6510             :     return 0;
    6511           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6512           0 :     return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6513             :   }
    6514             :   return 0;
    6515             : }
    6516             : 
    6517             : unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6518           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6519             :     return 0;
    6520           0 :   if ((Subtarget->hasAVX512())) {
    6521           0 :     return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6522             :   }
    6523             :   return 0;
    6524             : }
    6525             : 
    6526           0 : unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6527           0 :   switch (VT.SimpleTy) {
    6528           0 :   case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6529           0 :   case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6530           0 :   case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6531           0 :   case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6532           0 :   case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6533           0 :   case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6534           0 :   case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6535           0 :   case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6536           0 :   case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6537             :   default: return 0;
    6538             :   }
    6539             : }
    6540             : 
    6541             : // FastEmit functions for ISD::ROTR.
    6542             : 
    6543           0 : unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6544           0 :   if (RetVT.SimpleTy != MVT::i8)
    6545             :     return 0;
    6546           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6547           0 :   return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6548             : }
    6549             : 
    6550           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6551           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6552             :     return 0;
    6553           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6554           0 :     return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6555             :   }
    6556             :   return 0;
    6557             : }
    6558             : 
    6559           0 : unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6560           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6561             :     return 0;
    6562           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6563           0 :     return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6564             :   }
    6565             :   return 0;
    6566             : }
    6567             : 
    6568             : unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6569           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6570             :     return 0;
    6571           0 :   if ((Subtarget->hasAVX512())) {
    6572           0 :     return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6573             :   }
    6574             :   return 0;
    6575             : }
    6576             : 
    6577           0 : unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6578           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6579             :     return 0;
    6580           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6581           0 :     return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6582             :   }
    6583             :   return 0;
    6584             : }
    6585             : 
    6586           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6587           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6588             :     return 0;
    6589           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6590           0 :     return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6591             :   }
    6592             :   return 0;
    6593             : }
    6594             : 
    6595             : unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6596           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6597             :     return 0;
    6598           0 :   if ((Subtarget->hasAVX512())) {
    6599           0 :     return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6600             :   }
    6601             :   return 0;
    6602             : }
    6603             : 
    6604           0 : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6605           0 :   switch (VT.SimpleTy) {
    6606           0 :   case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6607           0 :   case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6608           0 :   case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6609           0 :   case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6610           0 :   case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6611           0 :   case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6612           0 :   case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6613             :   default: return 0;
    6614             :   }
    6615             : }
    6616             : 
    6617             : // FastEmit functions for ISD::SHL.
    6618             : 
    6619           3 : unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6620           3 :   if (RetVT.SimpleTy != MVT::i8)
    6621             :     return 0;
    6622           6 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6623           3 :   return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6624             : }
    6625             : 
    6626           0 : unsigned fastEmit_ISD_SHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6627           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6628             :     return 0;
    6629           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6630           0 :     return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6631             :   }
    6632             :   return 0;
    6633             : }
    6634             : 
    6635           0 : unsigned fastEmit_ISD_SHL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6636           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6637             :     return 0;
    6638           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6639           0 :     return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6640             :   }
    6641             :   return 0;
    6642             : }
    6643             : 
    6644             : unsigned fastEmit_ISD_SHL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6645           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6646             :     return 0;
    6647           0 :   if ((Subtarget->hasBWI())) {
    6648           0 :     return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6649             :   }
    6650             :   return 0;
    6651             : }
    6652             : 
    6653           0 : unsigned fastEmit_ISD_SHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6654           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6655             :     return 0;
    6656           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6657           0 :     return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6658             :   }
    6659           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6660           0 :     return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6661             :   }
    6662             :   return 0;
    6663             : }
    6664             : 
    6665           0 : unsigned fastEmit_ISD_SHL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6666           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6667             :     return 0;
    6668           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6669           0 :     return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6670             :   }
    6671           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6672           0 :     return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6673             :   }
    6674             :   return 0;
    6675             : }
    6676             : 
    6677             : unsigned fastEmit_ISD_SHL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6678           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6679             :     return 0;
    6680           0 :   if ((Subtarget->hasAVX512())) {
    6681           0 :     return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6682             :   }
    6683             :   return 0;
    6684             : }
    6685             : 
    6686           0 : unsigned fastEmit_ISD_SHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6687           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6688             :     return 0;
    6689           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6690           0 :     return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6691             :   }
    6692           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6693           0 :     return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6694             :   }
    6695             :   return 0;
    6696             : }
    6697             : 
    6698           0 : unsigned fastEmit_ISD_SHL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6699           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6700             :     return 0;
    6701           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6702           0 :     return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6703             :   }
    6704           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6705           0 :     return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6706             :   }
    6707             :   return 0;
    6708             : }
    6709             : 
    6710             : unsigned fastEmit_ISD_SHL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6711           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6712             :     return 0;
    6713           0 :   if ((Subtarget->hasAVX512())) {
    6714           0 :     return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6715             :   }
    6716             :   return 0;
    6717             : }
    6718             : 
    6719         107 : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6720         107 :   switch (VT.SimpleTy) {
    6721           3 :   case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6722           0 :   case MVT::v8i16: return fastEmit_ISD_SHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6723           0 :   case MVT::v16i16: return fastEmit_ISD_SHL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6724           0 :   case MVT::v32i16: return fastEmit_ISD_SHL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6725           0 :   case MVT::v4i32: return fastEmit_ISD_SHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6726           0 :   case MVT::v8i32: return fastEmit_ISD_SHL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6727           0 :   case MVT::v16i32: return fastEmit_ISD_SHL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6728           0 :   case MVT::v2i64: return fastEmit_ISD_SHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6729           0 :   case MVT::v4i64: return fastEmit_ISD_SHL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6730           0 :   case MVT::v8i64: return fastEmit_ISD_SHL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6731             :   default: return 0;
    6732             :   }
    6733             : }
    6734             : 
    6735             : // FastEmit functions for ISD::SMAX.
    6736             : 
    6737           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6738           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6739             :     return 0;
    6740           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6741           0 :     return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6742             :   }
    6743           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6744           0 :     return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6745             :   }
    6746           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6747           0 :     return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6748             :   }
    6749             :   return 0;
    6750             : }
    6751             : 
    6752           0 : unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6753           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    6754             :     return 0;
    6755           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6756           0 :     return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6757             :   }
    6758           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6759           0 :     return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6760             :   }
    6761             :   return 0;
    6762             : }
    6763             : 
    6764             : unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6765           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    6766             :     return 0;
    6767           0 :   if ((Subtarget->hasBWI())) {
    6768           0 :     return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6769             :   }
    6770             :   return 0;
    6771             : }
    6772             : 
    6773           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6774           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6775             :     return 0;
    6776           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6777           0 :     return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6778             :   }
    6779           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6780           0 :     return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6781             :   }
    6782           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6783           0 :     return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6784             :   }
    6785             :   return 0;
    6786             : }
    6787             : 
    6788           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6789           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6790             :     return 0;
    6791           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6792           0 :     return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6793             :   }
    6794           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6795           0 :     return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6796             :   }
    6797             :   return 0;
    6798             : }
    6799             : 
    6800             : unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6801           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6802             :     return 0;
    6803           0 :   if ((Subtarget->hasBWI())) {
    6804           0 :     return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6805             :   }
    6806             :   return 0;
    6807             : }
    6808             : 
    6809           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6810           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6811             :     return 0;
    6812           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6813           0 :     return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6814             :   }
    6815           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6816           0 :     return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6817             :   }
    6818           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6819           0 :     return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6820             :   }
    6821             :   return 0;
    6822             : }
    6823             : 
    6824           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6825           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6826             :     return 0;
    6827           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6828           0 :     return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6829             :   }
    6830           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6831           0 :     return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6832             :   }
    6833             :   return 0;
    6834             : }
    6835             : 
    6836             : unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6837           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6838             :     return 0;
    6839           0 :   if ((Subtarget->hasAVX512())) {
    6840           0 :     return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6841             :   }
    6842             :   return 0;
    6843             : }
    6844             : 
    6845           0 : unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6846           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6847             :     return 0;
    6848           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6849           0 :     return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6850             :   }
    6851             :   return 0;
    6852             : }
    6853             : 
    6854           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6855           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6856             :     return 0;
    6857           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6858           0 :     return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6859             :   }
    6860             :   return 0;
    6861             : }
    6862             : 
    6863             : unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6864           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6865             :     return 0;
    6866           0 :   if ((Subtarget->hasAVX512())) {
    6867           0 :     return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6868             :   }
    6869             :   return 0;
    6870             : }
    6871             : 
    6872           0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6873           0 :   switch (VT.SimpleTy) {
    6874           0 :   case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6875           0 :   case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6876           0 :   case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6877           0 :   case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6878           0 :   case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6879           0 :   case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6880           0 :   case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6881           0 :   case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6882           0 :   case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6883           0 :   case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6884           0 :   case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6885           0 :   case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6886             :   default: return 0;
    6887             :   }
    6888             : }
    6889             : 
    6890             : // FastEmit functions for ISD::SMIN.
    6891             : 
    6892           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6893           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6894             :     return 0;
    6895           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6896           0 :     return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6897             :   }
    6898           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6899           0 :     return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6900             :   }
    6901           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6902           0 :     return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6903             :   }
    6904             :   return 0;
    6905             : }
    6906             : 
    6907           0 : unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6908           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    6909             :     return 0;
    6910           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6911           0 :     return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6912             :   }
    6913           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6914           0 :     return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6915             :   }
    6916             :   return 0;
    6917             : }
    6918             : 
    6919             : unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6920           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    6921             :     return 0;
    6922           0 :   if ((Subtarget->hasBWI())) {
    6923           0 :     return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6924             :   }
    6925             :   return 0;
    6926             : }
    6927             : 
    6928           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6929           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6930             :     return 0;
    6931           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6932           0 :     return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6933             :   }
    6934           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6935           0 :     return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6936             :   }
    6937           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6938           0 :     return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6939             :   }
    6940             :   return 0;
    6941             : }
    6942             : 
    6943           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6944           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6945             :     return 0;
    6946           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6947           0 :     return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6948             :   }
    6949           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6950           0 :     return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6951             :   }
    6952             :   return 0;
    6953             : }
    6954             : 
    6955             : unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6956           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6957             :     return 0;
    6958           0 :   if ((Subtarget->hasBWI())) {
    6959           0 :     return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6960             :   }
    6961             :   return 0;
    6962             : }
    6963             : 
    6964           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6965           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6966             :     return 0;
    6967           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6968           0 :     return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6969             :   }
    6970           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6971           0 :     return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6972             :   }
    6973           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6974           0 :     return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6975             :   }
    6976             :   return 0;
    6977             : }
    6978             : 
    6979           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6980           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6981             :     return 0;
    6982           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6983           0 :     return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6984             :   }
    6985           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6986           0 :     return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6987             :   }
    6988             :   return 0;
    6989             : }
    6990             : 
    6991             : unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6992           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6993             :     return 0;
    6994           0 :   if ((Subtarget->hasAVX512())) {
    6995           0 :     return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6996             :   }
    6997             :   return 0;
    6998             : }
    6999             : 
    7000           0 : unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7001           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7002             :     return 0;
    7003           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7004           0 :     return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7005             :   }
    7006             :   return 0;
    7007             : }
    7008             : 
    7009           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7010           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7011             :     return 0;
    7012           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7013           0 :     return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7014             :   }
    7015             :   return 0;
    7016             : }
    7017             : 
    7018             : unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7019           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7020             :     return 0;
    7021           0 :   if ((Subtarget->hasAVX512())) {
    7022           0 :     return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7023             :   }
    7024             :   return 0;
    7025             : }
    7026             : 
    7027           0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7028           0 :   switch (VT.SimpleTy) {
    7029           0 :   case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7030           0 :   case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7031           0 :   case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7032           0 :   case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7033           0 :   case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7034           0 :   case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7035           0 :   case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7036           0 :   case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7037           0 :   case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7038           0 :   case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7039           0 :   case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7040           0 :   case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7041             :   default: return 0;
    7042             :   }
    7043             : }
    7044             : 
    7045             : // FastEmit functions for ISD::SRA.
    7046             : 
    7047           1 : unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7048           1 :   if (RetVT.SimpleTy != MVT::i8)
    7049             :     return 0;
    7050           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7051           1 :   return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7052             : }
    7053             : 
    7054           0 : unsigned fastEmit_ISD_SRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7055           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7056             :     return 0;
    7057           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7058           0 :     return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7059             :   }
    7060             :   return 0;
    7061             : }
    7062             : 
    7063           0 : unsigned fastEmit_ISD_SRA_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7064           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7065             :     return 0;
    7066           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7067           0 :     return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7068             :   }
    7069             :   return 0;
    7070             : }
    7071             : 
    7072             : unsigned fastEmit_ISD_SRA_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7073           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7074             :     return 0;
    7075           0 :   if ((Subtarget->hasBWI())) {
    7076           0 :     return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7077             :   }
    7078             :   return 0;
    7079             : }
    7080             : 
    7081           0 : unsigned fastEmit_ISD_SRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7082           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7083             :     return 0;
    7084           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7085           0 :     return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7086             :   }
    7087           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7088           0 :     return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7089             :   }
    7090             :   return 0;
    7091             : }
    7092             : 
    7093           0 : unsigned fastEmit_ISD_SRA_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7094           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7095             :     return 0;
    7096           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7097           0 :     return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7098             :   }
    7099           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7100           0 :     return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7101             :   }
    7102             :   return 0;
    7103             : }
    7104             : 
    7105             : unsigned fastEmit_ISD_SRA_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7106           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7107             :     return 0;
    7108           0 :   if ((Subtarget->hasAVX512())) {
    7109           0 :     return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7110             :   }
    7111             :   return 0;
    7112             : }
    7113             : 
    7114           0 : unsigned fastEmit_ISD_SRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7115           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7116             :     return 0;
    7117           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7118           0 :     return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7119             :   }
    7120             :   return 0;
    7121             : }
    7122             : 
    7123           0 : unsigned fastEmit_ISD_SRA_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7124           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7125             :     return 0;
    7126           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7127           0 :     return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7128             :   }
    7129             :   return 0;
    7130             : }
    7131             : 
    7132             : unsigned fastEmit_ISD_SRA_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7133           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7134             :     return 0;
    7135           0 :   if ((Subtarget->hasAVX512())) {
    7136           0 :     return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7137             :   }
    7138             :   return 0;
    7139             : }
    7140             : 
    7141           5 : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7142           5 :   switch (VT.SimpleTy) {
    7143           1 :   case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7144           0 :   case MVT::v8i16: return fastEmit_ISD_SRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7145           0 :   case MVT::v16i16: return fastEmit_ISD_SRA_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7146           0 :   case MVT::v32i16: return fastEmit_ISD_SRA_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7147           0 :   case MVT::v4i32: return fastEmit_ISD_SRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7148           0 :   case MVT::v8i32: return fastEmit_ISD_SRA_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7149           0 :   case MVT::v16i32: return fastEmit_ISD_SRA_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7150           0 :   case MVT::v2i64: return fastEmit_ISD_SRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7151           0 :   case MVT::v4i64: return fastEmit_ISD_SRA_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7152           0 :   case MVT::v8i64: return fastEmit_ISD_SRA_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7153             :   default: return 0;
    7154             :   }
    7155             : }
    7156             : 
    7157             : // FastEmit functions for ISD::SRL.
    7158             : 
    7159           1 : unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7160           1 :   if (RetVT.SimpleTy != MVT::i8)
    7161             :     return 0;
    7162           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7163           1 :   return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7164             : }
    7165             : 
    7166           0 : unsigned fastEmit_ISD_SRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7167           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7168             :     return 0;
    7169           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7170           0 :     return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7171             :   }
    7172             :   return 0;
    7173             : }
    7174             : 
    7175           0 : unsigned fastEmit_ISD_SRL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7176           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7177             :     return 0;
    7178           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7179           0 :     return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7180             :   }
    7181             :   return 0;
    7182             : }
    7183             : 
    7184             : unsigned fastEmit_ISD_SRL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7185           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7186             :     return 0;
    7187           0 :   if ((Subtarget->hasBWI())) {
    7188           0 :     return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7189             :   }
    7190             :   return 0;
    7191             : }
    7192             : 
    7193           0 : unsigned fastEmit_ISD_SRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7194           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7195             :     return 0;
    7196           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7197           0 :     return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7198             :   }
    7199           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7200           0 :     return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7201             :   }
    7202             :   return 0;
    7203             : }
    7204             : 
    7205           0 : unsigned fastEmit_ISD_SRL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7206           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7207             :     return 0;
    7208           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7209           0 :     return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7210             :   }
    7211           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7212           0 :     return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7213             :   }
    7214             :   return 0;
    7215             : }
    7216             : 
    7217             : unsigned fastEmit_ISD_SRL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7218           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7219             :     return 0;
    7220           0 :   if ((Subtarget->hasAVX512())) {
    7221           0 :     return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7222             :   }
    7223             :   return 0;
    7224             : }
    7225             : 
    7226           0 : unsigned fastEmit_ISD_SRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7227           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7228             :     return 0;
    7229           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7230           0 :     return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7231             :   }
    7232           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7233           0 :     return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7234             :   }
    7235             :   return 0;
    7236             : }
    7237             : 
    7238           0 : unsigned fastEmit_ISD_SRL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7239           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7240             :     return 0;
    7241           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7242           0 :     return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7243             :   }
    7244           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7245           0 :     return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7246             :   }
    7247             :   return 0;
    7248             : }
    7249             : 
    7250             : unsigned fastEmit_ISD_SRL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7251           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7252             :     return 0;
    7253           0 :   if ((Subtarget->hasAVX512())) {
    7254           0 :     return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7255             :   }
    7256             :   return 0;
    7257             : }
    7258             : 
    7259           4 : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7260           4 :   switch (VT.SimpleTy) {
    7261           1 :   case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7262           0 :   case MVT::v8i16: return fastEmit_ISD_SRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7263           0 :   case MVT::v16i16: return fastEmit_ISD_SRL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7264           0 :   case MVT::v32i16: return fastEmit_ISD_SRL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7265           0 :   case MVT::v4i32: return fastEmit_ISD_SRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7266           0 :   case MVT::v8i32: return fastEmit_ISD_SRL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7267           0 :   case MVT::v16i32: return fastEmit_ISD_SRL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7268           0 :   case MVT::v2i64: return fastEmit_ISD_SRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7269           0 :   case MVT::v4i64: return fastEmit_ISD_SRL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7270           0 :   case MVT::v8i64: return fastEmit_ISD_SRL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7271             :   default: return 0;
    7272             :   }
    7273             : }
    7274             : 
    7275             : // FastEmit functions for ISD::SUB.
    7276             : 
    7277             : unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7278           1 :   if (RetVT.SimpleTy != MVT::i8)
    7279             :     return 0;
    7280           1 :   return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7281             : }
    7282             : 
    7283             : unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7284           0 :   if (RetVT.SimpleTy != MVT::i16)
    7285             :     return 0;
    7286           0 :   return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7287             : }
    7288             : 
    7289             : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7290         115 :   if (RetVT.SimpleTy != MVT::i32)
    7291             :     return 0;
    7292         115 :   return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7293             : }
    7294             : 
    7295             : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7296        1279 :   if (RetVT.SimpleTy != MVT::i64)
    7297             :     return 0;
    7298        1279 :   return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7299             : }
    7300             : 
    7301           2 : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7302           2 :   if (RetVT.SimpleTy != MVT::v16i8)
    7303             :     return 0;
    7304           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7305           0 :     return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7306             :   }
    7307           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7308           2 :     return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7309             :   }
    7310           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7311           0 :     return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7312             :   }
    7313             :   return 0;
    7314             : }
    7315             : 
    7316           2 : unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7317           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    7318             :     return 0;
    7319           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7320           0 :     return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7321             :   }
    7322           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7323           2 :     return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7324             :   }
    7325             :   return 0;
    7326             : }
    7327             : 
    7328             : unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7329           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7330             :     return 0;
    7331           0 :   if ((Subtarget->hasBWI())) {
    7332           0 :     return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7333             :   }
    7334             :   return 0;
    7335             : }
    7336             : 
    7337           2 : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7338           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    7339             :     return 0;
    7340           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7341           0 :     return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7342             :   }
    7343           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7344           2 :     return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7345             :   }
    7346           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7347           0 :     return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7348             :   }
    7349             :   return 0;
    7350             : }
    7351             : 
    7352           2 : unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7353           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    7354             :     return 0;
    7355           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7356           0 :     return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7357             :   }
    7358           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7359           2 :     return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7360             :   }
    7361             :   return 0;
    7362             : }
    7363             : 
    7364             : unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7365           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7366             :     return 0;
    7367           0 :   if ((Subtarget->hasBWI())) {
    7368           0 :     return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7369             :   }
    7370             :   return 0;
    7371             : }
    7372             : 
    7373           2 : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7374           2 :   if (RetVT.SimpleTy != MVT::v4i32)
    7375             :     return 0;
    7376           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7377           0 :     return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7378             :   }
    7379           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7380           2 :     return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7381             :   }
    7382           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7383           0 :     return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7384             :   }
    7385             :   return 0;
    7386             : }
    7387             : 
    7388           2 : unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7389           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    7390             :     return 0;
    7391           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7392           0 :     return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7393             :   }
    7394           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7395           2 :     return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7396             :   }
    7397             :   return 0;
    7398             : }
    7399             : 
    7400             : unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7401           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7402             :     return 0;
    7403           0 :   if ((Subtarget->hasAVX512())) {
    7404           0 :     return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7405             :   }
    7406             :   return 0;
    7407             : }
    7408             : 
    7409           2 : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7410           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    7411             :     return 0;
    7412           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7413           0 :     return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7414             :   }
    7415           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7416           2 :     return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7417             :   }
    7418           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7419           0 :     return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7420             :   }
    7421             :   return 0;
    7422             : }
    7423             : 
    7424           2 : unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7425           2 :   if (RetVT.SimpleTy != MVT::v4i64)
    7426             :     return 0;
    7427           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7428           0 :     return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7429             :   }
    7430           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7431           2 :     return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7432             :   }
    7433             :   return 0;
    7434             : }
    7435             : 
    7436             : unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7437           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7438             :     return 0;
    7439           0 :   if ((Subtarget->hasAVX512())) {
    7440           0 :     return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7441             :   }
    7442             :   return 0;
    7443             : }
    7444             : 
    7445        1411 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7446        1411 :   switch (VT.SimpleTy) {
    7447           1 :   case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7448           0 :   case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7449         115 :   case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7450        1279 :   case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7451           2 :   case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7452           2 :   case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7453           0 :   case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7454           2 :   case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7455           2 :   case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7456           0 :   case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7457           2 :   case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7458           2 :   case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7459           0 :   case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7460           2 :   case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7461           2 :   case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7462           0 :   case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7463             :   default: return 0;
    7464             :   }
    7465             : }
    7466             : 
    7467             : // FastEmit functions for ISD::UMAX.
    7468             : 
    7469           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7470           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7471             :     return 0;
    7472           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7473           0 :     return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7474             :   }
    7475           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7476           0 :     return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7477             :   }
    7478           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7479           0 :     return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7480             :   }
    7481             :   return 0;
    7482             : }
    7483             : 
    7484           0 : unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7485           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7486             :     return 0;
    7487           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7488           0 :     return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7489             :   }
    7490           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7491           0 :     return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7492             :   }
    7493             :   return 0;
    7494             : }
    7495             : 
    7496             : unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7497           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7498             :     return 0;
    7499           0 :   if ((Subtarget->hasBWI())) {
    7500           0 :     return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7501             :   }
    7502             :   return 0;
    7503             : }
    7504             : 
    7505           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7506           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7507             :     return 0;
    7508           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7509           0 :     return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7510             :   }
    7511           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7512           0 :     return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7513             :   }
    7514           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7515           0 :     return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7516             :   }
    7517             :   return 0;
    7518             : }
    7519             : 
    7520           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7521           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7522             :     return 0;
    7523           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7524           0 :     return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7525             :   }
    7526           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7527           0 :     return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7528             :   }
    7529             :   return 0;
    7530             : }
    7531             : 
    7532             : unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7533           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7534             :     return 0;
    7535           0 :   if ((Subtarget->hasBWI())) {
    7536           0 :     return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7537             :   }
    7538             :   return 0;
    7539             : }
    7540             : 
    7541           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7542           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7543             :     return 0;
    7544           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7545           0 :     return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7546             :   }
    7547           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7548           0 :     return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7549             :   }
    7550           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7551           0 :     return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7552             :   }
    7553             :   return 0;
    7554             : }
    7555             : 
    7556           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7557           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7558             :     return 0;
    7559           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7560           0 :     return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7561             :   }
    7562           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7563           0 :     return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7564             :   }
    7565             :   return 0;
    7566             : }
    7567             : 
    7568             : unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7569           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7570             :     return 0;
    7571           0 :   if ((Subtarget->hasAVX512())) {
    7572           0 :     return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7573             :   }
    7574             :   return 0;
    7575             : }
    7576             : 
    7577           0 : unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7578           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7579             :     return 0;
    7580           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7581           0 :     return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7582             :   }
    7583             :   return 0;
    7584             : }
    7585             : 
    7586           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7587           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7588             :     return 0;
    7589           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7590           0 :     return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7591             :   }
    7592             :   return 0;
    7593             : }
    7594             : 
    7595             : unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7596           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7597             :     return 0;
    7598           0 :   if ((Subtarget->hasAVX512())) {
    7599           0 :     return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7600             :   }
    7601             :   return 0;
    7602             : }
    7603             : 
    7604           0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7605           0 :   switch (VT.SimpleTy) {
    7606           0 :   case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7607           0 :   case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7608           0 :   case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7609           0 :   case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7610           0 :   case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7611           0 :   case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7612           0 :   case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7613           0 :   case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7614           0 :   case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7615           0 :   case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7616           0 :   case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7617           0 :   case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7618             :   default: return 0;
    7619             :   }
    7620             : }
    7621             : 
    7622             : // FastEmit functions for ISD::UMIN.
    7623             : 
    7624           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7625           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7626             :     return 0;
    7627           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7628           0 :     return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7629             :   }
    7630           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7631           0 :     return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7632             :   }
    7633           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7634           0 :     return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7635             :   }
    7636             :   return 0;
    7637             : }
    7638             : 
    7639           0 : unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7640           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7641             :     return 0;
    7642           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7643           0 :     return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7644             :   }
    7645           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7646           0 :     return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7647             :   }
    7648             :   return 0;
    7649             : }
    7650             : 
    7651             : unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7652           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7653             :     return 0;
    7654           0 :   if ((Subtarget->hasBWI())) {
    7655           0 :     return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7656             :   }
    7657             :   return 0;
    7658             : }
    7659             : 
    7660           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7661           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7662             :     return 0;
    7663           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7664           0 :     return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7665             :   }
    7666           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7667           0 :     return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7668             :   }
    7669           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7670           0 :     return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7671             :   }
    7672             :   return 0;
    7673             : }
    7674             : 
    7675           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7676           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7677             :     return 0;
    7678           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7679           0 :     return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7680             :   }
    7681           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7682           0 :     return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7683             :   }
    7684             :   return 0;
    7685             : }
    7686             : 
    7687             : unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7688           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7689             :     return 0;
    7690           0 :   if ((Subtarget->hasBWI())) {
    7691           0 :     return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7692             :   }
    7693             :   return 0;
    7694             : }
    7695             : 
    7696           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7697           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7698             :     return 0;
    7699           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7700           0 :     return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7701             :   }
    7702           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7703           0 :     return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7704             :   }
    7705           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7706           0 :     return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7707             :   }
    7708             :   return 0;
    7709             : }
    7710             : 
    7711           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7712           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7713             :     return 0;
    7714           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7715           0 :     return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7716             :   }
    7717           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7718           0 :     return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7719             :   }
    7720             :   return 0;
    7721             : }
    7722             : 
    7723             : unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7724           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7725             :     return 0;
    7726           0 :   if ((Subtarget->hasAVX512())) {
    7727           0 :     return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7728             :   }
    7729             :   return 0;
    7730             : }
    7731             : 
    7732           0 : unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7733           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7734             :     return 0;
    7735           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7736           0 :     return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7737             :   }
    7738             :   return 0;
    7739             : }
    7740             : 
    7741           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7742           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7743             :     return 0;
    7744           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7745           0 :     return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7746             :   }
    7747             :   return 0;
    7748             : }
    7749             : 
    7750             : unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7751           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7752             :     return 0;
    7753           0 :   if ((Subtarget->hasAVX512())) {
    7754           0 :     return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7755             :   }
    7756             :   return 0;
    7757             : }
    7758             : 
    7759           0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7760           0 :   switch (VT.SimpleTy) {
    7761           0 :   case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7762           0 :   case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7763           0 :   case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7764           0 :   case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7765           0 :   case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7766           0 :   case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7767           0 :   case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7768           0 :   case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7769           0 :   case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7770           0 :   case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7771           0 :   case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7772           0 :   case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7773             :   default: return 0;
    7774             :   }
    7775             : }
    7776             : 
    7777             : // FastEmit functions for ISD::XOR.
    7778             : 
    7779             : unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7780           0 :   if (RetVT.SimpleTy != MVT::i8)
    7781             :     return 0;
    7782           0 :   return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7783             : }
    7784             : 
    7785             : unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7786           0 :   if (RetVT.SimpleTy != MVT::i16)
    7787             :     return 0;
    7788           0 :   return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7789             : }
    7790             : 
    7791             : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7792          17 :   if (RetVT.SimpleTy != MVT::i32)
    7793             :     return 0;
    7794          17 :   return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7795             : }
    7796             : 
    7797             : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7798           8 :   if (RetVT.SimpleTy != MVT::i64)
    7799             :     return 0;
    7800           8 :   return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7801             : }
    7802             : 
    7803             : unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7804           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    7805             :     return 0;
    7806           0 :   if ((Subtarget->hasDQI())) {
    7807           0 :     return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7808             :   }
    7809             :   return 0;
    7810             : }
    7811             : 
    7812             : unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7813           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    7814             :     return 0;
    7815           0 :   if ((Subtarget->hasAVX512())) {
    7816           0 :     return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7817             :   }
    7818             :   return 0;
    7819             : }
    7820             : 
    7821             : unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7822           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    7823             :     return 0;
    7824           0 :   if ((Subtarget->hasBWI())) {
    7825           0 :     return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7826             :   }
    7827             :   return 0;
    7828             : }
    7829             : 
    7830             : unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7831           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    7832             :     return 0;
    7833           0 :   if ((Subtarget->hasBWI())) {
    7834           0 :     return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7835             :   }
    7836             :   return 0;
    7837             : }
    7838             : 
    7839           2 : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7840           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    7841             :     return 0;
    7842           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7843           0 :     return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7844             :   }
    7845           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7846           2 :     return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7847             :   }
    7848           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7849           0 :     return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7850             :   }
    7851             :   return 0;
    7852             : }
    7853             : 
    7854           4 : unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7855           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    7856             :     return 0;
    7857           8 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    7858           2 :     return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7859             :   }
    7860           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7861           0 :     return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7862             :   }
    7863           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7864           2 :     return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7865             :   }
    7866             :   return 0;
    7867             : }
    7868             : 
    7869             : unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7870           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7871             :     return 0;
    7872           0 :   if ((Subtarget->hasAVX512())) {
    7873           0 :     return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7874             :   }
    7875             :   return 0;
    7876             : }
    7877             : 
    7878          35 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7879          35 :   switch (VT.SimpleTy) {
    7880           0 :   case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7881           0 :   case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7882          17 :   case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7883           8 :   case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7884           0 :   case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7885           0 :   case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7886           0 :   case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7887           0 :   case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7888           2 :   case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7889           4 :   case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7890           0 :   case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7891             :   default: return 0;
    7892             :   }
    7893             : }
    7894             : 
    7895             : // FastEmit functions for X86ISD::ADDS.
    7896             : 
    7897           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7898           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7899             :     return 0;
    7900           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7901           0 :     return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7902             :   }
    7903           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7904           0 :     return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7905             :   }
    7906           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7907           0 :     return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7908             :   }
    7909             :   return 0;
    7910             : }
    7911             : 
    7912           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7913           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7914             :     return 0;
    7915           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7916           0 :     return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7917             :   }
    7918           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7919           0 :     return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7920             :   }
    7921             :   return 0;
    7922             : }
    7923             : 
    7924             : unsigned fastEmit_X86ISD_ADDS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7925           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7926             :     return 0;
    7927           0 :   if ((Subtarget->hasBWI())) {
    7928           0 :     return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7929             :   }
    7930             :   return 0;
    7931             : }
    7932             : 
    7933           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7934           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7935             :     return 0;
    7936           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7937           0 :     return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7938             :   }
    7939           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7940           0 :     return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7941             :   }
    7942           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7943           0 :     return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7944             :   }
    7945             :   return 0;
    7946             : }
    7947             : 
    7948           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7949           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7950             :     return 0;
    7951           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7952           0 :     return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7953             :   }
    7954           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7955           0 :     return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7956             :   }
    7957             :   return 0;
    7958             : }
    7959             : 
    7960             : unsigned fastEmit_X86ISD_ADDS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7961           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7962             :     return 0;
    7963           0 :   if ((Subtarget->hasBWI())) {
    7964           0 :     return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7965             :   }
    7966             :   return 0;
    7967             : }
    7968             : 
    7969           0 : unsigned fastEmit_X86ISD_ADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7970           0 :   switch (VT.SimpleTy) {
    7971           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7972           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7973           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7974           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7975           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7976           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7977             :   default: return 0;
    7978             :   }
    7979             : }
    7980             : 
    7981             : // FastEmit functions for X86ISD::ADDSUB.
    7982             : 
    7983           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7984           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    7985             :     return 0;
    7986           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    7987           0 :     return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7988             :   }
    7989           0 :   if ((Subtarget->hasAVX())) {
    7990           0 :     return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7991             :   }
    7992             :   return 0;
    7993             : }
    7994             : 
    7995             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7996           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    7997             :     return 0;
    7998           0 :   if ((Subtarget->hasAVX())) {
    7999           0 :     return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8000             :   }
    8001             :   return 0;
    8002             : }
    8003             : 
    8004           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8005           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8006             :     return 0;
    8007           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8008           0 :     return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8009             :   }
    8010           0 :   if ((Subtarget->hasAVX())) {
    8011           0 :     return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8012             :   }
    8013             :   return 0;
    8014             : }
    8015             : 
    8016             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8017           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8018             :     return 0;
    8019           0 :   if ((Subtarget->hasAVX())) {
    8020           0 :     return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8021             :   }
    8022             :   return 0;
    8023             : }
    8024             : 
    8025           0 : unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8026           0 :   switch (VT.SimpleTy) {
    8027           0 :   case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8028           0 :   case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8029           0 :   case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8030           0 :   case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8031             :   default: return 0;
    8032             :   }
    8033             : }
    8034             : 
    8035             : // FastEmit functions for X86ISD::ADDUS.
    8036             : 
    8037           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8038           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8039             :     return 0;
    8040           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8041           0 :     return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8042             :   }
    8043           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8044           0 :     return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8045             :   }
    8046           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8047           0 :     return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8048             :   }
    8049             :   return 0;
    8050             : }
    8051             : 
    8052           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8053           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8054             :     return 0;
    8055           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8056           0 :     return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8057             :   }
    8058           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8059           0 :     return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8060             :   }
    8061             :   return 0;
    8062             : }
    8063             : 
    8064             : unsigned fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8065           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8066             :     return 0;
    8067           0 :   if ((Subtarget->hasBWI())) {
    8068           0 :     return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8069             :   }
    8070             :   return 0;
    8071             : }
    8072             : 
    8073           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8074           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8075             :     return 0;
    8076           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8077           0 :     return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8078             :   }
    8079           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8080           0 :     return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8081             :   }
    8082           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8083           0 :     return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8084             :   }
    8085             :   return 0;
    8086             : }
    8087             : 
    8088           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8089           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8090             :     return 0;
    8091           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8092           0 :     return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8093             :   }
    8094           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8095           0 :     return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8096             :   }
    8097             :   return 0;
    8098             : }
    8099             : 
    8100             : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8101           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8102             :     return 0;
    8103           0 :   if ((Subtarget->hasBWI())) {
    8104           0 :     return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8105             :   }
    8106             :   return 0;
    8107             : }
    8108             : 
    8109           0 : unsigned fastEmit_X86ISD_ADDUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8110           0 :   switch (VT.SimpleTy) {
    8111           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8112           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8113           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8114           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8115           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8116           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8117             :   default: return 0;
    8118             :   }
    8119             : }
    8120             : 
    8121             : // FastEmit functions for X86ISD::ANDNP.
    8122             : 
    8123           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8124           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    8125             :     return 0;
    8126           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8127           0 :     return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8128             :   }
    8129           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8130           0 :     return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8131             :   }
    8132           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8133           0 :     return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8134             :   }
    8135             :   return 0;
    8136             : }
    8137             : 
    8138           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8139           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    8140             :     return 0;
    8141           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    8142           0 :     return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8143             :   }
    8144           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8145           0 :     return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8146             :   }
    8147           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8148           0 :     return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8149             :   }
    8150             :   return 0;
    8151             : }
    8152             : 
    8153             : unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8154           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8155             :     return 0;
    8156           0 :   if ((Subtarget->hasAVX512())) {
    8157           0 :     return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8158             :   }
    8159             :   return 0;
    8160             : }
    8161             : 
    8162           0 : unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8163           0 :   switch (VT.SimpleTy) {
    8164           0 :   case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8165           0 :   case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8166           0 :   case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8167             :   default: return 0;
    8168             :   }
    8169             : }
    8170             : 
    8171             : // FastEmit functions for X86ISD::AVG.
    8172             : 
    8173           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8174           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8175             :     return 0;
    8176           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8177           0 :     return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8178             :   }
    8179           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8180           0 :     return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8181             :   }
    8182           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8183           0 :     return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8184             :   }
    8185             :   return 0;
    8186             : }
    8187             : 
    8188           0 : unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8189           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8190             :     return 0;
    8191           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8192           0 :     return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8193             :   }
    8194           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8195           0 :     return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8196             :   }
    8197             :   return 0;
    8198             : }
    8199             : 
    8200             : unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8201           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8202             :     return 0;
    8203           0 :   if ((Subtarget->hasBWI())) {
    8204           0 :     return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8205             :   }
    8206             :   return 0;
    8207             : }
    8208             : 
    8209           0 : unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8210           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8211             :     return 0;
    8212           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8213           0 :     return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8214             :   }
    8215           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8216           0 :     return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8217             :   }
    8218           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8219           0 :     return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8220             :   }
    8221             :   return 0;
    8222             : }
    8223             : 
    8224           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8225           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8226             :     return 0;
    8227           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8228           0 :     return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8229             :   }
    8230           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8231           0 :     return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8232             :   }
    8233             :   return 0;
    8234             : }
    8235             : 
    8236             : unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8237           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8238             :     return 0;
    8239           0 :   if ((Subtarget->hasBWI())) {
    8240           0 :     return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8241             :   }
    8242             :   return 0;
    8243             : }
    8244             : 
    8245           0 : unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8246           0 :   switch (VT.SimpleTy) {
    8247           0 :   case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8248           0 :   case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8249           0 :   case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8250           0 :   case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8251           0 :   case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8252           0 :   case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8253             :   default: return 0;
    8254             :   }
    8255             : }
    8256             : 
    8257             : // FastEmit functions for X86ISD::BT.
    8258             : 
    8259             : unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8260           0 :   if (RetVT.SimpleTy != MVT::i32)
    8261             :     return 0;
    8262           0 :   return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8263             : }
    8264             : 
    8265             : unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8266           0 :   if (RetVT.SimpleTy != MVT::i32)
    8267             :     return 0;
    8268           0 :   return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8269             : }
    8270             : 
    8271             : unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8272           0 :   if (RetVT.SimpleTy != MVT::i32)
    8273             :     return 0;
    8274           0 :   return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8275             : }
    8276             : 
    8277           0 : unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8278           0 :   switch (VT.SimpleTy) {
    8279           0 :   case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8280           0 :   case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8281           0 :   case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8282             :   default: return 0;
    8283             :   }
    8284             : }
    8285             : 
    8286             : // FastEmit functions for X86ISD::CMP.
    8287             : 
    8288             : unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8289           0 :   if (RetVT.SimpleTy != MVT::i32)
    8290             :     return 0;
    8291           0 :   return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8292             : }
    8293             : 
    8294             : unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8295           0 :   if (RetVT.SimpleTy != MVT::i32)
    8296             :     return 0;
    8297           0 :   return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8298             : }
    8299             : 
    8300             : unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8301           0 :   if (RetVT.SimpleTy != MVT::i32)
    8302             :     return 0;
    8303           0 :   return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8304             : }
    8305             : 
    8306             : unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8307           0 :   if (RetVT.SimpleTy != MVT::i32)
    8308             :     return 0;
    8309           0 :   return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8310             : }
    8311             : 
    8312           0 : unsigned fastEmit_X86ISD_CMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8313           0 :   if (RetVT.SimpleTy != MVT::i32)
    8314             :     return 0;
    8315           0 :   if ((Subtarget->hasAVX512())) {
    8316           0 :     return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8317             :   }
    8318           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8319           0 :     return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8320             :   }
    8321           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8322           0 :     return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8323             :   }
    8324           0 :   if ((!Subtarget->hasSSE1())) {
    8325           0 :     return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8326             :   }
    8327             :   return 0;
    8328             : }
    8329             : 
    8330           0 : unsigned fastEmit_X86ISD_CMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8331           0 :   if (RetVT.SimpleTy != MVT::i32)
    8332             :     return 0;
    8333           0 :   if ((Subtarget->hasAVX512())) {
    8334           0 :     return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8335             :   }
    8336           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8337           0 :     return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8338             :   }
    8339           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8340           0 :     return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8341             :   }
    8342           0 :   if ((!Subtarget->hasSSE2())) {
    8343           0 :     return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8344             :   }
    8345             :   return 0;
    8346             : }
    8347             : 
    8348             : unsigned fastEmit_X86ISD_CMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8349           0 :   if (RetVT.SimpleTy != MVT::i32)
    8350             :     return 0;
    8351           0 :   return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8352             : }
    8353             : 
    8354           0 : unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8355           0 :   switch (VT.SimpleTy) {
    8356           0 :   case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8357           0 :   case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8358           0 :   case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8359           0 :   case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8360           0 :   case MVT::f32: return fastEmit_X86ISD_CMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8361           0 :   case MVT::f64: return fastEmit_X86ISD_CMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8362           0 :   case MVT::f80: return fastEmit_X86ISD_CMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8363             :   default: return 0;
    8364             :   }
    8365             : }
    8366             : 
    8367             : // FastEmit functions for X86ISD::COMI.
    8368             : 
    8369           0 : unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8370           0 :   if (RetVT.SimpleTy != MVT::i32)
    8371             :     return 0;
    8372           0 :   if ((Subtarget->hasAVX512())) {
    8373           0 :     return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8374             :   }
    8375           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8376           0 :     return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8377             :   }
    8378           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8379           0 :     return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8380             :   }
    8381             :   return 0;
    8382             : }
    8383             : 
    8384           0 : unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8385           0 :   if (RetVT.SimpleTy != MVT::i32)
    8386             :     return 0;
    8387           0 :   if ((Subtarget->hasAVX512())) {
    8388           0 :     return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8389             :   }
    8390           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8391           0 :     return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8392             :   }
    8393           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8394           0 :     return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8395             :   }
    8396             :   return 0;
    8397             : }
    8398             : 
    8399           0 : unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8400           0 :   switch (VT.SimpleTy) {
    8401           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8402           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8403             :   default: return 0;
    8404             :   }
    8405             : }
    8406             : 
    8407             : // FastEmit functions for X86ISD::FAND.
    8408             : 
    8409             : unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8410           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8411             :     return 0;
    8412           0 :   return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8413             : }
    8414             : 
    8415             : unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8416           0 :   switch (VT.SimpleTy) {
    8417           0 :   case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8418             :   default: return 0;
    8419             :   }
    8420             : }
    8421             : 
    8422             : // FastEmit functions for X86ISD::FANDN.
    8423             : 
    8424             : unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8425           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8426             :     return 0;
    8427           0 :   return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8428             : }
    8429             : 
    8430             : unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8431           0 :   switch (VT.SimpleTy) {
    8432           0 :   case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8433             :   default: return 0;
    8434             :   }
    8435             : }
    8436             : 
    8437             : // FastEmit functions for X86ISD::FHADD.
    8438             : 
    8439           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8440           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8441             :     return 0;
    8442           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8443           0 :     return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8444             :   }
    8445           0 :   if ((Subtarget->hasAVX())) {
    8446           0 :     return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8447             :   }
    8448             :   return 0;
    8449             : }
    8450             : 
    8451             : unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8452           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8453             :     return 0;
    8454           0 :   if ((Subtarget->hasAVX())) {
    8455           0 :     return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8456             :   }
    8457             :   return 0;
    8458             : }
    8459             : 
    8460           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8461           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8462             :     return 0;
    8463           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8464           0 :     return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8465             :   }
    8466           0 :   if ((Subtarget->hasAVX())) {
    8467           0 :     return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8468             :   }
    8469             :   return 0;
    8470             : }
    8471             : 
    8472             : unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8473           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8474             :     return 0;
    8475           0 :   if ((Subtarget->hasAVX())) {
    8476           0 :     return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8477             :   }
    8478             :   return 0;
    8479             : }
    8480             : 
    8481           0 : unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8482           0 :   switch (VT.SimpleTy) {
    8483           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8484           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8485           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8486           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8487             :   default: return 0;
    8488             :   }
    8489             : }
    8490             : 
    8491             : // FastEmit functions for X86ISD::FHSUB.
    8492             : 
    8493           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8494           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8495             :     return 0;
    8496           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8497           0 :     return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8498             :   }
    8499           0 :   if ((Subtarget->hasAVX())) {
    8500           0 :     return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8501             :   }
    8502             :   return 0;
    8503             : }
    8504             : 
    8505             : unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8506           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8507             :     return 0;
    8508           0 :   if ((Subtarget->hasAVX())) {
    8509           0 :     return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8510             :   }
    8511             :   return 0;
    8512             : }
    8513             : 
    8514           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8515           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8516             :     return 0;
    8517           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8518           0 :     return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8519             :   }
    8520           0 :   if ((Subtarget->hasAVX())) {
    8521           0 :     return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8522             :   }
    8523             :   return 0;
    8524             : }
    8525             : 
    8526             : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8527           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8528             :     return 0;
    8529           0 :   if ((Subtarget->hasAVX())) {
    8530           0 :     return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8531             :   }
    8532             :   return 0;
    8533             : }
    8534             : 
    8535           0 : unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8536           0 :   switch (VT.SimpleTy) {
    8537           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8538           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8539           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8540           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8541             :   default: return 0;
    8542             :   }
    8543             : }
    8544             : 
    8545             : // FastEmit functions for X86ISD::FMAX.
    8546             : 
    8547           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8548           0 :   if (RetVT.SimpleTy != MVT::f32)
    8549             :     return 0;
    8550           0 :   if ((Subtarget->hasAVX512())) {
    8551           0 :     return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8552             :   }
    8553           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8554           0 :     return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8555             :   }
    8556           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8557           0 :     return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8558             :   }
    8559             :   return 0;
    8560             : }
    8561             : 
    8562           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8563           0 :   if (RetVT.SimpleTy != MVT::f64)
    8564             :     return 0;
    8565           0 :   if ((Subtarget->hasAVX512())) {
    8566           0 :     return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8567             :   }
    8568           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8569           0 :     return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8570             :   }
    8571           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8572           0 :     return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8573             :   }
    8574             :   return 0;
    8575             : }
    8576             : 
    8577           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8578           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8579             :     return 0;
    8580           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8581           0 :     return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8582             :   }
    8583           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8584           0 :     return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8585             :   }
    8586           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8587           0 :     return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8588             :   }
    8589             :   return 0;
    8590             : }
    8591             : 
    8592           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8593           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8594             :     return 0;
    8595           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8596           0 :     return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8597             :   }
    8598           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8599           0 :     return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8600             :   }
    8601             :   return 0;
    8602             : }
    8603             : 
    8604             : unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8605           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8606             :     return 0;
    8607           0 :   if ((Subtarget->hasAVX512())) {
    8608           0 :     return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8609             :   }
    8610             :   return 0;
    8611             : }
    8612             : 
    8613           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8614           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8615             :     return 0;
    8616           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8617           0 :     return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8618             :   }
    8619           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8620           0 :     return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8621             :   }
    8622           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8623           0 :     return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8624             :   }
    8625             :   return 0;
    8626             : }
    8627             : 
    8628           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8629           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8630             :     return 0;
    8631           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8632           0 :     return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8633             :   }
    8634           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8635           0 :     return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8636             :   }
    8637             :   return 0;
    8638             : }
    8639             : 
    8640             : unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8641           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8642             :     return 0;
    8643           0 :   if ((Subtarget->hasAVX512())) {
    8644           0 :     return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8645             :   }
    8646             :   return 0;
    8647             : }
    8648             : 
    8649           0 : unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8650           0 :   switch (VT.SimpleTy) {
    8651           0 :   case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8652           0 :   case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8653           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8654           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8655           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8656           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8657           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8658           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8659             :   default: return 0;
    8660             :   }
    8661             : }
    8662             : 
    8663             : // FastEmit functions for X86ISD::FMAXC.
    8664             : 
    8665           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8666           0 :   if (RetVT.SimpleTy != MVT::f32)
    8667             :     return 0;
    8668           0 :   if ((Subtarget->hasAVX512())) {
    8669           0 :     return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8670             :   }
    8671           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8672           0 :     return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8673             :   }
    8674           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8675           0 :     return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8676             :   }
    8677             :   return 0;
    8678             : }
    8679             : 
    8680           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8681           0 :   if (RetVT.SimpleTy != MVT::f64)
    8682             :     return 0;
    8683           0 :   if ((Subtarget->hasAVX512())) {
    8684           0 :     return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8685             :   }
    8686           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8687           0 :     return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8688             :   }
    8689           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8690           0 :     return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8691             :   }
    8692             :   return 0;
    8693             : }
    8694             : 
    8695           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8696           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8697             :     return 0;
    8698           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8699           0 :     return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8700             :   }
    8701           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8702           0 :     return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8703             :   }
    8704           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8705           0 :     return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8706             :   }
    8707             :   return 0;
    8708             : }
    8709             : 
    8710           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8711           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8712             :     return 0;
    8713           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8714           0 :     return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8715             :   }
    8716           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8717           0 :     return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8718             :   }
    8719             :   return 0;
    8720             : }
    8721             : 
    8722             : unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8723           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8724             :     return 0;
    8725           0 :   if ((Subtarget->hasAVX512())) {
    8726           0 :     return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8727             :   }
    8728             :   return 0;
    8729             : }
    8730             : 
    8731           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8732           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8733             :     return 0;
    8734           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8735           0 :     return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8736             :   }
    8737           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8738           0 :     return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8739             :   }
    8740           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8741           0 :     return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8742             :   }
    8743             :   return 0;
    8744             : }
    8745             : 
    8746           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8747           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8748             :     return 0;
    8749           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8750           0 :     return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8751             :   }
    8752           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8753           0 :     return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8754             :   }
    8755             :   return 0;
    8756             : }
    8757             : 
    8758             : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8759           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8760             :     return 0;
    8761           0 :   if ((Subtarget->hasAVX512())) {
    8762           0 :     return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8763             :   }
    8764             :   return 0;
    8765             : }
    8766             : 
    8767           0 : unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8768           0 :   switch (VT.SimpleTy) {
    8769           0 :   case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8770           0 :   case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8771           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8772           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8773           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8774           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8775           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8776           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8777             :   default: return 0;
    8778             :   }
    8779             : }
    8780             : 
    8781             : // FastEmit functions for X86ISD::FMAXS.
    8782             : 
    8783           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8784           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8785             :     return 0;
    8786           0 :   if ((Subtarget->hasAVX512())) {
    8787           0 :     return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8788             :   }
    8789           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8790           0 :     return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8791             :   }
    8792           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8793           0 :     return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8794             :   }
    8795             :   return 0;
    8796             : }
    8797             : 
    8798           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8799           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8800             :     return 0;
    8801           0 :   if ((Subtarget->hasAVX512())) {
    8802           0 :     return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8803             :   }
    8804           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8805           0 :     return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8806             :   }
    8807           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8808           0 :     return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8809             :   }
    8810             :   return 0;
    8811             : }
    8812             : 
    8813           0 : unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8814           0 :   switch (VT.SimpleTy) {
    8815           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8816           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8817             :   default: return 0;
    8818             :   }
    8819             : }
    8820             : 
    8821             : // FastEmit functions for X86ISD::FMIN.
    8822             : 
    8823           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8824           0 :   if (RetVT.SimpleTy != MVT::f32)
    8825             :     return 0;
    8826           0 :   if ((Subtarget->hasAVX512())) {
    8827           0 :     return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8828             :   }
    8829           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8830           0 :     return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8831             :   }
    8832           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8833           0 :     return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8834             :   }
    8835             :   return 0;
    8836             : }
    8837             : 
    8838           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8839           0 :   if (RetVT.SimpleTy != MVT::f64)
    8840             :     return 0;
    8841           0 :   if ((Subtarget->hasAVX512())) {
    8842           0 :     return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8843             :   }
    8844           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8845           0 :     return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8846             :   }
    8847           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8848           0 :     return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8849             :   }
    8850             :   return 0;
    8851             : }
    8852             : 
    8853           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8854           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8855             :     return 0;
    8856           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8857           0 :     return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8858             :   }
    8859           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8860           0 :     return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8861             :   }
    8862           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8863           0 :     return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8864             :   }
    8865             :   return 0;
    8866             : }
    8867             : 
    8868           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8869           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8870             :     return 0;
    8871           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8872           0 :     return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8873             :   }
    8874           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8875           0 :     return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8876             :   }
    8877             :   return 0;
    8878             : }
    8879             : 
    8880             : unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8881           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8882             :     return 0;
    8883           0 :   if ((Subtarget->hasAVX512())) {
    8884           0 :     return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8885             :   }
    8886             :   return 0;
    8887             : }
    8888             : 
    8889           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8890           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8891             :     return 0;
    8892           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8893           0 :     return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8894             :   }
    8895           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8896           0 :     return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8897             :   }
    8898           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8899           0 :     return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8900             :   }
    8901             :   return 0;
    8902             : }
    8903             : 
    8904           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8905           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8906             :     return 0;
    8907           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8908           0 :     return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8909             :   }
    8910           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8911           0 :     return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8912             :   }
    8913             :   return 0;
    8914             : }
    8915             : 
    8916             : unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8917           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8918             :     return 0;
    8919           0 :   if ((Subtarget->hasAVX512())) {
    8920           0 :     return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8921             :   }
    8922             :   return 0;
    8923             : }
    8924             : 
    8925           0 : unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8926           0 :   switch (VT.SimpleTy) {
    8927           0 :   case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8928           0 :   case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8929           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8930           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8931           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8932           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8933           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8934           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8935             :   default: return 0;
    8936             :   }
    8937             : }
    8938             : 
    8939             : // FastEmit functions for X86ISD::FMINC.
    8940             : 
    8941           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8942           0 :   if (RetVT.SimpleTy != MVT::f32)
    8943             :     return 0;
    8944           0 :   if ((Subtarget->hasAVX512())) {
    8945           0 :     return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8946             :   }
    8947           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8948           0 :     return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8949             :   }
    8950           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8951           0 :     return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8952             :   }
    8953             :   return 0;
    8954             : }
    8955             : 
    8956           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8957           0 :   if (RetVT.SimpleTy != MVT::f64)
    8958             :     return 0;
    8959           0 :   if ((Subtarget->hasAVX512())) {
    8960           0 :     return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8961             :   }
    8962           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8963           0 :     return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8964             :   }
    8965           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8966           0 :     return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8967             :   }
    8968             :   return 0;
    8969             : }
    8970             : 
    8971           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8972           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8973             :     return 0;
    8974           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8975           0 :     return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8976             :   }
    8977           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8978           0 :     return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8979             :   }
    8980           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8981           0 :     return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8982             :   }
    8983             :   return 0;
    8984             : }
    8985             : 
    8986           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8987           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8988             :     return 0;
    8989           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8990           0 :     return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8991             :   }
    8992           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8993           0 :     return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8994             :   }
    8995             :   return 0;
    8996             : }
    8997             : 
    8998             : unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8999           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9000             :     return 0;
    9001           0 :   if ((Subtarget->hasAVX512())) {
    9002           0 :     return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9003             :   }
    9004             :   return 0;
    9005             : }
    9006             : 
    9007           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9008           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9009             :     return 0;
    9010           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9011           0 :     return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9012             :   }
    9013           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9014           0 :     return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9015             :   }
    9016           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9017           0 :     return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9018             :   }
    9019             :   return 0;
    9020             : }
    9021             : 
    9022           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9023           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9024             :     return 0;
    9025           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9026           0 :     return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9027             :   }
    9028           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9029           0 :     return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9030             :   }
    9031             :   return 0;
    9032             : }
    9033             : 
    9034             : unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9035           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9036             :     return 0;
    9037           0 :   if ((Subtarget->hasAVX512())) {
    9038           0 :     return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9039             :   }
    9040             :   return 0;
    9041             : }
    9042             : 
    9043           0 : unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9044           0 :   switch (VT.SimpleTy) {
    9045           0 :   case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9046           0 :   case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9047           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9048           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9049           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9050           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9051           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9052           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9053             :   default: return 0;
    9054             :   }
    9055             : }
    9056             : 
    9057             : // FastEmit functions for X86ISD::FMINS.
    9058             : 
    9059           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9060           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9061             :     return 0;
    9062           0 :   if ((Subtarget->hasAVX512())) {
    9063           0 :     return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9064             :   }
    9065           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9066           0 :     return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9067             :   }
    9068           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9069           0 :     return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9070             :   }
    9071             :   return 0;
    9072             : }
    9073             : 
    9074           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9075           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9076             :     return 0;
    9077           0 :   if ((Subtarget->hasAVX512())) {
    9078           0 :     return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9079             :   }
    9080           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9081           0 :     return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9082             :   }
    9083           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9084           0 :     return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9085             :   }
    9086             :   return 0;
    9087             : }
    9088             : 
    9089           0 : unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9090           0 :   switch (VT.SimpleTy) {
    9091           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9092           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9093             :   default: return 0;
    9094             :   }
    9095             : }
    9096             : 
    9097             : // FastEmit functions for X86ISD::FOR.
    9098             : 
    9099             : unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9100           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9101             :     return 0;
    9102           0 :   return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9103             : }
    9104             : 
    9105             : unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9106           0 :   switch (VT.SimpleTy) {
    9107           0 :   case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9108             :   default: return 0;
    9109             :   }
    9110             : }
    9111             : 
    9112             : // FastEmit functions for X86ISD::FXOR.
    9113             : 
    9114             : unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9115           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9116             :     return 0;
    9117           0 :   return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9118             : }
    9119             : 
    9120             : unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9121           0 :   switch (VT.SimpleTy) {
    9122           0 :   case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9123             :   default: return 0;
    9124             :   }
    9125             : }
    9126             : 
    9127             : // FastEmit functions for X86ISD::GF2P8MULB.
    9128             : 
    9129           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9130           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    9131             :     return 0;
    9132           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
    9133           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9134             :   }
    9135           0 :   if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9136           0 :     return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9137             :   }
    9138           0 :   if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9139           0 :     return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9140             :   }
    9141             :   return 0;
    9142             : }
    9143             : 
    9144           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9145           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    9146             :     return 0;
    9147           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
    9148           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9149             :   }
    9150           0 :   if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9151           0 :     return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9152             :   }
    9153             :   return 0;
    9154             : }
    9155             : 
    9156           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9157           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    9158             :     return 0;
    9159           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasBWI()) && (Subtarget->hasGFNI())) {
    9160           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9161             :   }
    9162             :   return 0;
    9163             : }
    9164             : 
    9165           0 : unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9166           0 :   switch (VT.SimpleTy) {
    9167           0 :   case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9168           0 :   case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9169           0 :   case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9170             :   default: return 0;
    9171             :   }
    9172             : }
    9173             : 
    9174             : // FastEmit functions for X86ISD::HADD.
    9175             : 
    9176           0 : unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9177           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9178             :     return 0;
    9179           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9180           0 :     return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9181             :   }
    9182           0 :   if ((Subtarget->hasAVX())) {
    9183           0 :     return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9184             :   }
    9185             :   return 0;
    9186             : }
    9187             : 
    9188             : unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9189           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9190             :     return 0;
    9191           0 :   if ((Subtarget->hasAVX2())) {
    9192           0 :     return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9193             :   }
    9194             :   return 0;
    9195             : }
    9196             : 
    9197           0 : unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9198           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9199             :     return 0;
    9200           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9201           0 :     return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9202             :   }
    9203           0 :   if ((Subtarget->hasAVX())) {
    9204           0 :     return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9205             :   }
    9206             :   return 0;
    9207             : }
    9208             : 
    9209             : unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9210           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    9211             :     return 0;
    9212           0 :   if ((Subtarget->hasAVX2())) {
    9213           0 :     return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9214             :   }
    9215             :   return 0;
    9216             : }
    9217             : 
    9218           0 : unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9219           0 :   switch (VT.SimpleTy) {
    9220           0 :   case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9221           0 :   case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9222           0 :   case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9223           0 :   case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9224             :   default: return 0;
    9225             :   }
    9226             : }
    9227             : 
    9228             : // FastEmit functions for X86ISD::HSUB.
    9229             : 
    9230           0 : unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9231           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9232             :     return 0;
    9233           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9234           0 :     return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9235             :   }
    9236           0 :   if ((Subtarget->hasAVX())) {
    9237           0 :     return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9238             :   }
    9239             :   return 0;
    9240             : }
    9241             : 
    9242             : unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9243           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9244             :     return 0;
    9245           0 :   if ((Subtarget->hasAVX2())) {
    9246           0 :     return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9247             :   }
    9248             :   return 0;
    9249             : }
    9250             : 
    9251           0 : unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9252           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9253             :     return 0;
    9254           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9255           0 :     return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9256             :   }
    9257           0 :   if ((Subtarget->hasAVX())) {
    9258           0 :     return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9259             :   }
    9260             :   return 0;
    9261             : }
    9262             : 
    9263             : unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9264           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    9265             :     return 0;
    9266           0 :   if ((Subtarget->hasAVX2())) {
    9267           0 :     return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9268             :   }
    9269             :   return 0;
    9270             : }
    9271             : 
    9272           0 : unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9273           0 :   switch (VT.SimpleTy) {
    9274           0 :   case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9275           0 :   case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9276           0 :   case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9277           0 :   case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9278             :   default: return 0;
    9279             :   }
    9280             : }
    9281             : 
    9282             : // FastEmit functions for X86ISD::KADD.
    9283             : 
    9284             : unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9285           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    9286             :     return 0;
    9287           0 :   if ((Subtarget->hasDQI())) {
    9288           0 :     return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9289             :   }
    9290             :   return 0;
    9291             : }
    9292             : 
    9293             : unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9294           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    9295             :     return 0;
    9296           0 :   if ((Subtarget->hasDQI())) {
    9297           0 :     return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9298             :   }
    9299             :   return 0;
    9300             : }
    9301             : 
    9302             : unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9303           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    9304             :     return 0;
    9305           0 :   if ((Subtarget->hasBWI())) {
    9306           0 :     return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9307             :   }
    9308             :   return 0;
    9309             : }
    9310             : 
    9311             : unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9312           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    9313             :     return 0;
    9314           0 :   if ((Subtarget->hasBWI())) {
    9315           0 :     return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9316             :   }
    9317             :   return 0;
    9318             : }
    9319             : 
    9320           0 : unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9321           0 :   switch (VT.SimpleTy) {
    9322           0 :   case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9323           0 :   case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9324           0 :   case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9325           0 :   case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9326             :   default: return 0;
    9327             :   }
    9328             : }
    9329             : 
    9330             : // FastEmit functions for X86ISD::KORTEST.
    9331             : 
    9332             : unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9333           0 :   if (RetVT.SimpleTy != MVT::i32)
    9334             :     return 0;
    9335           0 :   if ((Subtarget->hasDQI())) {
    9336           0 :     return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9337             :   }
    9338             :   return 0;
    9339             : }
    9340             : 
    9341             : unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9342           0 :   if (RetVT.SimpleTy != MVT::i32)
    9343             :     return 0;
    9344           0 :   if ((Subtarget->hasAVX512())) {
    9345           0 :     return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9346             :   }
    9347             :   return 0;
    9348             : }
    9349             : 
    9350             : unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9351           0 :   if (RetVT.SimpleTy != MVT::i32)
    9352             :     return 0;
    9353           0 :   if ((Subtarget->hasBWI())) {
    9354           0 :     return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9355             :   }
    9356             :   return 0;
    9357             : }
    9358             : 
    9359             : unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9360           0 :   if (RetVT.SimpleTy != MVT::i32)
    9361             :     return 0;
    9362           0 :   if ((Subtarget->hasBWI())) {
    9363           0 :     return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9364             :   }
    9365             :   return 0;
    9366             : }
    9367             : 
    9368           0 : unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9369           0 :   switch (VT.SimpleTy) {
    9370           0 :   case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9371           0 :   case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9372           0 :   case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9373           0 :   case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9374             :   default: return 0;
    9375             :   }
    9376             : }
    9377             : 
    9378             : // FastEmit functions for X86ISD::KTEST.
    9379             : 
    9380             : unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9381           0 :   if (RetVT.SimpleTy != MVT::i32)
    9382             :     return 0;
    9383           0 :   if ((Subtarget->hasDQI())) {
    9384           0 :     return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9385             :   }
    9386             :   return 0;
    9387             : }
    9388             : 
    9389             : unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9390           0 :   if (RetVT.SimpleTy != MVT::i32)
    9391             :     return 0;
    9392           0 :   if ((Subtarget->hasDQI())) {
    9393           0 :     return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9394             :   }
    9395             :   return 0;
    9396             : }
    9397             : 
    9398             : unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9399           0 :   if (RetVT.SimpleTy != MVT::i32)
    9400             :     return 0;
    9401           0 :   if ((Subtarget->hasBWI())) {
    9402           0 :     return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9403             :   }
    9404             :   return 0;
    9405             : }
    9406             : 
    9407             : unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9408           0 :   if (RetVT.SimpleTy != MVT::i32)
    9409             :     return 0;
    9410           0 :   if ((Subtarget->hasBWI())) {
    9411           0 :     return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9412             :   }
    9413             :   return 0;
    9414             : }
    9415             : 
    9416           0 : unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9417           0 :   switch (VT.SimpleTy) {
    9418           0 :   case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9419           0 :   case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9420           0 :   case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9421           0 :   case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9422             :   default: return 0;
    9423             :   }
    9424             : }
    9425             : 
    9426             : // FastEmit functions for X86ISD::MOVHLPS.
    9427             : 
    9428           0 : unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9429           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9430             :     return 0;
    9431           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9432           0 :     return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9433             :   }
    9434           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9435           0 :     return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9436             :   }
    9437           0 :   if ((Subtarget->hasAVX512())) {
    9438           0 :     return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9439             :   }
    9440             :   return 0;
    9441             : }
    9442             : 
    9443             : unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9444           0 :   switch (VT.SimpleTy) {
    9445           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9446             :   default: return 0;
    9447             :   }
    9448             : }
    9449             : 
    9450             : // FastEmit functions for X86ISD::MOVLHPS.
    9451             : 
    9452           0 : unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9453           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9454             :     return 0;
    9455           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9456           0 :     return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9457             :   }
    9458           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9459           0 :     return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9460             :   }
    9461           0 :   if ((Subtarget->hasAVX512())) {
    9462           0 :     return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9463             :   }
    9464             :   return 0;
    9465             : }
    9466             : 
    9467             : unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9468           0 :   switch (VT.SimpleTy) {
    9469           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9470             :   default: return 0;
    9471             :   }
    9472             : }
    9473             : 
    9474             : // FastEmit functions for X86ISD::MOVLPD.
    9475             : 
    9476           0 : unsigned fastEmit_X86ISD_MOVLPD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9477           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    9478             :     return 0;
    9479           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9480           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9481             :   }
    9482           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9483           0 :     return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9484             :   }
    9485             :   return 0;
    9486             : }
    9487             : 
    9488           0 : unsigned fastEmit_X86ISD_MOVLPD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9489           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9490             :     return 0;
    9491           0 :   if ((Subtarget->hasAVX512())) {
    9492           0 :     return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9493             :   }
    9494           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9495           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9496             :   }
    9497           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9498           0 :     return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9499             :   }
    9500             :   return 0;
    9501             : }
    9502             : 
    9503           0 : unsigned fastEmit_X86ISD_MOVLPD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9504           0 :   switch (VT.SimpleTy) {
    9505           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVLPD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9506           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVLPD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9507             :   default: return 0;
    9508             :   }
    9509             : }
    9510             : 
    9511             : // FastEmit functions for X86ISD::MOVLPS.
    9512             : 
    9513           0 : unsigned fastEmit_X86ISD_MOVLPS_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9514           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9515             :     return 0;
    9516           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9517           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9518             :   }
    9519           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9520           0 :     return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9521             :   }
    9522             :   return 0;
    9523             : }
    9524             : 
    9525           0 : unsigned fastEmit_X86ISD_MOVLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9526           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9527             :     return 0;
    9528           0 :   if ((Subtarget->hasAVX512())) {
    9529           0 :     return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9530             :   }
    9531           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9532           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9533             :   }
    9534           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9535           0 :     return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9536             :   }
    9537             :   return 0;
    9538             : }
    9539             : 
    9540           0 : unsigned fastEmit_X86ISD_MOVLPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9541           0 :   switch (VT.SimpleTy) {
    9542           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVLPS_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9543           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVLPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9544             :   default: return 0;
    9545             :   }
    9546             : }
    9547             : 
    9548             : // FastEmit functions for X86ISD::MOVSD.
    9549             : 
    9550           0 : unsigned fastEmit_X86ISD_MOVSD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9551           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    9552             :     return 0;
    9553           0 :   if ((Subtarget->hasAVX512())) {
    9554           0 :     return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9555             :   }
    9556           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9557           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9558             :   }
    9559           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9560           0 :     return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9561             :   }
    9562             :   return 0;
    9563             : }
    9564             : 
    9565           0 : unsigned fastEmit_X86ISD_MOVSD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9566           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9567             :     return 0;
    9568           0 :   if ((Subtarget->hasAVX512())) {
    9569           0 :     return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9570             :   }
    9571           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9572           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9573             :   }
    9574           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9575