LCOV - code coverage report
Current view: top level - build-llvm/lib/Target/X86 - X86GenFastISel.inc (source / functions) Hit Total Coverage
Test: llvm-toolchain.info Lines: 836 6770 12.3 %
Date: 2018-05-20 00:06:23 Functions: 117 851 13.7 %
Legend: Lines: hit not hit

          Line data    Source code
       1             : /*===- TableGen'erated file -------------------------------------*- C++ -*-===*\
       2             : |*                                                                            *|
       3             : |* "Fast" Instruction Selector for the X86 target                             *|
       4             : |*                                                                            *|
       5             : |* Automatically generated file, do not edit!                                 *|
       6             : |*                                                                            *|
       7             : \*===----------------------------------------------------------------------===*/
       8             : 
       9             : 
      10             : // FastEmit Immediate Predicate functions.
      11             : static bool Predicate_i16immSExt8(int64_t Imm) {
      12             :  return isInt<8>(Imm); 
      13             : }
      14             : static bool Predicate_i32immSExt8(int64_t Imm) {
      15             :  return isInt<8>(Imm); 
      16             : }
      17             : static bool Predicate_i64immSExt8(int64_t Imm) {
      18             :  return isInt<8>(Imm); 
      19             : }
      20             : static bool Predicate_FROUND_NO_EXC(int64_t Imm) {
      21             :  return Imm == 8; 
      22             : }
      23             : static bool Predicate_FROUND_CURRENT(int64_t Imm) {
      24             : 
      25             :   return Imm == X86::STATIC_ROUNDING::CUR_DIRECTION;
      26             : 
      27             : }
      28             : static bool Predicate_BTRMask64(int64_t Imm) {
      29             : 
      30             :   return !isUInt<32>(Imm) && !isInt<32>(Imm) && isPowerOf2_64(~Imm);
      31             : 
      32             : }
      33             : static bool Predicate_BTCBTSMask64(int64_t Imm) {
      34             : 
      35             :   return !isInt<32>(Imm) && isPowerOf2_64(Imm);
      36             : 
      37             : }
      38             : static bool Predicate_i64immSExt32(int64_t Imm) {
      39             :  return isInt<32>(Imm); 
      40             : }
      41             : static bool Predicate_AndMask64(int64_t Imm) {
      42             : 
      43             :   return isMask_64(Imm) && !isUInt<32>(Imm);
      44             : 
      45             : }
      46             : 
      47             : 
      48             : // FastEmit functions for ISD::ABS.
      49             : 
      50           0 : unsigned fastEmit_ISD_ABS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      51           0 :   if (RetVT.SimpleTy != MVT::v16i8)
      52             :     return 0;
      53           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      54           0 :     return fastEmitInst_r(X86::VPABSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      55             :   }
      56           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      57           0 :     return fastEmitInst_r(X86::PABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      58             :   }
      59           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      60           0 :     return fastEmitInst_r(X86::VPABSBrr, &X86::VR128RegClass, Op0, Op0IsKill);
      61             :   }
      62             :   return 0;
      63             : }
      64             : 
      65           0 : unsigned fastEmit_ISD_ABS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      66           0 :   if (RetVT.SimpleTy != MVT::v32i8)
      67             :     return 0;
      68           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      69           0 :     return fastEmitInst_r(X86::VPABSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
      70             :   }
      71           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      72           0 :     return fastEmitInst_r(X86::VPABSBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
      73             :   }
      74             :   return 0;
      75             : }
      76             : 
      77             : unsigned fastEmit_ISD_ABS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      78           0 :   if (RetVT.SimpleTy != MVT::v64i8)
      79             :     return 0;
      80           0 :   if ((Subtarget->hasBWI())) {
      81           0 :     return fastEmitInst_r(X86::VPABSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
      82             :   }
      83             :   return 0;
      84             : }
      85             : 
      86           0 : unsigned fastEmit_ISD_ABS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
      87           0 :   if (RetVT.SimpleTy != MVT::v8i16)
      88             :     return 0;
      89           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
      90           0 :     return fastEmitInst_r(X86::VPABSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
      91             :   }
      92           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
      93           0 :     return fastEmitInst_r(X86::PABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      94             :   }
      95           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
      96           0 :     return fastEmitInst_r(X86::VPABSWrr, &X86::VR128RegClass, Op0, Op0IsKill);
      97             :   }
      98             :   return 0;
      99             : }
     100             : 
     101           0 : unsigned fastEmit_ISD_ABS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     102           0 :   if (RetVT.SimpleTy != MVT::v16i16)
     103             :     return 0;
     104           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
     105           0 :     return fastEmitInst_r(X86::VPABSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     106             :   }
     107           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
     108           0 :     return fastEmitInst_r(X86::VPABSWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     109             :   }
     110             :   return 0;
     111             : }
     112             : 
     113             : unsigned fastEmit_ISD_ABS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     114           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     115             :     return 0;
     116           0 :   if ((Subtarget->hasBWI())) {
     117           0 :     return fastEmitInst_r(X86::VPABSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     118             :   }
     119             :   return 0;
     120             : }
     121             : 
     122           0 : unsigned fastEmit_ISD_ABS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     123           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     124             :     return 0;
     125           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     126           0 :     return fastEmitInst_r(X86::VPABSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     127             :   }
     128           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
     129           0 :     return fastEmitInst_r(X86::PABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     130             :   }
     131           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     132           0 :     return fastEmitInst_r(X86::VPABSDrr, &X86::VR128RegClass, Op0, Op0IsKill);
     133             :   }
     134             :   return 0;
     135             : }
     136             : 
     137           0 : unsigned fastEmit_ISD_ABS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     138           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     139             :     return 0;
     140           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     141           0 :     return fastEmitInst_r(X86::VPABSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     142             :   }
     143           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
     144           0 :     return fastEmitInst_r(X86::VPABSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     145             :   }
     146             :   return 0;
     147             : }
     148             : 
     149             : unsigned fastEmit_ISD_ABS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     150           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     151             :     return 0;
     152           0 :   if ((Subtarget->hasAVX512())) {
     153           0 :     return fastEmitInst_r(X86::VPABSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     154             :   }
     155             :   return 0;
     156             : }
     157             : 
     158           0 : unsigned fastEmit_ISD_ABS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     159           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     160             :     return 0;
     161           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     162           0 :     return fastEmitInst_r(X86::VPABSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     163             :   }
     164             :   return 0;
     165             : }
     166             : 
     167           0 : unsigned fastEmit_ISD_ABS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     168           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     169             :     return 0;
     170           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
     171           0 :     return fastEmitInst_r(X86::VPABSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     172             :   }
     173             :   return 0;
     174             : }
     175             : 
     176             : unsigned fastEmit_ISD_ABS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     177           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     178             :     return 0;
     179           0 :   if ((Subtarget->hasAVX512())) {
     180           0 :     return fastEmitInst_r(X86::VPABSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     181             :   }
     182             :   return 0;
     183             : }
     184             : 
     185           0 : unsigned fastEmit_ISD_ABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     186           0 :   switch (VT.SimpleTy) {
     187           0 :   case MVT::v16i8: return fastEmit_ISD_ABS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     188           0 :   case MVT::v32i8: return fastEmit_ISD_ABS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     189           0 :   case MVT::v64i8: return fastEmit_ISD_ABS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     190           0 :   case MVT::v8i16: return fastEmit_ISD_ABS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     191           0 :   case MVT::v16i16: return fastEmit_ISD_ABS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     192           0 :   case MVT::v32i16: return fastEmit_ISD_ABS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     193           0 :   case MVT::v4i32: return fastEmit_ISD_ABS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     194           0 :   case MVT::v8i32: return fastEmit_ISD_ABS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     195           0 :   case MVT::v16i32: return fastEmit_ISD_ABS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     196           0 :   case MVT::v2i64: return fastEmit_ISD_ABS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     197           0 :   case MVT::v4i64: return fastEmit_ISD_ABS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     198           0 :   case MVT::v8i64: return fastEmit_ISD_ABS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     199             :   default: return 0;
     200             :   }
     201             : }
     202             : 
     203             : // FastEmit functions for ISD::ANY_EXTEND.
     204             : 
     205             : unsigned fastEmit_ISD_ANY_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     206          15 :   if (RetVT.SimpleTy != MVT::i32)
     207             :     return 0;
     208          15 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
     209             : }
     210             : 
     211             : unsigned fastEmit_ISD_ANY_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     212          18 :   switch (VT.SimpleTy) {
     213          15 :   case MVT::i8: return fastEmit_ISD_ANY_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
     214             :   default: return 0;
     215             :   }
     216             : }
     217             : 
     218             : // FastEmit functions for ISD::BITCAST.
     219             : 
     220           1 : unsigned fastEmit_ISD_BITCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     221           1 :   if (RetVT.SimpleTy != MVT::f32)
     222             :     return 0;
     223           2 :   if ((Subtarget->hasAVX512())) {
     224           0 :     return fastEmitInst_r(X86::VMOVDI2SSZrr, &X86::FR32XRegClass, Op0, Op0IsKill);
     225             :   }
     226           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     227           1 :     return fastEmitInst_r(X86::MOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     228             :   }
     229           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     230           0 :     return fastEmitInst_r(X86::VMOVDI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     231             :   }
     232             :   return 0;
     233             : }
     234             : 
     235           1 : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     236           2 :   if ((Subtarget->hasAVX512())) {
     237           0 :     return fastEmitInst_r(X86::VMOV64toSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill);
     238             :   }
     239           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     240           1 :     return fastEmitInst_r(X86::MOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     241             :   }
     242           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     243           0 :     return fastEmitInst_r(X86::VMOV64toSDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     244             :   }
     245             :   return 0;
     246             : }
     247             : 
     248             : unsigned fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     249           0 :   if ((Subtarget->hasMMX())) {
     250           0 :     return fastEmitInst_r(X86::MMX_MOVD64to64rr, &X86::VR64RegClass, Op0, Op0IsKill);
     251             :   }
     252             :   return 0;
     253             : }
     254             : 
     255           1 : unsigned fastEmit_ISD_BITCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     256           1 : switch (RetVT.SimpleTy) {
     257           1 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
     258           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_i64_MVT_x86mmx_r(Op0, Op0IsKill);
     259             :   default: return 0;
     260             : }
     261             : }
     262             : 
     263           3 : unsigned fastEmit_ISD_BITCAST_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     264           3 :   if (RetVT.SimpleTy != MVT::i32)
     265             :     return 0;
     266           6 :   if ((Subtarget->hasAVX512())) {
     267           0 :     return fastEmitInst_r(X86::VMOVSS2DIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     268             :   }
     269           3 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     270           3 :     return fastEmitInst_r(X86::MOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     271             :   }
     272           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     273           0 :     return fastEmitInst_r(X86::VMOVSS2DIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     274             :   }
     275             :   return 0;
     276             : }
     277             : 
     278           2 : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     279           4 :   if ((Subtarget->hasAVX512())) {
     280           0 :     return fastEmitInst_r(X86::VMOVSDto64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     281             :   }
     282           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     283           1 :     return fastEmitInst_r(X86::MOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     284             :   }
     285           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     286           1 :     return fastEmitInst_r(X86::VMOVSDto64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     287             :   }
     288             :   return 0;
     289             : }
     290             : 
     291             : unsigned fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
     292           0 :   if ((Subtarget->hasSSE2())) {
     293           0 :     return fastEmitInst_r(X86::MMX_MOVFR642Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
     294             :   }
     295             :   return 0;
     296             : }
     297             : 
     298           2 : unsigned fastEmit_ISD_BITCAST_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     299           2 : switch (RetVT.SimpleTy) {
     300           2 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     301           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_f64_MVT_x86mmx_r(Op0, Op0IsKill);
     302             :   default: return 0;
     303             : }
     304             : }
     305             : 
     306             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     307           0 :   if ((Subtarget->hasMMX())) {
     308           0 :     return fastEmitInst_r(X86::MMX_MOVD64from64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     309             :   }
     310             :   return 0;
     311             : }
     312             : 
     313             : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
     314           0 :   if ((Subtarget->hasSSE2())) {
     315           0 :     return fastEmitInst_r(X86::MMX_MOVQ2FR64rr, &X86::FR64RegClass, Op0, Op0IsKill);
     316             :   }
     317             :   return 0;
     318             : }
     319             : 
     320           0 : unsigned fastEmit_ISD_BITCAST_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     321           0 : switch (RetVT.SimpleTy) {
     322           0 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_i64_r(Op0, Op0IsKill);
     323           0 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_x86mmx_MVT_f64_r(Op0, Op0IsKill);
     324             :   default: return 0;
     325             : }
     326             : }
     327             : 
     328        1967 : unsigned fastEmit_ISD_BITCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     329        1967 :   switch (VT.SimpleTy) {
     330           1 :   case MVT::i32: return fastEmit_ISD_BITCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
     331           1 :   case MVT::i64: return fastEmit_ISD_BITCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
     332           3 :   case MVT::f32: return fastEmit_ISD_BITCAST_MVT_f32_r(RetVT, Op0, Op0IsKill);
     333           2 :   case MVT::f64: return fastEmit_ISD_BITCAST_MVT_f64_r(RetVT, Op0, Op0IsKill);
     334           0 :   case MVT::x86mmx: return fastEmit_ISD_BITCAST_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
     335             :   default: return 0;
     336             :   }
     337             : }
     338             : 
     339             : // FastEmit functions for ISD::BRIND.
     340             : 
     341             : unsigned fastEmit_ISD_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     342           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     343             :     return 0;
     344           0 :   if ((!Subtarget->is64Bit())) {
     345           0 :     return fastEmitInst_r(X86::JMP16r, &X86::GR16RegClass, Op0, Op0IsKill);
     346             :   }
     347             :   return 0;
     348             : }
     349             : 
     350             : unsigned fastEmit_ISD_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     351           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     352             :     return 0;
     353           0 :   if ((!Subtarget->is64Bit())) {
     354           0 :     return fastEmitInst_r(X86::JMP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     355             :   }
     356             :   return 0;
     357             : }
     358             : 
     359             : unsigned fastEmit_ISD_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     360           0 :   if (RetVT.SimpleTy != MVT::isVoid)
     361             :     return 0;
     362           0 :   if ((Subtarget->is64Bit())) {
     363           0 :     return fastEmitInst_r(X86::JMP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     364             :   }
     365             :   return 0;
     366             : }
     367             : 
     368           0 : unsigned fastEmit_ISD_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     369           0 :   switch (VT.SimpleTy) {
     370           0 :   case MVT::i16: return fastEmit_ISD_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
     371           0 :   case MVT::i32: return fastEmit_ISD_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
     372           0 :   case MVT::i64: return fastEmit_ISD_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
     373             :   default: return 0;
     374             :   }
     375             : }
     376             : 
     377             : // FastEmit functions for ISD::BSWAP.
     378             : 
     379             : unsigned fastEmit_ISD_BSWAP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     380           0 :   if (RetVT.SimpleTy != MVT::i32)
     381             :     return 0;
     382           0 :   return fastEmitInst_r(X86::BSWAP32r, &X86::GR32RegClass, Op0, Op0IsKill);
     383             : }
     384             : 
     385             : unsigned fastEmit_ISD_BSWAP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     386           0 :   if (RetVT.SimpleTy != MVT::i64)
     387             :     return 0;
     388           0 :   return fastEmitInst_r(X86::BSWAP64r, &X86::GR64RegClass, Op0, Op0IsKill);
     389             : }
     390             : 
     391           0 : unsigned fastEmit_ISD_BSWAP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     392           0 :   switch (VT.SimpleTy) {
     393           0 :   case MVT::i32: return fastEmit_ISD_BSWAP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     394           0 :   case MVT::i64: return fastEmit_ISD_BSWAP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     395             :   default: return 0;
     396             :   }
     397             : }
     398             : 
     399             : // FastEmit functions for ISD::CTLZ.
     400             : 
     401             : unsigned fastEmit_ISD_CTLZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     402           0 :   if (RetVT.SimpleTy != MVT::i16)
     403             :     return 0;
     404           0 :   if ((Subtarget->hasLZCNT())) {
     405           0 :     return fastEmitInst_r(X86::LZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     406             :   }
     407             :   return 0;
     408             : }
     409             : 
     410             : unsigned fastEmit_ISD_CTLZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     411           0 :   if (RetVT.SimpleTy != MVT::i32)
     412             :     return 0;
     413           0 :   if ((Subtarget->hasLZCNT())) {
     414           0 :     return fastEmitInst_r(X86::LZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     415             :   }
     416             :   return 0;
     417             : }
     418             : 
     419             : unsigned fastEmit_ISD_CTLZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     420           0 :   if (RetVT.SimpleTy != MVT::i64)
     421             :     return 0;
     422           0 :   if ((Subtarget->hasLZCNT())) {
     423           0 :     return fastEmitInst_r(X86::LZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     424             :   }
     425             :   return 0;
     426             : }
     427             : 
     428           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     429           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     430             :     return 0;
     431           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     432           0 :     return fastEmitInst_r(X86::VPLZCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     433             :   }
     434             :   return 0;
     435             : }
     436             : 
     437           0 : unsigned fastEmit_ISD_CTLZ_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     438           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     439             :     return 0;
     440           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     441           0 :     return fastEmitInst_r(X86::VPLZCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     442             :   }
     443             :   return 0;
     444             : }
     445             : 
     446             : unsigned fastEmit_ISD_CTLZ_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     447           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     448             :     return 0;
     449           0 :   if ((Subtarget->hasCDI())) {
     450           0 :     return fastEmitInst_r(X86::VPLZCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     451             :   }
     452             :   return 0;
     453             : }
     454             : 
     455           0 : unsigned fastEmit_ISD_CTLZ_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     456           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     457             :     return 0;
     458           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     459           0 :     return fastEmitInst_r(X86::VPLZCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     460             :   }
     461             :   return 0;
     462             : }
     463             : 
     464           0 : unsigned fastEmit_ISD_CTLZ_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     465           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     466             :     return 0;
     467           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
     468           0 :     return fastEmitInst_r(X86::VPLZCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     469             :   }
     470             :   return 0;
     471             : }
     472             : 
     473             : unsigned fastEmit_ISD_CTLZ_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     474           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     475             :     return 0;
     476           0 :   if ((Subtarget->hasCDI())) {
     477           0 :     return fastEmitInst_r(X86::VPLZCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     478             :   }
     479             :   return 0;
     480             : }
     481             : 
     482           0 : unsigned fastEmit_ISD_CTLZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     483           0 :   switch (VT.SimpleTy) {
     484           0 :   case MVT::i16: return fastEmit_ISD_CTLZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     485           0 :   case MVT::i32: return fastEmit_ISD_CTLZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     486           0 :   case MVT::i64: return fastEmit_ISD_CTLZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     487           0 :   case MVT::v4i32: return fastEmit_ISD_CTLZ_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     488           0 :   case MVT::v8i32: return fastEmit_ISD_CTLZ_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     489           0 :   case MVT::v16i32: return fastEmit_ISD_CTLZ_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     490           0 :   case MVT::v2i64: return fastEmit_ISD_CTLZ_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     491           0 :   case MVT::v4i64: return fastEmit_ISD_CTLZ_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     492           0 :   case MVT::v8i64: return fastEmit_ISD_CTLZ_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     493             :   default: return 0;
     494             :   }
     495             : }
     496             : 
     497             : // FastEmit functions for ISD::CTPOP.
     498             : 
     499             : unsigned fastEmit_ISD_CTPOP_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     500           0 :   if (RetVT.SimpleTy != MVT::i16)
     501             :     return 0;
     502           0 :   if ((Subtarget->hasPOPCNT())) {
     503           0 :     return fastEmitInst_r(X86::POPCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     504             :   }
     505             :   return 0;
     506             : }
     507             : 
     508             : unsigned fastEmit_ISD_CTPOP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     509           0 :   if (RetVT.SimpleTy != MVT::i32)
     510             :     return 0;
     511           0 :   if ((Subtarget->hasPOPCNT())) {
     512           0 :     return fastEmitInst_r(X86::POPCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     513             :   }
     514             :   return 0;
     515             : }
     516             : 
     517             : unsigned fastEmit_ISD_CTPOP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     518           0 :   if (RetVT.SimpleTy != MVT::i64)
     519             :     return 0;
     520           0 :   if ((Subtarget->hasPOPCNT())) {
     521           0 :     return fastEmitInst_r(X86::POPCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     522             :   }
     523             :   return 0;
     524             : }
     525             : 
     526           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     527           0 :   if (RetVT.SimpleTy != MVT::v16i8)
     528             :     return 0;
     529           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     530           0 :     return fastEmitInst_r(X86::VPOPCNTBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     531             :   }
     532             :   return 0;
     533             : }
     534             : 
     535           0 : unsigned fastEmit_ISD_CTPOP_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     536           0 :   if (RetVT.SimpleTy != MVT::v32i8)
     537             :     return 0;
     538           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     539           0 :     return fastEmitInst_r(X86::VPOPCNTBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     540             :   }
     541             :   return 0;
     542             : }
     543             : 
     544             : unsigned fastEmit_ISD_CTPOP_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     545           0 :   if (RetVT.SimpleTy != MVT::v64i8)
     546             :     return 0;
     547           0 :   if ((Subtarget->hasBITALG())) {
     548           0 :     return fastEmitInst_r(X86::VPOPCNTBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     549             :   }
     550             :   return 0;
     551             : }
     552             : 
     553           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     554           0 :   if (RetVT.SimpleTy != MVT::v8i16)
     555             :     return 0;
     556           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     557           0 :     return fastEmitInst_r(X86::VPOPCNTWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     558             :   }
     559             :   return 0;
     560             : }
     561             : 
     562           0 : unsigned fastEmit_ISD_CTPOP_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     563           0 :   if (RetVT.SimpleTy != MVT::v16i16)
     564             :     return 0;
     565           0 :   if ((Subtarget->hasBITALG()) && (Subtarget->hasVLX())) {
     566           0 :     return fastEmitInst_r(X86::VPOPCNTWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     567             :   }
     568             :   return 0;
     569             : }
     570             : 
     571             : unsigned fastEmit_ISD_CTPOP_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     572           0 :   if (RetVT.SimpleTy != MVT::v32i16)
     573             :     return 0;
     574           0 :   if ((Subtarget->hasBITALG())) {
     575           0 :     return fastEmitInst_r(X86::VPOPCNTWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     576             :   }
     577             :   return 0;
     578             : }
     579             : 
     580           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     581           0 :   if (RetVT.SimpleTy != MVT::v4i32)
     582             :     return 0;
     583           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     584           0 :     return fastEmitInst_r(X86::VPOPCNTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     585             :   }
     586             :   return 0;
     587             : }
     588             : 
     589           0 : unsigned fastEmit_ISD_CTPOP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     590           0 :   if (RetVT.SimpleTy != MVT::v8i32)
     591             :     return 0;
     592           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     593           0 :     return fastEmitInst_r(X86::VPOPCNTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     594             :   }
     595             :   return 0;
     596             : }
     597             : 
     598             : unsigned fastEmit_ISD_CTPOP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     599           0 :   if (RetVT.SimpleTy != MVT::v16i32)
     600             :     return 0;
     601           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     602           0 :     return fastEmitInst_r(X86::VPOPCNTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     603             :   }
     604             :   return 0;
     605             : }
     606             : 
     607           0 : unsigned fastEmit_ISD_CTPOP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     608           0 :   if (RetVT.SimpleTy != MVT::v2i64)
     609             :     return 0;
     610           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     611           0 :     return fastEmitInst_r(X86::VPOPCNTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     612             :   }
     613             :   return 0;
     614             : }
     615             : 
     616           0 : unsigned fastEmit_ISD_CTPOP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     617           0 :   if (RetVT.SimpleTy != MVT::v4i64)
     618             :     return 0;
     619           0 :   if ((Subtarget->hasVLX()) && (Subtarget->hasVPOPCNTDQ())) {
     620           0 :     return fastEmitInst_r(X86::VPOPCNTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     621             :   }
     622             :   return 0;
     623             : }
     624             : 
     625             : unsigned fastEmit_ISD_CTPOP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     626           0 :   if (RetVT.SimpleTy != MVT::v8i64)
     627             :     return 0;
     628           0 :   if ((Subtarget->hasVPOPCNTDQ())) {
     629           0 :     return fastEmitInst_r(X86::VPOPCNTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     630             :   }
     631             :   return 0;
     632             : }
     633             : 
     634           0 : unsigned fastEmit_ISD_CTPOP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     635           0 :   switch (VT.SimpleTy) {
     636           0 :   case MVT::i16: return fastEmit_ISD_CTPOP_MVT_i16_r(RetVT, Op0, Op0IsKill);
     637           0 :   case MVT::i32: return fastEmit_ISD_CTPOP_MVT_i32_r(RetVT, Op0, Op0IsKill);
     638           0 :   case MVT::i64: return fastEmit_ISD_CTPOP_MVT_i64_r(RetVT, Op0, Op0IsKill);
     639           0 :   case MVT::v16i8: return fastEmit_ISD_CTPOP_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
     640           0 :   case MVT::v32i8: return fastEmit_ISD_CTPOP_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
     641           0 :   case MVT::v64i8: return fastEmit_ISD_CTPOP_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
     642           0 :   case MVT::v8i16: return fastEmit_ISD_CTPOP_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
     643           0 :   case MVT::v16i16: return fastEmit_ISD_CTPOP_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
     644           0 :   case MVT::v32i16: return fastEmit_ISD_CTPOP_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
     645           0 :   case MVT::v4i32: return fastEmit_ISD_CTPOP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
     646           0 :   case MVT::v8i32: return fastEmit_ISD_CTPOP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
     647           0 :   case MVT::v16i32: return fastEmit_ISD_CTPOP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
     648           0 :   case MVT::v2i64: return fastEmit_ISD_CTPOP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
     649           0 :   case MVT::v4i64: return fastEmit_ISD_CTPOP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
     650           0 :   case MVT::v8i64: return fastEmit_ISD_CTPOP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
     651             :   default: return 0;
     652             :   }
     653             : }
     654             : 
     655             : // FastEmit functions for ISD::CTTZ.
     656             : 
     657             : unsigned fastEmit_ISD_CTTZ_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     658           0 :   if (RetVT.SimpleTy != MVT::i16)
     659             :     return 0;
     660           0 :   if ((Subtarget->hasBMI())) {
     661           0 :     return fastEmitInst_r(X86::TZCNT16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     662             :   }
     663             :   return 0;
     664             : }
     665             : 
     666             : unsigned fastEmit_ISD_CTTZ_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     667           0 :   if (RetVT.SimpleTy != MVT::i32)
     668             :     return 0;
     669           0 :   if ((Subtarget->hasBMI())) {
     670           0 :     return fastEmitInst_r(X86::TZCNT32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     671             :   }
     672             :   return 0;
     673             : }
     674             : 
     675             : unsigned fastEmit_ISD_CTTZ_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     676           0 :   if (RetVT.SimpleTy != MVT::i64)
     677             :     return 0;
     678           0 :   if ((Subtarget->hasBMI())) {
     679           0 :     return fastEmitInst_r(X86::TZCNT64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     680             :   }
     681             :   return 0;
     682             : }
     683             : 
     684           0 : unsigned fastEmit_ISD_CTTZ_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     685           0 :   switch (VT.SimpleTy) {
     686           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_MVT_i16_r(RetVT, Op0, Op0IsKill);
     687           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_MVT_i32_r(RetVT, Op0, Op0IsKill);
     688           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_MVT_i64_r(RetVT, Op0, Op0IsKill);
     689             :   default: return 0;
     690             :   }
     691             : }
     692             : 
     693             : // FastEmit functions for ISD::CTTZ_ZERO_UNDEF.
     694             : 
     695             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     696           0 :   if (RetVT.SimpleTy != MVT::i16)
     697             :     return 0;
     698           0 :   return fastEmitInst_r(X86::BSF16rr, &X86::GR16RegClass, Op0, Op0IsKill);
     699             : }
     700             : 
     701             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     702           0 :   if (RetVT.SimpleTy != MVT::i32)
     703             :     return 0;
     704           0 :   return fastEmitInst_r(X86::BSF32rr, &X86::GR32RegClass, Op0, Op0IsKill);
     705             : }
     706             : 
     707             : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     708           0 :   if (RetVT.SimpleTy != MVT::i64)
     709             :     return 0;
     710           0 :   return fastEmitInst_r(X86::BSF64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     711             : }
     712             : 
     713           0 : unsigned fastEmit_ISD_CTTZ_ZERO_UNDEF_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     714           0 :   switch (VT.SimpleTy) {
     715           0 :   case MVT::i16: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i16_r(RetVT, Op0, Op0IsKill);
     716           0 :   case MVT::i32: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i32_r(RetVT, Op0, Op0IsKill);
     717           0 :   case MVT::i64: return fastEmit_ISD_CTTZ_ZERO_UNDEF_MVT_i64_r(RetVT, Op0, Op0IsKill);
     718             :   default: return 0;
     719             :   }
     720             : }
     721             : 
     722             : // FastEmit functions for ISD::FABS.
     723             : 
     724             : unsigned fastEmit_ISD_FABS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     725           0 :   if (RetVT.SimpleTy != MVT::f32)
     726             :     return 0;
     727           0 :   if ((!Subtarget->hasSSE1())) {
     728           0 :     return fastEmitInst_r(X86::ABS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     729             :   }
     730             :   return 0;
     731             : }
     732             : 
     733             : unsigned fastEmit_ISD_FABS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     734           0 :   if (RetVT.SimpleTy != MVT::f64)
     735             :     return 0;
     736           0 :   if ((!Subtarget->hasSSE2())) {
     737           0 :     return fastEmitInst_r(X86::ABS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     738             :   }
     739             :   return 0;
     740             : }
     741             : 
     742             : unsigned fastEmit_ISD_FABS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     743           0 :   if (RetVT.SimpleTy != MVT::f80)
     744             :     return 0;
     745           0 :   return fastEmitInst_r(X86::ABS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     746             : }
     747             : 
     748           0 : unsigned fastEmit_ISD_FABS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     749           0 :   switch (VT.SimpleTy) {
     750           0 :   case MVT::f32: return fastEmit_ISD_FABS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     751           0 :   case MVT::f64: return fastEmit_ISD_FABS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     752           0 :   case MVT::f80: return fastEmit_ISD_FABS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     753             :   default: return 0;
     754             :   }
     755             : }
     756             : 
     757             : // FastEmit functions for ISD::FCOS.
     758             : 
     759             : unsigned fastEmit_ISD_FCOS_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     760           0 :   if (RetVT.SimpleTy != MVT::f32)
     761             :     return 0;
     762           0 :   if ((!Subtarget->hasSSE1())) {
     763           0 :     return fastEmitInst_r(X86::COS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     764             :   }
     765             :   return 0;
     766             : }
     767             : 
     768             : unsigned fastEmit_ISD_FCOS_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     769           0 :   if (RetVT.SimpleTy != MVT::f64)
     770             :     return 0;
     771           0 :   if ((!Subtarget->hasSSE2())) {
     772           0 :     return fastEmitInst_r(X86::COS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     773             :   }
     774             :   return 0;
     775             : }
     776             : 
     777             : unsigned fastEmit_ISD_FCOS_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     778           0 :   if (RetVT.SimpleTy != MVT::f80)
     779             :     return 0;
     780           0 :   return fastEmitInst_r(X86::COS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     781             : }
     782             : 
     783           0 : unsigned fastEmit_ISD_FCOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     784           0 :   switch (VT.SimpleTy) {
     785           0 :   case MVT::f32: return fastEmit_ISD_FCOS_MVT_f32_r(RetVT, Op0, Op0IsKill);
     786           0 :   case MVT::f64: return fastEmit_ISD_FCOS_MVT_f64_r(RetVT, Op0, Op0IsKill);
     787           0 :   case MVT::f80: return fastEmit_ISD_FCOS_MVT_f80_r(RetVT, Op0, Op0IsKill);
     788             :   default: return 0;
     789             :   }
     790             : }
     791             : 
     792             : // FastEmit functions for ISD::FNEG.
     793             : 
     794             : unsigned fastEmit_ISD_FNEG_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     795           1 :   if (RetVT.SimpleTy != MVT::f32)
     796             :     return 0;
     797           2 :   if ((!Subtarget->hasSSE1())) {
     798           0 :     return fastEmitInst_r(X86::CHS_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
     799             :   }
     800             :   return 0;
     801             : }
     802             : 
     803             : unsigned fastEmit_ISD_FNEG_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     804           1 :   if (RetVT.SimpleTy != MVT::f64)
     805             :     return 0;
     806           2 :   if ((!Subtarget->hasSSE2())) {
     807           0 :     return fastEmitInst_r(X86::CHS_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
     808             :   }
     809             :   return 0;
     810             : }
     811             : 
     812             : unsigned fastEmit_ISD_FNEG_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     813           0 :   if (RetVT.SimpleTy != MVT::f80)
     814             :     return 0;
     815           0 :   return fastEmitInst_r(X86::CHS_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
     816             : }
     817             : 
     818           2 : unsigned fastEmit_ISD_FNEG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     819           2 :   switch (VT.SimpleTy) {
     820           1 :   case MVT::f32: return fastEmit_ISD_FNEG_MVT_f32_r(RetVT, Op0, Op0IsKill);
     821           1 :   case MVT::f64: return fastEmit_ISD_FNEG_MVT_f64_r(RetVT, Op0, Op0IsKill);
     822           0 :   case MVT::f80: return fastEmit_ISD_FNEG_MVT_f80_r(RetVT, Op0, Op0IsKill);
     823             :   default: return 0;
     824             :   }
     825             : }
     826             : 
     827             : // FastEmit functions for ISD::FP_EXTEND.
     828             : 
     829           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     830           0 :   if (RetVT.SimpleTy != MVT::f64)
     831             :     return 0;
     832           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     833           0 :     return fastEmitInst_r(X86::CVTSS2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
     834             :   }
     835             :   return 0;
     836             : }
     837             : 
     838           0 : unsigned fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     839           0 :   if (RetVT.SimpleTy != MVT::v4f64)
     840             :     return 0;
     841           0 :   if ((Subtarget->hasVLX())) {
     842           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     843             :   }
     844           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     845           0 :     return fastEmitInst_r(X86::VCVTPS2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
     846             :   }
     847             :   return 0;
     848             : }
     849             : 
     850             : unsigned fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     851           0 :   if (RetVT.SimpleTy != MVT::v8f64)
     852             :     return 0;
     853           0 :   if ((Subtarget->hasAVX512())) {
     854           0 :     return fastEmitInst_r(X86::VCVTPS2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
     855             :   }
     856             :   return 0;
     857             : }
     858             : 
     859           0 : unsigned fastEmit_ISD_FP_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     860           0 :   switch (VT.SimpleTy) {
     861           0 :   case MVT::f32: return fastEmit_ISD_FP_EXTEND_MVT_f32_r(RetVT, Op0, Op0IsKill);
     862           0 :   case MVT::v4f32: return fastEmit_ISD_FP_EXTEND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
     863           0 :   case MVT::v8f32: return fastEmit_ISD_FP_EXTEND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
     864             :   default: return 0;
     865             :   }
     866             : }
     867             : 
     868             : // FastEmit functions for ISD::FP_ROUND.
     869             : 
     870           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     871           0 :   if (RetVT.SimpleTy != MVT::f32)
     872             :     return 0;
     873           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     874           0 :     return fastEmitInst_r(X86::CVTSD2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
     875             :   }
     876             :   return 0;
     877             : }
     878             : 
     879           0 : unsigned fastEmit_ISD_FP_ROUND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     880           0 :   if (RetVT.SimpleTy != MVT::v4f32)
     881             :     return 0;
     882           0 :   if ((Subtarget->hasVLX())) {
     883           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     884             :   }
     885           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     886           0 :     return fastEmitInst_r(X86::VCVTPD2PSYrr, &X86::VR128RegClass, Op0, Op0IsKill);
     887             :   }
     888             :   return 0;
     889             : }
     890             : 
     891             : unsigned fastEmit_ISD_FP_ROUND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     892           0 :   if (RetVT.SimpleTy != MVT::v8f32)
     893             :     return 0;
     894           0 :   if ((Subtarget->hasAVX512())) {
     895           0 :     return fastEmitInst_r(X86::VCVTPD2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
     896             :   }
     897             :   return 0;
     898             : }
     899             : 
     900           0 : unsigned fastEmit_ISD_FP_ROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
     901           0 :   switch (VT.SimpleTy) {
     902           0 :   case MVT::f64: return fastEmit_ISD_FP_ROUND_MVT_f64_r(RetVT, Op0, Op0IsKill);
     903           0 :   case MVT::v4f64: return fastEmit_ISD_FP_ROUND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
     904           0 :   case MVT::v8f64: return fastEmit_ISD_FP_ROUND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
     905             :   default: return 0;
     906             :   }
     907             : }
     908             : 
     909             : // FastEmit functions for ISD::FP_TO_SINT.
     910             : 
     911           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     912           0 :   if ((Subtarget->hasAVX512())) {
     913           0 :     return fastEmitInst_r(X86::VCVTTSS2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     914             :   }
     915           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     916           0 :     return fastEmitInst_r(X86::CVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     917             :   }
     918           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     919           0 :     return fastEmitInst_r(X86::VCVTTSS2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     920             :   }
     921             :   return 0;
     922             : }
     923             : 
     924           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     925           0 :   if ((Subtarget->hasAVX512())) {
     926           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     927             :   }
     928           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
     929           0 :     return fastEmitInst_r(X86::CVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     930             :   }
     931           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     932           0 :     return fastEmitInst_r(X86::VCVTTSS2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     933             :   }
     934             :   return 0;
     935             : }
     936             : 
     937           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     938           0 : switch (RetVT.SimpleTy) {
     939           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
     940           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
     941             :   default: return 0;
     942             : }
     943             : }
     944             : 
     945           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
     946           0 :   if ((Subtarget->hasAVX512())) {
     947           0 :     return fastEmitInst_r(X86::VCVTTSD2SIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
     948             :   }
     949           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     950           0 :     return fastEmitInst_r(X86::CVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     951             :   }
     952           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     953           0 :     return fastEmitInst_r(X86::VCVTTSD2SIrr, &X86::GR32RegClass, Op0, Op0IsKill);
     954             :   }
     955             :   return 0;
     956             : }
     957             : 
     958           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
     959           0 :   if ((Subtarget->hasAVX512())) {
     960           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
     961             :   }
     962           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     963           0 :     return fastEmitInst_r(X86::CVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     964             :   }
     965           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
     966           0 :     return fastEmitInst_r(X86::VCVTTSD2SI64rr, &X86::GR64RegClass, Op0, Op0IsKill);
     967             :   }
     968             :   return 0;
     969             : }
     970             : 
     971           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
     972           0 : switch (RetVT.SimpleTy) {
     973           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
     974           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
     975             :   default: return 0;
     976             : }
     977             : }
     978             : 
     979           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
     980           0 :   if ((Subtarget->hasVLX())) {
     981           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
     982             :   }
     983           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
     984           0 :     return fastEmitInst_r(X86::CVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     985             :   }
     986           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
     987           0 :     return fastEmitInst_r(X86::VCVTTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
     988             :   }
     989             :   return 0;
     990             : }
     991             : 
     992           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
     993           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
     994           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
     995             :   }
     996             :   return 0;
     997             : }
     998             : 
     999           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1000           0 : switch (RetVT.SimpleTy) {
    1001           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    1002           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    1003             :   default: return 0;
    1004             : }
    1005             : }
    1006             : 
    1007           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1008           0 :   if ((Subtarget->hasVLX())) {
    1009           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1010             :   }
    1011           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1012           0 :     return fastEmitInst_r(X86::VCVTTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1013             :   }
    1014             :   return 0;
    1015             : }
    1016             : 
    1017             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1018           0 :   if ((Subtarget->hasDQI())) {
    1019           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1020             :   }
    1021             :   return 0;
    1022             : }
    1023             : 
    1024           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1025           0 : switch (RetVT.SimpleTy) {
    1026           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1027           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1028             :   default: return 0;
    1029             : }
    1030             : }
    1031             : 
    1032             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1033           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1034             :     return 0;
    1035           0 :   if ((Subtarget->hasAVX512())) {
    1036           0 :     return fastEmitInst_r(X86::VCVTTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1037             :   }
    1038             :   return 0;
    1039             : }
    1040             : 
    1041           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1042           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1043             :     return 0;
    1044           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1045           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1046             :   }
    1047             :   return 0;
    1048             : }
    1049             : 
    1050           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1051           0 :   if ((Subtarget->hasVLX())) {
    1052           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1053             :   }
    1054           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1055           0 :     return fastEmitInst_r(X86::VCVTTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1056             :   }
    1057             :   return 0;
    1058             : }
    1059             : 
    1060           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1061           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1062           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1063             :   }
    1064             :   return 0;
    1065             : }
    1066             : 
    1067           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1068           0 : switch (RetVT.SimpleTy) {
    1069           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1070           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1071             :   default: return 0;
    1072             : }
    1073             : }
    1074             : 
    1075             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1076           0 :   if ((Subtarget->hasAVX512())) {
    1077           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1078             :   }
    1079             :   return 0;
    1080             : }
    1081             : 
    1082             : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1083           0 :   if ((Subtarget->hasDQI())) {
    1084           0 :     return fastEmitInst_r(X86::VCVTTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1085             :   }
    1086             :   return 0;
    1087             : }
    1088             : 
    1089           0 : unsigned fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1090           0 : switch (RetVT.SimpleTy) {
    1091           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1092           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1093             :   default: return 0;
    1094             : }
    1095             : }
    1096             : 
    1097           0 : unsigned fastEmit_ISD_FP_TO_SINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1098           0 :   switch (VT.SimpleTy) {
    1099           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_SINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1100           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_SINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1101           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_SINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1102           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_SINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1103           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_SINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1104           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_SINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1105           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_SINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1106           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_SINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1107             :   default: return 0;
    1108             :   }
    1109             : }
    1110             : 
    1111             : // FastEmit functions for ISD::FP_TO_UINT.
    1112             : 
    1113             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1114           0 :   if ((Subtarget->hasAVX512())) {
    1115           0 :     return fastEmitInst_r(X86::VCVTTSS2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1116             :   }
    1117             :   return 0;
    1118             : }
    1119             : 
    1120             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1121           0 :   if ((Subtarget->hasAVX512())) {
    1122           0 :     return fastEmitInst_r(X86::VCVTTSS2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1123             :   }
    1124             :   return 0;
    1125             : }
    1126             : 
    1127           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1128           0 : switch (RetVT.SimpleTy) {
    1129           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i32_r(Op0, Op0IsKill);
    1130           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f32_MVT_i64_r(Op0, Op0IsKill);
    1131             :   default: return 0;
    1132             : }
    1133             : }
    1134             : 
    1135             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1136           0 :   if ((Subtarget->hasAVX512())) {
    1137           0 :     return fastEmitInst_r(X86::VCVTTSD2USIZrr, &X86::GR32RegClass, Op0, Op0IsKill);
    1138             :   }
    1139             :   return 0;
    1140             : }
    1141             : 
    1142             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1143           0 :   if ((Subtarget->hasAVX512())) {
    1144           0 :     return fastEmitInst_r(X86::VCVTTSD2USI64Zrr, &X86::GR64RegClass, Op0, Op0IsKill);
    1145             :   }
    1146             :   return 0;
    1147             : }
    1148             : 
    1149           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1150           0 : switch (RetVT.SimpleTy) {
    1151           0 :   case MVT::i32: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i32_r(Op0, Op0IsKill);
    1152           0 :   case MVT::i64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_MVT_i64_r(Op0, Op0IsKill);
    1153             :   default: return 0;
    1154             : }
    1155             : }
    1156             : 
    1157             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1158           0 :   if ((Subtarget->hasVLX())) {
    1159           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1160             :   }
    1161             :   return 0;
    1162             : }
    1163             : 
    1164           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1165           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1166           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1167             :   }
    1168             :   return 0;
    1169             : }
    1170             : 
    1171           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1172           0 : switch (RetVT.SimpleTy) {
    1173           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    1174           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    1175             :   default: return 0;
    1176             : }
    1177             : }
    1178             : 
    1179             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1180           0 :   if ((Subtarget->hasVLX())) {
    1181           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1182             :   }
    1183             :   return 0;
    1184             : }
    1185             : 
    1186             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1187           0 :   if ((Subtarget->hasDQI())) {
    1188           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1189             :   }
    1190             :   return 0;
    1191             : }
    1192             : 
    1193           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1194           0 : switch (RetVT.SimpleTy) {
    1195           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    1196           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    1197             :   default: return 0;
    1198             : }
    1199             : }
    1200             : 
    1201             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1202           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    1203             :     return 0;
    1204           0 :   if ((Subtarget->hasAVX512())) {
    1205           0 :     return fastEmitInst_r(X86::VCVTTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1206             :   }
    1207             :   return 0;
    1208             : }
    1209             : 
    1210           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1211           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1212             :     return 0;
    1213           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1214           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1215             :   }
    1216             :   return 0;
    1217             : }
    1218             : 
    1219             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1220           0 :   if ((Subtarget->hasVLX())) {
    1221           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1222             :   }
    1223             :   return 0;
    1224             : }
    1225             : 
    1226           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1227           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1228           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1229             :   }
    1230             :   return 0;
    1231             : }
    1232             : 
    1233           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1234           0 : switch (RetVT.SimpleTy) {
    1235           0 :   case MVT::v4i32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    1236           0 :   case MVT::v4i64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    1237             :   default: return 0;
    1238             : }
    1239             : }
    1240             : 
    1241             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1242           0 :   if ((Subtarget->hasAVX512())) {
    1243           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1244             :   }
    1245             :   return 0;
    1246             : }
    1247             : 
    1248             : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1249           0 :   if ((Subtarget->hasDQI())) {
    1250           0 :     return fastEmitInst_r(X86::VCVTTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1251             :   }
    1252             :   return 0;
    1253             : }
    1254             : 
    1255           0 : unsigned fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1256           0 : switch (RetVT.SimpleTy) {
    1257           0 :   case MVT::v8i32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    1258           0 :   case MVT::v8i64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    1259             :   default: return 0;
    1260             : }
    1261             : }
    1262             : 
    1263           0 : unsigned fastEmit_ISD_FP_TO_UINT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1264           0 :   switch (VT.SimpleTy) {
    1265           0 :   case MVT::f32: return fastEmit_ISD_FP_TO_UINT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1266           0 :   case MVT::f64: return fastEmit_ISD_FP_TO_UINT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1267           0 :   case MVT::v4f32: return fastEmit_ISD_FP_TO_UINT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1268           0 :   case MVT::v8f32: return fastEmit_ISD_FP_TO_UINT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1269           0 :   case MVT::v16f32: return fastEmit_ISD_FP_TO_UINT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1270           0 :   case MVT::v2f64: return fastEmit_ISD_FP_TO_UINT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1271           0 :   case MVT::v4f64: return fastEmit_ISD_FP_TO_UINT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1272           0 :   case MVT::v8f64: return fastEmit_ISD_FP_TO_UINT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1273             :   default: return 0;
    1274             :   }
    1275             : }
    1276             : 
    1277             : // FastEmit functions for ISD::FSIN.
    1278             : 
    1279             : unsigned fastEmit_ISD_FSIN_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1280           0 :   if (RetVT.SimpleTy != MVT::f32)
    1281             :     return 0;
    1282           0 :   if ((!Subtarget->hasSSE1())) {
    1283           0 :     return fastEmitInst_r(X86::SIN_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1284             :   }
    1285             :   return 0;
    1286             : }
    1287             : 
    1288             : unsigned fastEmit_ISD_FSIN_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1289           0 :   if (RetVT.SimpleTy != MVT::f64)
    1290             :     return 0;
    1291           0 :   if ((!Subtarget->hasSSE2())) {
    1292           0 :     return fastEmitInst_r(X86::SIN_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1293             :   }
    1294             :   return 0;
    1295             : }
    1296             : 
    1297             : unsigned fastEmit_ISD_FSIN_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1298           0 :   if (RetVT.SimpleTy != MVT::f80)
    1299             :     return 0;
    1300           0 :   return fastEmitInst_r(X86::SIN_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1301             : }
    1302             : 
    1303           0 : unsigned fastEmit_ISD_FSIN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1304           0 :   switch (VT.SimpleTy) {
    1305           0 :   case MVT::f32: return fastEmit_ISD_FSIN_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1306           0 :   case MVT::f64: return fastEmit_ISD_FSIN_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1307           0 :   case MVT::f80: return fastEmit_ISD_FSIN_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1308             :   default: return 0;
    1309             :   }
    1310             : }
    1311             : 
    1312             : // FastEmit functions for ISD::FSQRT.
    1313             : 
    1314           0 : unsigned fastEmit_ISD_FSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1315           0 :   if (RetVT.SimpleTy != MVT::f32)
    1316             :     return 0;
    1317           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1318           0 :     return fastEmitInst_r(X86::SQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    1319             :   }
    1320           0 :   if ((!Subtarget->hasSSE1())) {
    1321           0 :     return fastEmitInst_r(X86::SQRT_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill);
    1322             :   }
    1323             :   return 0;
    1324             : }
    1325             : 
    1326           0 : unsigned fastEmit_ISD_FSQRT_MVT_f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1327           0 :   if (RetVT.SimpleTy != MVT::f64)
    1328             :     return 0;
    1329           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1330           0 :     return fastEmitInst_r(X86::SQRTSDr, &X86::FR64RegClass, Op0, Op0IsKill);
    1331             :   }
    1332           0 :   if ((!Subtarget->hasSSE2())) {
    1333           0 :     return fastEmitInst_r(X86::SQRT_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill);
    1334             :   }
    1335             :   return 0;
    1336             : }
    1337             : 
    1338             : unsigned fastEmit_ISD_FSQRT_MVT_f80_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1339           0 :   if (RetVT.SimpleTy != MVT::f80)
    1340             :     return 0;
    1341           0 :   return fastEmitInst_r(X86::SQRT_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill);
    1342             : }
    1343             : 
    1344           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1345           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    1346             :     return 0;
    1347           0 :   if ((Subtarget->hasVLX())) {
    1348           0 :     return fastEmitInst_r(X86::VSQRTPSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1349             :   }
    1350           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1351           0 :     return fastEmitInst_r(X86::SQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1352             :   }
    1353           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1354           0 :     return fastEmitInst_r(X86::VSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    1355             :   }
    1356             :   return 0;
    1357             : }
    1358             : 
    1359           0 : unsigned fastEmit_ISD_FSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1360           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    1361             :     return 0;
    1362           0 :   if ((Subtarget->hasVLX())) {
    1363           0 :     return fastEmitInst_r(X86::VSQRTPSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1364             :   }
    1365           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1366           0 :     return fastEmitInst_r(X86::VSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1367             :   }
    1368             :   return 0;
    1369             : }
    1370             : 
    1371             : unsigned fastEmit_ISD_FSQRT_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1372           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1373             :     return 0;
    1374           0 :   if ((Subtarget->hasAVX512())) {
    1375           0 :     return fastEmitInst_r(X86::VSQRTPSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1376             :   }
    1377             :   return 0;
    1378             : }
    1379             : 
    1380           0 : unsigned fastEmit_ISD_FSQRT_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1381           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1382             :     return 0;
    1383           0 :   if ((Subtarget->hasVLX())) {
    1384           0 :     return fastEmitInst_r(X86::VSQRTPDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    1385             :   }
    1386           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1387           0 :     return fastEmitInst_r(X86::SQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1388             :   }
    1389           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1390           0 :     return fastEmitInst_r(X86::VSQRTPDr, &X86::VR128RegClass, Op0, Op0IsKill);
    1391             :   }
    1392             :   return 0;
    1393             : }
    1394             : 
    1395           0 : unsigned fastEmit_ISD_FSQRT_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1396           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    1397             :     return 0;
    1398           0 :   if ((Subtarget->hasVLX())) {
    1399           0 :     return fastEmitInst_r(X86::VSQRTPDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    1400             :   }
    1401           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1402           0 :     return fastEmitInst_r(X86::VSQRTPDYr, &X86::VR256RegClass, Op0, Op0IsKill);
    1403             :   }
    1404             :   return 0;
    1405             : }
    1406             : 
    1407             : unsigned fastEmit_ISD_FSQRT_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1408           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    1409             :     return 0;
    1410           0 :   if ((Subtarget->hasAVX512())) {
    1411           0 :     return fastEmitInst_r(X86::VSQRTPDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    1412             :   }
    1413             :   return 0;
    1414             : }
    1415             : 
    1416           0 : unsigned fastEmit_ISD_FSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1417           0 :   switch (VT.SimpleTy) {
    1418           0 :   case MVT::f32: return fastEmit_ISD_FSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    1419           0 :   case MVT::f64: return fastEmit_ISD_FSQRT_MVT_f64_r(RetVT, Op0, Op0IsKill);
    1420           0 :   case MVT::f80: return fastEmit_ISD_FSQRT_MVT_f80_r(RetVT, Op0, Op0IsKill);
    1421           0 :   case MVT::v4f32: return fastEmit_ISD_FSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    1422           0 :   case MVT::v8f32: return fastEmit_ISD_FSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    1423           0 :   case MVT::v16f32: return fastEmit_ISD_FSQRT_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    1424           0 :   case MVT::v2f64: return fastEmit_ISD_FSQRT_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    1425           0 :   case MVT::v4f64: return fastEmit_ISD_FSQRT_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    1426           0 :   case MVT::v8f64: return fastEmit_ISD_FSQRT_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    1427             :   default: return 0;
    1428             :   }
    1429             : }
    1430             : 
    1431             : // FastEmit functions for ISD::SCALAR_TO_VECTOR.
    1432             : 
    1433           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1434           2 :   if ((Subtarget->hasAVX512())) {
    1435           0 :     return fastEmitInst_r(X86::VMOVDI2PDIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1436             :   }
    1437           1 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1438           0 :     return fastEmitInst_r(X86::MOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1439             :   }
    1440           1 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1441           1 :     return fastEmitInst_r(X86::VMOVDI2PDIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1442             :   }
    1443             :   return 0;
    1444             : }
    1445             : 
    1446             : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(unsigned Op0, bool Op0IsKill) {
    1447           0 :   if ((Subtarget->hasMMX())) {
    1448           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    1449             :   }
    1450             :   return 0;
    1451             : }
    1452             : 
    1453           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1454           1 : switch (RetVT.SimpleTy) {
    1455           1 :   case MVT::v4i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    1456           0 :   case MVT::x86mmx: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_MVT_x86mmx_r(Op0, Op0IsKill);
    1457             :   default: return 0;
    1458             : }
    1459             : }
    1460             : 
    1461           0 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1462           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1463             :     return 0;
    1464           0 :   if ((Subtarget->hasAVX512())) {
    1465           0 :     return fastEmitInst_r(X86::VMOV64toPQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1466             :   }
    1467           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1468           0 :     return fastEmitInst_r(X86::MOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1469             :   }
    1470           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    1471           0 :     return fastEmitInst_r(X86::VMOV64toPQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1472             :   }
    1473             :   return 0;
    1474             : }
    1475             : 
    1476           1 : unsigned fastEmit_ISD_SCALAR_TO_VECTOR_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1477           1 :   switch (VT.SimpleTy) {
    1478           1 :   case MVT::i32: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1479           0 :   case MVT::i64: return fastEmit_ISD_SCALAR_TO_VECTOR_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1480             :   default: return 0;
    1481             :   }
    1482             : }
    1483             : 
    1484             : // FastEmit functions for ISD::SIGN_EXTEND.
    1485             : 
    1486             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1487         420 :   return fastEmitInst_r(X86::MOVSX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    1488             : }
    1489             : 
    1490             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1491           6 :   return fastEmitInst_r(X86::MOVSX64rr8, &X86::GR64RegClass, Op0, Op0IsKill);
    1492             : }
    1493             : 
    1494         434 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1495         434 : switch (RetVT.SimpleTy) {
    1496         840 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i32_r(Op0, Op0IsKill);
    1497          12 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_MVT_i64_r(Op0, Op0IsKill);
    1498             :   default: return 0;
    1499             : }
    1500             : }
    1501             : 
    1502             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1503          68 :   return fastEmitInst_r(X86::MOVSX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    1504             : }
    1505             : 
    1506             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(unsigned Op0, bool Op0IsKill) {
    1507           4 :   return fastEmitInst_r(X86::MOVSX64rr16, &X86::GR64RegClass, Op0, Op0IsKill);
    1508             : }
    1509             : 
    1510          72 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1511          72 : switch (RetVT.SimpleTy) {
    1512         136 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i32_r(Op0, Op0IsKill);
    1513           8 :   case MVT::i64: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_MVT_i64_r(Op0, Op0IsKill);
    1514             :   default: return 0;
    1515             : }
    1516             : }
    1517             : 
    1518             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1519         849 :   if (RetVT.SimpleTy != MVT::i64)
    1520             :     return 0;
    1521         849 :   if ((Subtarget->is64Bit())) {
    1522         849 :     return fastEmitInst_r(X86::MOVSX64rr32, &X86::GR64RegClass, Op0, Op0IsKill);
    1523             :   }
    1524             :   return 0;
    1525             : }
    1526             : 
    1527           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1528           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1529             :     return 0;
    1530           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1531           0 :     return fastEmitInst_r(X86::VPMOVM2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1532             :   }
    1533             :   return 0;
    1534             : }
    1535             : 
    1536           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1537           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1538           0 :     return fastEmitInst_r(X86::VPMOVM2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1539             :   }
    1540             :   return 0;
    1541             : }
    1542             : 
    1543           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    1544           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1545           0 :     return fastEmitInst_r(X86::VPMOVM2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1546             :   }
    1547             :   return 0;
    1548             : }
    1549             : 
    1550           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1551           0 : switch (RetVT.SimpleTy) {
    1552           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i32_r(Op0, Op0IsKill);
    1553           0 :   case MVT::v4i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_MVT_v4i64_r(Op0, Op0IsKill);
    1554             :   default: return 0;
    1555             : }
    1556             : }
    1557             : 
    1558           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1559           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1560           0 :     return fastEmitInst_r(X86::VPMOVM2WZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1561             :   }
    1562             :   return 0;
    1563             : }
    1564             : 
    1565           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    1566           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1567           0 :     return fastEmitInst_r(X86::VPMOVM2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1568             :   }
    1569             :   return 0;
    1570             : }
    1571             : 
    1572             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    1573           0 :   if ((Subtarget->hasDQI())) {
    1574           0 :     return fastEmitInst_r(X86::VPMOVM2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1575             :   }
    1576             :   return 0;
    1577             : }
    1578             : 
    1579           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1580           0 : switch (RetVT.SimpleTy) {
    1581           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i16_r(Op0, Op0IsKill);
    1582           0 :   case MVT::v8i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i32_r(Op0, Op0IsKill);
    1583           0 :   case MVT::v8i64: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    1584             :   default: return 0;
    1585             : }
    1586             : }
    1587             : 
    1588           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    1589           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1590           0 :     return fastEmitInst_r(X86::VPMOVM2BZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1591             :   }
    1592             :   return 0;
    1593             : }
    1594             : 
    1595           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    1596           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1597           0 :     return fastEmitInst_r(X86::VPMOVM2WZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1598             :   }
    1599             :   return 0;
    1600             : }
    1601             : 
    1602             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    1603           0 :   if ((Subtarget->hasDQI())) {
    1604           0 :     return fastEmitInst_r(X86::VPMOVM2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1605             :   }
    1606             :   return 0;
    1607             : }
    1608             : 
    1609           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1610           0 : switch (RetVT.SimpleTy) {
    1611           0 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i8_r(Op0, Op0IsKill);
    1612           0 :   case MVT::v16i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i16_r(Op0, Op0IsKill);
    1613           0 :   case MVT::v16i32: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    1614             :   default: return 0;
    1615             : }
    1616             : }
    1617             : 
    1618           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    1619           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1620           0 :     return fastEmitInst_r(X86::VPMOVM2BZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1621             :   }
    1622             :   return 0;
    1623             : }
    1624             : 
    1625             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    1626           0 :   if ((Subtarget->hasBWI())) {
    1627           0 :     return fastEmitInst_r(X86::VPMOVM2WZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1628             :   }
    1629             :   return 0;
    1630             : }
    1631             : 
    1632           0 : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1633           0 : switch (RetVT.SimpleTy) {
    1634           0 :   case MVT::v32i8: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i8_r(Op0, Op0IsKill);
    1635           0 :   case MVT::v32i16: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_MVT_v32i16_r(Op0, Op0IsKill);
    1636             :   default: return 0;
    1637             : }
    1638             : }
    1639             : 
    1640             : unsigned fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1641           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    1642             :     return 0;
    1643           0 :   if ((Subtarget->hasBWI())) {
    1644           0 :     return fastEmitInst_r(X86::VPMOVM2BZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1645             :   }
    1646             :   return 0;
    1647             : }
    1648             : 
    1649        1361 : unsigned fastEmit_ISD_SIGN_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1650        1361 :   switch (VT.SimpleTy) {
    1651         434 :   case MVT::i8: return fastEmit_ISD_SIGN_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    1652          72 :   case MVT::i16: return fastEmit_ISD_SIGN_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    1653         849 :   case MVT::i32: return fastEmit_ISD_SIGN_EXTEND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1654           0 :   case MVT::v2i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v2i1_r(RetVT, Op0, Op0IsKill);
    1655           0 :   case MVT::v4i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v4i1_r(RetVT, Op0, Op0IsKill);
    1656           0 :   case MVT::v8i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    1657           0 :   case MVT::v16i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    1658           0 :   case MVT::v32i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v32i1_r(RetVT, Op0, Op0IsKill);
    1659           0 :   case MVT::v64i1: return fastEmit_ISD_SIGN_EXTEND_MVT_v64i1_r(RetVT, Op0, Op0IsKill);
    1660             :   default: return 0;
    1661             :   }
    1662             : }
    1663             : 
    1664             : // FastEmit functions for ISD::SIGN_EXTEND_VECTOR_INREG.
    1665             : 
    1666           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    1667           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1668           0 :     return fastEmitInst_r(X86::PMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1669             :   }
    1670           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    1671           0 :     return fastEmitInst_r(X86::VPMOVSXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1672             :   }
    1673           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    1674           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1675             :   }
    1676             :   return 0;
    1677             : }
    1678             : 
    1679           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1680           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1681           0 :     return fastEmitInst_r(X86::PMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1682             :   }
    1683           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1684           0 :     return fastEmitInst_r(X86::VPMOVSXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1685             :   }
    1686           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1687           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1688             :   }
    1689             :   return 0;
    1690             : }
    1691             : 
    1692           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1693           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1694           0 :     return fastEmitInst_r(X86::PMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1695             :   }
    1696           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1697           0 :     return fastEmitInst_r(X86::VPMOVSXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1698             :   }
    1699           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1700           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1701             :   }
    1702             :   return 0;
    1703             : }
    1704             : 
    1705           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1706           0 : switch (RetVT.SimpleTy) {
    1707           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    1708           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    1709           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    1710             :   default: return 0;
    1711             : }
    1712             : }
    1713             : 
    1714           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    1715           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1716           0 :     return fastEmitInst_r(X86::PMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1717             :   }
    1718           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1719           0 :     return fastEmitInst_r(X86::VPMOVSXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1720             :   }
    1721           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1722           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1723             :   }
    1724             :   return 0;
    1725             : }
    1726             : 
    1727           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    1728           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1729           0 :     return fastEmitInst_r(X86::PMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1730             :   }
    1731           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1732           0 :     return fastEmitInst_r(X86::VPMOVSXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1733             :   }
    1734           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1735           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1736             :   }
    1737             :   return 0;
    1738             : }
    1739             : 
    1740           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1741           0 : switch (RetVT.SimpleTy) {
    1742           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    1743           0 :   case MVT::v2i64: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    1744             :   default: return 0;
    1745             : }
    1746             : }
    1747             : 
    1748           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1749           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    1750             :     return 0;
    1751           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    1752           0 :     return fastEmitInst_r(X86::PMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1753             :   }
    1754           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1755           0 :     return fastEmitInst_r(X86::VPMOVSXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1756             :   }
    1757           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    1758           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1759             :   }
    1760             :   return 0;
    1761             : }
    1762             : 
    1763           0 : unsigned fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1764           0 :   switch (VT.SimpleTy) {
    1765           0 :   case MVT::v16i8: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    1766           0 :   case MVT::v8i16: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    1767           0 :   case MVT::v4i32: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1768             :   default: return 0;
    1769             :   }
    1770             : }
    1771             : 
    1772             : // FastEmit functions for ISD::SINT_TO_FP.
    1773             : 
    1774          14 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1775          28 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1776          11 :     return fastEmitInst_r(X86::CVTSI2SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1777             :   }
    1778             :   return 0;
    1779             : }
    1780             : 
    1781          16 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1782          32 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1783          11 :     return fastEmitInst_r(X86::CVTSI2SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1784             :   }
    1785             :   return 0;
    1786             : }
    1787             : 
    1788          30 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1789          30 : switch (RetVT.SimpleTy) {
    1790          14 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f32_r(Op0, Op0IsKill);
    1791          16 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i32_MVT_f64_r(Op0, Op0IsKill);
    1792             :   default: return 0;
    1793             : }
    1794             : }
    1795             : 
    1796           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(unsigned Op0, bool Op0IsKill) {
    1797          12 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    1798           3 :     return fastEmitInst_r(X86::CVTSI642SSrr, &X86::FR32RegClass, Op0, Op0IsKill);
    1799             :   }
    1800             :   return 0;
    1801             : }
    1802             : 
    1803           6 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(unsigned Op0, bool Op0IsKill) {
    1804          12 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1805           3 :     return fastEmitInst_r(X86::CVTSI642SDrr, &X86::FR64RegClass, Op0, Op0IsKill);
    1806             :   }
    1807             :   return 0;
    1808             : }
    1809             : 
    1810          12 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1811          12 : switch (RetVT.SimpleTy) {
    1812           6 :   case MVT::f32: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f32_r(Op0, Op0IsKill);
    1813           6 :   case MVT::f64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_MVT_f64_r(Op0, Op0IsKill);
    1814             :   default: return 0;
    1815             : }
    1816             : }
    1817             : 
    1818           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1819           0 :   if ((Subtarget->hasVLX())) {
    1820           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1821             :   }
    1822           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    1823           0 :     return fastEmitInst_r(X86::CVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1824             :   }
    1825           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1826           0 :     return fastEmitInst_r(X86::VCVTDQ2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    1827             :   }
    1828             :   return 0;
    1829             : }
    1830             : 
    1831           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1832           2 :   if ((Subtarget->hasVLX())) {
    1833           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1834             :   }
    1835           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1836           2 :     return fastEmitInst_r(X86::VCVTDQ2PDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1837             :   }
    1838             :   return 0;
    1839             : }
    1840             : 
    1841           2 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1842           2 : switch (RetVT.SimpleTy) {
    1843           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    1844           2 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    1845             :   default: return 0;
    1846             : }
    1847             : }
    1848             : 
    1849           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1850           0 :   if ((Subtarget->hasVLX())) {
    1851           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1852             :   }
    1853           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    1854           0 :     return fastEmitInst_r(X86::VCVTDQ2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    1855             :   }
    1856             :   return 0;
    1857             : }
    1858             : 
    1859             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1860           0 :   if ((Subtarget->hasAVX512())) {
    1861           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1862             :   }
    1863             :   return 0;
    1864             : }
    1865             : 
    1866           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1867           0 : switch (RetVT.SimpleTy) {
    1868           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    1869           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    1870             :   default: return 0;
    1871             : }
    1872             : }
    1873             : 
    1874             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1875           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    1876             :     return 0;
    1877           0 :   if ((Subtarget->hasAVX512())) {
    1878           0 :     return fastEmitInst_r(X86::VCVTDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1879             :   }
    1880             :   return 0;
    1881             : }
    1882             : 
    1883           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1884           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    1885             :     return 0;
    1886           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1887           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1888             :   }
    1889             :   return 0;
    1890             : }
    1891             : 
    1892           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    1893           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1894           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    1895             :   }
    1896             :   return 0;
    1897             : }
    1898             : 
    1899           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    1900           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    1901           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1902             :   }
    1903             :   return 0;
    1904             : }
    1905             : 
    1906           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1907           0 : switch (RetVT.SimpleTy) {
    1908           0 :   case MVT::v4f32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    1909           0 :   case MVT::v4f64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    1910             :   default: return 0;
    1911             : }
    1912             : }
    1913             : 
    1914             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    1915           0 :   if ((Subtarget->hasDQI())) {
    1916           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    1917             :   }
    1918             :   return 0;
    1919             : }
    1920             : 
    1921             : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    1922           0 :   if ((Subtarget->hasDQI())) {
    1923           0 :     return fastEmitInst_r(X86::VCVTQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    1924             :   }
    1925             :   return 0;
    1926             : }
    1927             : 
    1928           0 : unsigned fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1929           0 : switch (RetVT.SimpleTy) {
    1930           0 :   case MVT::v8f32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    1931           0 :   case MVT::v8f64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    1932             :   default: return 0;
    1933             : }
    1934             : }
    1935             : 
    1936          44 : unsigned fastEmit_ISD_SINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1937          44 :   switch (VT.SimpleTy) {
    1938          30 :   case MVT::i32: return fastEmit_ISD_SINT_TO_FP_MVT_i32_r(RetVT, Op0, Op0IsKill);
    1939          12 :   case MVT::i64: return fastEmit_ISD_SINT_TO_FP_MVT_i64_r(RetVT, Op0, Op0IsKill);
    1940           2 :   case MVT::v4i32: return fastEmit_ISD_SINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    1941           0 :   case MVT::v8i32: return fastEmit_ISD_SINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    1942           0 :   case MVT::v16i32: return fastEmit_ISD_SINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    1943           0 :   case MVT::v2i64: return fastEmit_ISD_SINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    1944           0 :   case MVT::v4i64: return fastEmit_ISD_SINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    1945           0 :   case MVT::v8i64: return fastEmit_ISD_SINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    1946             :   default: return 0;
    1947             :   }
    1948             : }
    1949             : 
    1950             : // FastEmit functions for ISD::TRUNCATE.
    1951             : 
    1952             : unsigned fastEmit_ISD_TRUNCATE_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1953           2 :   if (RetVT.SimpleTy != MVT::i8)
    1954             :     return 0;
    1955           2 :   if ((Subtarget->is64Bit())) {
    1956           1 :     return fastEmitInst_extractsubreg(RetVT, Op0, Op0IsKill, X86::sub_8bit);
    1957             :   }
    1958             :   return 0;
    1959             : }
    1960             : 
    1961             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1962          60 :   if ((Subtarget->is64Bit())) {
    1963         112 :     return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1964             :   }
    1965             :   return 0;
    1966             : }
    1967             : 
    1968             : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1969          62 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1970             : }
    1971             : 
    1972          91 : unsigned fastEmit_ISD_TRUNCATE_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1973          91 : switch (RetVT.SimpleTy) {
    1974          60 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i8_r(Op0, Op0IsKill);
    1975          62 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i32_MVT_i16_r(Op0, Op0IsKill);
    1976             :   default: return 0;
    1977             : }
    1978             : }
    1979             : 
    1980             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(unsigned Op0, bool Op0IsKill) {
    1981          18 :   return fastEmitInst_extractsubreg(MVT::i8, Op0, Op0IsKill, X86::sub_8bit);
    1982             : }
    1983             : 
    1984             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(unsigned Op0, bool Op0IsKill) {
    1985           4 :   return fastEmitInst_extractsubreg(MVT::i16, Op0, Op0IsKill, X86::sub_16bit);
    1986             : }
    1987             : 
    1988             : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(unsigned Op0, bool Op0IsKill) {
    1989         272 :   return fastEmitInst_extractsubreg(MVT::i32, Op0, Op0IsKill, X86::sub_32bit);
    1990             : }
    1991             : 
    1992         147 : unsigned fastEmit_ISD_TRUNCATE_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    1993         147 : switch (RetVT.SimpleTy) {
    1994          18 :   case MVT::i8: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i8_r(Op0, Op0IsKill);
    1995           4 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i16_r(Op0, Op0IsKill);
    1996         272 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i64_MVT_i32_r(Op0, Op0IsKill);
    1997             :   default: return 0;
    1998             : }
    1999             : }
    2000             : 
    2001           2 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2002           2 :   if (RetVT.SimpleTy != MVT::v16i8)
    2003             :     return 0;
    2004           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2005           2 :     return fastEmitInst_r(X86::VPMOVWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2006             :   }
    2007             :   return 0;
    2008             : }
    2009             : 
    2010             : unsigned fastEmit_ISD_TRUNCATE_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2011           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    2012             :     return 0;
    2013           2 :   if ((Subtarget->hasBWI())) {
    2014           2 :     return fastEmitInst_r(X86::VPMOVWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2015             :   }
    2016             :   return 0;
    2017             : }
    2018             : 
    2019           2 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2020           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    2021             :     return 0;
    2022           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2023           2 :     return fastEmitInst_r(X86::VPMOVDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2024             :   }
    2025             :   return 0;
    2026             : }
    2027             : 
    2028             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    2029           4 :   if ((Subtarget->hasAVX512())) {
    2030           2 :     return fastEmitInst_r(X86::VPMOVDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2031             :   }
    2032             :   return 0;
    2033             : }
    2034             : 
    2035             : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    2036           0 :   if ((Subtarget->hasAVX512())) {
    2037           0 :     return fastEmitInst_r(X86::VPMOVDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2038             :   }
    2039             :   return 0;
    2040             : }
    2041             : 
    2042           2 : unsigned fastEmit_ISD_TRUNCATE_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2043           2 : switch (RetVT.SimpleTy) {
    2044           2 :   case MVT::v16i8: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    2045           0 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    2046             :   default: return 0;
    2047             : }
    2048             : }
    2049             : 
    2050           4 : unsigned fastEmit_ISD_TRUNCATE_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2051           4 :   if (RetVT.SimpleTy != MVT::v4i32)
    2052             :     return 0;
    2053           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2054           2 :     return fastEmitInst_r(X86::VPMOVQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2055             :   }
    2056             :   return 0;
    2057             : }
    2058             : 
    2059             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2060           4 :   if ((Subtarget->hasAVX512())) {
    2061           2 :     return fastEmitInst_r(X86::VPMOVQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2062             :   }
    2063             :   return 0;
    2064             : }
    2065             : 
    2066             : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2067           4 :   if ((Subtarget->hasAVX512())) {
    2068           2 :     return fastEmitInst_r(X86::VPMOVQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2069             :   }
    2070             :   return 0;
    2071             : }
    2072             : 
    2073           8 : unsigned fastEmit_ISD_TRUNCATE_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2074           8 : switch (RetVT.SimpleTy) {
    2075           2 :   case MVT::v8i16: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    2076           2 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    2077             :   default: return 0;
    2078             : }
    2079             : }
    2080             : 
    2081         263 : unsigned fastEmit_ISD_TRUNCATE_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2082         263 :   switch (VT.SimpleTy) {
    2083           2 :   case MVT::i16: return fastEmit_ISD_TRUNCATE_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2084          91 :   case MVT::i32: return fastEmit_ISD_TRUNCATE_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2085         147 :   case MVT::i64: return fastEmit_ISD_TRUNCATE_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2086           2 :   case MVT::v16i16: return fastEmit_ISD_TRUNCATE_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2087           2 :   case MVT::v32i16: return fastEmit_ISD_TRUNCATE_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2088           2 :   case MVT::v8i32: return fastEmit_ISD_TRUNCATE_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2089           2 :   case MVT::v16i32: return fastEmit_ISD_TRUNCATE_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2090           4 :   case MVT::v4i64: return fastEmit_ISD_TRUNCATE_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2091           8 :   case MVT::v8i64: return fastEmit_ISD_TRUNCATE_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2092             :   default: return 0;
    2093             :   }
    2094             : }
    2095             : 
    2096             : // FastEmit functions for ISD::UINT_TO_FP.
    2097             : 
    2098             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2099           0 :   if ((Subtarget->hasVLX())) {
    2100           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2101             :   }
    2102             :   return 0;
    2103             : }
    2104             : 
    2105             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2106           0 :   if ((Subtarget->hasVLX())) {
    2107           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2108             :   }
    2109             :   return 0;
    2110             : }
    2111             : 
    2112           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2113           0 : switch (RetVT.SimpleTy) {
    2114           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f32_r(Op0, Op0IsKill);
    2115           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_MVT_v4f64_r(Op0, Op0IsKill);
    2116             :   default: return 0;
    2117             : }
    2118             : }
    2119             : 
    2120             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2121           0 :   if ((Subtarget->hasVLX())) {
    2122           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2123             :   }
    2124             :   return 0;
    2125             : }
    2126             : 
    2127             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2128           0 :   if ((Subtarget->hasAVX512())) {
    2129           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2130             :   }
    2131             :   return 0;
    2132             : }
    2133             : 
    2134           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2135           0 : switch (RetVT.SimpleTy) {
    2136           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f32_r(Op0, Op0IsKill);
    2137           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_MVT_v8f64_r(Op0, Op0IsKill);
    2138             :   default: return 0;
    2139             : }
    2140             : }
    2141             : 
    2142             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2143           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2144             :     return 0;
    2145           0 :   if ((Subtarget->hasAVX512())) {
    2146           0 :     return fastEmitInst_r(X86::VCVTUDQ2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2147             :   }
    2148             :   return 0;
    2149             : }
    2150             : 
    2151           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2152           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2153             :     return 0;
    2154           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2155           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2156             :   }
    2157             :   return 0;
    2158             : }
    2159             : 
    2160           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2161           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2162           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2163             :   }
    2164             :   return 0;
    2165             : }
    2166             : 
    2167           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    2168           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2169           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2170             :   }
    2171             :   return 0;
    2172             : }
    2173             : 
    2174           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2175           0 : switch (RetVT.SimpleTy) {
    2176           0 :   case MVT::v4f32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f32_r(Op0, Op0IsKill);
    2177           0 :   case MVT::v4f64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_MVT_v4f64_r(Op0, Op0IsKill);
    2178             :   default: return 0;
    2179             : }
    2180             : }
    2181             : 
    2182             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2183           0 :   if ((Subtarget->hasDQI())) {
    2184           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2185             :   }
    2186             :   return 0;
    2187             : }
    2188             : 
    2189             : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    2190           0 :   if ((Subtarget->hasDQI())) {
    2191           0 :     return fastEmitInst_r(X86::VCVTUQQ2PDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2192             :   }
    2193             :   return 0;
    2194             : }
    2195             : 
    2196           0 : unsigned fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2197           0 : switch (RetVT.SimpleTy) {
    2198           0 :   case MVT::v8f32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f32_r(Op0, Op0IsKill);
    2199           0 :   case MVT::v8f64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_MVT_v8f64_r(Op0, Op0IsKill);
    2200             :   default: return 0;
    2201             : }
    2202             : }
    2203             : 
    2204           0 : unsigned fastEmit_ISD_UINT_TO_FP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2205           0 :   switch (VT.SimpleTy) {
    2206           0 :   case MVT::v4i32: return fastEmit_ISD_UINT_TO_FP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2207           0 :   case MVT::v8i32: return fastEmit_ISD_UINT_TO_FP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2208           0 :   case MVT::v16i32: return fastEmit_ISD_UINT_TO_FP_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2209           0 :   case MVT::v2i64: return fastEmit_ISD_UINT_TO_FP_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2210           0 :   case MVT::v4i64: return fastEmit_ISD_UINT_TO_FP_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2211           0 :   case MVT::v8i64: return fastEmit_ISD_UINT_TO_FP_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2212             :   default: return 0;
    2213             :   }
    2214             : }
    2215             : 
    2216             : // FastEmit functions for ISD::ZERO_EXTEND.
    2217             : 
    2218             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2219        1489 :   if (RetVT.SimpleTy != MVT::i32)
    2220             :     return 0;
    2221        1475 :   return fastEmitInst_r(X86::MOVZX32rr8, &X86::GR32RegClass, Op0, Op0IsKill);
    2222             : }
    2223             : 
    2224             : unsigned fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2225          76 :   if (RetVT.SimpleTy != MVT::i32)
    2226             :     return 0;
    2227          74 :   return fastEmitInst_r(X86::MOVZX32rr16, &X86::GR32RegClass, Op0, Op0IsKill);
    2228             : }
    2229             : 
    2230        1623 : unsigned fastEmit_ISD_ZERO_EXTEND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2231        1623 :   switch (VT.SimpleTy) {
    2232        1489 :   case MVT::i8: return fastEmit_ISD_ZERO_EXTEND_MVT_i8_r(RetVT, Op0, Op0IsKill);
    2233          76 :   case MVT::i16: return fastEmit_ISD_ZERO_EXTEND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2234             :   default: return 0;
    2235             :   }
    2236             : }
    2237             : 
    2238             : // FastEmit functions for ISD::ZERO_EXTEND_VECTOR_INREG.
    2239             : 
    2240           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    2241           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2242           0 :     return fastEmitInst_r(X86::PMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2243             :   }
    2244           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    2245           0 :     return fastEmitInst_r(X86::VPMOVZXBWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2246             :   }
    2247           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    2248           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2249             :   }
    2250             :   return 0;
    2251             : }
    2252             : 
    2253           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2254           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2255           0 :     return fastEmitInst_r(X86::PMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2256             :   }
    2257           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2258           0 :     return fastEmitInst_r(X86::VPMOVZXBDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2259             :   }
    2260           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2261           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2262             :   }
    2263             :   return 0;
    2264             : }
    2265             : 
    2266           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2267           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2268           0 :     return fastEmitInst_r(X86::PMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2269             :   }
    2270           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2271           0 :     return fastEmitInst_r(X86::VPMOVZXBQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2272             :   }
    2273           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2274           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2275             :   }
    2276             :   return 0;
    2277             : }
    2278             : 
    2279           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2280           0 : switch (RetVT.SimpleTy) {
    2281           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v8i16_r(Op0, Op0IsKill);
    2282           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v4i32_r(Op0, Op0IsKill);
    2283           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_MVT_v2i64_r(Op0, Op0IsKill);
    2284             :   default: return 0;
    2285             : }
    2286             : }
    2287             : 
    2288           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2289           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2290           0 :     return fastEmitInst_r(X86::PMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2291             :   }
    2292           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2293           0 :     return fastEmitInst_r(X86::VPMOVZXWDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2294             :   }
    2295           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2296           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2297             :   }
    2298             :   return 0;
    2299             : }
    2300             : 
    2301           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2302           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2303           0 :     return fastEmitInst_r(X86::PMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2304             :   }
    2305           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2306           0 :     return fastEmitInst_r(X86::VPMOVZXWQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2307             :   }
    2308           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2309           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2310             :   }
    2311             :   return 0;
    2312             : }
    2313             : 
    2314           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2315           0 : switch (RetVT.SimpleTy) {
    2316           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v4i32_r(Op0, Op0IsKill);
    2317           0 :   case MVT::v2i64: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_MVT_v2i64_r(Op0, Op0IsKill);
    2318             :   default: return 0;
    2319             : }
    2320             : }
    2321             : 
    2322           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2323           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2324             :     return 0;
    2325           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    2326           0 :     return fastEmitInst_r(X86::PMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2327             :   }
    2328           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2329           0 :     return fastEmitInst_r(X86::VPMOVZXDQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2330             :   }
    2331           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2332           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2333             :   }
    2334             :   return 0;
    2335             : }
    2336             : 
    2337           0 : unsigned fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2338           0 :   switch (VT.SimpleTy) {
    2339           0 :   case MVT::v16i8: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2340           0 :   case MVT::v8i16: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2341           0 :   case MVT::v4i32: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2342             :   default: return 0;
    2343             :   }
    2344             : }
    2345             : 
    2346             : // FastEmit functions for X86ISD::CALL.
    2347             : 
    2348             : unsigned fastEmit_X86ISD_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2349           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2350             :     return 0;
    2351           0 :   if ((!Subtarget->is64Bit())) {
    2352           0 :     return fastEmitInst_r(X86::CALL16r, &X86::GR16RegClass, Op0, Op0IsKill);
    2353             :   }
    2354             :   return 0;
    2355             : }
    2356             : 
    2357           0 : unsigned fastEmit_X86ISD_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2358           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2359             :     return 0;
    2360           0 :   if ((!Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
    2361           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL32, &X86::GR32RegClass, Op0, Op0IsKill);
    2362             :   }
    2363           0 :   if ((!Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
    2364           0 :     return fastEmitInst_r(X86::CALL32r, &X86::GR32RegClass, Op0, Op0IsKill);
    2365             :   }
    2366             :   return 0;
    2367             : }
    2368             : 
    2369           0 : unsigned fastEmit_X86ISD_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2370           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    2371             :     return 0;
    2372           0 :   if ((Subtarget->is64Bit()) && (Subtarget->useRetpoline())) {
    2373           0 :     return fastEmitInst_r(X86::RETPOLINE_CALL64, &X86::GR64RegClass, Op0, Op0IsKill);
    2374             :   }
    2375           0 :   if ((Subtarget->is64Bit()) && (!Subtarget->useRetpoline())) {
    2376           0 :     return fastEmitInst_r(X86::CALL64r, &X86::GR64RegClass, Op0, Op0IsKill);
    2377             :   }
    2378             :   return 0;
    2379             : }
    2380             : 
    2381           0 : unsigned fastEmit_X86ISD_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2382           0 :   switch (VT.SimpleTy) {
    2383           0 :   case MVT::i16: return fastEmit_X86ISD_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    2384           0 :   case MVT::i32: return fastEmit_X86ISD_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    2385           0 :   case MVT::i64: return fastEmit_X86ISD_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    2386             :   default: return 0;
    2387             :   }
    2388             : }
    2389             : 
    2390             : // FastEmit functions for X86ISD::COMPRESS.
    2391             : 
    2392           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2393           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    2394             :     return 0;
    2395           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2396           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2397             :   }
    2398             :   return 0;
    2399             : }
    2400             : 
    2401           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2402           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    2403             :     return 0;
    2404           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2405           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2406             :   }
    2407             :   return 0;
    2408             : }
    2409             : 
    2410             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2411           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    2412             :     return 0;
    2413           0 :   if ((Subtarget->hasVBMI2())) {
    2414           0 :     return fastEmitInst_r(X86::VPCOMPRESSBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2415             :   }
    2416             :   return 0;
    2417             : }
    2418             : 
    2419           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2420           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    2421             :     return 0;
    2422           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2423           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2424             :   }
    2425             :   return 0;
    2426             : }
    2427             : 
    2428           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2429           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    2430             :     return 0;
    2431           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    2432           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2433             :   }
    2434             :   return 0;
    2435             : }
    2436             : 
    2437             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2438           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    2439             :     return 0;
    2440           0 :   if ((Subtarget->hasVBMI2())) {
    2441           0 :     return fastEmitInst_r(X86::VPCOMPRESSWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2442             :   }
    2443             :   return 0;
    2444             : }
    2445             : 
    2446           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2447           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2448             :     return 0;
    2449           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2450           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2451             :   }
    2452             :   return 0;
    2453             : }
    2454             : 
    2455           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2456           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2457             :     return 0;
    2458           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2459           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2460             :   }
    2461             :   return 0;
    2462             : }
    2463             : 
    2464             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2465           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2466             :     return 0;
    2467           0 :   if ((Subtarget->hasAVX512())) {
    2468           0 :     return fastEmitInst_r(X86::VPCOMPRESSDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2469             :   }
    2470             :   return 0;
    2471             : }
    2472             : 
    2473           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2474           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2475             :     return 0;
    2476           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2477           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2478             :   }
    2479             :   return 0;
    2480             : }
    2481             : 
    2482           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2483           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2484             :     return 0;
    2485           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2486           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2487             :   }
    2488             :   return 0;
    2489             : }
    2490             : 
    2491             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2492           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2493             :     return 0;
    2494           0 :   if ((Subtarget->hasAVX512())) {
    2495           0 :     return fastEmitInst_r(X86::VPCOMPRESSQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2496             :   }
    2497             :   return 0;
    2498             : }
    2499             : 
    2500           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2501           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    2502             :     return 0;
    2503           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2504           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2505             :   }
    2506             :   return 0;
    2507             : }
    2508             : 
    2509           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2510           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    2511             :     return 0;
    2512           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2513           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2514             :   }
    2515             :   return 0;
    2516             : }
    2517             : 
    2518             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2519           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2520             :     return 0;
    2521           0 :   if ((Subtarget->hasAVX512())) {
    2522           0 :     return fastEmitInst_r(X86::VCOMPRESSPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2523             :   }
    2524             :   return 0;
    2525             : }
    2526             : 
    2527           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2528           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2529             :     return 0;
    2530           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2531           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2532             :   }
    2533             :   return 0;
    2534             : }
    2535             : 
    2536           0 : unsigned fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2537           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    2538             :     return 0;
    2539           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    2540           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2541             :   }
    2542             :   return 0;
    2543             : }
    2544             : 
    2545             : unsigned fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2546           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    2547             :     return 0;
    2548           0 :   if ((Subtarget->hasAVX512())) {
    2549           0 :     return fastEmitInst_r(X86::VCOMPRESSPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2550             :   }
    2551             :   return 0;
    2552             : }
    2553             : 
    2554           0 : unsigned fastEmit_X86ISD_COMPRESS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2555           0 :   switch (VT.SimpleTy) {
    2556           0 :   case MVT::v16i8: return fastEmit_X86ISD_COMPRESS_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    2557           0 :   case MVT::v32i8: return fastEmit_X86ISD_COMPRESS_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    2558           0 :   case MVT::v64i8: return fastEmit_X86ISD_COMPRESS_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    2559           0 :   case MVT::v8i16: return fastEmit_X86ISD_COMPRESS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2560           0 :   case MVT::v16i16: return fastEmit_X86ISD_COMPRESS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2561           0 :   case MVT::v32i16: return fastEmit_X86ISD_COMPRESS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    2562           0 :   case MVT::v4i32: return fastEmit_X86ISD_COMPRESS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2563           0 :   case MVT::v8i32: return fastEmit_X86ISD_COMPRESS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2564           0 :   case MVT::v16i32: return fastEmit_X86ISD_COMPRESS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2565           0 :   case MVT::v2i64: return fastEmit_X86ISD_COMPRESS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2566           0 :   case MVT::v4i64: return fastEmit_X86ISD_COMPRESS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2567           0 :   case MVT::v8i64: return fastEmit_X86ISD_COMPRESS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2568           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMPRESS_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2569           0 :   case MVT::v8f32: return fastEmit_X86ISD_COMPRESS_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2570           0 :   case MVT::v16f32: return fastEmit_X86ISD_COMPRESS_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2571           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMPRESS_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2572           0 :   case MVT::v4f64: return fastEmit_X86ISD_COMPRESS_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2573           0 :   case MVT::v8f64: return fastEmit_X86ISD_COMPRESS_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2574             :   default: return 0;
    2575             :   }
    2576             : }
    2577             : 
    2578             : // FastEmit functions for X86ISD::CONFLICT.
    2579             : 
    2580           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2581           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    2582             :     return 0;
    2583           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2584           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2585             :   }
    2586             :   return 0;
    2587             : }
    2588             : 
    2589           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2590           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    2591             :     return 0;
    2592           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2593           0 :     return fastEmitInst_r(X86::VPCONFLICTDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2594             :   }
    2595             :   return 0;
    2596             : }
    2597             : 
    2598             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2599           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2600             :     return 0;
    2601           0 :   if ((Subtarget->hasCDI())) {
    2602           0 :     return fastEmitInst_r(X86::VPCONFLICTDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2603             :   }
    2604             :   return 0;
    2605             : }
    2606             : 
    2607           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2608           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    2609             :     return 0;
    2610           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2611           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2612             :   }
    2613             :   return 0;
    2614             : }
    2615             : 
    2616           0 : unsigned fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2617           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    2618             :     return 0;
    2619           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    2620           0 :     return fastEmitInst_r(X86::VPCONFLICTQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2621             :   }
    2622             :   return 0;
    2623             : }
    2624             : 
    2625             : unsigned fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2626           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    2627             :     return 0;
    2628           0 :   if ((Subtarget->hasCDI())) {
    2629           0 :     return fastEmitInst_r(X86::VPCONFLICTQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2630             :   }
    2631             :   return 0;
    2632             : }
    2633             : 
    2634           0 : unsigned fastEmit_X86ISD_CONFLICT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2635           0 :   switch (VT.SimpleTy) {
    2636           0 :   case MVT::v4i32: return fastEmit_X86ISD_CONFLICT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    2637           0 :   case MVT::v8i32: return fastEmit_X86ISD_CONFLICT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    2638           0 :   case MVT::v16i32: return fastEmit_X86ISD_CONFLICT_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    2639           0 :   case MVT::v2i64: return fastEmit_X86ISD_CONFLICT_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    2640           0 :   case MVT::v4i64: return fastEmit_X86ISD_CONFLICT_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    2641           0 :   case MVT::v8i64: return fastEmit_X86ISD_CONFLICT_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    2642             :   default: return 0;
    2643             :   }
    2644             : }
    2645             : 
    2646             : // FastEmit functions for X86ISD::CVTP2SI.
    2647             : 
    2648           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2649           0 :   if ((Subtarget->hasVLX())) {
    2650           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2651             :   }
    2652           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2653           0 :     return fastEmitInst_r(X86::CVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2654             :   }
    2655           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2656           0 :     return fastEmitInst_r(X86::VCVTPS2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2657             :   }
    2658             :   return 0;
    2659             : }
    2660             : 
    2661           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2662           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2663           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2664             :   }
    2665             :   return 0;
    2666             : }
    2667             : 
    2668           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2669           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2670           0 :     return fastEmitInst_r(X86::VCVTPS2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2671             :   }
    2672             :   return 0;
    2673             : }
    2674             : 
    2675           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2676           0 : switch (RetVT.SimpleTy) {
    2677           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2678           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2679           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2680             :   default: return 0;
    2681             : }
    2682             : }
    2683             : 
    2684           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2685           0 :   if ((Subtarget->hasVLX())) {
    2686           0 :     return fastEmitInst_r(X86::VCVTPS2DQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2687             :   }
    2688           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2689           0 :     return fastEmitInst_r(X86::VCVTPS2DQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    2690             :   }
    2691             :   return 0;
    2692             : }
    2693             : 
    2694             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2695           0 :   if ((Subtarget->hasDQI())) {
    2696           0 :     return fastEmitInst_r(X86::VCVTPS2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2697             :   }
    2698             :   return 0;
    2699             : }
    2700             : 
    2701           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2702           0 : switch (RetVT.SimpleTy) {
    2703           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2704           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2705             :   default: return 0;
    2706             : }
    2707             : }
    2708             : 
    2709             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2710           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2711             :     return 0;
    2712           0 :   if ((Subtarget->hasAVX512())) {
    2713           0 :     return fastEmitInst_r(X86::VCVTPS2DQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2714             :   }
    2715             :   return 0;
    2716             : }
    2717             : 
    2718           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2719           0 :   if ((Subtarget->hasVLX())) {
    2720           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2721             :   }
    2722           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    2723           0 :     return fastEmitInst_r(X86::CVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2724             :   }
    2725           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2726           0 :     return fastEmitInst_r(X86::VCVTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2727             :   }
    2728             :   return 0;
    2729             : }
    2730             : 
    2731           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2732           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2733           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2734             :   }
    2735             :   return 0;
    2736             : }
    2737             : 
    2738           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2739           0 : switch (RetVT.SimpleTy) {
    2740           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2741           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2742             :   default: return 0;
    2743             : }
    2744             : }
    2745             : 
    2746           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2747           0 :   if ((Subtarget->hasVLX())) {
    2748           0 :     return fastEmitInst_r(X86::VCVTPD2DQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2749             :   }
    2750           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    2751           0 :     return fastEmitInst_r(X86::VCVTPD2DQYrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2752             :   }
    2753             :   return 0;
    2754             : }
    2755             : 
    2756           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2757           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2758           0 :     return fastEmitInst_r(X86::VCVTPD2QQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2759             :   }
    2760             :   return 0;
    2761             : }
    2762             : 
    2763           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2764           0 : switch (RetVT.SimpleTy) {
    2765           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2766           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2767             :   default: return 0;
    2768             : }
    2769             : }
    2770             : 
    2771             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2772           0 :   if ((Subtarget->hasAVX512())) {
    2773           0 :     return fastEmitInst_r(X86::VCVTPD2DQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2774             :   }
    2775             :   return 0;
    2776             : }
    2777             : 
    2778             : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2779           0 :   if ((Subtarget->hasDQI())) {
    2780           0 :     return fastEmitInst_r(X86::VCVTPD2QQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2781             :   }
    2782             :   return 0;
    2783             : }
    2784             : 
    2785           0 : unsigned fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2786           0 : switch (RetVT.SimpleTy) {
    2787           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2788           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2789             :   default: return 0;
    2790             : }
    2791             : }
    2792             : 
    2793           0 : unsigned fastEmit_X86ISD_CVTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2794           0 :   switch (VT.SimpleTy) {
    2795           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2796           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2SI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2797           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2SI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2798           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2799           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2SI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2800           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2SI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2801             :   default: return 0;
    2802             :   }
    2803             : }
    2804             : 
    2805             : // FastEmit functions for X86ISD::CVTP2UI.
    2806             : 
    2807             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2808           0 :   if ((Subtarget->hasVLX())) {
    2809           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2810             :   }
    2811             :   return 0;
    2812             : }
    2813             : 
    2814           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2815           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2816           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2817             :   }
    2818             :   return 0;
    2819             : }
    2820             : 
    2821           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2822           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2823           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2824             :   }
    2825             :   return 0;
    2826             : }
    2827             : 
    2828           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2829           0 : switch (RetVT.SimpleTy) {
    2830           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i32_r(Op0, Op0IsKill);
    2831           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v2i64_r(Op0, Op0IsKill);
    2832           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_MVT_v4i64_r(Op0, Op0IsKill);
    2833             :   default: return 0;
    2834             : }
    2835             : }
    2836             : 
    2837             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2838           0 :   if ((Subtarget->hasVLX())) {
    2839           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2840             :   }
    2841             :   return 0;
    2842             : }
    2843             : 
    2844             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2845           0 :   if ((Subtarget->hasDQI())) {
    2846           0 :     return fastEmitInst_r(X86::VCVTPS2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2847             :   }
    2848             :   return 0;
    2849             : }
    2850             : 
    2851           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2852           0 : switch (RetVT.SimpleTy) {
    2853           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i32_r(Op0, Op0IsKill);
    2854           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_MVT_v8i64_r(Op0, Op0IsKill);
    2855             :   default: return 0;
    2856             : }
    2857             : }
    2858             : 
    2859             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2860           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    2861             :     return 0;
    2862           0 :   if ((Subtarget->hasAVX512())) {
    2863           0 :     return fastEmitInst_r(X86::VCVTPS2UDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2864             :   }
    2865             :   return 0;
    2866             : }
    2867             : 
    2868             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2869           0 :   if ((Subtarget->hasVLX())) {
    2870           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2871             :   }
    2872             :   return 0;
    2873             : }
    2874             : 
    2875           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    2876           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2877           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2878             :   }
    2879             :   return 0;
    2880             : }
    2881             : 
    2882           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2883           0 : switch (RetVT.SimpleTy) {
    2884           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v4i32_r(Op0, Op0IsKill);
    2885           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_MVT_v2i64_r(Op0, Op0IsKill);
    2886             :   default: return 0;
    2887             : }
    2888             : }
    2889             : 
    2890             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    2891           0 :   if ((Subtarget->hasVLX())) {
    2892           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2893             :   }
    2894             :   return 0;
    2895             : }
    2896             : 
    2897           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    2898           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    2899           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2900             :   }
    2901             :   return 0;
    2902             : }
    2903             : 
    2904           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2905           0 : switch (RetVT.SimpleTy) {
    2906           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i32_r(Op0, Op0IsKill);
    2907           0 :   case MVT::v4i64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_MVT_v4i64_r(Op0, Op0IsKill);
    2908             :   default: return 0;
    2909             : }
    2910             : }
    2911             : 
    2912             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    2913           0 :   if ((Subtarget->hasAVX512())) {
    2914           0 :     return fastEmitInst_r(X86::VCVTPD2UDQZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2915             :   }
    2916             :   return 0;
    2917             : }
    2918             : 
    2919             : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    2920           0 :   if ((Subtarget->hasDQI())) {
    2921           0 :     return fastEmitInst_r(X86::VCVTPD2UQQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2922             :   }
    2923             :   return 0;
    2924             : }
    2925             : 
    2926           0 : unsigned fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2927           0 : switch (RetVT.SimpleTy) {
    2928           0 :   case MVT::v8i32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i32_r(Op0, Op0IsKill);
    2929           0 :   case MVT::v8i64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_MVT_v8i64_r(Op0, Op0IsKill);
    2930             :   default: return 0;
    2931             : }
    2932             : }
    2933             : 
    2934           0 : unsigned fastEmit_X86ISD_CVTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2935           0 :   switch (VT.SimpleTy) {
    2936           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    2937           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTP2UI_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    2938           0 :   case MVT::v16f32: return fastEmit_X86ISD_CVTP2UI_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    2939           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    2940           0 :   case MVT::v4f64: return fastEmit_X86ISD_CVTP2UI_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    2941           0 :   case MVT::v8f64: return fastEmit_X86ISD_CVTP2UI_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    2942             :   default: return 0;
    2943             :   }
    2944             : }
    2945             : 
    2946             : // FastEmit functions for X86ISD::CVTPH2PS.
    2947             : 
    2948           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    2949           0 :   if ((Subtarget->hasVLX())) {
    2950           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    2951             :   }
    2952           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2953           0 :     return fastEmitInst_r(X86::VCVTPH2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    2954             :   }
    2955             :   return 0;
    2956             : }
    2957             : 
    2958           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    2959           0 :   if ((Subtarget->hasVLX())) {
    2960           0 :     return fastEmitInst_r(X86::VCVTPH2PSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    2961             :   }
    2962           0 :   if ((Subtarget->hasF16C()) && (!Subtarget->hasVLX())) {
    2963           0 :     return fastEmitInst_r(X86::VCVTPH2PSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    2964             :   }
    2965             :   return 0;
    2966             : }
    2967             : 
    2968           0 : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2969           0 : switch (RetVT.SimpleTy) {
    2970           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v4f32_r(Op0, Op0IsKill);
    2971           0 :   case MVT::v8f32: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_MVT_v8f32_r(Op0, Op0IsKill);
    2972             :   default: return 0;
    2973             : }
    2974             : }
    2975             : 
    2976             : unsigned fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2977           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    2978             :     return 0;
    2979           0 :   if ((Subtarget->hasAVX512())) {
    2980           0 :     return fastEmitInst_r(X86::VCVTPH2PSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    2981             :   }
    2982             :   return 0;
    2983             : }
    2984             : 
    2985           0 : unsigned fastEmit_X86ISD_CVTPH2PS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2986           0 :   switch (VT.SimpleTy) {
    2987           0 :   case MVT::v8i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    2988           0 :   case MVT::v16i16: return fastEmit_X86ISD_CVTPH2PS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    2989             :   default: return 0;
    2990             :   }
    2991             : }
    2992             : 
    2993             : // FastEmit functions for X86ISD::CVTSI2P.
    2994             : 
    2995           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    2996           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    2997             :     return 0;
    2998           0 :   if ((Subtarget->hasVLX())) {
    2999           0 :     return fastEmitInst_r(X86::VCVTDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3000             :   }
    3001           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3002           0 :     return fastEmitInst_r(X86::CVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3003             :   }
    3004           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3005           0 :     return fastEmitInst_r(X86::VCVTDQ2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3006             :   }
    3007             :   return 0;
    3008             : }
    3009             : 
    3010           0 : unsigned fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3011           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3012             :     return 0;
    3013           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3014           0 :     return fastEmitInst_r(X86::VCVTQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3015             :   }
    3016             :   return 0;
    3017             : }
    3018             : 
    3019           0 : unsigned fastEmit_X86ISD_CVTSI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3020           0 :   switch (VT.SimpleTy) {
    3021           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTSI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3022           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTSI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3023             :   default: return 0;
    3024             :   }
    3025             : }
    3026             : 
    3027             : // FastEmit functions for X86ISD::CVTTP2SI.
    3028             : 
    3029           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3030           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3031             :     return 0;
    3032           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3033           0 :     return fastEmitInst_r(X86::VCVTTPS2QQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3034             :   }
    3035             :   return 0;
    3036             : }
    3037             : 
    3038           0 : unsigned fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3039           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3040             :     return 0;
    3041           0 :   if ((Subtarget->hasVLX())) {
    3042           0 :     return fastEmitInst_r(X86::VCVTTPD2DQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3043             :   }
    3044           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3045           0 :     return fastEmitInst_r(X86::CVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3046             :   }
    3047           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3048           0 :     return fastEmitInst_r(X86::VCVTTPD2DQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3049             :   }
    3050             :   return 0;
    3051             : }
    3052             : 
    3053           0 : unsigned fastEmit_X86ISD_CVTTP2SI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3054           0 :   switch (VT.SimpleTy) {
    3055           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2SI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3056           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2SI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3057             :   default: return 0;
    3058             :   }
    3059             : }
    3060             : 
    3061             : // FastEmit functions for X86ISD::CVTTP2UI.
    3062             : 
    3063           0 : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3064           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3065             :     return 0;
    3066           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3067           0 :     return fastEmitInst_r(X86::VCVTTPS2UQQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3068             :   }
    3069             :   return 0;
    3070             : }
    3071             : 
    3072             : unsigned fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3073           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3074             :     return 0;
    3075           0 :   if ((Subtarget->hasVLX())) {
    3076           0 :     return fastEmitInst_r(X86::VCVTTPD2UDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3077             :   }
    3078             :   return 0;
    3079             : }
    3080             : 
    3081           0 : unsigned fastEmit_X86ISD_CVTTP2UI_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3082           0 :   switch (VT.SimpleTy) {
    3083           0 :   case MVT::v4f32: return fastEmit_X86ISD_CVTTP2UI_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3084           0 :   case MVT::v2f64: return fastEmit_X86ISD_CVTTP2UI_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3085             :   default: return 0;
    3086             :   }
    3087             : }
    3088             : 
    3089             : // FastEmit functions for X86ISD::CVTUI2P.
    3090             : 
    3091             : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3092           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3093             :     return 0;
    3094           0 :   if ((Subtarget->hasVLX())) {
    3095           0 :     return fastEmitInst_r(X86::VCVTUDQ2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3096             :   }
    3097             :   return 0;
    3098             : }
    3099             : 
    3100           0 : unsigned fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3101           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3102             :     return 0;
    3103           0 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    3104           0 :     return fastEmitInst_r(X86::VCVTUQQ2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3105             :   }
    3106             :   return 0;
    3107             : }
    3108             : 
    3109           0 : unsigned fastEmit_X86ISD_CVTUI2P_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3110           0 :   switch (VT.SimpleTy) {
    3111           0 :   case MVT::v4i32: return fastEmit_X86ISD_CVTUI2P_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3112           0 :   case MVT::v2i64: return fastEmit_X86ISD_CVTUI2P_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3113             :   default: return 0;
    3114             :   }
    3115             : }
    3116             : 
    3117             : // FastEmit functions for X86ISD::EH_RETURN.
    3118             : 
    3119             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3120           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3121             :     return 0;
    3122           0 :   return fastEmitInst_r(X86::EH_RETURN, &X86::GR32RegClass, Op0, Op0IsKill);
    3123             : }
    3124             : 
    3125             : unsigned fastEmit_X86ISD_EH_RETURN_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3126           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3127             :     return 0;
    3128           0 :   return fastEmitInst_r(X86::EH_RETURN64, &X86::GR64RegClass, Op0, Op0IsKill);
    3129             : }
    3130             : 
    3131           0 : unsigned fastEmit_X86ISD_EH_RETURN_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3132           0 :   switch (VT.SimpleTy) {
    3133           0 :   case MVT::i32: return fastEmit_X86ISD_EH_RETURN_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3134           0 :   case MVT::i64: return fastEmit_X86ISD_EH_RETURN_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3135             :   default: return 0;
    3136             :   }
    3137             : }
    3138             : 
    3139             : // FastEmit functions for X86ISD::EXPAND.
    3140             : 
    3141           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3142           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    3143             :     return 0;
    3144           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3145           0 :     return fastEmitInst_r(X86::VPEXPANDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3146             :   }
    3147             :   return 0;
    3148             : }
    3149             : 
    3150           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3151           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    3152             :     return 0;
    3153           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3154           0 :     return fastEmitInst_r(X86::VPEXPANDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3155             :   }
    3156             :   return 0;
    3157             : }
    3158             : 
    3159             : unsigned fastEmit_X86ISD_EXPAND_MVT_v64i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3160           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    3161             :     return 0;
    3162           0 :   if ((Subtarget->hasVBMI2())) {
    3163           0 :     return fastEmitInst_r(X86::VPEXPANDBZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3164             :   }
    3165             :   return 0;
    3166             : }
    3167             : 
    3168           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3169           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3170             :     return 0;
    3171           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3172           0 :     return fastEmitInst_r(X86::VPEXPANDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3173             :   }
    3174             :   return 0;
    3175             : }
    3176             : 
    3177           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3178           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    3179             :     return 0;
    3180           0 :   if ((Subtarget->hasVBMI2()) && (Subtarget->hasVLX())) {
    3181           0 :     return fastEmitInst_r(X86::VPEXPANDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3182             :   }
    3183             :   return 0;
    3184             : }
    3185             : 
    3186             : unsigned fastEmit_X86ISD_EXPAND_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3187           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    3188             :     return 0;
    3189           0 :   if ((Subtarget->hasVBMI2())) {
    3190           0 :     return fastEmitInst_r(X86::VPEXPANDWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3191             :   }
    3192             :   return 0;
    3193             : }
    3194             : 
    3195           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3196           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3197             :     return 0;
    3198           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3199           0 :     return fastEmitInst_r(X86::VPEXPANDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3200             :   }
    3201             :   return 0;
    3202             : }
    3203             : 
    3204           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3205           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3206             :     return 0;
    3207           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3208           0 :     return fastEmitInst_r(X86::VPEXPANDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3209             :   }
    3210             :   return 0;
    3211             : }
    3212             : 
    3213             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3214           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    3215             :     return 0;
    3216           0 :   if ((Subtarget->hasAVX512())) {
    3217           0 :     return fastEmitInst_r(X86::VPEXPANDDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3218             :   }
    3219             :   return 0;
    3220             : }
    3221             : 
    3222           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3223           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    3224             :     return 0;
    3225           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3226           0 :     return fastEmitInst_r(X86::VPEXPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3227             :   }
    3228             :   return 0;
    3229             : }
    3230             : 
    3231           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3232           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    3233             :     return 0;
    3234           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3235           0 :     return fastEmitInst_r(X86::VPEXPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3236             :   }
    3237             :   return 0;
    3238             : }
    3239             : 
    3240             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3241           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    3242             :     return 0;
    3243           0 :   if ((Subtarget->hasAVX512())) {
    3244           0 :     return fastEmitInst_r(X86::VPEXPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3245             :   }
    3246             :   return 0;
    3247             : }
    3248             : 
    3249           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3250           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3251             :     return 0;
    3252           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3253           0 :     return fastEmitInst_r(X86::VEXPANDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3254             :   }
    3255             :   return 0;
    3256             : }
    3257             : 
    3258           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3259           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3260             :     return 0;
    3261           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3262           0 :     return fastEmitInst_r(X86::VEXPANDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3263             :   }
    3264             :   return 0;
    3265             : }
    3266             : 
    3267             : unsigned fastEmit_X86ISD_EXPAND_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3268           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3269             :     return 0;
    3270           0 :   if ((Subtarget->hasAVX512())) {
    3271           0 :     return fastEmitInst_r(X86::VEXPANDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3272             :   }
    3273             :   return 0;
    3274             : }
    3275             : 
    3276           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3277           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3278             :     return 0;
    3279           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3280           0 :     return fastEmitInst_r(X86::VEXPANDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3281             :   }
    3282             :   return 0;
    3283             : }
    3284             : 
    3285           0 : unsigned fastEmit_X86ISD_EXPAND_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3286           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3287             :     return 0;
    3288           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3289           0 :     return fastEmitInst_r(X86::VEXPANDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3290             :   }
    3291             :   return 0;
    3292             : }
    3293             : 
    3294             : unsigned fastEmit_X86ISD_EXPAND_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3295           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3296             :     return 0;
    3297           0 :   if ((Subtarget->hasAVX512())) {
    3298           0 :     return fastEmitInst_r(X86::VEXPANDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3299             :   }
    3300             :   return 0;
    3301             : }
    3302             : 
    3303           0 : unsigned fastEmit_X86ISD_EXPAND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3304           0 :   switch (VT.SimpleTy) {
    3305           0 :   case MVT::v16i8: return fastEmit_X86ISD_EXPAND_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3306           0 :   case MVT::v32i8: return fastEmit_X86ISD_EXPAND_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3307           0 :   case MVT::v64i8: return fastEmit_X86ISD_EXPAND_MVT_v64i8_r(RetVT, Op0, Op0IsKill);
    3308           0 :   case MVT::v8i16: return fastEmit_X86ISD_EXPAND_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3309           0 :   case MVT::v16i16: return fastEmit_X86ISD_EXPAND_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    3310           0 :   case MVT::v32i16: return fastEmit_X86ISD_EXPAND_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    3311           0 :   case MVT::v4i32: return fastEmit_X86ISD_EXPAND_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3312           0 :   case MVT::v8i32: return fastEmit_X86ISD_EXPAND_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3313           0 :   case MVT::v16i32: return fastEmit_X86ISD_EXPAND_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    3314           0 :   case MVT::v2i64: return fastEmit_X86ISD_EXPAND_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3315           0 :   case MVT::v4i64: return fastEmit_X86ISD_EXPAND_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    3316           0 :   case MVT::v8i64: return fastEmit_X86ISD_EXPAND_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    3317           0 :   case MVT::v4f32: return fastEmit_X86ISD_EXPAND_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3318           0 :   case MVT::v8f32: return fastEmit_X86ISD_EXPAND_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3319           0 :   case MVT::v16f32: return fastEmit_X86ISD_EXPAND_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3320           0 :   case MVT::v2f64: return fastEmit_X86ISD_EXPAND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3321           0 :   case MVT::v4f64: return fastEmit_X86ISD_EXPAND_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3322           0 :   case MVT::v8f64: return fastEmit_X86ISD_EXPAND_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3323             :   default: return 0;
    3324             :   }
    3325             : }
    3326             : 
    3327             : // FastEmit functions for X86ISD::FRCP.
    3328             : 
    3329           0 : unsigned fastEmit_X86ISD_FRCP_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3330           0 :   if (RetVT.SimpleTy != MVT::f32)
    3331             :     return 0;
    3332           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3333           0 :     return fastEmitInst_r(X86::RCPSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3334             :   }
    3335             :   return 0;
    3336             : }
    3337             : 
    3338           0 : unsigned fastEmit_X86ISD_FRCP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3339           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3340             :     return 0;
    3341           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3342           0 :     return fastEmitInst_r(X86::RCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3343             :   }
    3344           0 :   if ((Subtarget->hasAVX())) {
    3345           0 :     return fastEmitInst_r(X86::VRCPPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3346             :   }
    3347             :   return 0;
    3348             : }
    3349             : 
    3350             : unsigned fastEmit_X86ISD_FRCP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3351           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3352             :     return 0;
    3353           0 :   if ((Subtarget->hasAVX())) {
    3354           0 :     return fastEmitInst_r(X86::VRCPPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3355             :   }
    3356             :   return 0;
    3357             : }
    3358             : 
    3359           0 : unsigned fastEmit_X86ISD_FRCP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3360           0 :   switch (VT.SimpleTy) {
    3361           0 :   case MVT::f32: return fastEmit_X86ISD_FRCP_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3362           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRCP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3363           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRCP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3364             :   default: return 0;
    3365             :   }
    3366             : }
    3367             : 
    3368             : // FastEmit functions for X86ISD::FRSQRT.
    3369             : 
    3370           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3371           0 :   if (RetVT.SimpleTy != MVT::f32)
    3372             :     return 0;
    3373           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3374           0 :     return fastEmitInst_r(X86::RSQRTSSr, &X86::FR32RegClass, Op0, Op0IsKill);
    3375             :   }
    3376             :   return 0;
    3377             : }
    3378             : 
    3379           0 : unsigned fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3380           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3381             :     return 0;
    3382           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3383           0 :     return fastEmitInst_r(X86::RSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3384             :   }
    3385           0 :   if ((Subtarget->hasAVX())) {
    3386           0 :     return fastEmitInst_r(X86::VRSQRTPSr, &X86::VR128RegClass, Op0, Op0IsKill);
    3387             :   }
    3388             :   return 0;
    3389             : }
    3390             : 
    3391             : unsigned fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3392           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3393             :     return 0;
    3394           0 :   if ((Subtarget->hasAVX())) {
    3395           0 :     return fastEmitInst_r(X86::VRSQRTPSYr, &X86::VR256RegClass, Op0, Op0IsKill);
    3396             :   }
    3397             :   return 0;
    3398             : }
    3399             : 
    3400           0 : unsigned fastEmit_X86ISD_FRSQRT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3401           0 :   switch (VT.SimpleTy) {
    3402           0 :   case MVT::f32: return fastEmit_X86ISD_FRSQRT_MVT_f32_r(RetVT, Op0, Op0IsKill);
    3403           0 :   case MVT::v4f32: return fastEmit_X86ISD_FRSQRT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3404           0 :   case MVT::v8f32: return fastEmit_X86ISD_FRSQRT_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3405             :   default: return 0;
    3406             :   }
    3407             : }
    3408             : 
    3409             : // FastEmit functions for X86ISD::MMX_MOVD2W.
    3410             : 
    3411             : unsigned fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3412           0 :   if (RetVT.SimpleTy != MVT::i32)
    3413             :     return 0;
    3414           0 :   if ((Subtarget->hasMMX())) {
    3415           0 :     return fastEmitInst_r(X86::MMX_MOVD64grr, &X86::GR32RegClass, Op0, Op0IsKill);
    3416             :   }
    3417             :   return 0;
    3418             : }
    3419             : 
    3420           0 : unsigned fastEmit_X86ISD_MMX_MOVD2W_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3421           0 :   switch (VT.SimpleTy) {
    3422           0 :   case MVT::x86mmx: return fastEmit_X86ISD_MMX_MOVD2W_MVT_x86mmx_r(RetVT, Op0, Op0IsKill);
    3423             :   default: return 0;
    3424             :   }
    3425             : }
    3426             : 
    3427             : // FastEmit functions for X86ISD::MMX_MOVW2D.
    3428             : 
    3429             : unsigned fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3430           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3431             :     return 0;
    3432           0 :   if ((Subtarget->hasMMX())) {
    3433           0 :     return fastEmitInst_r(X86::MMX_MOVD64rr, &X86::VR64RegClass, Op0, Op0IsKill);
    3434             :   }
    3435             :   return 0;
    3436             : }
    3437             : 
    3438           0 : unsigned fastEmit_X86ISD_MMX_MOVW2D_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3439           0 :   switch (VT.SimpleTy) {
    3440           0 :   case MVT::i32: return fastEmit_X86ISD_MMX_MOVW2D_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3441             :   default: return 0;
    3442             :   }
    3443             : }
    3444             : 
    3445             : // FastEmit functions for X86ISD::MOVDDUP.
    3446             : 
    3447           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3448           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3449             :     return 0;
    3450           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3451           0 :     return fastEmitInst_r(X86::MOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3452             :   }
    3453           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3454           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3455             :   }
    3456             :   return 0;
    3457             : }
    3458             : 
    3459           0 : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3460           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3461             :     return 0;
    3462           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3463           0 :     return fastEmitInst_r(X86::VMOVDDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3464             :   }
    3465           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3466           0 :     return fastEmitInst_r(X86::VMOVDDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3467             :   }
    3468             :   return 0;
    3469             : }
    3470             : 
    3471             : unsigned fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3472           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3473             :     return 0;
    3474           0 :   if ((Subtarget->hasAVX512())) {
    3475           0 :     return fastEmitInst_r(X86::VMOVDDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3476             :   }
    3477             :   return 0;
    3478             : }
    3479             : 
    3480           0 : unsigned fastEmit_X86ISD_MOVDDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3481           0 :   switch (VT.SimpleTy) {
    3482           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVDDUP_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3483           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVDDUP_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3484           0 :   case MVT::v8f64: return fastEmit_X86ISD_MOVDDUP_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3485             :   default: return 0;
    3486             :   }
    3487             : }
    3488             : 
    3489             : // FastEmit functions for X86ISD::MOVDQ2Q.
    3490             : 
    3491             : unsigned fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3492           0 :   if (RetVT.SimpleTy != MVT::x86mmx)
    3493             :     return 0;
    3494           0 :   return fastEmitInst_r(X86::MMX_MOVDQ2Qrr, &X86::VR64RegClass, Op0, Op0IsKill);
    3495             : }
    3496             : 
    3497             : unsigned fastEmit_X86ISD_MOVDQ2Q_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3498           0 :   switch (VT.SimpleTy) {
    3499           0 :   case MVT::v2i64: return fastEmit_X86ISD_MOVDQ2Q_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    3500             :   default: return 0;
    3501             :   }
    3502             : }
    3503             : 
    3504             : // FastEmit functions for X86ISD::MOVMSK.
    3505             : 
    3506           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3507           0 :   if (RetVT.SimpleTy != MVT::i32)
    3508             :     return 0;
    3509           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3510           0 :     return fastEmitInst_r(X86::PMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3511             :   }
    3512           0 :   if ((Subtarget->hasAVX())) {
    3513           0 :     return fastEmitInst_r(X86::VPMOVMSKBrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3514             :   }
    3515             :   return 0;
    3516             : }
    3517             : 
    3518             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3519           0 :   if (RetVT.SimpleTy != MVT::i32)
    3520             :     return 0;
    3521           0 :   if ((Subtarget->hasAVX2())) {
    3522           0 :     return fastEmitInst_r(X86::VPMOVMSKBYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3523             :   }
    3524             :   return 0;
    3525             : }
    3526             : 
    3527           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3528           0 :   if (RetVT.SimpleTy != MVT::i32)
    3529             :     return 0;
    3530           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    3531           0 :     return fastEmitInst_r(X86::MOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3532             :   }
    3533           0 :   if ((Subtarget->hasAVX())) {
    3534           0 :     return fastEmitInst_r(X86::VMOVMSKPSrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3535             :   }
    3536             :   return 0;
    3537             : }
    3538             : 
    3539             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3540           0 :   if (RetVT.SimpleTy != MVT::i32)
    3541             :     return 0;
    3542           0 :   if ((Subtarget->hasAVX())) {
    3543           0 :     return fastEmitInst_r(X86::VMOVMSKPSYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3544             :   }
    3545             :   return 0;
    3546             : }
    3547             : 
    3548           0 : unsigned fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3549           0 :   if (RetVT.SimpleTy != MVT::i32)
    3550             :     return 0;
    3551           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    3552           0 :     return fastEmitInst_r(X86::MOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3553             :   }
    3554           0 :   if ((Subtarget->hasAVX())) {
    3555           0 :     return fastEmitInst_r(X86::VMOVMSKPDrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3556             :   }
    3557             :   return 0;
    3558             : }
    3559             : 
    3560             : unsigned fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3561           0 :   if (RetVT.SimpleTy != MVT::i32)
    3562             :     return 0;
    3563           0 :   if ((Subtarget->hasAVX())) {
    3564           0 :     return fastEmitInst_r(X86::VMOVMSKPDYrr, &X86::GR32RegClass, Op0, Op0IsKill);
    3565             :   }
    3566             :   return 0;
    3567             : }
    3568             : 
    3569           0 : unsigned fastEmit_X86ISD_MOVMSK_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3570           0 :   switch (VT.SimpleTy) {
    3571           0 :   case MVT::v16i8: return fastEmit_X86ISD_MOVMSK_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    3572           0 :   case MVT::v32i8: return fastEmit_X86ISD_MOVMSK_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    3573           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVMSK_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3574           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVMSK_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3575           0 :   case MVT::v2f64: return fastEmit_X86ISD_MOVMSK_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3576           0 :   case MVT::v4f64: return fastEmit_X86ISD_MOVMSK_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3577             :   default: return 0;
    3578             :   }
    3579             : }
    3580             : 
    3581             : // FastEmit functions for X86ISD::MOVSHDUP.
    3582             : 
    3583           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3584           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3585             :     return 0;
    3586           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3587           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3588             :   }
    3589           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3590           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3591             :   }
    3592             :   return 0;
    3593             : }
    3594             : 
    3595           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3596           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3597             :     return 0;
    3598           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3599           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3600             :   }
    3601             :   return 0;
    3602             : }
    3603             : 
    3604           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3605           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3606             :     return 0;
    3607           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3608           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3609             :   }
    3610           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3611           0 :     return fastEmitInst_r(X86::MOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3612             :   }
    3613           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3614           0 :     return fastEmitInst_r(X86::VMOVSHDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3615             :   }
    3616             :   return 0;
    3617             : }
    3618             : 
    3619           0 : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3620           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3621             :     return 0;
    3622           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3623           0 :     return fastEmitInst_r(X86::VMOVSHDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3624             :   }
    3625           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3626           0 :     return fastEmitInst_r(X86::VMOVSHDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3627             :   }
    3628             :   return 0;
    3629             : }
    3630             : 
    3631             : unsigned fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3632           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3633             :     return 0;
    3634           0 :   if ((Subtarget->hasAVX512())) {
    3635           0 :     return fastEmitInst_r(X86::VMOVSHDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3636             :   }
    3637             :   return 0;
    3638             : }
    3639             : 
    3640           0 : unsigned fastEmit_X86ISD_MOVSHDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3641           0 :   switch (VT.SimpleTy) {
    3642           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3643           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3644           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3645           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3646           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSHDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3647             :   default: return 0;
    3648             :   }
    3649             : }
    3650             : 
    3651             : // FastEmit functions for X86ISD::MOVSLDUP.
    3652             : 
    3653           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3654           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    3655             :     return 0;
    3656           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3657           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3658             :   }
    3659           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3660           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3661             :   }
    3662             :   return 0;
    3663             : }
    3664             : 
    3665           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3666           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    3667             :     return 0;
    3668           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3669           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3670             :   }
    3671             :   return 0;
    3672             : }
    3673             : 
    3674           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3675           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3676             :     return 0;
    3677           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3678           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    3679             :   }
    3680           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    3681           0 :     return fastEmitInst_r(X86::MOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3682             :   }
    3683           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3684           0 :     return fastEmitInst_r(X86::VMOVSLDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3685             :   }
    3686             :   return 0;
    3687             : }
    3688             : 
    3689           0 : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3690           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3691             :     return 0;
    3692           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3693           0 :     return fastEmitInst_r(X86::VMOVSLDUPZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    3694             :   }
    3695           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    3696           0 :     return fastEmitInst_r(X86::VMOVSLDUPYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    3697             :   }
    3698             :   return 0;
    3699             : }
    3700             : 
    3701             : unsigned fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3702           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3703             :     return 0;
    3704           0 :   if ((Subtarget->hasAVX512())) {
    3705           0 :     return fastEmitInst_r(X86::VMOVSLDUPZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    3706             :   }
    3707             :   return 0;
    3708             : }
    3709             : 
    3710           0 : unsigned fastEmit_X86ISD_MOVSLDUP_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3711           0 :   switch (VT.SimpleTy) {
    3712           0 :   case MVT::v4i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    3713           0 :   case MVT::v8i32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    3714           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3715           0 :   case MVT::v8f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3716           0 :   case MVT::v16f32: return fastEmit_X86ISD_MOVSLDUP_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3717             :   default: return 0;
    3718             :   }
    3719             : }
    3720             : 
    3721             : // FastEmit functions for X86ISD::NT_BRIND.
    3722             : 
    3723             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3724           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3725             :     return 0;
    3726           0 :   if ((Subtarget->hasIBT())) {
    3727           0 :     return fastEmitInst_r(X86::JMP16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
    3728             :   }
    3729             :   return 0;
    3730             : }
    3731             : 
    3732             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3733           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3734             :     return 0;
    3735           0 :   if ((Subtarget->hasIBT())) {
    3736           0 :     return fastEmitInst_r(X86::JMP32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
    3737             :   }
    3738             :   return 0;
    3739             : }
    3740             : 
    3741             : unsigned fastEmit_X86ISD_NT_BRIND_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3742           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3743             :     return 0;
    3744           0 :   if ((Subtarget->hasIBT())) {
    3745           0 :     return fastEmitInst_r(X86::JMP64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
    3746             :   }
    3747             :   return 0;
    3748             : }
    3749             : 
    3750           0 : unsigned fastEmit_X86ISD_NT_BRIND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3751           0 :   switch (VT.SimpleTy) {
    3752           0 :   case MVT::i16: return fastEmit_X86ISD_NT_BRIND_MVT_i16_r(RetVT, Op0, Op0IsKill);
    3753           0 :   case MVT::i32: return fastEmit_X86ISD_NT_BRIND_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3754           0 :   case MVT::i64: return fastEmit_X86ISD_NT_BRIND_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3755             :   default: return 0;
    3756             :   }
    3757             : }
    3758             : 
    3759             : // FastEmit functions for X86ISD::NT_CALL.
    3760             : 
    3761             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3762           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3763             :     return 0;
    3764           0 :   if ((Subtarget->hasIBT())) {
    3765           0 :     return fastEmitInst_r(X86::CALL16r_NT, &X86::GR16RegClass, Op0, Op0IsKill);
    3766             :   }
    3767             :   return 0;
    3768             : }
    3769             : 
    3770             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3771           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3772             :     return 0;
    3773           0 :   if ((Subtarget->hasIBT())) {
    3774           0 :     return fastEmitInst_r(X86::CALL32r_NT, &X86::GR32RegClass, Op0, Op0IsKill);
    3775             :   }
    3776             :   return 0;
    3777             : }
    3778             : 
    3779             : unsigned fastEmit_X86ISD_NT_CALL_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3780           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    3781             :     return 0;
    3782           0 :   if ((Subtarget->hasIBT())) {
    3783           0 :     return fastEmitInst_r(X86::CALL64r_NT, &X86::GR64RegClass, Op0, Op0IsKill);
    3784             :   }
    3785             :   return 0;
    3786             : }
    3787             : 
    3788           0 : unsigned fastEmit_X86ISD_NT_CALL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3789           0 :   switch (VT.SimpleTy) {
    3790           0 :   case MVT::i16: return fastEmit_X86ISD_NT_CALL_MVT_i16_r(RetVT, Op0, Op0IsKill);
    3791           0 :   case MVT::i32: return fastEmit_X86ISD_NT_CALL_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3792           0 :   case MVT::i64: return fastEmit_X86ISD_NT_CALL_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3793             :   default: return 0;
    3794             :   }
    3795             : }
    3796             : 
    3797             : // FastEmit functions for X86ISD::PHMINPOS.
    3798             : 
    3799           0 : unsigned fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3800           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    3801             :     return 0;
    3802           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    3803           0 :     return fastEmitInst_r(X86::PHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3804             :   }
    3805           0 :   if ((Subtarget->hasAVX())) {
    3806           0 :     return fastEmitInst_r(X86::VPHMINPOSUWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    3807             :   }
    3808             :   return 0;
    3809             : }
    3810             : 
    3811             : unsigned fastEmit_X86ISD_PHMINPOS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3812           0 :   switch (VT.SimpleTy) {
    3813           0 :   case MVT::v8i16: return fastEmit_X86ISD_PHMINPOS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    3814             :   default: return 0;
    3815             :   }
    3816             : }
    3817             : 
    3818             : // FastEmit functions for X86ISD::RCP14.
    3819             : 
    3820             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3821           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3822             :     return 0;
    3823           0 :   if ((Subtarget->hasVLX())) {
    3824           0 :     return fastEmitInst_r(X86::VRCP14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3825             :   }
    3826             :   return 0;
    3827             : }
    3828             : 
    3829             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3830           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3831             :     return 0;
    3832           0 :   if ((Subtarget->hasVLX())) {
    3833           0 :     return fastEmitInst_r(X86::VRCP14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3834             :   }
    3835             :   return 0;
    3836             : }
    3837             : 
    3838             : unsigned fastEmit_X86ISD_RCP14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3839           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3840             :     return 0;
    3841           0 :   if ((Subtarget->hasAVX512())) {
    3842           0 :     return fastEmitInst_r(X86::VRCP14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3843             :   }
    3844             :   return 0;
    3845             : }
    3846             : 
    3847             : unsigned fastEmit_X86ISD_RCP14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3848           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3849             :     return 0;
    3850           0 :   if ((Subtarget->hasVLX())) {
    3851           0 :     return fastEmitInst_r(X86::VRCP14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3852             :   }
    3853             :   return 0;
    3854             : }
    3855             : 
    3856             : unsigned fastEmit_X86ISD_RCP14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3857           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3858             :     return 0;
    3859           0 :   if ((Subtarget->hasVLX())) {
    3860           0 :     return fastEmitInst_r(X86::VRCP14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3861             :   }
    3862             :   return 0;
    3863             : }
    3864             : 
    3865             : unsigned fastEmit_X86ISD_RCP14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3866           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3867             :     return 0;
    3868           0 :   if ((Subtarget->hasAVX512())) {
    3869           0 :     return fastEmitInst_r(X86::VRCP14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3870             :   }
    3871             :   return 0;
    3872             : }
    3873             : 
    3874           0 : unsigned fastEmit_X86ISD_RCP14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3875           0 :   switch (VT.SimpleTy) {
    3876           0 :   case MVT::v4f32: return fastEmit_X86ISD_RCP14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3877           0 :   case MVT::v8f32: return fastEmit_X86ISD_RCP14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3878           0 :   case MVT::v16f32: return fastEmit_X86ISD_RCP14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3879           0 :   case MVT::v2f64: return fastEmit_X86ISD_RCP14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3880           0 :   case MVT::v4f64: return fastEmit_X86ISD_RCP14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3881           0 :   case MVT::v8f64: return fastEmit_X86ISD_RCP14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3882             :   default: return 0;
    3883             :   }
    3884             : }
    3885             : 
    3886             : // FastEmit functions for X86ISD::RSQRT14.
    3887             : 
    3888             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3889           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    3890             :     return 0;
    3891           0 :   if ((Subtarget->hasVLX())) {
    3892           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3893             :   }
    3894             :   return 0;
    3895             : }
    3896             : 
    3897             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3898           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    3899             :     return 0;
    3900           0 :   if ((Subtarget->hasVLX())) {
    3901           0 :     return fastEmitInst_r(X86::VRSQRT14PSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3902             :   }
    3903             :   return 0;
    3904             : }
    3905             : 
    3906             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3907           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    3908             :     return 0;
    3909           0 :   if ((Subtarget->hasAVX512())) {
    3910           0 :     return fastEmitInst_r(X86::VRSQRT14PSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3911             :   }
    3912             :   return 0;
    3913             : }
    3914             : 
    3915             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3916           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    3917             :     return 0;
    3918           0 :   if ((Subtarget->hasVLX())) {
    3919           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3920             :   }
    3921             :   return 0;
    3922             : }
    3923             : 
    3924             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3925           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    3926             :     return 0;
    3927           0 :   if ((Subtarget->hasVLX())) {
    3928           0 :     return fastEmitInst_r(X86::VRSQRT14PDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3929             :   }
    3930             :   return 0;
    3931             : }
    3932             : 
    3933             : unsigned fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3934           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    3935             :     return 0;
    3936           0 :   if ((Subtarget->hasAVX512())) {
    3937           0 :     return fastEmitInst_r(X86::VRSQRT14PDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    3938             :   }
    3939             :   return 0;
    3940             : }
    3941             : 
    3942           0 : unsigned fastEmit_X86ISD_RSQRT14_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3943           0 :   switch (VT.SimpleTy) {
    3944           0 :   case MVT::v4f32: return fastEmit_X86ISD_RSQRT14_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    3945           0 :   case MVT::v8f32: return fastEmit_X86ISD_RSQRT14_MVT_v8f32_r(RetVT, Op0, Op0IsKill);
    3946           0 :   case MVT::v16f32: return fastEmit_X86ISD_RSQRT14_MVT_v16f32_r(RetVT, Op0, Op0IsKill);
    3947           0 :   case MVT::v2f64: return fastEmit_X86ISD_RSQRT14_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    3948           0 :   case MVT::v4f64: return fastEmit_X86ISD_RSQRT14_MVT_v4f64_r(RetVT, Op0, Op0IsKill);
    3949           0 :   case MVT::v8f64: return fastEmit_X86ISD_RSQRT14_MVT_v8f64_r(RetVT, Op0, Op0IsKill);
    3950             :   default: return 0;
    3951             :   }
    3952             : }
    3953             : 
    3954             : // FastEmit functions for X86ISD::SEG_ALLOCA.
    3955             : 
    3956           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3957           0 :   if (RetVT.SimpleTy != MVT::i32)
    3958             :     return 0;
    3959           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    3960           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    3961             :   }
    3962             :   return 0;
    3963             : }
    3964             : 
    3965             : unsigned fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3966           0 :   if (RetVT.SimpleTy != MVT::i64)
    3967             :     return 0;
    3968           0 :   if ((Subtarget->is64Bit())) {
    3969           0 :     return fastEmitInst_r(X86::SEG_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    3970             :   }
    3971             :   return 0;
    3972             : }
    3973             : 
    3974           0 : unsigned fastEmit_X86ISD_SEG_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    3975           0 :   switch (VT.SimpleTy) {
    3976           0 :   case MVT::i32: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    3977           0 :   case MVT::i64: return fastEmit_X86ISD_SEG_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    3978             :   default: return 0;
    3979             :   }
    3980             : }
    3981             : 
    3982             : // FastEmit functions for X86ISD::VBROADCAST.
    3983             : 
    3984           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    3985           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3986           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    3987             :   }
    3988             :   return 0;
    3989             : }
    3990             : 
    3991           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    3992           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    3993           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    3994             :   }
    3995             :   return 0;
    3996             : }
    3997             : 
    3998             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    3999           0 :   if ((Subtarget->hasAVX512())) {
    4000           0 :     return fastEmitInst_r(X86::VPBROADCASTDrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4001             :   }
    4002             :   return 0;
    4003             : }
    4004             : 
    4005           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4006           0 : switch (RetVT.SimpleTy) {
    4007           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v4i32_r(Op0, Op0IsKill);
    4008           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v8i32_r(Op0, Op0IsKill);
    4009           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_MVT_v16i32_r(Op0, Op0IsKill);
    4010             :   default: return 0;
    4011             : }
    4012             : }
    4013             : 
    4014           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4015           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4016           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4017             :   }
    4018             :   return 0;
    4019             : }
    4020             : 
    4021           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4022           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4023           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4024             :   }
    4025             :   return 0;
    4026             : }
    4027             : 
    4028             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4029           0 :   if ((Subtarget->hasAVX512())) {
    4030           0 :     return fastEmitInst_r(X86::VPBROADCASTQrZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4031             :   }
    4032             :   return 0;
    4033             : }
    4034             : 
    4035           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4036           0 : switch (RetVT.SimpleTy) {
    4037           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v2i64_r(Op0, Op0IsKill);
    4038           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v4i64_r(Op0, Op0IsKill);
    4039           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_MVT_v8i64_r(Op0, Op0IsKill);
    4040             :   default: return 0;
    4041             : }
    4042             : }
    4043             : 
    4044           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4045           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4046           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4047             :   }
    4048           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4049           0 :     return fastEmitInst_r(X86::VPBROADCASTBrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4050             :   }
    4051             :   return 0;
    4052             : }
    4053             : 
    4054           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(unsigned Op0, bool Op0IsKill) {
    4055           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4056           0 :     return fastEmitInst_r(X86::VPBROADCASTBZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4057             :   }
    4058           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4059           0 :     return fastEmitInst_r(X86::VPBROADCASTBYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4060             :   }
    4061             :   return 0;
    4062             : }
    4063             : 
    4064             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(unsigned Op0, bool Op0IsKill) {
    4065           0 :   if ((Subtarget->hasBWI())) {
    4066           0 :     return fastEmitInst_r(X86::VPBROADCASTBZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4067             :   }
    4068             :   return 0;
    4069             : }
    4070             : 
    4071           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4072           0 : switch (RetVT.SimpleTy) {
    4073           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v16i8_r(Op0, Op0IsKill);
    4074           0 :   case MVT::v32i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v32i8_r(Op0, Op0IsKill);
    4075           0 :   case MVT::v64i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_MVT_v64i8_r(Op0, Op0IsKill);
    4076             :   default: return 0;
    4077             : }
    4078             : }
    4079             : 
    4080           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4081           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4082           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4083             :   }
    4084           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4085           0 :     return fastEmitInst_r(X86::VPBROADCASTWrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4086             :   }
    4087             :   return 0;
    4088             : }
    4089             : 
    4090           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4091           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4092           0 :     return fastEmitInst_r(X86::VPBROADCASTWZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4093             :   }
    4094           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4095           0 :     return fastEmitInst_r(X86::VPBROADCASTWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4096             :   }
    4097             :   return 0;
    4098             : }
    4099             : 
    4100             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(unsigned Op0, bool Op0IsKill) {
    4101           0 :   if ((Subtarget->hasBWI())) {
    4102           0 :     return fastEmitInst_r(X86::VPBROADCASTWZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4103             :   }
    4104             :   return 0;
    4105             : }
    4106             : 
    4107           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4108           0 : switch (RetVT.SimpleTy) {
    4109           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v8i16_r(Op0, Op0IsKill);
    4110           0 :   case MVT::v16i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v16i16_r(Op0, Op0IsKill);
    4111           0 :   case MVT::v32i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_MVT_v32i16_r(Op0, Op0IsKill);
    4112             :   default: return 0;
    4113             : }
    4114             : }
    4115             : 
    4116           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4117           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4118           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4119             :   }
    4120           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4121           0 :     return fastEmitInst_r(X86::VPBROADCASTDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4122             :   }
    4123             :   return 0;
    4124             : }
    4125             : 
    4126           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4127           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4128           0 :     return fastEmitInst_r(X86::VPBROADCASTDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4129             :   }
    4130           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4131           0 :     return fastEmitInst_r(X86::VPBROADCASTDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4132             :   }
    4133             :   return 0;
    4134             : }
    4135             : 
    4136             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4137           0 :   if ((Subtarget->hasAVX512())) {
    4138           0 :     return fastEmitInst_r(X86::VPBROADCASTDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4139             :   }
    4140             :   return 0;
    4141             : }
    4142             : 
    4143           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4144           0 : switch (RetVT.SimpleTy) {
    4145           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v4i32_r(Op0, Op0IsKill);
    4146           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v8i32_r(Op0, Op0IsKill);
    4147           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_MVT_v16i32_r(Op0, Op0IsKill);
    4148             :   default: return 0;
    4149             : }
    4150             : }
    4151             : 
    4152           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4153           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4154           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4155             :   }
    4156           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4157           0 :     return fastEmitInst_r(X86::VPBROADCASTQrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4158             :   }
    4159             :   return 0;
    4160             : }
    4161             : 
    4162           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4163           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4164           0 :     return fastEmitInst_r(X86::VPBROADCASTQZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4165             :   }
    4166           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4167           0 :     return fastEmitInst_r(X86::VPBROADCASTQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4168             :   }
    4169             :   return 0;
    4170             : }
    4171             : 
    4172             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4173           0 :   if ((Subtarget->hasAVX512())) {
    4174           0 :     return fastEmitInst_r(X86::VPBROADCASTQZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4175             :   }
    4176             :   return 0;
    4177             : }
    4178             : 
    4179           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4180           0 : switch (RetVT.SimpleTy) {
    4181           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v2i64_r(Op0, Op0IsKill);
    4182           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v4i64_r(Op0, Op0IsKill);
    4183           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_MVT_v8i64_r(Op0, Op0IsKill);
    4184             :   default: return 0;
    4185             : }
    4186             : }
    4187             : 
    4188           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(unsigned Op0, bool Op0IsKill) {
    4189           0 :   if ((Subtarget->hasVLX())) {
    4190           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ128r, &X86::VR128XRegClass, Op0, Op0IsKill);
    4191             :   }
    4192           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4193           0 :     return fastEmitInst_r(X86::VBROADCASTSSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4194             :   }
    4195             :   return 0;
    4196             : }
    4197             : 
    4198           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(unsigned Op0, bool Op0IsKill) {
    4199           0 :   if ((Subtarget->hasVLX())) {
    4200           0 :     return fastEmitInst_r(X86::VBROADCASTSSZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4201             :   }
    4202           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4203           0 :     return fastEmitInst_r(X86::VBROADCASTSSYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4204             :   }
    4205             :   return 0;
    4206             : }
    4207             : 
    4208             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(unsigned Op0, bool Op0IsKill) {
    4209           0 :   if ((Subtarget->hasAVX512())) {
    4210           0 :     return fastEmitInst_r(X86::VBROADCASTSSZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4211             :   }
    4212             :   return 0;
    4213             : }
    4214             : 
    4215           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4216           0 : switch (RetVT.SimpleTy) {
    4217           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v4f32_r(Op0, Op0IsKill);
    4218           0 :   case MVT::v8f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v8f32_r(Op0, Op0IsKill);
    4219           0 :   case MVT::v16f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_MVT_v16f32_r(Op0, Op0IsKill);
    4220             :   default: return 0;
    4221             : }
    4222             : }
    4223             : 
    4224           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(unsigned Op0, bool Op0IsKill) {
    4225           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4226           0 :     return fastEmitInst_r(X86::VMOVDDUPrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4227             :   }
    4228           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4229           0 :     return fastEmitInst_r(X86::VMOVDDUPZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4230             :   }
    4231             :   return 0;
    4232             : }
    4233             : 
    4234           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(unsigned Op0, bool Op0IsKill) {
    4235           0 :   if ((Subtarget->hasVLX())) {
    4236           0 :     return fastEmitInst_r(X86::VBROADCASTSDZ256r, &X86::VR256XRegClass, Op0, Op0IsKill);
    4237             :   }
    4238           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    4239           0 :     return fastEmitInst_r(X86::VBROADCASTSDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4240             :   }
    4241             :   return 0;
    4242             : }
    4243             : 
    4244             : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(unsigned Op0, bool Op0IsKill) {
    4245           0 :   if ((Subtarget->hasAVX512())) {
    4246           0 :     return fastEmitInst_r(X86::VBROADCASTSDZr, &X86::VR512RegClass, Op0, Op0IsKill);
    4247             :   }
    4248             :   return 0;
    4249             : }
    4250             : 
    4251           0 : unsigned fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4252           0 : switch (RetVT.SimpleTy) {
    4253           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v2f64_r(Op0, Op0IsKill);
    4254           0 :   case MVT::v4f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v4f64_r(Op0, Op0IsKill);
    4255           0 :   case MVT::v8f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_MVT_v8f64_r(Op0, Op0IsKill);
    4256             :   default: return 0;
    4257             : }
    4258             : }
    4259             : 
    4260           0 : unsigned fastEmit_X86ISD_VBROADCAST_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4261           0 :   switch (VT.SimpleTy) {
    4262           0 :   case MVT::i32: return fastEmit_X86ISD_VBROADCAST_MVT_i32_r(RetVT, Op0, Op0IsKill);
    4263           0 :   case MVT::i64: return fastEmit_X86ISD_VBROADCAST_MVT_i64_r(RetVT, Op0, Op0IsKill);
    4264           0 :   case MVT::v16i8: return fastEmit_X86ISD_VBROADCAST_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4265           0 :   case MVT::v8i16: return fastEmit_X86ISD_VBROADCAST_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4266           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCAST_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4267           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCAST_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4268           0 :   case MVT::v4f32: return fastEmit_X86ISD_VBROADCAST_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4269           0 :   case MVT::v2f64: return fastEmit_X86ISD_VBROADCAST_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4270             :   default: return 0;
    4271             :   }
    4272             : }
    4273             : 
    4274             : // FastEmit functions for X86ISD::VBROADCASTM.
    4275             : 
    4276           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(unsigned Op0, bool Op0IsKill) {
    4277           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4278           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4279             :   }
    4280             :   return 0;
    4281             : }
    4282             : 
    4283           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4284           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4285           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4286             :   }
    4287             :   return 0;
    4288             : }
    4289             : 
    4290             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4291           0 :   if ((Subtarget->hasCDI())) {
    4292           0 :     return fastEmitInst_r(X86::VPBROADCASTMB2QZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4293             :   }
    4294             :   return 0;
    4295             : }
    4296             : 
    4297           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4298           0 : switch (RetVT.SimpleTy) {
    4299           0 :   case MVT::v2i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v2i64_r(Op0, Op0IsKill);
    4300           0 :   case MVT::v4i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v4i64_r(Op0, Op0IsKill);
    4301           0 :   case MVT::v8i64: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_MVT_v8i64_r(Op0, Op0IsKill);
    4302             :   default: return 0;
    4303             : }
    4304             : }
    4305             : 
    4306           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4307           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4308           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4309             :   }
    4310             :   return 0;
    4311             : }
    4312             : 
    4313           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4314           0 :   if ((Subtarget->hasCDI()) && (Subtarget->hasVLX())) {
    4315           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4316             :   }
    4317             :   return 0;
    4318             : }
    4319             : 
    4320             : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4321           0 :   if ((Subtarget->hasCDI())) {
    4322           0 :     return fastEmitInst_r(X86::VPBROADCASTMW2DZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4323             :   }
    4324             :   return 0;
    4325             : }
    4326             : 
    4327           0 : unsigned fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4328           0 : switch (RetVT.SimpleTy) {
    4329           0 :   case MVT::v4i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v4i32_r(Op0, Op0IsKill);
    4330           0 :   case MVT::v8i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v8i32_r(Op0, Op0IsKill);
    4331           0 :   case MVT::v16i32: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_MVT_v16i32_r(Op0, Op0IsKill);
    4332             :   default: return 0;
    4333             : }
    4334             : }
    4335             : 
    4336           0 : unsigned fastEmit_X86ISD_VBROADCASTM_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4337           0 :   switch (VT.SimpleTy) {
    4338           0 :   case MVT::v8i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v8i1_r(RetVT, Op0, Op0IsKill);
    4339           0 :   case MVT::v16i1: return fastEmit_X86ISD_VBROADCASTM_MVT_v16i1_r(RetVT, Op0, Op0IsKill);
    4340             :   default: return 0;
    4341             :   }
    4342             : }
    4343             : 
    4344             : // FastEmit functions for X86ISD::VFPEXT.
    4345             : 
    4346           0 : unsigned fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4347           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    4348             :     return 0;
    4349           0 :   if ((Subtarget->hasVLX())) {
    4350           0 :     return fastEmitInst_r(X86::VCVTPS2PDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4351             :   }
    4352           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4353           0 :     return fastEmitInst_r(X86::CVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4354             :   }
    4355           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4356           0 :     return fastEmitInst_r(X86::VCVTPS2PDrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4357             :   }
    4358             :   return 0;
    4359             : }
    4360             : 
    4361             : unsigned fastEmit_X86ISD_VFPEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4362           0 :   switch (VT.SimpleTy) {
    4363           0 :   case MVT::v4f32: return fastEmit_X86ISD_VFPEXT_MVT_v4f32_r(RetVT, Op0, Op0IsKill);
    4364             :   default: return 0;
    4365             :   }
    4366             : }
    4367             : 
    4368             : // FastEmit functions for X86ISD::VFPROUND.
    4369             : 
    4370           0 : unsigned fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4371           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    4372             :     return 0;
    4373           0 :   if ((Subtarget->hasVLX())) {
    4374           0 :     return fastEmitInst_r(X86::VCVTPD2PSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4375             :   }
    4376           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    4377           0 :     return fastEmitInst_r(X86::CVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4378             :   }
    4379           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4380           0 :     return fastEmitInst_r(X86::VCVTPD2PSrr, &X86::VR128RegClass, Op0, Op0IsKill);
    4381             :   }
    4382             :   return 0;
    4383             : }
    4384             : 
    4385             : unsigned fastEmit_X86ISD_VFPROUND_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4386           0 :   switch (VT.SimpleTy) {
    4387           0 :   case MVT::v2f64: return fastEmit_X86ISD_VFPROUND_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    4388             :   default: return 0;
    4389             :   }
    4390             : }
    4391             : 
    4392             : // FastEmit functions for X86ISD::VSEXT.
    4393             : 
    4394           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4395           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    4396           0 :     return fastEmitInst_r(X86::VPMOVSXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4397             :   }
    4398           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4399           0 :     return fastEmitInst_r(X86::VPMOVSXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4400             :   }
    4401             :   return 0;
    4402             : }
    4403             : 
    4404           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4405           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4406           0 :     return fastEmitInst_r(X86::VPMOVSXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4407             :   }
    4408           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4409           0 :     return fastEmitInst_r(X86::VPMOVSXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4410             :   }
    4411             :   return 0;
    4412             : }
    4413             : 
    4414             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    4415           0 :   if ((Subtarget->hasAVX512())) {
    4416           0 :     return fastEmitInst_r(X86::VPMOVSXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4417             :   }
    4418             :   return 0;
    4419             : }
    4420             : 
    4421           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4422           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4423           0 :     return fastEmitInst_r(X86::VPMOVSXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4424             :   }
    4425           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4426           0 :     return fastEmitInst_r(X86::VPMOVSXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4427             :   }
    4428             :   return 0;
    4429             : }
    4430             : 
    4431             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4432           0 :   if ((Subtarget->hasAVX512())) {
    4433           0 :     return fastEmitInst_r(X86::VPMOVSXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4434             :   }
    4435             :   return 0;
    4436             : }
    4437             : 
    4438           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4439           0 : switch (RetVT.SimpleTy) {
    4440           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    4441           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    4442           0 :   case MVT::v16i32: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    4443           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    4444           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    4445             :   default: return 0;
    4446             : }
    4447             : }
    4448             : 
    4449             : unsigned fastEmit_X86ISD_VSEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4450           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    4451             :     return 0;
    4452           0 :   if ((Subtarget->hasBWI())) {
    4453           0 :     return fastEmitInst_r(X86::VPMOVSXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4454             :   }
    4455             :   return 0;
    4456             : }
    4457             : 
    4458           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4459           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4460           0 :     return fastEmitInst_r(X86::VPMOVSXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4461             :   }
    4462           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4463           0 :     return fastEmitInst_r(X86::VPMOVSXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4464             :   }
    4465             :   return 0;
    4466             : }
    4467             : 
    4468           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    4469           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4470           0 :     return fastEmitInst_r(X86::VPMOVSXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4471             :   }
    4472           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4473           0 :     return fastEmitInst_r(X86::VPMOVSXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4474             :   }
    4475             :   return 0;
    4476             : }
    4477             : 
    4478             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    4479           0 :   if ((Subtarget->hasAVX512())) {
    4480           0 :     return fastEmitInst_r(X86::VPMOVSXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4481             :   }
    4482             :   return 0;
    4483             : }
    4484             : 
    4485           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4486           0 : switch (RetVT.SimpleTy) {
    4487           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    4488           0 :   case MVT::v4i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    4489           0 :   case MVT::v8i64: return fastEmit_X86ISD_VSEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    4490             :   default: return 0;
    4491             : }
    4492             : }
    4493             : 
    4494             : unsigned fastEmit_X86ISD_VSEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4495           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    4496             :     return 0;
    4497           0 :   if ((Subtarget->hasAVX512())) {
    4498           0 :     return fastEmitInst_r(X86::VPMOVSXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4499             :   }
    4500             :   return 0;
    4501             : }
    4502             : 
    4503           0 : unsigned fastEmit_X86ISD_VSEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4504           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    4505             :     return 0;
    4506           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    4507           0 :     return fastEmitInst_r(X86::VPMOVSXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    4508             :   }
    4509           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4510           0 :     return fastEmitInst_r(X86::VPMOVSXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4511             :   }
    4512             :   return 0;
    4513             : }
    4514             : 
    4515             : unsigned fastEmit_X86ISD_VSEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4516           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    4517             :     return 0;
    4518           0 :   if ((Subtarget->hasAVX512())) {
    4519           0 :     return fastEmitInst_r(X86::VPMOVSXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    4520             :   }
    4521             :   return 0;
    4522             : }
    4523             : 
    4524           0 : unsigned fastEmit_X86ISD_VSEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4525           0 :   switch (VT.SimpleTy) {
    4526           0 :   case MVT::v16i8: return fastEmit_X86ISD_VSEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    4527           0 :   case MVT::v32i8: return fastEmit_X86ISD_VSEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    4528           0 :   case MVT::v8i16: return fastEmit_X86ISD_VSEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4529           0 :   case MVT::v16i16: return fastEmit_X86ISD_VSEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4530           0 :   case MVT::v4i32: return fastEmit_X86ISD_VSEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4531           0 :   case MVT::v8i32: return fastEmit_X86ISD_VSEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4532             :   default: return 0;
    4533             :   }
    4534             : }
    4535             : 
    4536             : // FastEmit functions for X86ISD::VTRUNC.
    4537             : 
    4538           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4539           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4540             :     return 0;
    4541           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4542           0 :     return fastEmitInst_r(X86::VPMOVWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4543             :   }
    4544             :   return 0;
    4545             : }
    4546             : 
    4547           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4548           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4549           0 :     return fastEmitInst_r(X86::VPMOVDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4550             :   }
    4551             :   return 0;
    4552             : }
    4553             : 
    4554           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4555           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4556           0 :     return fastEmitInst_r(X86::VPMOVDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4557             :   }
    4558             :   return 0;
    4559             : }
    4560             : 
    4561           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4562           0 : switch (RetVT.SimpleTy) {
    4563           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4564           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4565             :   default: return 0;
    4566             : }
    4567             : }
    4568             : 
    4569           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4570           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4571             :     return 0;
    4572           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4573           0 :     return fastEmitInst_r(X86::VPMOVDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4574             :   }
    4575             :   return 0;
    4576             : }
    4577             : 
    4578           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4579           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4580           0 :     return fastEmitInst_r(X86::VPMOVQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4581             :   }
    4582             :   return 0;
    4583             : }
    4584             : 
    4585           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4586           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4587           0 :     return fastEmitInst_r(X86::VPMOVQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4588             :   }
    4589             :   return 0;
    4590             : }
    4591             : 
    4592           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4593           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4594           0 :     return fastEmitInst_r(X86::VPMOVQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4595             :   }
    4596             :   return 0;
    4597             : }
    4598             : 
    4599           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4600           0 : switch (RetVT.SimpleTy) {
    4601           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4602           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4603           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4604             :   default: return 0;
    4605             : }
    4606             : }
    4607             : 
    4608           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4609           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4610           0 :     return fastEmitInst_r(X86::VPMOVQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4611             :   }
    4612             :   return 0;
    4613             : }
    4614             : 
    4615           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4616           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4617           0 :     return fastEmitInst_r(X86::VPMOVQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4618             :   }
    4619             :   return 0;
    4620             : }
    4621             : 
    4622           0 : unsigned fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4623           0 : switch (RetVT.SimpleTy) {
    4624           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4625           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4626             :   default: return 0;
    4627             : }
    4628             : }
    4629             : 
    4630             : unsigned fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4631           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4632             :     return 0;
    4633           0 :   if ((Subtarget->hasAVX512())) {
    4634           0 :     return fastEmitInst_r(X86::VPMOVQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4635             :   }
    4636             :   return 0;
    4637             : }
    4638             : 
    4639           0 : unsigned fastEmit_X86ISD_VTRUNC_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4640           0 :   switch (VT.SimpleTy) {
    4641           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNC_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4642           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNC_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4643           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNC_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4644           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNC_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4645           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNC_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4646           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNC_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4647             :   default: return 0;
    4648             :   }
    4649             : }
    4650             : 
    4651             : // FastEmit functions for X86ISD::VTRUNCS.
    4652             : 
    4653           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4654           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4655             :     return 0;
    4656           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4657           0 :     return fastEmitInst_r(X86::VPMOVSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4658             :   }
    4659             :   return 0;
    4660             : }
    4661             : 
    4662           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4663           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4664             :     return 0;
    4665           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4666           0 :     return fastEmitInst_r(X86::VPMOVSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4667             :   }
    4668             :   return 0;
    4669             : }
    4670             : 
    4671             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4672           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4673             :     return 0;
    4674           0 :   if ((Subtarget->hasBWI())) {
    4675           0 :     return fastEmitInst_r(X86::VPMOVSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4676             :   }
    4677             :   return 0;
    4678             : }
    4679             : 
    4680           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4681           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4682           0 :     return fastEmitInst_r(X86::VPMOVSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4683             :   }
    4684             :   return 0;
    4685             : }
    4686             : 
    4687           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4688           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4689           0 :     return fastEmitInst_r(X86::VPMOVSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4690             :   }
    4691             :   return 0;
    4692             : }
    4693             : 
    4694           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4695           0 : switch (RetVT.SimpleTy) {
    4696           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4697           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4698             :   default: return 0;
    4699             : }
    4700             : }
    4701             : 
    4702           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4703           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4704           0 :     return fastEmitInst_r(X86::VPMOVSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4705             :   }
    4706             :   return 0;
    4707             : }
    4708             : 
    4709           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4710           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4711           0 :     return fastEmitInst_r(X86::VPMOVSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4712             :   }
    4713             :   return 0;
    4714             : }
    4715             : 
    4716           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4717           0 : switch (RetVT.SimpleTy) {
    4718           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4719           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4720             :   default: return 0;
    4721             : }
    4722             : }
    4723             : 
    4724             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4725           0 :   if ((Subtarget->hasAVX512())) {
    4726           0 :     return fastEmitInst_r(X86::VPMOVSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4727             :   }
    4728             :   return 0;
    4729             : }
    4730             : 
    4731             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4732           0 :   if ((Subtarget->hasAVX512())) {
    4733           0 :     return fastEmitInst_r(X86::VPMOVSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4734             :   }
    4735             :   return 0;
    4736             : }
    4737             : 
    4738           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4739           0 : switch (RetVT.SimpleTy) {
    4740           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4741           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4742             :   default: return 0;
    4743             : }
    4744             : }
    4745             : 
    4746           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4747           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4748           0 :     return fastEmitInst_r(X86::VPMOVSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4749             :   }
    4750             :   return 0;
    4751             : }
    4752             : 
    4753           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4754           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4755           0 :     return fastEmitInst_r(X86::VPMOVSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4756             :   }
    4757             :   return 0;
    4758             : }
    4759             : 
    4760           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4761           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4762           0 :     return fastEmitInst_r(X86::VPMOVSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4763             :   }
    4764             :   return 0;
    4765             : }
    4766             : 
    4767           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4768           0 : switch (RetVT.SimpleTy) {
    4769           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4770           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4771           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4772             :   default: return 0;
    4773             : }
    4774             : }
    4775             : 
    4776           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4777           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4778           0 :     return fastEmitInst_r(X86::VPMOVSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4779             :   }
    4780             :   return 0;
    4781             : }
    4782             : 
    4783           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4784           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4785           0 :     return fastEmitInst_r(X86::VPMOVSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4786             :   }
    4787             :   return 0;
    4788             : }
    4789             : 
    4790           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4791           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4792           0 :     return fastEmitInst_r(X86::VPMOVSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4793             :   }
    4794             :   return 0;
    4795             : }
    4796             : 
    4797           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4798           0 : switch (RetVT.SimpleTy) {
    4799           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    4800           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    4801           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    4802             :   default: return 0;
    4803             : }
    4804             : }
    4805             : 
    4806             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4807           0 :   if ((Subtarget->hasAVX512())) {
    4808           0 :     return fastEmitInst_r(X86::VPMOVSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4809             :   }
    4810             :   return 0;
    4811             : }
    4812             : 
    4813             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4814           0 :   if ((Subtarget->hasAVX512())) {
    4815           0 :     return fastEmitInst_r(X86::VPMOVSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4816             :   }
    4817             :   return 0;
    4818             : }
    4819             : 
    4820             : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    4821           0 :   if ((Subtarget->hasAVX512())) {
    4822           0 :     return fastEmitInst_r(X86::VPMOVSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4823             :   }
    4824             :   return 0;
    4825             : }
    4826             : 
    4827           0 : unsigned fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4828           0 : switch (RetVT.SimpleTy) {
    4829           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    4830           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    4831           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    4832             :   default: return 0;
    4833             : }
    4834             : }
    4835             : 
    4836           0 : unsigned fastEmit_X86ISD_VTRUNCS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4837           0 :   switch (VT.SimpleTy) {
    4838           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    4839           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    4840           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    4841           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    4842           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    4843           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    4844           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    4845           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    4846           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    4847             :   default: return 0;
    4848             :   }
    4849             : }
    4850             : 
    4851             : // FastEmit functions for X86ISD::VTRUNCUS.
    4852             : 
    4853           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4854           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4855             :     return 0;
    4856           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4857           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4858             :   }
    4859             :   return 0;
    4860             : }
    4861             : 
    4862           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4863           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    4864             :     return 0;
    4865           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    4866           0 :     return fastEmitInst_r(X86::VPMOVUSWBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4867             :   }
    4868             :   return 0;
    4869             : }
    4870             : 
    4871             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4872           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    4873             :     return 0;
    4874           0 :   if ((Subtarget->hasBWI())) {
    4875           0 :     return fastEmitInst_r(X86::VPMOVUSWBZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4876             :   }
    4877             :   return 0;
    4878             : }
    4879             : 
    4880           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4881           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4882           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4883             :   }
    4884             :   return 0;
    4885             : }
    4886             : 
    4887           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4888           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4889           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4890             :   }
    4891             :   return 0;
    4892             : }
    4893             : 
    4894           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4895           0 : switch (RetVT.SimpleTy) {
    4896           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v16i8_r(Op0, Op0IsKill);
    4897           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_MVT_v8i16_r(Op0, Op0IsKill);
    4898             :   default: return 0;
    4899             : }
    4900             : }
    4901             : 
    4902           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4903           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4904           0 :     return fastEmitInst_r(X86::VPMOVUSDBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4905             :   }
    4906             :   return 0;
    4907             : }
    4908             : 
    4909           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4910           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4911           0 :     return fastEmitInst_r(X86::VPMOVUSDWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4912             :   }
    4913             :   return 0;
    4914             : }
    4915             : 
    4916           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4917           0 : switch (RetVT.SimpleTy) {
    4918           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v16i8_r(Op0, Op0IsKill);
    4919           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_MVT_v8i16_r(Op0, Op0IsKill);
    4920             :   default: return 0;
    4921             : }
    4922             : }
    4923             : 
    4924             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4925           0 :   if ((Subtarget->hasAVX512())) {
    4926           0 :     return fastEmitInst_r(X86::VPMOVUSDBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4927             :   }
    4928             :   return 0;
    4929             : }
    4930             : 
    4931             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    4932           0 :   if ((Subtarget->hasAVX512())) {
    4933           0 :     return fastEmitInst_r(X86::VPMOVUSDWZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    4934             :   }
    4935             :   return 0;
    4936             : }
    4937             : 
    4938           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4939           0 : switch (RetVT.SimpleTy) {
    4940           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i8_r(Op0, Op0IsKill);
    4941           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_MVT_v16i16_r(Op0, Op0IsKill);
    4942             :   default: return 0;
    4943             : }
    4944             : }
    4945             : 
    4946           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4947           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4948           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4949             :   }
    4950             :   return 0;
    4951             : }
    4952             : 
    4953           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4954           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4955           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4956             :   }
    4957             :   return 0;
    4958             : }
    4959             : 
    4960           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4961           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4962           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4963             :   }
    4964             :   return 0;
    4965             : }
    4966             : 
    4967           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4968           0 : switch (RetVT.SimpleTy) {
    4969           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v16i8_r(Op0, Op0IsKill);
    4970           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v8i16_r(Op0, Op0IsKill);
    4971           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_MVT_v4i32_r(Op0, Op0IsKill);
    4972             :   default: return 0;
    4973             : }
    4974             : }
    4975             : 
    4976           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    4977           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4978           0 :     return fastEmitInst_r(X86::VPMOVUSQBZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4979             :   }
    4980             :   return 0;
    4981             : }
    4982             : 
    4983           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    4984           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4985           0 :     return fastEmitInst_r(X86::VPMOVUSQWZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4986             :   }
    4987             :   return 0;
    4988             : }
    4989             : 
    4990           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(unsigned Op0, bool Op0IsKill) {
    4991           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    4992           0 :     return fastEmitInst_r(X86::VPMOVUSQDZ256rr, &X86::VR128XRegClass, Op0, Op0IsKill);
    4993             :   }
    4994             :   return 0;
    4995             : }
    4996             : 
    4997           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    4998           0 : switch (RetVT.SimpleTy) {
    4999           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v16i8_r(Op0, Op0IsKill);
    5000           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v8i16_r(Op0, Op0IsKill);
    5001           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_MVT_v4i32_r(Op0, Op0IsKill);
    5002             :   default: return 0;
    5003             : }
    5004             : }
    5005             : 
    5006             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(unsigned Op0, bool Op0IsKill) {
    5007           0 :   if ((Subtarget->hasAVX512())) {
    5008           0 :     return fastEmitInst_r(X86::VPMOVUSQBZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5009             :   }
    5010             :   return 0;
    5011             : }
    5012             : 
    5013             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(unsigned Op0, bool Op0IsKill) {
    5014           0 :   if ((Subtarget->hasAVX512())) {
    5015           0 :     return fastEmitInst_r(X86::VPMOVUSQWZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5016             :   }
    5017             :   return 0;
    5018             : }
    5019             : 
    5020             : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5021           0 :   if ((Subtarget->hasAVX512())) {
    5022           0 :     return fastEmitInst_r(X86::VPMOVUSQDZrr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5023             :   }
    5024             :   return 0;
    5025             : }
    5026             : 
    5027           0 : unsigned fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5028           0 : switch (RetVT.SimpleTy) {
    5029           0 :   case MVT::v16i8: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v16i8_r(Op0, Op0IsKill);
    5030           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i16_r(Op0, Op0IsKill);
    5031           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_MVT_v8i32_r(Op0, Op0IsKill);
    5032             :   default: return 0;
    5033             : }
    5034             : }
    5035             : 
    5036           0 : unsigned fastEmit_X86ISD_VTRUNCUS_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5037           0 :   switch (VT.SimpleTy) {
    5038           0 :   case MVT::v8i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5039           0 :   case MVT::v16i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5040           0 :   case MVT::v32i16: return fastEmit_X86ISD_VTRUNCUS_MVT_v32i16_r(RetVT, Op0, Op0IsKill);
    5041           0 :   case MVT::v4i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5042           0 :   case MVT::v8i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5043           0 :   case MVT::v16i32: return fastEmit_X86ISD_VTRUNCUS_MVT_v16i32_r(RetVT, Op0, Op0IsKill);
    5044           0 :   case MVT::v2i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5045           0 :   case MVT::v4i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v4i64_r(RetVT, Op0, Op0IsKill);
    5046           0 :   case MVT::v8i64: return fastEmit_X86ISD_VTRUNCUS_MVT_v8i64_r(RetVT, Op0, Op0IsKill);
    5047             :   default: return 0;
    5048             :   }
    5049             : }
    5050             : 
    5051             : // FastEmit functions for X86ISD::VZEXT.
    5052             : 
    5053           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(unsigned Op0, bool Op0IsKill) {
    5054           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5055           0 :     return fastEmitInst_r(X86::VPMOVZXBWYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5056             :   }
    5057           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5058           0 :     return fastEmitInst_r(X86::VPMOVZXBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5059             :   }
    5060             :   return 0;
    5061             : }
    5062             : 
    5063           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5064           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5065           0 :     return fastEmitInst_r(X86::VPMOVZXBDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5066             :   }
    5067           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5068           0 :     return fastEmitInst_r(X86::VPMOVZXBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5069             :   }
    5070             :   return 0;
    5071             : }
    5072             : 
    5073             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(unsigned Op0, bool Op0IsKill) {
    5074           0 :   if ((Subtarget->hasAVX512())) {
    5075           0 :     return fastEmitInst_r(X86::VPMOVZXBDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5076             :   }
    5077             :   return 0;
    5078             : }
    5079             : 
    5080           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5081           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5082           0 :     return fastEmitInst_r(X86::VPMOVZXBQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5083             :   }
    5084           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5085           0 :     return fastEmitInst_r(X86::VPMOVZXBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5086             :   }
    5087             :   return 0;
    5088             : }
    5089             : 
    5090             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5091           0 :   if ((Subtarget->hasAVX512())) {
    5092           0 :     return fastEmitInst_r(X86::VPMOVZXBQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5093             :   }
    5094             :   return 0;
    5095             : }
    5096             : 
    5097           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5098           0 : switch (RetVT.SimpleTy) {
    5099           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i16_r(Op0, Op0IsKill);
    5100           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i32_r(Op0, Op0IsKill);
    5101           0 :   case MVT::v16i32: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v16i32_r(Op0, Op0IsKill);
    5102           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v4i64_r(Op0, Op0IsKill);
    5103           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v16i8_MVT_v8i64_r(Op0, Op0IsKill);
    5104             :   default: return 0;
    5105             : }
    5106             : }
    5107             : 
    5108             : unsigned fastEmit_X86ISD_VZEXT_MVT_v32i8_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5109           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5110             :     return 0;
    5111           0 :   if ((Subtarget->hasBWI())) {
    5112           0 :     return fastEmitInst_r(X86::VPMOVZXBWZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5113             :   }
    5114             :   return 0;
    5115             : }
    5116             : 
    5117           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(unsigned Op0, bool Op0IsKill) {
    5118           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5119           0 :     return fastEmitInst_r(X86::VPMOVZXWDYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5120             :   }
    5121           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5122           0 :     return fastEmitInst_r(X86::VPMOVZXWDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5123             :   }
    5124             :   return 0;
    5125             : }
    5126             : 
    5127           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(unsigned Op0, bool Op0IsKill) {
    5128           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5129           0 :     return fastEmitInst_r(X86::VPMOVZXWQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5130             :   }
    5131           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5132           0 :     return fastEmitInst_r(X86::VPMOVZXWQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5133             :   }
    5134             :   return 0;
    5135             : }
    5136             : 
    5137             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(unsigned Op0, bool Op0IsKill) {
    5138           0 :   if ((Subtarget->hasAVX512())) {
    5139           0 :     return fastEmitInst_r(X86::VPMOVZXWQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5140             :   }
    5141             :   return 0;
    5142             : }
    5143             : 
    5144           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5145           0 : switch (RetVT.SimpleTy) {
    5146           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i32_r(Op0, Op0IsKill);
    5147           0 :   case MVT::v4i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v4i64_r(Op0, Op0IsKill);
    5148           0 :   case MVT::v8i64: return fastEmit_X86ISD_VZEXT_MVT_v8i16_MVT_v8i64_r(Op0, Op0IsKill);
    5149             :   default: return 0;
    5150             : }
    5151             : }
    5152             : 
    5153             : unsigned fastEmit_X86ISD_VZEXT_MVT_v16i16_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5154           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    5155             :     return 0;
    5156           0 :   if ((Subtarget->hasAVX512())) {
    5157           0 :     return fastEmitInst_r(X86::VPMOVZXWDZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5158             :   }
    5159             :   return 0;
    5160             : }
    5161             : 
    5162           0 : unsigned fastEmit_X86ISD_VZEXT_MVT_v4i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5163           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    5164             :     return 0;
    5165           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5166           0 :     return fastEmitInst_r(X86::VPMOVZXDQYrr, &X86::VR256RegClass, Op0, Op0IsKill);
    5167             :   }
    5168           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5169           0 :     return fastEmitInst_r(X86::VPMOVZXDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill);
    5170             :   }
    5171             :   return 0;
    5172             : }
    5173             : 
    5174             : unsigned fastEmit_X86ISD_VZEXT_MVT_v8i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5175           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5176             :     return 0;
    5177           0 :   if ((Subtarget->hasAVX512())) {
    5178           0 :     return fastEmitInst_r(X86::VPMOVZXDQZrr, &X86::VR512RegClass, Op0, Op0IsKill);
    5179             :   }
    5180             :   return 0;
    5181             : }
    5182             : 
    5183           0 : unsigned fastEmit_X86ISD_VZEXT_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5184           0 :   switch (VT.SimpleTy) {
    5185           0 :   case MVT::v16i8: return fastEmit_X86ISD_VZEXT_MVT_v16i8_r(RetVT, Op0, Op0IsKill);
    5186           0 :   case MVT::v32i8: return fastEmit_X86ISD_VZEXT_MVT_v32i8_r(RetVT, Op0, Op0IsKill);
    5187           0 :   case MVT::v8i16: return fastEmit_X86ISD_VZEXT_MVT_v8i16_r(RetVT, Op0, Op0IsKill);
    5188           0 :   case MVT::v16i16: return fastEmit_X86ISD_VZEXT_MVT_v16i16_r(RetVT, Op0, Op0IsKill);
    5189           0 :   case MVT::v4i32: return fastEmit_X86ISD_VZEXT_MVT_v4i32_r(RetVT, Op0, Op0IsKill);
    5190           0 :   case MVT::v8i32: return fastEmit_X86ISD_VZEXT_MVT_v8i32_r(RetVT, Op0, Op0IsKill);
    5191             :   default: return 0;
    5192             :   }
    5193             : }
    5194             : 
    5195             : // FastEmit functions for X86ISD::VZEXT_MOVL.
    5196             : 
    5197           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5198           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    5199             :     return 0;
    5200           0 :   if ((Subtarget->hasAVX512())) {
    5201           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5202             :   }
    5203           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5204           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5205             :   }
    5206           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5207           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5208             :   }
    5209             :   return 0;
    5210             : }
    5211             : 
    5212           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5213           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    5214             :     return 0;
    5215           0 :   if ((Subtarget->hasAVX512())) {
    5216           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIZrr, &X86::VR128XRegClass, Op0, Op0IsKill);
    5217             :   }
    5218           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5219           0 :     return fastEmitInst_r(X86::MOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5220             :   }
    5221           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5222           0 :     return fastEmitInst_r(X86::VMOVZPQILo2PQIrr, &X86::VR128RegClass, Op0, Op0IsKill);
    5223             :   }
    5224             :   return 0;
    5225             : }
    5226             : 
    5227           0 : unsigned fastEmit_X86ISD_VZEXT_MOVL_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5228           0 :   switch (VT.SimpleTy) {
    5229           0 :   case MVT::v2i64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2i64_r(RetVT, Op0, Op0IsKill);
    5230           0 :   case MVT::v2f64: return fastEmit_X86ISD_VZEXT_MOVL_MVT_v2f64_r(RetVT, Op0, Op0IsKill);
    5231             :   default: return 0;
    5232             :   }
    5233             : }
    5234             : 
    5235             : // FastEmit functions for X86ISD::WIN_ALLOCA.
    5236             : 
    5237           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5238           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5239             :     return 0;
    5240           0 :   if ((!Subtarget->isTarget64BitLP64())) {
    5241           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_32, &X86::GR32RegClass, Op0, Op0IsKill);
    5242             :   }
    5243             :   return 0;
    5244             : }
    5245             : 
    5246             : unsigned fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5247           0 :   if (RetVT.SimpleTy != MVT::isVoid)
    5248             :     return 0;
    5249           0 :   if ((Subtarget->is64Bit())) {
    5250           0 :     return fastEmitInst_r(X86::WIN_ALLOCA_64, &X86::GR64RegClass, Op0, Op0IsKill);
    5251             :   }
    5252             :   return 0;
    5253             : }
    5254             : 
    5255           0 : unsigned fastEmit_X86ISD_WIN_ALLOCA_r(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill) {
    5256           0 :   switch (VT.SimpleTy) {
    5257           0 :   case MVT::i32: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i32_r(RetVT, Op0, Op0IsKill);
    5258           0 :   case MVT::i64: return fastEmit_X86ISD_WIN_ALLOCA_MVT_i64_r(RetVT, Op0, Op0IsKill);
    5259             :   default: return 0;
    5260             :   }
    5261             : }
    5262             : 
    5263             : // Top-level FastEmit function.
    5264             : 
    5265        5279 : unsigned fastEmit_r(MVT VT, MVT RetVT, unsigned Opcode, unsigned Op0, bool Op0IsKill) override {
    5266        5279 :   switch (Opcode) {
    5267           0 :   case ISD::ABS: return fastEmit_ISD_ABS_r(VT, RetVT, Op0, Op0IsKill);
    5268          18 :   case ISD::ANY_EXTEND: return fastEmit_ISD_ANY_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5269        1967 :   case ISD::BITCAST: return fastEmit_ISD_BITCAST_r(VT, RetVT, Op0, Op0IsKill);
    5270           0 :   case ISD::BRIND: return fastEmit_ISD_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    5271           0 :   case ISD::BSWAP: return fastEmit_ISD_BSWAP_r(VT, RetVT, Op0, Op0IsKill);
    5272           0 :   case ISD::CTLZ: return fastEmit_ISD_CTLZ_r(VT, RetVT, Op0, Op0IsKill);
    5273           0 :   case ISD::CTPOP: return fastEmit_ISD_CTPOP_r(VT, RetVT, Op0, Op0IsKill);
    5274           0 :   case ISD::CTTZ: return fastEmit_ISD_CTTZ_r(VT, RetVT, Op0, Op0IsKill);
    5275           0 :   case ISD::CTTZ_ZERO_UNDEF: return fastEmit_ISD_CTTZ_ZERO_UNDEF_r(VT, RetVT, Op0, Op0IsKill);
    5276           0 :   case ISD::FABS: return fastEmit_ISD_FABS_r(VT, RetVT, Op0, Op0IsKill);
    5277           0 :   case ISD::FCOS: return fastEmit_ISD_FCOS_r(VT, RetVT, Op0, Op0IsKill);
    5278           2 :   case ISD::FNEG: return fastEmit_ISD_FNEG_r(VT, RetVT, Op0, Op0IsKill);
    5279           0 :   case ISD::FP_EXTEND: return fastEmit_ISD_FP_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5280           0 :   case ISD::FP_ROUND: return fastEmit_ISD_FP_ROUND_r(VT, RetVT, Op0, Op0IsKill);
    5281           0 :   case ISD::FP_TO_SINT: return fastEmit_ISD_FP_TO_SINT_r(VT, RetVT, Op0, Op0IsKill);
    5282           0 :   case ISD::FP_TO_UINT: return fastEmit_ISD_FP_TO_UINT_r(VT, RetVT, Op0, Op0IsKill);
    5283           0 :   case ISD::FSIN: return fastEmit_ISD_FSIN_r(VT, RetVT, Op0, Op0IsKill);
    5284           0 :   case ISD::FSQRT: return fastEmit_ISD_FSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5285           1 :   case ISD::SCALAR_TO_VECTOR: return fastEmit_ISD_SCALAR_TO_VECTOR_r(VT, RetVT, Op0, Op0IsKill);
    5286        1361 :   case ISD::SIGN_EXTEND: return fastEmit_ISD_SIGN_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5287           0 :   case ISD::SIGN_EXTEND_VECTOR_INREG: return fastEmit_ISD_SIGN_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5288          44 :   case ISD::SINT_TO_FP: return fastEmit_ISD_SINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5289         263 :   case ISD::TRUNCATE: return fastEmit_ISD_TRUNCATE_r(VT, RetVT, Op0, Op0IsKill);
    5290           0 :   case ISD::UINT_TO_FP: return fastEmit_ISD_UINT_TO_FP_r(VT, RetVT, Op0, Op0IsKill);
    5291        1623 :   case ISD::ZERO_EXTEND: return fastEmit_ISD_ZERO_EXTEND_r(VT, RetVT, Op0, Op0IsKill);
    5292           0 :   case ISD::ZERO_EXTEND_VECTOR_INREG: return fastEmit_ISD_ZERO_EXTEND_VECTOR_INREG_r(VT, RetVT, Op0, Op0IsKill);
    5293           0 :   case X86ISD::CALL: return fastEmit_X86ISD_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5294           0 :   case X86ISD::COMPRESS: return fastEmit_X86ISD_COMPRESS_r(VT, RetVT, Op0, Op0IsKill);
    5295           0 :   case X86ISD::CONFLICT: return fastEmit_X86ISD_CONFLICT_r(VT, RetVT, Op0, Op0IsKill);
    5296           0 :   case X86ISD::CVTP2SI: return fastEmit_X86ISD_CVTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5297           0 :   case X86ISD::CVTP2UI: return fastEmit_X86ISD_CVTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5298           0 :   case X86ISD::CVTPH2PS: return fastEmit_X86ISD_CVTPH2PS_r(VT, RetVT, Op0, Op0IsKill);
    5299           0 :   case X86ISD::CVTSI2P: return fastEmit_X86ISD_CVTSI2P_r(VT, RetVT, Op0, Op0IsKill);
    5300           0 :   case X86ISD::CVTTP2SI: return fastEmit_X86ISD_CVTTP2SI_r(VT, RetVT, Op0, Op0IsKill);
    5301           0 :   case X86ISD::CVTTP2UI: return fastEmit_X86ISD_CVTTP2UI_r(VT, RetVT, Op0, Op0IsKill);
    5302           0 :   case X86ISD::CVTUI2P: return fastEmit_X86ISD_CVTUI2P_r(VT, RetVT, Op0, Op0IsKill);
    5303           0 :   case X86ISD::EH_RETURN: return fastEmit_X86ISD_EH_RETURN_r(VT, RetVT, Op0, Op0IsKill);
    5304           0 :   case X86ISD::EXPAND: return fastEmit_X86ISD_EXPAND_r(VT, RetVT, Op0, Op0IsKill);
    5305           0 :   case X86ISD::FRCP: return fastEmit_X86ISD_FRCP_r(VT, RetVT, Op0, Op0IsKill);
    5306           0 :   case X86ISD::FRSQRT: return fastEmit_X86ISD_FRSQRT_r(VT, RetVT, Op0, Op0IsKill);
    5307           0 :   case X86ISD::MMX_MOVD2W: return fastEmit_X86ISD_MMX_MOVD2W_r(VT, RetVT, Op0, Op0IsKill);
    5308           0 :   case X86ISD::MMX_MOVW2D: return fastEmit_X86ISD_MMX_MOVW2D_r(VT, RetVT, Op0, Op0IsKill);
    5309           0 :   case X86ISD::MOVDDUP: return fastEmit_X86ISD_MOVDDUP_r(VT, RetVT, Op0, Op0IsKill);
    5310           0 :   case X86ISD::MOVDQ2Q: return fastEmit_X86ISD_MOVDQ2Q_r(VT, RetVT, Op0, Op0IsKill);
    5311           0 :   case X86ISD::MOVMSK: return fastEmit_X86ISD_MOVMSK_r(VT, RetVT, Op0, Op0IsKill);
    5312           0 :   case X86ISD::MOVSHDUP: return fastEmit_X86ISD_MOVSHDUP_r(VT, RetVT, Op0, Op0IsKill);
    5313           0 :   case X86ISD::MOVSLDUP: return fastEmit_X86ISD_MOVSLDUP_r(VT, RetVT, Op0, Op0IsKill);
    5314           0 :   case X86ISD::NT_BRIND: return fastEmit_X86ISD_NT_BRIND_r(VT, RetVT, Op0, Op0IsKill);
    5315           0 :   case X86ISD::NT_CALL: return fastEmit_X86ISD_NT_CALL_r(VT, RetVT, Op0, Op0IsKill);
    5316           0 :   case X86ISD::PHMINPOS: return fastEmit_X86ISD_PHMINPOS_r(VT, RetVT, Op0, Op0IsKill);
    5317           0 :   case X86ISD::RCP14: return fastEmit_X86ISD_RCP14_r(VT, RetVT, Op0, Op0IsKill);
    5318           0 :   case X86ISD::RSQRT14: return fastEmit_X86ISD_RSQRT14_r(VT, RetVT, Op0, Op0IsKill);
    5319           0 :   case X86ISD::SEG_ALLOCA: return fastEmit_X86ISD_SEG_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5320           0 :   case X86ISD::VBROADCAST: return fastEmit_X86ISD_VBROADCAST_r(VT, RetVT, Op0, Op0IsKill);
    5321           0 :   case X86ISD::VBROADCASTM: return fastEmit_X86ISD_VBROADCASTM_r(VT, RetVT, Op0, Op0IsKill);
    5322           0 :   case X86ISD::VFPEXT: return fastEmit_X86ISD_VFPEXT_r(VT, RetVT, Op0, Op0IsKill);
    5323           0 :   case X86ISD::VFPROUND: return fastEmit_X86ISD_VFPROUND_r(VT, RetVT, Op0, Op0IsKill);
    5324           0 :   case X86ISD::VSEXT: return fastEmit_X86ISD_VSEXT_r(VT, RetVT, Op0, Op0IsKill);
    5325           0 :   case X86ISD::VTRUNC: return fastEmit_X86ISD_VTRUNC_r(VT, RetVT, Op0, Op0IsKill);
    5326           0 :   case X86ISD::VTRUNCS: return fastEmit_X86ISD_VTRUNCS_r(VT, RetVT, Op0, Op0IsKill);
    5327           0 :   case X86ISD::VTRUNCUS: return fastEmit_X86ISD_VTRUNCUS_r(VT, RetVT, Op0, Op0IsKill);
    5328           0 :   case X86ISD::VZEXT: return fastEmit_X86ISD_VZEXT_r(VT, RetVT, Op0, Op0IsKill);
    5329           0 :   case X86ISD::VZEXT_MOVL: return fastEmit_X86ISD_VZEXT_MOVL_r(VT, RetVT, Op0, Op0IsKill);
    5330           0 :   case X86ISD::WIN_ALLOCA: return fastEmit_X86ISD_WIN_ALLOCA_r(VT, RetVT, Op0, Op0IsKill);
    5331             :   default: return 0;
    5332             :   }
    5333             : }
    5334             : 
    5335             : // FastEmit functions for ISD::ADD.
    5336             : 
    5337             : unsigned fastEmit_ISD_ADD_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5338           1 :   if (RetVT.SimpleTy != MVT::i8)
    5339             :     return 0;
    5340           1 :   return fastEmitInst_rr(X86::ADD8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5341             : }
    5342             : 
    5343             : unsigned fastEmit_ISD_ADD_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5344           1 :   if (RetVT.SimpleTy != MVT::i16)
    5345             :     return 0;
    5346           1 :   return fastEmitInst_rr(X86::ADD16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5347             : }
    5348             : 
    5349             : unsigned fastEmit_ISD_ADD_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5350        1085 :   if (RetVT.SimpleTy != MVT::i32)
    5351             :     return 0;
    5352        1085 :   return fastEmitInst_rr(X86::ADD32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5353             : }
    5354             : 
    5355             : unsigned fastEmit_ISD_ADD_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5356        1166 :   if (RetVT.SimpleTy != MVT::i64)
    5357             :     return 0;
    5358        1166 :   return fastEmitInst_rr(X86::ADD64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5359             : }
    5360             : 
    5361           2 : unsigned fastEmit_ISD_ADD_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5362           2 :   if (RetVT.SimpleTy != MVT::v16i8)
    5363             :     return 0;
    5364           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5365           0 :     return fastEmitInst_rr(X86::VPADDBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5366             :   }
    5367           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5368           2 :     return fastEmitInst_rr(X86::PADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5369             :   }
    5370           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5371           0 :     return fastEmitInst_rr(X86::VPADDBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5372             :   }
    5373             :   return 0;
    5374             : }
    5375             : 
    5376           2 : unsigned fastEmit_ISD_ADD_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5377           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    5378             :     return 0;
    5379           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5380           0 :     return fastEmitInst_rr(X86::VPADDBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5381             :   }
    5382           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5383           2 :     return fastEmitInst_rr(X86::VPADDBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5384             :   }
    5385             :   return 0;
    5386             : }
    5387             : 
    5388             : unsigned fastEmit_ISD_ADD_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5389           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    5390             :     return 0;
    5391           0 :   if ((Subtarget->hasBWI())) {
    5392           0 :     return fastEmitInst_rr(X86::VPADDBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5393             :   }
    5394             :   return 0;
    5395             : }
    5396             : 
    5397           2 : unsigned fastEmit_ISD_ADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5398           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    5399             :     return 0;
    5400           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5401           0 :     return fastEmitInst_rr(X86::VPADDWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5402             :   }
    5403           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5404           2 :     return fastEmitInst_rr(X86::PADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5405             :   }
    5406           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5407           0 :     return fastEmitInst_rr(X86::VPADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5408             :   }
    5409             :   return 0;
    5410             : }
    5411             : 
    5412           2 : unsigned fastEmit_ISD_ADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5413           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    5414             :     return 0;
    5415           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    5416           0 :     return fastEmitInst_rr(X86::VPADDWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5417             :   }
    5418           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    5419           2 :     return fastEmitInst_rr(X86::VPADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5420             :   }
    5421             :   return 0;
    5422             : }
    5423             : 
    5424             : unsigned fastEmit_ISD_ADD_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5425           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    5426             :     return 0;
    5427           0 :   if ((Subtarget->hasBWI())) {
    5428           0 :     return fastEmitInst_rr(X86::VPADDWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5429             :   }
    5430             :   return 0;
    5431             : }
    5432             : 
    5433          18 : unsigned fastEmit_ISD_ADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5434          18 :   if (RetVT.SimpleTy != MVT::v4i32)
    5435             :     return 0;
    5436          36 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5437           2 :     return fastEmitInst_rr(X86::VPADDDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5438             :   }
    5439          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5440           6 :     return fastEmitInst_rr(X86::PADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5441             :   }
    5442          20 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5443          10 :     return fastEmitInst_rr(X86::VPADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5444             :   }
    5445             :   return 0;
    5446             : }
    5447             : 
    5448           3 : unsigned fastEmit_ISD_ADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5449           3 :   if (RetVT.SimpleTy != MVT::v8i32)
    5450             :     return 0;
    5451           6 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5452           1 :     return fastEmitInst_rr(X86::VPADDDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5453             :   }
    5454           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5455           2 :     return fastEmitInst_rr(X86::VPADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5456             :   }
    5457             :   return 0;
    5458             : }
    5459             : 
    5460             : unsigned fastEmit_ISD_ADD_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5461           2 :   if (RetVT.SimpleTy != MVT::v16i32)
    5462             :     return 0;
    5463           4 :   if ((Subtarget->hasAVX512())) {
    5464           2 :     return fastEmitInst_rr(X86::VPADDDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5465             :   }
    5466             :   return 0;
    5467             : }
    5468             : 
    5469           2 : unsigned fastEmit_ISD_ADD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5470           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    5471             :     return 0;
    5472           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5473           0 :     return fastEmitInst_rr(X86::VPADDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5474             :   }
    5475           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5476           2 :     return fastEmitInst_rr(X86::PADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5477             :   }
    5478           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5479           0 :     return fastEmitInst_rr(X86::VPADDQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5480             :   }
    5481             :   return 0;
    5482             : }
    5483             : 
    5484           4 : unsigned fastEmit_ISD_ADD_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5485           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    5486             :     return 0;
    5487           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5488           2 :     return fastEmitInst_rr(X86::VPADDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5489             :   }
    5490           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5491           2 :     return fastEmitInst_rr(X86::VPADDQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5492             :   }
    5493             :   return 0;
    5494             : }
    5495             : 
    5496             : unsigned fastEmit_ISD_ADD_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5497           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5498             :     return 0;
    5499           0 :   if ((Subtarget->hasAVX512())) {
    5500           0 :     return fastEmitInst_rr(X86::VPADDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5501             :   }
    5502             :   return 0;
    5503             : }
    5504             : 
    5505        2290 : unsigned fastEmit_ISD_ADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5506        2290 :   switch (VT.SimpleTy) {
    5507           1 :   case MVT::i8: return fastEmit_ISD_ADD_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5508           1 :   case MVT::i16: return fastEmit_ISD_ADD_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5509        1085 :   case MVT::i32: return fastEmit_ISD_ADD_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5510        1166 :   case MVT::i64: return fastEmit_ISD_ADD_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5511           2 :   case MVT::v16i8: return fastEmit_ISD_ADD_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5512           2 :   case MVT::v32i8: return fastEmit_ISD_ADD_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5513           0 :   case MVT::v64i8: return fastEmit_ISD_ADD_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5514           2 :   case MVT::v8i16: return fastEmit_ISD_ADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5515           2 :   case MVT::v16i16: return fastEmit_ISD_ADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5516           0 :   case MVT::v32i16: return fastEmit_ISD_ADD_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5517          18 :   case MVT::v4i32: return fastEmit_ISD_ADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5518           3 :   case MVT::v8i32: return fastEmit_ISD_ADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5519           2 :   case MVT::v16i32: return fastEmit_ISD_ADD_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5520           2 :   case MVT::v2i64: return fastEmit_ISD_ADD_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5521           4 :   case MVT::v4i64: return fastEmit_ISD_ADD_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5522           0 :   case MVT::v8i64: return fastEmit_ISD_ADD_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5523             :   default: return 0;
    5524             :   }
    5525             : }
    5526             : 
    5527             : // FastEmit functions for ISD::AND.
    5528             : 
    5529             : unsigned fastEmit_ISD_AND_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5530           6 :   if (RetVT.SimpleTy != MVT::i8)
    5531             :     return 0;
    5532           6 :   return fastEmitInst_rr(X86::AND8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5533             : }
    5534             : 
    5535             : unsigned fastEmit_ISD_AND_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5536           0 :   if (RetVT.SimpleTy != MVT::i16)
    5537             :     return 0;
    5538           0 :   return fastEmitInst_rr(X86::AND16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5539             : }
    5540             : 
    5541             : unsigned fastEmit_ISD_AND_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5542         148 :   if (RetVT.SimpleTy != MVT::i32)
    5543             :     return 0;
    5544         148 :   return fastEmitInst_rr(X86::AND32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5545             : }
    5546             : 
    5547             : unsigned fastEmit_ISD_AND_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5548          16 :   if (RetVT.SimpleTy != MVT::i64)
    5549             :     return 0;
    5550          16 :   return fastEmitInst_rr(X86::AND64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5551             : }
    5552             : 
    5553             : unsigned fastEmit_ISD_AND_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5554           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    5555             :     return 0;
    5556           0 :   if ((Subtarget->hasDQI())) {
    5557           0 :     return fastEmitInst_rr(X86::KANDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5558             :   }
    5559             :   return 0;
    5560             : }
    5561             : 
    5562             : unsigned fastEmit_ISD_AND_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5563           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    5564             :     return 0;
    5565           0 :   if ((Subtarget->hasAVX512())) {
    5566           0 :     return fastEmitInst_rr(X86::KANDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5567             :   }
    5568             :   return 0;
    5569             : }
    5570             : 
    5571             : unsigned fastEmit_ISD_AND_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5572           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    5573             :     return 0;
    5574           0 :   if ((Subtarget->hasBWI())) {
    5575           0 :     return fastEmitInst_rr(X86::KANDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5576             :   }
    5577             :   return 0;
    5578             : }
    5579             : 
    5580             : unsigned fastEmit_ISD_AND_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5581           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    5582             :     return 0;
    5583           0 :   if ((Subtarget->hasBWI())) {
    5584           0 :     return fastEmitInst_rr(X86::KANDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5585             :   }
    5586             :   return 0;
    5587             : }
    5588             : 
    5589           8 : unsigned fastEmit_ISD_AND_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5590           8 :   if (RetVT.SimpleTy != MVT::v2i64)
    5591             :     return 0;
    5592          16 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5593           0 :     return fastEmitInst_rr(X86::VPANDQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5594             :   }
    5595           8 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5596           4 :     return fastEmitInst_rr(X86::PANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5597             :   }
    5598           8 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5599           4 :     return fastEmitInst_rr(X86::VPANDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5600             :   }
    5601             :   return 0;
    5602             : }
    5603             : 
    5604          12 : unsigned fastEmit_ISD_AND_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5605          12 :   if (RetVT.SimpleTy != MVT::v4i64)
    5606             :     return 0;
    5607          24 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    5608           8 :     return fastEmitInst_rr(X86::VANDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5609             :   }
    5610           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5611           0 :     return fastEmitInst_rr(X86::VPANDQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5612             :   }
    5613           8 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    5614           4 :     return fastEmitInst_rr(X86::VPANDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5615             :   }
    5616             :   return 0;
    5617             : }
    5618             : 
    5619             : unsigned fastEmit_ISD_AND_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5620           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    5621             :     return 0;
    5622           0 :   if ((Subtarget->hasAVX512())) {
    5623           0 :     return fastEmitInst_rr(X86::VPANDQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5624             :   }
    5625             :   return 0;
    5626             : }
    5627             : 
    5628         198 : unsigned fastEmit_ISD_AND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5629         198 :   switch (VT.SimpleTy) {
    5630           6 :   case MVT::i8: return fastEmit_ISD_AND_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5631           0 :   case MVT::i16: return fastEmit_ISD_AND_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5632         148 :   case MVT::i32: return fastEmit_ISD_AND_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5633          16 :   case MVT::i64: return fastEmit_ISD_AND_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5634           0 :   case MVT::v8i1: return fastEmit_ISD_AND_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5635           0 :   case MVT::v16i1: return fastEmit_ISD_AND_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5636           0 :   case MVT::v32i1: return fastEmit_ISD_AND_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5637           0 :   case MVT::v64i1: return fastEmit_ISD_AND_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5638           8 :   case MVT::v2i64: return fastEmit_ISD_AND_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5639          12 :   case MVT::v4i64: return fastEmit_ISD_AND_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5640           0 :   case MVT::v8i64: return fastEmit_ISD_AND_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5641             :   default: return 0;
    5642             :   }
    5643             : }
    5644             : 
    5645             : // FastEmit functions for ISD::FADD.
    5646             : 
    5647          25 : unsigned fastEmit_ISD_FADD_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5648          25 :   if (RetVT.SimpleTy != MVT::f32)
    5649             :     return 0;
    5650          50 :   if ((Subtarget->hasAVX512())) {
    5651           2 :     return fastEmitInst_rr(X86::VADDSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5652             :   }
    5653          23 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5654          21 :     return fastEmitInst_rr(X86::ADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5655             :   }
    5656           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5657           2 :     return fastEmitInst_rr(X86::VADDSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5658             :   }
    5659           0 :   if ((!Subtarget->hasSSE1())) {
    5660           0 :     return fastEmitInst_rr(X86::ADD_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5661             :   }
    5662             :   return 0;
    5663             : }
    5664             : 
    5665          26 : unsigned fastEmit_ISD_FADD_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5666          26 :   if (RetVT.SimpleTy != MVT::f64)
    5667             :     return 0;
    5668          52 :   if ((Subtarget->hasAVX512())) {
    5669           2 :     return fastEmitInst_rr(X86::VADDSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5670             :   }
    5671          24 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5672          22 :     return fastEmitInst_rr(X86::ADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5673             :   }
    5674           2 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5675           2 :     return fastEmitInst_rr(X86::VADDSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5676             :   }
    5677           0 :   if ((!Subtarget->hasSSE2())) {
    5678           0 :     return fastEmitInst_rr(X86::ADD_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5679             :   }
    5680             :   return 0;
    5681             : }
    5682             : 
    5683             : unsigned fastEmit_ISD_FADD_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5684           0 :   if (RetVT.SimpleTy != MVT::f80)
    5685             :     return 0;
    5686           0 :   return fastEmitInst_rr(X86::ADD_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5687             : }
    5688             : 
    5689           2 : unsigned fastEmit_ISD_FADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5690           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5691             :     return 0;
    5692           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5693           0 :     return fastEmitInst_rr(X86::VADDPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5694             :   }
    5695           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5696           2 :     return fastEmitInst_rr(X86::ADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5697             :   }
    5698           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5699           0 :     return fastEmitInst_rr(X86::VADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5700             :   }
    5701             :   return 0;
    5702             : }
    5703             : 
    5704           2 : unsigned fastEmit_ISD_FADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5705           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5706             :     return 0;
    5707           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5708           0 :     return fastEmitInst_rr(X86::VADDPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5709             :   }
    5710           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5711           2 :     return fastEmitInst_rr(X86::VADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5712             :   }
    5713             :   return 0;
    5714             : }
    5715             : 
    5716             : unsigned fastEmit_ISD_FADD_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5717           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5718             :     return 0;
    5719           0 :   if ((Subtarget->hasAVX512())) {
    5720           0 :     return fastEmitInst_rr(X86::VADDPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5721             :   }
    5722             :   return 0;
    5723             : }
    5724             : 
    5725          18 : unsigned fastEmit_ISD_FADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5726          18 :   if (RetVT.SimpleTy != MVT::v2f64)
    5727             :     return 0;
    5728          36 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5729           2 :     return fastEmitInst_rr(X86::VADDPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5730             :   }
    5731          16 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5732           6 :     return fastEmitInst_rr(X86::ADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5733             :   }
    5734          20 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5735          10 :     return fastEmitInst_rr(X86::VADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5736             :   }
    5737             :   return 0;
    5738             : }
    5739             : 
    5740          14 : unsigned fastEmit_ISD_FADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5741          14 :   if (RetVT.SimpleTy != MVT::v4f64)
    5742             :     return 0;
    5743          28 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5744           2 :     return fastEmitInst_rr(X86::VADDPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5745             :   }
    5746          12 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5747          12 :     return fastEmitInst_rr(X86::VADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5748             :   }
    5749             :   return 0;
    5750             : }
    5751             : 
    5752             : unsigned fastEmit_ISD_FADD_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5753           8 :   if (RetVT.SimpleTy != MVT::v8f64)
    5754             :     return 0;
    5755          16 :   if ((Subtarget->hasAVX512())) {
    5756           8 :     return fastEmitInst_rr(X86::VADDPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5757             :   }
    5758             :   return 0;
    5759             : }
    5760             : 
    5761          95 : unsigned fastEmit_ISD_FADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5762          95 :   switch (VT.SimpleTy) {
    5763          25 :   case MVT::f32: return fastEmit_ISD_FADD_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5764          26 :   case MVT::f64: return fastEmit_ISD_FADD_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5765           0 :   case MVT::f80: return fastEmit_ISD_FADD_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5766           2 :   case MVT::v4f32: return fastEmit_ISD_FADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5767           2 :   case MVT::v8f32: return fastEmit_ISD_FADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5768           0 :   case MVT::v16f32: return fastEmit_ISD_FADD_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5769          18 :   case MVT::v2f64: return fastEmit_ISD_FADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5770          14 :   case MVT::v4f64: return fastEmit_ISD_FADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5771           8 :   case MVT::v8f64: return fastEmit_ISD_FADD_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5772             :   default: return 0;
    5773             :   }
    5774             : }
    5775             : 
    5776             : // FastEmit functions for ISD::FDIV.
    5777             : 
    5778           8 : unsigned fastEmit_ISD_FDIV_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5779           8 :   if (RetVT.SimpleTy != MVT::f32)
    5780             :     return 0;
    5781          16 :   if ((Subtarget->hasAVX512())) {
    5782           0 :     return fastEmitInst_rr(X86::VDIVSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5783             :   }
    5784           8 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5785           8 :     return fastEmitInst_rr(X86::DIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5786             :   }
    5787           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5788           0 :     return fastEmitInst_rr(X86::VDIVSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5789             :   }
    5790           0 :   if ((!Subtarget->hasSSE1())) {
    5791           0 :     return fastEmitInst_rr(X86::DIV_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5792             :   }
    5793             :   return 0;
    5794             : }
    5795             : 
    5796           0 : unsigned fastEmit_ISD_FDIV_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5797           0 :   if (RetVT.SimpleTy != MVT::f64)
    5798             :     return 0;
    5799           0 :   if ((Subtarget->hasAVX512())) {
    5800           0 :     return fastEmitInst_rr(X86::VDIVSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5801             :   }
    5802           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5803           0 :     return fastEmitInst_rr(X86::DIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5804             :   }
    5805           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5806           0 :     return fastEmitInst_rr(X86::VDIVSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5807             :   }
    5808           0 :   if ((!Subtarget->hasSSE2())) {
    5809           0 :     return fastEmitInst_rr(X86::DIV_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5810             :   }
    5811             :   return 0;
    5812             : }
    5813             : 
    5814             : unsigned fastEmit_ISD_FDIV_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5815           0 :   if (RetVT.SimpleTy != MVT::f80)
    5816             :     return 0;
    5817           0 :   return fastEmitInst_rr(X86::DIV_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5818             : }
    5819             : 
    5820           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5821           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5822             :     return 0;
    5823           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5824           0 :     return fastEmitInst_rr(X86::VDIVPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5825             :   }
    5826           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5827           2 :     return fastEmitInst_rr(X86::DIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5828             :   }
    5829           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5830           0 :     return fastEmitInst_rr(X86::VDIVPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5831             :   }
    5832             :   return 0;
    5833             : }
    5834             : 
    5835           2 : unsigned fastEmit_ISD_FDIV_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5836           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5837             :     return 0;
    5838           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5839           0 :     return fastEmitInst_rr(X86::VDIVPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5840             :   }
    5841           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5842           2 :     return fastEmitInst_rr(X86::VDIVPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5843             :   }
    5844             :   return 0;
    5845             : }
    5846             : 
    5847             : unsigned fastEmit_ISD_FDIV_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5848           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5849             :     return 0;
    5850           0 :   if ((Subtarget->hasAVX512())) {
    5851           0 :     return fastEmitInst_rr(X86::VDIVPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5852             :   }
    5853             :   return 0;
    5854             : }
    5855             : 
    5856           2 : unsigned fastEmit_ISD_FDIV_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5857           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    5858             :     return 0;
    5859           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5860           0 :     return fastEmitInst_rr(X86::VDIVPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5861             :   }
    5862           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5863           2 :     return fastEmitInst_rr(X86::DIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5864             :   }
    5865           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5866           0 :     return fastEmitInst_rr(X86::VDIVPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5867             :   }
    5868             :   return 0;
    5869             : }
    5870             : 
    5871           2 : unsigned fastEmit_ISD_FDIV_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5872           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    5873             :     return 0;
    5874           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5875           0 :     return fastEmitInst_rr(X86::VDIVPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5876             :   }
    5877           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5878           2 :     return fastEmitInst_rr(X86::VDIVPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5879             :   }
    5880             :   return 0;
    5881             : }
    5882             : 
    5883             : unsigned fastEmit_ISD_FDIV_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5884           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    5885             :     return 0;
    5886           0 :   if ((Subtarget->hasAVX512())) {
    5887           0 :     return fastEmitInst_rr(X86::VDIVPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5888             :   }
    5889             :   return 0;
    5890             : }
    5891             : 
    5892          16 : unsigned fastEmit_ISD_FDIV_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5893          16 :   switch (VT.SimpleTy) {
    5894           8 :   case MVT::f32: return fastEmit_ISD_FDIV_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5895           0 :   case MVT::f64: return fastEmit_ISD_FDIV_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5896           0 :   case MVT::f80: return fastEmit_ISD_FDIV_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5897           2 :   case MVT::v4f32: return fastEmit_ISD_FDIV_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5898           2 :   case MVT::v8f32: return fastEmit_ISD_FDIV_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5899           0 :   case MVT::v16f32: return fastEmit_ISD_FDIV_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5900           2 :   case MVT::v2f64: return fastEmit_ISD_FDIV_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5901           2 :   case MVT::v4f64: return fastEmit_ISD_FDIV_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5902           0 :   case MVT::v8f64: return fastEmit_ISD_FDIV_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    5903             :   default: return 0;
    5904             :   }
    5905             : }
    5906             : 
    5907             : // FastEmit functions for ISD::FMUL.
    5908             : 
    5909          10 : unsigned fastEmit_ISD_FMUL_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5910          10 :   if (RetVT.SimpleTy != MVT::f32)
    5911             :     return 0;
    5912          20 :   if ((Subtarget->hasAVX512())) {
    5913           0 :     return fastEmitInst_rr(X86::VMULSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5914             :   }
    5915          10 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5916          10 :     return fastEmitInst_rr(X86::MULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5917             :   }
    5918           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5919           0 :     return fastEmitInst_rr(X86::VMULSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5920             :   }
    5921           0 :   if ((!Subtarget->hasSSE1())) {
    5922           0 :     return fastEmitInst_rr(X86::MUL_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5923             :   }
    5924             :   return 0;
    5925             : }
    5926             : 
    5927           6 : unsigned fastEmit_ISD_FMUL_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5928           6 :   if (RetVT.SimpleTy != MVT::f64)
    5929             :     return 0;
    5930          12 :   if ((Subtarget->hasAVX512())) {
    5931           0 :     return fastEmitInst_rr(X86::VMULSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5932             :   }
    5933           6 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5934           6 :     return fastEmitInst_rr(X86::MULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5935             :   }
    5936           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    5937           0 :     return fastEmitInst_rr(X86::VMULSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5938             :   }
    5939           0 :   if ((!Subtarget->hasSSE2())) {
    5940           0 :     return fastEmitInst_rr(X86::MUL_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5941             :   }
    5942             :   return 0;
    5943             : }
    5944             : 
    5945             : unsigned fastEmit_ISD_FMUL_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5946           0 :   if (RetVT.SimpleTy != MVT::f80)
    5947             :     return 0;
    5948           0 :   return fastEmitInst_rr(X86::MUL_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5949             : }
    5950             : 
    5951           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5952           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    5953             :     return 0;
    5954           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5955           0 :     return fastEmitInst_rr(X86::VMULPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5956             :   }
    5957           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    5958           2 :     return fastEmitInst_rr(X86::MULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5959             :   }
    5960           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5961           0 :     return fastEmitInst_rr(X86::VMULPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5962             :   }
    5963             :   return 0;
    5964             : }
    5965             : 
    5966           2 : unsigned fastEmit_ISD_FMUL_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5967           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    5968             :     return 0;
    5969           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5970           0 :     return fastEmitInst_rr(X86::VMULPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5971             :   }
    5972           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5973           2 :     return fastEmitInst_rr(X86::VMULPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5974             :   }
    5975             :   return 0;
    5976             : }
    5977             : 
    5978             : unsigned fastEmit_ISD_FMUL_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5979           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    5980             :     return 0;
    5981           0 :   if ((Subtarget->hasAVX512())) {
    5982           0 :     return fastEmitInst_rr(X86::VMULPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5983             :   }
    5984             :   return 0;
    5985             : }
    5986             : 
    5987           2 : unsigned fastEmit_ISD_FMUL_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    5988           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    5989             :     return 0;
    5990           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    5991           0 :     return fastEmitInst_rr(X86::VMULPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5992             :   }
    5993           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    5994           2 :     return fastEmitInst_rr(X86::MULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5995             :   }
    5996           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    5997           0 :     return fastEmitInst_rr(X86::VMULPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    5998             :   }
    5999             :   return 0;
    6000             : }
    6001             : 
    6002           2 : unsigned fastEmit_ISD_FMUL_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6003           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6004             :     return 0;
    6005           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6006           0 :     return fastEmitInst_rr(X86::VMULPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6007             :   }
    6008           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6009           2 :     return fastEmitInst_rr(X86::VMULPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6010             :   }
    6011             :   return 0;
    6012             : }
    6013             : 
    6014             : unsigned fastEmit_ISD_FMUL_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6015           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6016             :     return 0;
    6017           0 :   if ((Subtarget->hasAVX512())) {
    6018           0 :     return fastEmitInst_rr(X86::VMULPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6019             :   }
    6020             :   return 0;
    6021             : }
    6022             : 
    6023          24 : unsigned fastEmit_ISD_FMUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6024          24 :   switch (VT.SimpleTy) {
    6025          10 :   case MVT::f32: return fastEmit_ISD_FMUL_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6026           6 :   case MVT::f64: return fastEmit_ISD_FMUL_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6027           0 :   case MVT::f80: return fastEmit_ISD_FMUL_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6028           2 :   case MVT::v4f32: return fastEmit_ISD_FMUL_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6029           2 :   case MVT::v8f32: return fastEmit_ISD_FMUL_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6030           0 :   case MVT::v16f32: return fastEmit_ISD_FMUL_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6031           2 :   case MVT::v2f64: return fastEmit_ISD_FMUL_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6032           2 :   case MVT::v4f64: return fastEmit_ISD_FMUL_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6033           0 :   case MVT::v8f64: return fastEmit_ISD_FMUL_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6034             :   default: return 0;
    6035             :   }
    6036             : }
    6037             : 
    6038             : // FastEmit functions for ISD::FSUB.
    6039             : 
    6040           0 : unsigned fastEmit_ISD_FSUB_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6041           0 :   if (RetVT.SimpleTy != MVT::f32)
    6042             :     return 0;
    6043           0 :   if ((Subtarget->hasAVX512())) {
    6044           0 :     return fastEmitInst_rr(X86::VSUBSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6045             :   }
    6046           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6047           0 :     return fastEmitInst_rr(X86::SUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6048             :   }
    6049           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6050           0 :     return fastEmitInst_rr(X86::VSUBSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6051             :   }
    6052           0 :   if ((!Subtarget->hasSSE1())) {
    6053           0 :     return fastEmitInst_rr(X86::SUB_Fp32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6054             :   }
    6055             :   return 0;
    6056             : }
    6057             : 
    6058          38 : unsigned fastEmit_ISD_FSUB_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6059          38 :   if (RetVT.SimpleTy != MVT::f64)
    6060             :     return 0;
    6061          76 :   if ((Subtarget->hasAVX512())) {
    6062           0 :     return fastEmitInst_rr(X86::VSUBSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6063             :   }
    6064          38 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6065          38 :     return fastEmitInst_rr(X86::SUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6066             :   }
    6067           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    6068           0 :     return fastEmitInst_rr(X86::VSUBSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6069             :   }
    6070           0 :   if ((!Subtarget->hasSSE2())) {
    6071           0 :     return fastEmitInst_rr(X86::SUB_Fp64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6072             :   }
    6073             :   return 0;
    6074             : }
    6075             : 
    6076             : unsigned fastEmit_ISD_FSUB_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6077           0 :   if (RetVT.SimpleTy != MVT::f80)
    6078             :     return 0;
    6079           0 :   return fastEmitInst_rr(X86::SUB_Fp80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6080             : }
    6081             : 
    6082           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6083           2 :   if (RetVT.SimpleTy != MVT::v4f32)
    6084             :     return 0;
    6085           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6086           0 :     return fastEmitInst_rr(X86::VSUBPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6087             :   }
    6088           2 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    6089           2 :     return fastEmitInst_rr(X86::SUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6090             :   }
    6091           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6092           0 :     return fastEmitInst_rr(X86::VSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6093             :   }
    6094             :   return 0;
    6095             : }
    6096             : 
    6097           2 : unsigned fastEmit_ISD_FSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6098           2 :   if (RetVT.SimpleTy != MVT::v8f32)
    6099             :     return 0;
    6100           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6101           0 :     return fastEmitInst_rr(X86::VSUBPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6102             :   }
    6103           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6104           2 :     return fastEmitInst_rr(X86::VSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6105             :   }
    6106             :   return 0;
    6107             : }
    6108             : 
    6109             : unsigned fastEmit_ISD_FSUB_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6110           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    6111             :     return 0;
    6112           0 :   if ((Subtarget->hasAVX512())) {
    6113           0 :     return fastEmitInst_rr(X86::VSUBPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6114             :   }
    6115             :   return 0;
    6116             : }
    6117             : 
    6118           2 : unsigned fastEmit_ISD_FSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6119           2 :   if (RetVT.SimpleTy != MVT::v2f64)
    6120             :     return 0;
    6121           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6122           0 :     return fastEmitInst_rr(X86::VSUBPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6123             :   }
    6124           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6125           2 :     return fastEmitInst_rr(X86::SUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6126             :   }
    6127           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6128           0 :     return fastEmitInst_rr(X86::VSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6129             :   }
    6130             :   return 0;
    6131             : }
    6132             : 
    6133           2 : unsigned fastEmit_ISD_FSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6134           2 :   if (RetVT.SimpleTy != MVT::v4f64)
    6135             :     return 0;
    6136           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6137           0 :     return fastEmitInst_rr(X86::VSUBPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6138             :   }
    6139           2 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6140           2 :     return fastEmitInst_rr(X86::VSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6141             :   }
    6142             :   return 0;
    6143             : }
    6144             : 
    6145             : unsigned fastEmit_ISD_FSUB_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6146           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    6147             :     return 0;
    6148           0 :   if ((Subtarget->hasAVX512())) {
    6149           0 :     return fastEmitInst_rr(X86::VSUBPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6150             :   }
    6151             :   return 0;
    6152             : }
    6153             : 
    6154          46 : unsigned fastEmit_ISD_FSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6155          46 :   switch (VT.SimpleTy) {
    6156           0 :   case MVT::f32: return fastEmit_ISD_FSUB_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6157          38 :   case MVT::f64: return fastEmit_ISD_FSUB_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6158           0 :   case MVT::f80: return fastEmit_ISD_FSUB_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6159           2 :   case MVT::v4f32: return fastEmit_ISD_FSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6160           2 :   case MVT::v8f32: return fastEmit_ISD_FSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6161           0 :   case MVT::v16f32: return fastEmit_ISD_FSUB_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6162           2 :   case MVT::v2f64: return fastEmit_ISD_FSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6163           2 :   case MVT::v4f64: return fastEmit_ISD_FSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6164           0 :   case MVT::v8f64: return fastEmit_ISD_FSUB_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6165             :   default: return 0;
    6166             :   }
    6167             : }
    6168             : 
    6169             : // FastEmit functions for ISD::MUL.
    6170             : 
    6171           2 : unsigned fastEmit_ISD_MUL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6172           2 :   if (RetVT.SimpleTy != MVT::i8)
    6173             :     return 0;
    6174           4 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::AL).addReg(Op0);
    6175           2 :   return fastEmitInst_r(X86::MUL8r, &X86::GR8RegClass, Op1, Op1IsKill);
    6176             : }
    6177             : 
    6178             : unsigned fastEmit_ISD_MUL_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6179           0 :   if (RetVT.SimpleTy != MVT::i16)
    6180             :     return 0;
    6181           0 :   return fastEmitInst_rr(X86::IMUL16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6182             : }
    6183             : 
    6184             : unsigned fastEmit_ISD_MUL_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6185          22 :   if (RetVT.SimpleTy != MVT::i32)
    6186             :     return 0;
    6187          22 :   return fastEmitInst_rr(X86::IMUL32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6188             : }
    6189             : 
    6190             : unsigned fastEmit_ISD_MUL_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6191          90 :   if (RetVT.SimpleTy != MVT::i64)
    6192             :     return 0;
    6193          90 :   return fastEmitInst_rr(X86::IMUL64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6194             : }
    6195             : 
    6196           2 : unsigned fastEmit_ISD_MUL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6197           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    6198             :     return 0;
    6199           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6200           0 :     return fastEmitInst_rr(X86::VPMULLWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6201             :   }
    6202           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6203           2 :     return fastEmitInst_rr(X86::PMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6204             :   }
    6205           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6206           0 :     return fastEmitInst_rr(X86::VPMULLWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6207             :   }
    6208             :   return 0;
    6209             : }
    6210             : 
    6211           2 : unsigned fastEmit_ISD_MUL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6212           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    6213             :     return 0;
    6214           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6215           0 :     return fastEmitInst_rr(X86::VPMULLWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6216             :   }
    6217           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6218           2 :     return fastEmitInst_rr(X86::VPMULLWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6219             :   }
    6220             :   return 0;
    6221             : }
    6222             : 
    6223             : unsigned fastEmit_ISD_MUL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6224           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6225             :     return 0;
    6226           0 :   if ((Subtarget->hasBWI())) {
    6227           0 :     return fastEmitInst_rr(X86::VPMULLWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6228             :   }
    6229             :   return 0;
    6230             : }
    6231             : 
    6232           2 : unsigned fastEmit_ISD_MUL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6233           2 :   if (RetVT.SimpleTy != MVT::v4i32)
    6234             :     return 0;
    6235           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6236           0 :     return fastEmitInst_rr(X86::VPMULLDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6237             :   }
    6238           2 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6239           2 :     return fastEmitInst_rr(X86::PMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6240             :   }
    6241           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6242           0 :     return fastEmitInst_rr(X86::VPMULLDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6243             :   }
    6244             :   return 0;
    6245             : }
    6246             : 
    6247           2 : unsigned fastEmit_ISD_MUL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6248           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    6249             :     return 0;
    6250           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6251           0 :     return fastEmitInst_rr(X86::VPMULLDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6252             :   }
    6253           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6254           2 :     return fastEmitInst_rr(X86::VPMULLDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6255             :   }
    6256             :   return 0;
    6257             : }
    6258             : 
    6259             : unsigned fastEmit_ISD_MUL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6260           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6261             :     return 0;
    6262           0 :   if ((Subtarget->hasAVX512())) {
    6263           0 :     return fastEmitInst_rr(X86::VPMULLDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6264             :   }
    6265             :   return 0;
    6266             : }
    6267             : 
    6268           4 : unsigned fastEmit_ISD_MUL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6269           4 :   if (RetVT.SimpleTy != MVT::v2i64)
    6270             :     return 0;
    6271           4 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6272           0 :     return fastEmitInst_rr(X86::VPMULLQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6273             :   }
    6274             :   return 0;
    6275             : }
    6276             : 
    6277           4 : unsigned fastEmit_ISD_MUL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6278           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    6279             :     return 0;
    6280           4 :   if ((Subtarget->hasDQI()) && (Subtarget->hasVLX())) {
    6281           0 :     return fastEmitInst_rr(X86::VPMULLQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6282             :   }
    6283             :   return 0;
    6284             : }
    6285             : 
    6286             : unsigned fastEmit_ISD_MUL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6287           4 :   if (RetVT.SimpleTy != MVT::v8i64)
    6288             :     return 0;
    6289           4 :   if ((Subtarget->hasDQI())) {
    6290           0 :     return fastEmitInst_rr(X86::VPMULLQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6291             :   }
    6292             :   return 0;
    6293             : }
    6294             : 
    6295         134 : unsigned fastEmit_ISD_MUL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6296         134 :   switch (VT.SimpleTy) {
    6297           2 :   case MVT::i8: return fastEmit_ISD_MUL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6298           0 :   case MVT::i16: return fastEmit_ISD_MUL_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6299          22 :   case MVT::i32: return fastEmit_ISD_MUL_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6300          90 :   case MVT::i64: return fastEmit_ISD_MUL_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6301           2 :   case MVT::v8i16: return fastEmit_ISD_MUL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6302           2 :   case MVT::v16i16: return fastEmit_ISD_MUL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6303           0 :   case MVT::v32i16: return fastEmit_ISD_MUL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6304           2 :   case MVT::v4i32: return fastEmit_ISD_MUL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6305           2 :   case MVT::v8i32: return fastEmit_ISD_MUL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6306           0 :   case MVT::v16i32: return fastEmit_ISD_MUL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6307           4 :   case MVT::v2i64: return fastEmit_ISD_MUL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6308           4 :   case MVT::v4i64: return fastEmit_ISD_MUL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6309           4 :   case MVT::v8i64: return fastEmit_ISD_MUL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6310             :   default: return 0;
    6311             :   }
    6312             : }
    6313             : 
    6314             : // FastEmit functions for ISD::MULHS.
    6315             : 
    6316           0 : unsigned fastEmit_ISD_MULHS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6317           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6318             :     return 0;
    6319           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6320           0 :     return fastEmitInst_rr(X86::VPMULHWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6321             :   }
    6322           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6323           0 :     return fastEmitInst_rr(X86::PMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6324             :   }
    6325           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6326           0 :     return fastEmitInst_rr(X86::VPMULHWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6327             :   }
    6328             :   return 0;
    6329             : }
    6330             : 
    6331           0 : unsigned fastEmit_ISD_MULHS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6332           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6333             :     return 0;
    6334           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6335           0 :     return fastEmitInst_rr(X86::VPMULHWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6336             :   }
    6337           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6338           0 :     return fastEmitInst_rr(X86::VPMULHWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6339             :   }
    6340             :   return 0;
    6341             : }
    6342             : 
    6343             : unsigned fastEmit_ISD_MULHS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6344           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6345             :     return 0;
    6346           0 :   if ((Subtarget->hasBWI())) {
    6347           0 :     return fastEmitInst_rr(X86::VPMULHWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6348             :   }
    6349             :   return 0;
    6350             : }
    6351             : 
    6352           0 : unsigned fastEmit_ISD_MULHS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6353           0 :   switch (VT.SimpleTy) {
    6354           0 :   case MVT::v8i16: return fastEmit_ISD_MULHS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6355           0 :   case MVT::v16i16: return fastEmit_ISD_MULHS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6356           0 :   case MVT::v32i16: return fastEmit_ISD_MULHS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6357             :   default: return 0;
    6358             :   }
    6359             : }
    6360             : 
    6361             : // FastEmit functions for ISD::MULHU.
    6362             : 
    6363           0 : unsigned fastEmit_ISD_MULHU_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6364           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6365             :     return 0;
    6366           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6367           0 :     return fastEmitInst_rr(X86::VPMULHUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6368             :   }
    6369           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6370           0 :     return fastEmitInst_rr(X86::PMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6371             :   }
    6372           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6373           0 :     return fastEmitInst_rr(X86::VPMULHUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6374             :   }
    6375             :   return 0;
    6376             : }
    6377             : 
    6378           0 : unsigned fastEmit_ISD_MULHU_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6379           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6380             :     return 0;
    6381           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6382           0 :     return fastEmitInst_rr(X86::VPMULHUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6383             :   }
    6384           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6385           0 :     return fastEmitInst_rr(X86::VPMULHUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6386             :   }
    6387             :   return 0;
    6388             : }
    6389             : 
    6390             : unsigned fastEmit_ISD_MULHU_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6391           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6392             :     return 0;
    6393           0 :   if ((Subtarget->hasBWI())) {
    6394           0 :     return fastEmitInst_rr(X86::VPMULHUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6395             :   }
    6396             :   return 0;
    6397             : }
    6398             : 
    6399           0 : unsigned fastEmit_ISD_MULHU_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6400           0 :   switch (VT.SimpleTy) {
    6401           0 :   case MVT::v8i16: return fastEmit_ISD_MULHU_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6402           0 :   case MVT::v16i16: return fastEmit_ISD_MULHU_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6403           0 :   case MVT::v32i16: return fastEmit_ISD_MULHU_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6404             :   default: return 0;
    6405             :   }
    6406             : }
    6407             : 
    6408             : // FastEmit functions for ISD::OR.
    6409             : 
    6410             : unsigned fastEmit_ISD_OR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6411           6 :   if (RetVT.SimpleTy != MVT::i8)
    6412             :     return 0;
    6413           6 :   return fastEmitInst_rr(X86::OR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6414             : }
    6415             : 
    6416             : unsigned fastEmit_ISD_OR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6417          20 :   if (RetVT.SimpleTy != MVT::i16)
    6418             :     return 0;
    6419          20 :   return fastEmitInst_rr(X86::OR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6420             : }
    6421             : 
    6422             : unsigned fastEmit_ISD_OR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6423         134 :   if (RetVT.SimpleTy != MVT::i32)
    6424             :     return 0;
    6425         134 :   return fastEmitInst_rr(X86::OR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6426             : }
    6427             : 
    6428             : unsigned fastEmit_ISD_OR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6429          13 :   if (RetVT.SimpleTy != MVT::i64)
    6430             :     return 0;
    6431          13 :   return fastEmitInst_rr(X86::OR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6432             : }
    6433             : 
    6434             : unsigned fastEmit_ISD_OR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6435           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    6436             :     return 0;
    6437           0 :   if ((Subtarget->hasDQI())) {
    6438           0 :     return fastEmitInst_rr(X86::KORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6439             :   }
    6440             :   return 0;
    6441             : }
    6442             : 
    6443             : unsigned fastEmit_ISD_OR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6444           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    6445             :     return 0;
    6446           0 :   if ((Subtarget->hasAVX512())) {
    6447           0 :     return fastEmitInst_rr(X86::KORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6448             :   }
    6449             :   return 0;
    6450             : }
    6451             : 
    6452             : unsigned fastEmit_ISD_OR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6453           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    6454             :     return 0;
    6455           0 :   if ((Subtarget->hasBWI())) {
    6456           0 :     return fastEmitInst_rr(X86::KORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6457             :   }
    6458             :   return 0;
    6459             : }
    6460             : 
    6461             : unsigned fastEmit_ISD_OR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6462           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    6463             :     return 0;
    6464           0 :   if ((Subtarget->hasBWI())) {
    6465           0 :     return fastEmitInst_rr(X86::KORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6466             :   }
    6467             :   return 0;
    6468             : }
    6469             : 
    6470           4 : unsigned fastEmit_ISD_OR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6471           4 :   if (RetVT.SimpleTy != MVT::v2i64)
    6472             :     return 0;
    6473           8 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6474           0 :     return fastEmitInst_rr(X86::VPORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6475             :   }
    6476           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6477           2 :     return fastEmitInst_rr(X86::PORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6478             :   }
    6479           4 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6480           2 :     return fastEmitInst_rr(X86::VPORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6481             :   }
    6482             :   return 0;
    6483             : }
    6484             : 
    6485           6 : unsigned fastEmit_ISD_OR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6486           6 :   if (RetVT.SimpleTy != MVT::v4i64)
    6487             :     return 0;
    6488          12 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    6489           4 :     return fastEmitInst_rr(X86::VORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6490             :   }
    6491           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6492           0 :     return fastEmitInst_rr(X86::VPORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6493             :   }
    6494           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6495           2 :     return fastEmitInst_rr(X86::VPORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6496             :   }
    6497             :   return 0;
    6498             : }
    6499             : 
    6500             : unsigned fastEmit_ISD_OR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6501           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6502             :     return 0;
    6503           0 :   if ((Subtarget->hasAVX512())) {
    6504           0 :     return fastEmitInst_rr(X86::VPORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6505             :   }
    6506             :   return 0;
    6507             : }
    6508             : 
    6509         187 : unsigned fastEmit_ISD_OR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6510         187 :   switch (VT.SimpleTy) {
    6511           6 :   case MVT::i8: return fastEmit_ISD_OR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6512          20 :   case MVT::i16: return fastEmit_ISD_OR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6513         134 :   case MVT::i32: return fastEmit_ISD_OR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6514          13 :   case MVT::i64: return fastEmit_ISD_OR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6515           0 :   case MVT::v8i1: return fastEmit_ISD_OR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6516           0 :   case MVT::v16i1: return fastEmit_ISD_OR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6517           0 :   case MVT::v32i1: return fastEmit_ISD_OR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6518           0 :   case MVT::v64i1: return fastEmit_ISD_OR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6519           4 :   case MVT::v2i64: return fastEmit_ISD_OR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6520           6 :   case MVT::v4i64: return fastEmit_ISD_OR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6521           0 :   case MVT::v8i64: return fastEmit_ISD_OR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6522             :   default: return 0;
    6523             :   }
    6524             : }
    6525             : 
    6526             : // FastEmit functions for ISD::ROTL.
    6527             : 
    6528           0 : unsigned fastEmit_ISD_ROTL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6529           0 :   if (RetVT.SimpleTy != MVT::i8)
    6530             :     return 0;
    6531           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6532           0 :   return fastEmitInst_r(X86::ROL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6533             : }
    6534             : 
    6535             : unsigned fastEmit_ISD_ROTL_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6536           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6537             :     return 0;
    6538           0 :   if ((Subtarget->hasXOP())) {
    6539           0 :     return fastEmitInst_rr(X86::VPROTBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6540             :   }
    6541             :   return 0;
    6542             : }
    6543             : 
    6544             : unsigned fastEmit_ISD_ROTL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6545           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6546             :     return 0;
    6547           0 :   if ((Subtarget->hasXOP())) {
    6548           0 :     return fastEmitInst_rr(X86::VPROTWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6549             :   }
    6550             :   return 0;
    6551             : }
    6552             : 
    6553           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6554           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6555             :     return 0;
    6556           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6557           0 :     return fastEmitInst_rr(X86::VPROLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6558             :   }
    6559           0 :   if ((Subtarget->hasXOP())) {
    6560           0 :     return fastEmitInst_rr(X86::VPROTDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6561             :   }
    6562             :   return 0;
    6563             : }
    6564             : 
    6565           0 : unsigned fastEmit_ISD_ROTL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6566           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6567             :     return 0;
    6568           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6569           0 :     return fastEmitInst_rr(X86::VPROLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6570             :   }
    6571             :   return 0;
    6572             : }
    6573             : 
    6574             : unsigned fastEmit_ISD_ROTL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6575           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6576             :     return 0;
    6577           0 :   if ((Subtarget->hasAVX512())) {
    6578           0 :     return fastEmitInst_rr(X86::VPROLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6579             :   }
    6580             :   return 0;
    6581             : }
    6582             : 
    6583           0 : unsigned fastEmit_ISD_ROTL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6584           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6585             :     return 0;
    6586           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6587           0 :     return fastEmitInst_rr(X86::VPROLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6588             :   }
    6589           0 :   if ((Subtarget->hasXOP())) {
    6590           0 :     return fastEmitInst_rr(X86::VPROTQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6591             :   }
    6592             :   return 0;
    6593             : }
    6594             : 
    6595           0 : unsigned fastEmit_ISD_ROTL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6596           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6597             :     return 0;
    6598           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6599           0 :     return fastEmitInst_rr(X86::VPROLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6600             :   }
    6601             :   return 0;
    6602             : }
    6603             : 
    6604             : unsigned fastEmit_ISD_ROTL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6605           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6606             :     return 0;
    6607           0 :   if ((Subtarget->hasAVX512())) {
    6608           0 :     return fastEmitInst_rr(X86::VPROLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6609             :   }
    6610             :   return 0;
    6611             : }
    6612             : 
    6613           0 : unsigned fastEmit_ISD_ROTL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6614           0 :   switch (VT.SimpleTy) {
    6615           0 :   case MVT::i8: return fastEmit_ISD_ROTL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6616           0 :   case MVT::v16i8: return fastEmit_ISD_ROTL_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6617           0 :   case MVT::v8i16: return fastEmit_ISD_ROTL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6618           0 :   case MVT::v4i32: return fastEmit_ISD_ROTL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6619           0 :   case MVT::v8i32: return fastEmit_ISD_ROTL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6620           0 :   case MVT::v16i32: return fastEmit_ISD_ROTL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6621           0 :   case MVT::v2i64: return fastEmit_ISD_ROTL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6622           0 :   case MVT::v4i64: return fastEmit_ISD_ROTL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6623           0 :   case MVT::v8i64: return fastEmit_ISD_ROTL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6624             :   default: return 0;
    6625             :   }
    6626             : }
    6627             : 
    6628             : // FastEmit functions for ISD::ROTR.
    6629             : 
    6630           0 : unsigned fastEmit_ISD_ROTR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6631           0 :   if (RetVT.SimpleTy != MVT::i8)
    6632             :     return 0;
    6633           0 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6634           0 :   return fastEmitInst_r(X86::ROR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6635             : }
    6636             : 
    6637           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6638           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6639             :     return 0;
    6640           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6641           0 :     return fastEmitInst_rr(X86::VPRORVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6642             :   }
    6643             :   return 0;
    6644             : }
    6645             : 
    6646           0 : unsigned fastEmit_ISD_ROTR_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6647           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6648             :     return 0;
    6649           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6650           0 :     return fastEmitInst_rr(X86::VPRORVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6651             :   }
    6652             :   return 0;
    6653             : }
    6654             : 
    6655             : unsigned fastEmit_ISD_ROTR_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6656           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6657             :     return 0;
    6658           0 :   if ((Subtarget->hasAVX512())) {
    6659           0 :     return fastEmitInst_rr(X86::VPRORVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6660             :   }
    6661             :   return 0;
    6662             : }
    6663             : 
    6664           0 : unsigned fastEmit_ISD_ROTR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6665           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6666             :     return 0;
    6667           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6668           0 :     return fastEmitInst_rr(X86::VPRORVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6669             :   }
    6670             :   return 0;
    6671             : }
    6672             : 
    6673           0 : unsigned fastEmit_ISD_ROTR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6674           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6675             :     return 0;
    6676           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6677           0 :     return fastEmitInst_rr(X86::VPRORVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6678             :   }
    6679             :   return 0;
    6680             : }
    6681             : 
    6682             : unsigned fastEmit_ISD_ROTR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6683           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6684             :     return 0;
    6685           0 :   if ((Subtarget->hasAVX512())) {
    6686           0 :     return fastEmitInst_rr(X86::VPRORVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6687             :   }
    6688             :   return 0;
    6689             : }
    6690             : 
    6691           0 : unsigned fastEmit_ISD_ROTR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6692           0 :   switch (VT.SimpleTy) {
    6693           0 :   case MVT::i8: return fastEmit_ISD_ROTR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6694           0 :   case MVT::v4i32: return fastEmit_ISD_ROTR_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6695           0 :   case MVT::v8i32: return fastEmit_ISD_ROTR_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6696           0 :   case MVT::v16i32: return fastEmit_ISD_ROTR_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6697           0 :   case MVT::v2i64: return fastEmit_ISD_ROTR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6698           0 :   case MVT::v4i64: return fastEmit_ISD_ROTR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6699           0 :   case MVT::v8i64: return fastEmit_ISD_ROTR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6700             :   default: return 0;
    6701             :   }
    6702             : }
    6703             : 
    6704             : // FastEmit functions for ISD::SHL.
    6705             : 
    6706           3 : unsigned fastEmit_ISD_SHL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6707           3 :   if (RetVT.SimpleTy != MVT::i8)
    6708             :     return 0;
    6709           6 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    6710           3 :   return fastEmitInst_r(X86::SHL8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    6711             : }
    6712             : 
    6713           0 : unsigned fastEmit_ISD_SHL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6714           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6715             :     return 0;
    6716           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6717           0 :     return fastEmitInst_rr(X86::VPSLLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6718             :   }
    6719             :   return 0;
    6720             : }
    6721             : 
    6722           0 : unsigned fastEmit_ISD_SHL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6723           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6724             :     return 0;
    6725           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6726           0 :     return fastEmitInst_rr(X86::VPSLLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6727             :   }
    6728             :   return 0;
    6729             : }
    6730             : 
    6731             : unsigned fastEmit_ISD_SHL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6732           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6733             :     return 0;
    6734           0 :   if ((Subtarget->hasBWI())) {
    6735           0 :     return fastEmitInst_rr(X86::VPSLLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6736             :   }
    6737             :   return 0;
    6738             : }
    6739             : 
    6740           0 : unsigned fastEmit_ISD_SHL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6741           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6742             :     return 0;
    6743           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6744           0 :     return fastEmitInst_rr(X86::VPSLLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6745             :   }
    6746           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6747           0 :     return fastEmitInst_rr(X86::VPSLLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6748             :   }
    6749             :   return 0;
    6750             : }
    6751             : 
    6752           0 : unsigned fastEmit_ISD_SHL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6753           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6754             :     return 0;
    6755           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6756           0 :     return fastEmitInst_rr(X86::VPSLLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6757             :   }
    6758           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6759           0 :     return fastEmitInst_rr(X86::VPSLLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6760             :   }
    6761             :   return 0;
    6762             : }
    6763             : 
    6764             : unsigned fastEmit_ISD_SHL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6765           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6766             :     return 0;
    6767           0 :   if ((Subtarget->hasAVX512())) {
    6768           0 :     return fastEmitInst_rr(X86::VPSLLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6769             :   }
    6770             :   return 0;
    6771             : }
    6772             : 
    6773           0 : unsigned fastEmit_ISD_SHL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6774           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6775             :     return 0;
    6776           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6777           0 :     return fastEmitInst_rr(X86::VPSLLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6778             :   }
    6779           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6780           0 :     return fastEmitInst_rr(X86::VPSLLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6781             :   }
    6782             :   return 0;
    6783             : }
    6784             : 
    6785           0 : unsigned fastEmit_ISD_SHL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6786           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6787             :     return 0;
    6788           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6789           0 :     return fastEmitInst_rr(X86::VPSLLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6790             :   }
    6791           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6792           0 :     return fastEmitInst_rr(X86::VPSLLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6793             :   }
    6794             :   return 0;
    6795             : }
    6796             : 
    6797             : unsigned fastEmit_ISD_SHL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6798           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6799             :     return 0;
    6800           0 :   if ((Subtarget->hasAVX512())) {
    6801           0 :     return fastEmitInst_rr(X86::VPSLLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6802             :   }
    6803             :   return 0;
    6804             : }
    6805             : 
    6806         107 : unsigned fastEmit_ISD_SHL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6807         107 :   switch (VT.SimpleTy) {
    6808           3 :   case MVT::i8: return fastEmit_ISD_SHL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6809           0 :   case MVT::v8i16: return fastEmit_ISD_SHL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6810           0 :   case MVT::v16i16: return fastEmit_ISD_SHL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6811           0 :   case MVT::v32i16: return fastEmit_ISD_SHL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6812           0 :   case MVT::v4i32: return fastEmit_ISD_SHL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6813           0 :   case MVT::v8i32: return fastEmit_ISD_SHL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6814           0 :   case MVT::v16i32: return fastEmit_ISD_SHL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6815           0 :   case MVT::v2i64: return fastEmit_ISD_SHL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6816           0 :   case MVT::v4i64: return fastEmit_ISD_SHL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6817           0 :   case MVT::v8i64: return fastEmit_ISD_SHL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6818             :   default: return 0;
    6819             :   }
    6820             : }
    6821             : 
    6822             : // FastEmit functions for ISD::SMAX.
    6823             : 
    6824           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6825           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6826             :     return 0;
    6827           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6828           0 :     return fastEmitInst_rr(X86::VPMAXSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6829             :   }
    6830           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6831           0 :     return fastEmitInst_rr(X86::PMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6832             :   }
    6833           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6834           0 :     return fastEmitInst_rr(X86::VPMAXSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6835             :   }
    6836             :   return 0;
    6837             : }
    6838             : 
    6839           0 : unsigned fastEmit_ISD_SMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6840           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    6841             :     return 0;
    6842           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6843           0 :     return fastEmitInst_rr(X86::VPMAXSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6844             :   }
    6845           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6846           0 :     return fastEmitInst_rr(X86::VPMAXSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6847             :   }
    6848             :   return 0;
    6849             : }
    6850             : 
    6851             : unsigned fastEmit_ISD_SMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6852           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    6853             :     return 0;
    6854           0 :   if ((Subtarget->hasBWI())) {
    6855           0 :     return fastEmitInst_rr(X86::VPMAXSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6856             :   }
    6857             :   return 0;
    6858             : }
    6859             : 
    6860           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6861           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    6862             :     return 0;
    6863           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6864           0 :     return fastEmitInst_rr(X86::VPMAXSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6865             :   }
    6866           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    6867           0 :     return fastEmitInst_rr(X86::PMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6868             :   }
    6869           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6870           0 :     return fastEmitInst_rr(X86::VPMAXSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6871             :   }
    6872             :   return 0;
    6873             : }
    6874             : 
    6875           0 : unsigned fastEmit_ISD_SMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6876           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    6877             :     return 0;
    6878           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6879           0 :     return fastEmitInst_rr(X86::VPMAXSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6880             :   }
    6881           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6882           0 :     return fastEmitInst_rr(X86::VPMAXSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6883             :   }
    6884             :   return 0;
    6885             : }
    6886             : 
    6887             : unsigned fastEmit_ISD_SMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6888           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    6889             :     return 0;
    6890           0 :   if ((Subtarget->hasBWI())) {
    6891           0 :     return fastEmitInst_rr(X86::VPMAXSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6892             :   }
    6893             :   return 0;
    6894             : }
    6895             : 
    6896           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6897           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    6898             :     return 0;
    6899           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6900           0 :     return fastEmitInst_rr(X86::VPMAXSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6901             :   }
    6902           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6903           0 :     return fastEmitInst_rr(X86::PMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6904             :   }
    6905           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    6906           0 :     return fastEmitInst_rr(X86::VPMAXSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6907             :   }
    6908             :   return 0;
    6909             : }
    6910             : 
    6911           0 : unsigned fastEmit_ISD_SMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6912           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    6913             :     return 0;
    6914           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6915           0 :     return fastEmitInst_rr(X86::VPMAXSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6916             :   }
    6917           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    6918           0 :     return fastEmitInst_rr(X86::VPMAXSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6919             :   }
    6920             :   return 0;
    6921             : }
    6922             : 
    6923             : unsigned fastEmit_ISD_SMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6924           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    6925             :     return 0;
    6926           0 :   if ((Subtarget->hasAVX512())) {
    6927           0 :     return fastEmitInst_rr(X86::VPMAXSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6928             :   }
    6929             :   return 0;
    6930             : }
    6931             : 
    6932           0 : unsigned fastEmit_ISD_SMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6933           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    6934             :     return 0;
    6935           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6936           0 :     return fastEmitInst_rr(X86::VPMAXSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6937             :   }
    6938             :   return 0;
    6939             : }
    6940             : 
    6941           0 : unsigned fastEmit_ISD_SMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6942           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    6943             :     return 0;
    6944           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    6945           0 :     return fastEmitInst_rr(X86::VPMAXSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6946             :   }
    6947             :   return 0;
    6948             : }
    6949             : 
    6950             : unsigned fastEmit_ISD_SMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6951           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    6952             :     return 0;
    6953           0 :   if ((Subtarget->hasAVX512())) {
    6954           0 :     return fastEmitInst_rr(X86::VPMAXSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6955             :   }
    6956             :   return 0;
    6957             : }
    6958             : 
    6959           0 : unsigned fastEmit_ISD_SMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6960           0 :   switch (VT.SimpleTy) {
    6961           0 :   case MVT::v16i8: return fastEmit_ISD_SMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6962           0 :   case MVT::v32i8: return fastEmit_ISD_SMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6963           0 :   case MVT::v64i8: return fastEmit_ISD_SMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6964           0 :   case MVT::v8i16: return fastEmit_ISD_SMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6965           0 :   case MVT::v16i16: return fastEmit_ISD_SMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6966           0 :   case MVT::v32i16: return fastEmit_ISD_SMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6967           0 :   case MVT::v4i32: return fastEmit_ISD_SMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6968           0 :   case MVT::v8i32: return fastEmit_ISD_SMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6969           0 :   case MVT::v16i32: return fastEmit_ISD_SMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6970           0 :   case MVT::v2i64: return fastEmit_ISD_SMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6971           0 :   case MVT::v4i64: return fastEmit_ISD_SMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6972           0 :   case MVT::v8i64: return fastEmit_ISD_SMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    6973             :   default: return 0;
    6974             :   }
    6975             : }
    6976             : 
    6977             : // FastEmit functions for ISD::SMIN.
    6978             : 
    6979           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6980           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    6981             :     return 0;
    6982           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6983           0 :     return fastEmitInst_rr(X86::VPMINSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6984             :   }
    6985           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    6986           0 :     return fastEmitInst_rr(X86::PMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6987             :   }
    6988           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    6989           0 :     return fastEmitInst_rr(X86::VPMINSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6990             :   }
    6991             :   return 0;
    6992             : }
    6993             : 
    6994           0 : unsigned fastEmit_ISD_SMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    6995           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    6996             :     return 0;
    6997           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    6998           0 :     return fastEmitInst_rr(X86::VPMINSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    6999             :   }
    7000           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7001           0 :     return fastEmitInst_rr(X86::VPMINSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7002             :   }
    7003             :   return 0;
    7004             : }
    7005             : 
    7006             : unsigned fastEmit_ISD_SMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7007           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7008             :     return 0;
    7009           0 :   if ((Subtarget->hasBWI())) {
    7010           0 :     return fastEmitInst_rr(X86::VPMINSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7011             :   }
    7012             :   return 0;
    7013             : }
    7014             : 
    7015           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7016           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7017             :     return 0;
    7018           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7019           0 :     return fastEmitInst_rr(X86::VPMINSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7020             :   }
    7021           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7022           0 :     return fastEmitInst_rr(X86::PMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7023             :   }
    7024           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7025           0 :     return fastEmitInst_rr(X86::VPMINSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7026             :   }
    7027             :   return 0;
    7028             : }
    7029             : 
    7030           0 : unsigned fastEmit_ISD_SMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7031           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7032             :     return 0;
    7033           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7034           0 :     return fastEmitInst_rr(X86::VPMINSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7035             :   }
    7036           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7037           0 :     return fastEmitInst_rr(X86::VPMINSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7038             :   }
    7039             :   return 0;
    7040             : }
    7041             : 
    7042             : unsigned fastEmit_ISD_SMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7043           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7044             :     return 0;
    7045           0 :   if ((Subtarget->hasBWI())) {
    7046           0 :     return fastEmitInst_rr(X86::VPMINSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7047             :   }
    7048             :   return 0;
    7049             : }
    7050             : 
    7051           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7052           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7053             :     return 0;
    7054           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7055           0 :     return fastEmitInst_rr(X86::VPMINSDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7056             :   }
    7057           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7058           0 :     return fastEmitInst_rr(X86::PMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7059             :   }
    7060           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7061           0 :     return fastEmitInst_rr(X86::VPMINSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7062             :   }
    7063             :   return 0;
    7064             : }
    7065             : 
    7066           0 : unsigned fastEmit_ISD_SMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7067           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7068             :     return 0;
    7069           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7070           0 :     return fastEmitInst_rr(X86::VPMINSDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7071             :   }
    7072           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7073           0 :     return fastEmitInst_rr(X86::VPMINSDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7074             :   }
    7075             :   return 0;
    7076             : }
    7077             : 
    7078             : unsigned fastEmit_ISD_SMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7079           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7080             :     return 0;
    7081           0 :   if ((Subtarget->hasAVX512())) {
    7082           0 :     return fastEmitInst_rr(X86::VPMINSDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7083             :   }
    7084             :   return 0;
    7085             : }
    7086             : 
    7087           0 : unsigned fastEmit_ISD_SMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7088           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7089             :     return 0;
    7090           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7091           0 :     return fastEmitInst_rr(X86::VPMINSQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7092             :   }
    7093             :   return 0;
    7094             : }
    7095             : 
    7096           0 : unsigned fastEmit_ISD_SMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7097           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7098             :     return 0;
    7099           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7100           0 :     return fastEmitInst_rr(X86::VPMINSQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7101             :   }
    7102             :   return 0;
    7103             : }
    7104             : 
    7105             : unsigned fastEmit_ISD_SMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7106           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7107             :     return 0;
    7108           0 :   if ((Subtarget->hasAVX512())) {
    7109           0 :     return fastEmitInst_rr(X86::VPMINSQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7110             :   }
    7111             :   return 0;
    7112             : }
    7113             : 
    7114           0 : unsigned fastEmit_ISD_SMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7115           0 :   switch (VT.SimpleTy) {
    7116           0 :   case MVT::v16i8: return fastEmit_ISD_SMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7117           0 :   case MVT::v32i8: return fastEmit_ISD_SMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7118           0 :   case MVT::v64i8: return fastEmit_ISD_SMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7119           0 :   case MVT::v8i16: return fastEmit_ISD_SMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7120           0 :   case MVT::v16i16: return fastEmit_ISD_SMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7121           0 :   case MVT::v32i16: return fastEmit_ISD_SMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7122           0 :   case MVT::v4i32: return fastEmit_ISD_SMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7123           0 :   case MVT::v8i32: return fastEmit_ISD_SMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7124           0 :   case MVT::v16i32: return fastEmit_ISD_SMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7125           0 :   case MVT::v2i64: return fastEmit_ISD_SMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7126           0 :   case MVT::v4i64: return fastEmit_ISD_SMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7127           0 :   case MVT::v8i64: return fastEmit_ISD_SMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7128             :   default: return 0;
    7129             :   }
    7130             : }
    7131             : 
    7132             : // FastEmit functions for ISD::SRA.
    7133             : 
    7134           1 : unsigned fastEmit_ISD_SRA_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7135           1 :   if (RetVT.SimpleTy != MVT::i8)
    7136             :     return 0;
    7137           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7138           1 :   return fastEmitInst_r(X86::SAR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7139             : }
    7140             : 
    7141           0 : unsigned fastEmit_ISD_SRA_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7142           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7143             :     return 0;
    7144           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7145           0 :     return fastEmitInst_rr(X86::VPSRAVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7146             :   }
    7147             :   return 0;
    7148             : }
    7149             : 
    7150           0 : unsigned fastEmit_ISD_SRA_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7151           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7152             :     return 0;
    7153           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7154           0 :     return fastEmitInst_rr(X86::VPSRAVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7155             :   }
    7156             :   return 0;
    7157             : }
    7158             : 
    7159             : unsigned fastEmit_ISD_SRA_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7160           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7161             :     return 0;
    7162           0 :   if ((Subtarget->hasBWI())) {
    7163           0 :     return fastEmitInst_rr(X86::VPSRAVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7164             :   }
    7165             :   return 0;
    7166             : }
    7167             : 
    7168           0 : unsigned fastEmit_ISD_SRA_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7169           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7170             :     return 0;
    7171           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7172           0 :     return fastEmitInst_rr(X86::VPSRAVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7173             :   }
    7174           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7175           0 :     return fastEmitInst_rr(X86::VPSRAVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7176             :   }
    7177             :   return 0;
    7178             : }
    7179             : 
    7180           0 : unsigned fastEmit_ISD_SRA_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7181           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7182             :     return 0;
    7183           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7184           0 :     return fastEmitInst_rr(X86::VPSRAVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7185             :   }
    7186           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7187           0 :     return fastEmitInst_rr(X86::VPSRAVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7188             :   }
    7189             :   return 0;
    7190             : }
    7191             : 
    7192             : unsigned fastEmit_ISD_SRA_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7193           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7194             :     return 0;
    7195           0 :   if ((Subtarget->hasAVX512())) {
    7196           0 :     return fastEmitInst_rr(X86::VPSRAVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7197             :   }
    7198             :   return 0;
    7199             : }
    7200             : 
    7201           0 : unsigned fastEmit_ISD_SRA_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7202           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7203             :     return 0;
    7204           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7205           0 :     return fastEmitInst_rr(X86::VPSRAVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7206             :   }
    7207             :   return 0;
    7208             : }
    7209             : 
    7210           0 : unsigned fastEmit_ISD_SRA_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7211           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7212             :     return 0;
    7213           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7214           0 :     return fastEmitInst_rr(X86::VPSRAVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7215             :   }
    7216             :   return 0;
    7217             : }
    7218             : 
    7219             : unsigned fastEmit_ISD_SRA_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7220           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7221             :     return 0;
    7222           0 :   if ((Subtarget->hasAVX512())) {
    7223           0 :     return fastEmitInst_rr(X86::VPSRAVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7224             :   }
    7225             :   return 0;
    7226             : }
    7227             : 
    7228           5 : unsigned fastEmit_ISD_SRA_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7229           5 :   switch (VT.SimpleTy) {
    7230           1 :   case MVT::i8: return fastEmit_ISD_SRA_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7231           0 :   case MVT::v8i16: return fastEmit_ISD_SRA_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7232           0 :   case MVT::v16i16: return fastEmit_ISD_SRA_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7233           0 :   case MVT::v32i16: return fastEmit_ISD_SRA_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7234           0 :   case MVT::v4i32: return fastEmit_ISD_SRA_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7235           0 :   case MVT::v8i32: return fastEmit_ISD_SRA_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7236           0 :   case MVT::v16i32: return fastEmit_ISD_SRA_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7237           0 :   case MVT::v2i64: return fastEmit_ISD_SRA_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7238           0 :   case MVT::v4i64: return fastEmit_ISD_SRA_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7239           0 :   case MVT::v8i64: return fastEmit_ISD_SRA_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7240             :   default: return 0;
    7241             :   }
    7242             : }
    7243             : 
    7244             : // FastEmit functions for ISD::SRL.
    7245             : 
    7246           1 : unsigned fastEmit_ISD_SRL_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7247           1 :   if (RetVT.SimpleTy != MVT::i8)
    7248             :     return 0;
    7249           2 :   BuildMI(*FuncInfo.MBB, FuncInfo.InsertPt, DbgLoc, TII.get(TargetOpcode::COPY), X86::CL).addReg(Op1);
    7250           1 :   return fastEmitInst_r(X86::SHR8rCL, &X86::GR8RegClass, Op0, Op0IsKill);
    7251             : }
    7252             : 
    7253           0 : unsigned fastEmit_ISD_SRL_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7254           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7255             :     return 0;
    7256           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7257           0 :     return fastEmitInst_rr(X86::VPSRLVWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7258             :   }
    7259             :   return 0;
    7260             : }
    7261             : 
    7262           0 : unsigned fastEmit_ISD_SRL_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7263           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7264             :     return 0;
    7265           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7266           0 :     return fastEmitInst_rr(X86::VPSRLVWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7267             :   }
    7268             :   return 0;
    7269             : }
    7270             : 
    7271             : unsigned fastEmit_ISD_SRL_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7272           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7273             :     return 0;
    7274           0 :   if ((Subtarget->hasBWI())) {
    7275           0 :     return fastEmitInst_rr(X86::VPSRLVWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7276             :   }
    7277             :   return 0;
    7278             : }
    7279             : 
    7280           0 : unsigned fastEmit_ISD_SRL_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7281           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7282             :     return 0;
    7283           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7284           0 :     return fastEmitInst_rr(X86::VPSRLVDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7285             :   }
    7286           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7287           0 :     return fastEmitInst_rr(X86::VPSRLVDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7288             :   }
    7289             :   return 0;
    7290             : }
    7291             : 
    7292           0 : unsigned fastEmit_ISD_SRL_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7293           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7294             :     return 0;
    7295           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7296           0 :     return fastEmitInst_rr(X86::VPSRLVDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7297             :   }
    7298           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7299           0 :     return fastEmitInst_rr(X86::VPSRLVDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7300             :   }
    7301             :   return 0;
    7302             : }
    7303             : 
    7304             : unsigned fastEmit_ISD_SRL_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7305           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7306             :     return 0;
    7307           0 :   if ((Subtarget->hasAVX512())) {
    7308           0 :     return fastEmitInst_rr(X86::VPSRLVDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7309             :   }
    7310             :   return 0;
    7311             : }
    7312             : 
    7313           0 : unsigned fastEmit_ISD_SRL_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7314           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7315             :     return 0;
    7316           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7317           0 :     return fastEmitInst_rr(X86::VPSRLVQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7318             :   }
    7319           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7320           0 :     return fastEmitInst_rr(X86::VPSRLVQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7321             :   }
    7322             :   return 0;
    7323             : }
    7324             : 
    7325           0 : unsigned fastEmit_ISD_SRL_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7326           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7327             :     return 0;
    7328           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7329           0 :     return fastEmitInst_rr(X86::VPSRLVQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7330             :   }
    7331           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7332           0 :     return fastEmitInst_rr(X86::VPSRLVQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7333             :   }
    7334             :   return 0;
    7335             : }
    7336             : 
    7337             : unsigned fastEmit_ISD_SRL_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7338           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7339             :     return 0;
    7340           0 :   if ((Subtarget->hasAVX512())) {
    7341           0 :     return fastEmitInst_rr(X86::VPSRLVQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7342             :   }
    7343             :   return 0;
    7344             : }
    7345             : 
    7346           4 : unsigned fastEmit_ISD_SRL_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7347           4 :   switch (VT.SimpleTy) {
    7348           1 :   case MVT::i8: return fastEmit_ISD_SRL_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7349           0 :   case MVT::v8i16: return fastEmit_ISD_SRL_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7350           0 :   case MVT::v16i16: return fastEmit_ISD_SRL_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7351           0 :   case MVT::v32i16: return fastEmit_ISD_SRL_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7352           0 :   case MVT::v4i32: return fastEmit_ISD_SRL_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7353           0 :   case MVT::v8i32: return fastEmit_ISD_SRL_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7354           0 :   case MVT::v16i32: return fastEmit_ISD_SRL_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7355           0 :   case MVT::v2i64: return fastEmit_ISD_SRL_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7356           0 :   case MVT::v4i64: return fastEmit_ISD_SRL_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7357           0 :   case MVT::v8i64: return fastEmit_ISD_SRL_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7358             :   default: return 0;
    7359             :   }
    7360             : }
    7361             : 
    7362             : // FastEmit functions for ISD::SUB.
    7363             : 
    7364             : unsigned fastEmit_ISD_SUB_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7365           1 :   if (RetVT.SimpleTy != MVT::i8)
    7366             :     return 0;
    7367           1 :   return fastEmitInst_rr(X86::SUB8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7368             : }
    7369             : 
    7370             : unsigned fastEmit_ISD_SUB_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7371           0 :   if (RetVT.SimpleTy != MVT::i16)
    7372             :     return 0;
    7373           0 :   return fastEmitInst_rr(X86::SUB16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7374             : }
    7375             : 
    7376             : unsigned fastEmit_ISD_SUB_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7377         124 :   if (RetVT.SimpleTy != MVT::i32)
    7378             :     return 0;
    7379         124 :   return fastEmitInst_rr(X86::SUB32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7380             : }
    7381             : 
    7382             : unsigned fastEmit_ISD_SUB_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7383        1286 :   if (RetVT.SimpleTy != MVT::i64)
    7384             :     return 0;
    7385        1286 :   return fastEmitInst_rr(X86::SUB64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7386             : }
    7387             : 
    7388           2 : unsigned fastEmit_ISD_SUB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7389           2 :   if (RetVT.SimpleTy != MVT::v16i8)
    7390             :     return 0;
    7391           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7392           0 :     return fastEmitInst_rr(X86::VPSUBBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7393             :   }
    7394           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7395           2 :     return fastEmitInst_rr(X86::PSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7396             :   }
    7397           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7398           0 :     return fastEmitInst_rr(X86::VPSUBBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7399             :   }
    7400             :   return 0;
    7401             : }
    7402             : 
    7403           2 : unsigned fastEmit_ISD_SUB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7404           2 :   if (RetVT.SimpleTy != MVT::v32i8)
    7405             :     return 0;
    7406           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7407           0 :     return fastEmitInst_rr(X86::VPSUBBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7408             :   }
    7409           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7410           2 :     return fastEmitInst_rr(X86::VPSUBBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7411             :   }
    7412             :   return 0;
    7413             : }
    7414             : 
    7415             : unsigned fastEmit_ISD_SUB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7416           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7417             :     return 0;
    7418           0 :   if ((Subtarget->hasBWI())) {
    7419           0 :     return fastEmitInst_rr(X86::VPSUBBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7420             :   }
    7421             :   return 0;
    7422             : }
    7423             : 
    7424           2 : unsigned fastEmit_ISD_SUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7425           2 :   if (RetVT.SimpleTy != MVT::v8i16)
    7426             :     return 0;
    7427           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7428           0 :     return fastEmitInst_rr(X86::VPSUBWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7429             :   }
    7430           4 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7431           2 :     return fastEmitInst_rr(X86::PSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7432             :   }
    7433           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7434           0 :     return fastEmitInst_rr(X86::VPSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7435             :   }
    7436             :   return 0;
    7437             : }
    7438             : 
    7439           2 : unsigned fastEmit_ISD_SUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7440           2 :   if (RetVT.SimpleTy != MVT::v16i16)
    7441             :     return 0;
    7442           2 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7443           0 :     return fastEmitInst_rr(X86::VPSUBWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7444             :   }
    7445           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7446           2 :     return fastEmitInst_rr(X86::VPSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7447             :   }
    7448             :   return 0;
    7449             : }
    7450             : 
    7451             : unsigned fastEmit_ISD_SUB_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7452           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7453             :     return 0;
    7454           0 :   if ((Subtarget->hasBWI())) {
    7455           0 :     return fastEmitInst_rr(X86::VPSUBWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7456             :   }
    7457             :   return 0;
    7458             : }
    7459             : 
    7460           2 : unsigned fastEmit_ISD_SUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7461           2 :   if (RetVT.SimpleTy != MVT::v4i32)
    7462             :     return 0;
    7463           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7464           0 :     return fastEmitInst_rr(X86::VPSUBDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7465             :   }
    7466           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7467           2 :     return fastEmitInst_rr(X86::PSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7468             :   }
    7469           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7470           0 :     return fastEmitInst_rr(X86::VPSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7471             :   }
    7472             :   return 0;
    7473             : }
    7474             : 
    7475           2 : unsigned fastEmit_ISD_SUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7476           2 :   if (RetVT.SimpleTy != MVT::v8i32)
    7477             :     return 0;
    7478           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7479           0 :     return fastEmitInst_rr(X86::VPSUBDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7480             :   }
    7481           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7482           2 :     return fastEmitInst_rr(X86::VPSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7483             :   }
    7484             :   return 0;
    7485             : }
    7486             : 
    7487             : unsigned fastEmit_ISD_SUB_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7488           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7489             :     return 0;
    7490           0 :   if ((Subtarget->hasAVX512())) {
    7491           0 :     return fastEmitInst_rr(X86::VPSUBDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7492             :   }
    7493             :   return 0;
    7494             : }
    7495             : 
    7496           2 : unsigned fastEmit_ISD_SUB_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7497           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    7498             :     return 0;
    7499           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7500           0 :     return fastEmitInst_rr(X86::VPSUBQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7501             :   }
    7502           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7503           2 :     return fastEmitInst_rr(X86::PSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7504             :   }
    7505           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7506           0 :     return fastEmitInst_rr(X86::VPSUBQrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7507             :   }
    7508             :   return 0;
    7509             : }
    7510             : 
    7511           2 : unsigned fastEmit_ISD_SUB_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7512           2 :   if (RetVT.SimpleTy != MVT::v4i64)
    7513             :     return 0;
    7514           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7515           0 :     return fastEmitInst_rr(X86::VPSUBQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7516             :   }
    7517           2 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7518           2 :     return fastEmitInst_rr(X86::VPSUBQYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7519             :   }
    7520             :   return 0;
    7521             : }
    7522             : 
    7523             : unsigned fastEmit_ISD_SUB_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7524           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7525             :     return 0;
    7526           0 :   if ((Subtarget->hasAVX512())) {
    7527           0 :     return fastEmitInst_rr(X86::VPSUBQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7528             :   }
    7529             :   return 0;
    7530             : }
    7531             : 
    7532        1427 : unsigned fastEmit_ISD_SUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7533        1427 :   switch (VT.SimpleTy) {
    7534           1 :   case MVT::i8: return fastEmit_ISD_SUB_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7535           0 :   case MVT::i16: return fastEmit_ISD_SUB_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7536         124 :   case MVT::i32: return fastEmit_ISD_SUB_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7537        1286 :   case MVT::i64: return fastEmit_ISD_SUB_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7538           2 :   case MVT::v16i8: return fastEmit_ISD_SUB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7539           2 :   case MVT::v32i8: return fastEmit_ISD_SUB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7540           0 :   case MVT::v64i8: return fastEmit_ISD_SUB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7541           2 :   case MVT::v8i16: return fastEmit_ISD_SUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7542           2 :   case MVT::v16i16: return fastEmit_ISD_SUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7543           0 :   case MVT::v32i16: return fastEmit_ISD_SUB_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7544           2 :   case MVT::v4i32: return fastEmit_ISD_SUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7545           2 :   case MVT::v8i32: return fastEmit_ISD_SUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7546           0 :   case MVT::v16i32: return fastEmit_ISD_SUB_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7547           2 :   case MVT::v2i64: return fastEmit_ISD_SUB_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7548           2 :   case MVT::v4i64: return fastEmit_ISD_SUB_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7549           0 :   case MVT::v8i64: return fastEmit_ISD_SUB_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7550             :   default: return 0;
    7551             :   }
    7552             : }
    7553             : 
    7554             : // FastEmit functions for ISD::UMAX.
    7555             : 
    7556           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7557           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7558             :     return 0;
    7559           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7560           0 :     return fastEmitInst_rr(X86::VPMAXUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7561             :   }
    7562           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7563           0 :     return fastEmitInst_rr(X86::PMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7564             :   }
    7565           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7566           0 :     return fastEmitInst_rr(X86::VPMAXUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7567             :   }
    7568             :   return 0;
    7569             : }
    7570             : 
    7571           0 : unsigned fastEmit_ISD_UMAX_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7572           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7573             :     return 0;
    7574           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7575           0 :     return fastEmitInst_rr(X86::VPMAXUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7576             :   }
    7577           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7578           0 :     return fastEmitInst_rr(X86::VPMAXUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7579             :   }
    7580             :   return 0;
    7581             : }
    7582             : 
    7583             : unsigned fastEmit_ISD_UMAX_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7584           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7585             :     return 0;
    7586           0 :   if ((Subtarget->hasBWI())) {
    7587           0 :     return fastEmitInst_rr(X86::VPMAXUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7588             :   }
    7589             :   return 0;
    7590             : }
    7591             : 
    7592           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7593           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7594             :     return 0;
    7595           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7596           0 :     return fastEmitInst_rr(X86::VPMAXUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7597             :   }
    7598           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7599           0 :     return fastEmitInst_rr(X86::PMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7600             :   }
    7601           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7602           0 :     return fastEmitInst_rr(X86::VPMAXUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7603             :   }
    7604             :   return 0;
    7605             : }
    7606             : 
    7607           0 : unsigned fastEmit_ISD_UMAX_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7608           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7609             :     return 0;
    7610           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7611           0 :     return fastEmitInst_rr(X86::VPMAXUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7612             :   }
    7613           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7614           0 :     return fastEmitInst_rr(X86::VPMAXUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7615             :   }
    7616             :   return 0;
    7617             : }
    7618             : 
    7619             : unsigned fastEmit_ISD_UMAX_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7620           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7621             :     return 0;
    7622           0 :   if ((Subtarget->hasBWI())) {
    7623           0 :     return fastEmitInst_rr(X86::VPMAXUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7624             :   }
    7625             :   return 0;
    7626             : }
    7627             : 
    7628           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7629           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7630             :     return 0;
    7631           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7632           0 :     return fastEmitInst_rr(X86::VPMAXUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7633             :   }
    7634           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7635           0 :     return fastEmitInst_rr(X86::PMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7636             :   }
    7637           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7638           0 :     return fastEmitInst_rr(X86::VPMAXUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7639             :   }
    7640             :   return 0;
    7641             : }
    7642             : 
    7643           0 : unsigned fastEmit_ISD_UMAX_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7644           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7645             :     return 0;
    7646           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7647           0 :     return fastEmitInst_rr(X86::VPMAXUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7648             :   }
    7649           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7650           0 :     return fastEmitInst_rr(X86::VPMAXUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7651             :   }
    7652             :   return 0;
    7653             : }
    7654             : 
    7655             : unsigned fastEmit_ISD_UMAX_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7656           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7657             :     return 0;
    7658           0 :   if ((Subtarget->hasAVX512())) {
    7659           0 :     return fastEmitInst_rr(X86::VPMAXUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7660             :   }
    7661             :   return 0;
    7662             : }
    7663             : 
    7664           0 : unsigned fastEmit_ISD_UMAX_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7665           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7666             :     return 0;
    7667           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7668           0 :     return fastEmitInst_rr(X86::VPMAXUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7669             :   }
    7670             :   return 0;
    7671             : }
    7672             : 
    7673           0 : unsigned fastEmit_ISD_UMAX_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7674           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7675             :     return 0;
    7676           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7677           0 :     return fastEmitInst_rr(X86::VPMAXUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7678             :   }
    7679             :   return 0;
    7680             : }
    7681             : 
    7682             : unsigned fastEmit_ISD_UMAX_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7683           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7684             :     return 0;
    7685           0 :   if ((Subtarget->hasAVX512())) {
    7686           0 :     return fastEmitInst_rr(X86::VPMAXUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7687             :   }
    7688             :   return 0;
    7689             : }
    7690             : 
    7691           0 : unsigned fastEmit_ISD_UMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7692           0 :   switch (VT.SimpleTy) {
    7693           0 :   case MVT::v16i8: return fastEmit_ISD_UMAX_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7694           0 :   case MVT::v32i8: return fastEmit_ISD_UMAX_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7695           0 :   case MVT::v64i8: return fastEmit_ISD_UMAX_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7696           0 :   case MVT::v8i16: return fastEmit_ISD_UMAX_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7697           0 :   case MVT::v16i16: return fastEmit_ISD_UMAX_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7698           0 :   case MVT::v32i16: return fastEmit_ISD_UMAX_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7699           0 :   case MVT::v4i32: return fastEmit_ISD_UMAX_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7700           0 :   case MVT::v8i32: return fastEmit_ISD_UMAX_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7701           0 :   case MVT::v16i32: return fastEmit_ISD_UMAX_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7702           0 :   case MVT::v2i64: return fastEmit_ISD_UMAX_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7703           0 :   case MVT::v4i64: return fastEmit_ISD_UMAX_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7704           0 :   case MVT::v8i64: return fastEmit_ISD_UMAX_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7705             :   default: return 0;
    7706             :   }
    7707             : }
    7708             : 
    7709             : // FastEmit functions for ISD::UMIN.
    7710             : 
    7711           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7712           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7713             :     return 0;
    7714           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7715           0 :     return fastEmitInst_rr(X86::VPMINUBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7716             :   }
    7717           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7718           0 :     return fastEmitInst_rr(X86::PMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7719             :   }
    7720           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7721           0 :     return fastEmitInst_rr(X86::VPMINUBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7722             :   }
    7723             :   return 0;
    7724             : }
    7725             : 
    7726           0 : unsigned fastEmit_ISD_UMIN_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7727           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    7728             :     return 0;
    7729           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7730           0 :     return fastEmitInst_rr(X86::VPMINUBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7731             :   }
    7732           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7733           0 :     return fastEmitInst_rr(X86::VPMINUBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7734             :   }
    7735             :   return 0;
    7736             : }
    7737             : 
    7738             : unsigned fastEmit_ISD_UMIN_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7739           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    7740             :     return 0;
    7741           0 :   if ((Subtarget->hasBWI())) {
    7742           0 :     return fastEmitInst_rr(X86::VPMINUBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7743             :   }
    7744             :   return 0;
    7745             : }
    7746             : 
    7747           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7748           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    7749             :     return 0;
    7750           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7751           0 :     return fastEmitInst_rr(X86::VPMINUWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7752             :   }
    7753           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7754           0 :     return fastEmitInst_rr(X86::PMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7755             :   }
    7756           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7757           0 :     return fastEmitInst_rr(X86::VPMINUWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7758             :   }
    7759             :   return 0;
    7760             : }
    7761             : 
    7762           0 : unsigned fastEmit_ISD_UMIN_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7763           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    7764             :     return 0;
    7765           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7766           0 :     return fastEmitInst_rr(X86::VPMINUWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7767             :   }
    7768           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7769           0 :     return fastEmitInst_rr(X86::VPMINUWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7770             :   }
    7771             :   return 0;
    7772             : }
    7773             : 
    7774             : unsigned fastEmit_ISD_UMIN_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7775           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    7776             :     return 0;
    7777           0 :   if ((Subtarget->hasBWI())) {
    7778           0 :     return fastEmitInst_rr(X86::VPMINUWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7779             :   }
    7780             :   return 0;
    7781             : }
    7782             : 
    7783           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7784           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    7785             :     return 0;
    7786           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7787           0 :     return fastEmitInst_rr(X86::VPMINUDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7788             :   }
    7789           0 :   if ((Subtarget->hasSSE41() && !Subtarget->hasAVX())) {
    7790           0 :     return fastEmitInst_rr(X86::PMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7791             :   }
    7792           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7793           0 :     return fastEmitInst_rr(X86::VPMINUDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7794             :   }
    7795             :   return 0;
    7796             : }
    7797             : 
    7798           0 : unsigned fastEmit_ISD_UMIN_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7799           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    7800             :     return 0;
    7801           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7802           0 :     return fastEmitInst_rr(X86::VPMINUDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7803             :   }
    7804           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7805           0 :     return fastEmitInst_rr(X86::VPMINUDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7806             :   }
    7807             :   return 0;
    7808             : }
    7809             : 
    7810             : unsigned fastEmit_ISD_UMIN_MVT_v16i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7811           0 :   if (RetVT.SimpleTy != MVT::v16i32)
    7812             :     return 0;
    7813           0 :   if ((Subtarget->hasAVX512())) {
    7814           0 :     return fastEmitInst_rr(X86::VPMINUDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7815             :   }
    7816             :   return 0;
    7817             : }
    7818             : 
    7819           0 : unsigned fastEmit_ISD_UMIN_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7820           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    7821             :     return 0;
    7822           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7823           0 :     return fastEmitInst_rr(X86::VPMINUQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7824             :   }
    7825             :   return 0;
    7826             : }
    7827             : 
    7828           0 : unsigned fastEmit_ISD_UMIN_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7829           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    7830             :     return 0;
    7831           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7832           0 :     return fastEmitInst_rr(X86::VPMINUQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7833             :   }
    7834             :   return 0;
    7835             : }
    7836             : 
    7837             : unsigned fastEmit_ISD_UMIN_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7838           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7839             :     return 0;
    7840           0 :   if ((Subtarget->hasAVX512())) {
    7841           0 :     return fastEmitInst_rr(X86::VPMINUQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7842             :   }
    7843             :   return 0;
    7844             : }
    7845             : 
    7846           0 : unsigned fastEmit_ISD_UMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7847           0 :   switch (VT.SimpleTy) {
    7848           0 :   case MVT::v16i8: return fastEmit_ISD_UMIN_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7849           0 :   case MVT::v32i8: return fastEmit_ISD_UMIN_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7850           0 :   case MVT::v64i8: return fastEmit_ISD_UMIN_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7851           0 :   case MVT::v8i16: return fastEmit_ISD_UMIN_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7852           0 :   case MVT::v16i16: return fastEmit_ISD_UMIN_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7853           0 :   case MVT::v32i16: return fastEmit_ISD_UMIN_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7854           0 :   case MVT::v4i32: return fastEmit_ISD_UMIN_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7855           0 :   case MVT::v8i32: return fastEmit_ISD_UMIN_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7856           0 :   case MVT::v16i32: return fastEmit_ISD_UMIN_MVT_v16i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7857           0 :   case MVT::v2i64: return fastEmit_ISD_UMIN_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7858           0 :   case MVT::v4i64: return fastEmit_ISD_UMIN_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7859           0 :   case MVT::v8i64: return fastEmit_ISD_UMIN_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7860             :   default: return 0;
    7861             :   }
    7862             : }
    7863             : 
    7864             : // FastEmit functions for ISD::XOR.
    7865             : 
    7866             : unsigned fastEmit_ISD_XOR_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7867           0 :   if (RetVT.SimpleTy != MVT::i8)
    7868             :     return 0;
    7869           0 :   return fastEmitInst_rr(X86::XOR8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7870             : }
    7871             : 
    7872             : unsigned fastEmit_ISD_XOR_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7873           0 :   if (RetVT.SimpleTy != MVT::i16)
    7874             :     return 0;
    7875           0 :   return fastEmitInst_rr(X86::XOR16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7876             : }
    7877             : 
    7878             : unsigned fastEmit_ISD_XOR_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7879          17 :   if (RetVT.SimpleTy != MVT::i32)
    7880             :     return 0;
    7881          17 :   return fastEmitInst_rr(X86::XOR32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7882             : }
    7883             : 
    7884             : unsigned fastEmit_ISD_XOR_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7885           7 :   if (RetVT.SimpleTy != MVT::i64)
    7886             :     return 0;
    7887           7 :   return fastEmitInst_rr(X86::XOR64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7888             : }
    7889             : 
    7890             : unsigned fastEmit_ISD_XOR_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7891           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    7892             :     return 0;
    7893           0 :   if ((Subtarget->hasDQI())) {
    7894           0 :     return fastEmitInst_rr(X86::KXORBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7895             :   }
    7896             :   return 0;
    7897             : }
    7898             : 
    7899             : unsigned fastEmit_ISD_XOR_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7900           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    7901             :     return 0;
    7902           0 :   if ((Subtarget->hasAVX512())) {
    7903           0 :     return fastEmitInst_rr(X86::KXORWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7904             :   }
    7905             :   return 0;
    7906             : }
    7907             : 
    7908             : unsigned fastEmit_ISD_XOR_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7909           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    7910             :     return 0;
    7911           0 :   if ((Subtarget->hasBWI())) {
    7912           0 :     return fastEmitInst_rr(X86::KXORDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7913             :   }
    7914             :   return 0;
    7915             : }
    7916             : 
    7917             : unsigned fastEmit_ISD_XOR_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7918           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    7919             :     return 0;
    7920           0 :   if ((Subtarget->hasBWI())) {
    7921           0 :     return fastEmitInst_rr(X86::KXORQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7922             :   }
    7923             :   return 0;
    7924             : }
    7925             : 
    7926           2 : unsigned fastEmit_ISD_XOR_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7927           2 :   if (RetVT.SimpleTy != MVT::v2i64)
    7928             :     return 0;
    7929           4 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7930           0 :     return fastEmitInst_rr(X86::VPXORQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7931             :   }
    7932           2 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7933           2 :     return fastEmitInst_rr(X86::PXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7934             :   }
    7935           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    7936           0 :     return fastEmitInst_rr(X86::VPXORrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7937             :   }
    7938             :   return 0;
    7939             : }
    7940             : 
    7941           4 : unsigned fastEmit_ISD_XOR_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7942           4 :   if (RetVT.SimpleTy != MVT::v4i64)
    7943             :     return 0;
    7944           8 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    7945           2 :     return fastEmitInst_rr(X86::VXORPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7946             :   }
    7947           2 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    7948           0 :     return fastEmitInst_rr(X86::VPXORQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7949             :   }
    7950           4 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    7951           2 :     return fastEmitInst_rr(X86::VPXORYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7952             :   }
    7953             :   return 0;
    7954             : }
    7955             : 
    7956             : unsigned fastEmit_ISD_XOR_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7957           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    7958             :     return 0;
    7959           0 :   if ((Subtarget->hasAVX512())) {
    7960           0 :     return fastEmitInst_rr(X86::VPXORQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7961             :   }
    7962             :   return 0;
    7963             : }
    7964             : 
    7965          34 : unsigned fastEmit_ISD_XOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7966          34 :   switch (VT.SimpleTy) {
    7967           0 :   case MVT::i8: return fastEmit_ISD_XOR_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7968           0 :   case MVT::i16: return fastEmit_ISD_XOR_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7969          17 :   case MVT::i32: return fastEmit_ISD_XOR_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7970           7 :   case MVT::i64: return fastEmit_ISD_XOR_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7971           0 :   case MVT::v8i1: return fastEmit_ISD_XOR_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7972           0 :   case MVT::v16i1: return fastEmit_ISD_XOR_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7973           0 :   case MVT::v32i1: return fastEmit_ISD_XOR_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7974           0 :   case MVT::v64i1: return fastEmit_ISD_XOR_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7975           2 :   case MVT::v2i64: return fastEmit_ISD_XOR_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7976           4 :   case MVT::v4i64: return fastEmit_ISD_XOR_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7977           0 :   case MVT::v8i64: return fastEmit_ISD_XOR_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    7978             :   default: return 0;
    7979             :   }
    7980             : }
    7981             : 
    7982             : // FastEmit functions for X86ISD::ADDS.
    7983             : 
    7984           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    7985           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    7986             :     return 0;
    7987           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    7988           0 :     return fastEmitInst_rr(X86::VPADDSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7989             :   }
    7990           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    7991           0 :     return fastEmitInst_rr(X86::PADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7992             :   }
    7993           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    7994           0 :     return fastEmitInst_rr(X86::VPADDSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    7995             :   }
    7996             :   return 0;
    7997             : }
    7998             : 
    7999           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8000           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8001             :     return 0;
    8002           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8003           0 :     return fastEmitInst_rr(X86::VPADDSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8004             :   }
    8005           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8006           0 :     return fastEmitInst_rr(X86::VPADDSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8007             :   }
    8008             :   return 0;
    8009             : }
    8010             : 
    8011             : unsigned fastEmit_X86ISD_ADDS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8012           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8013             :     return 0;
    8014           0 :   if ((Subtarget->hasBWI())) {
    8015           0 :     return fastEmitInst_rr(X86::VPADDSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8016             :   }
    8017             :   return 0;
    8018             : }
    8019             : 
    8020           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8021           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8022             :     return 0;
    8023           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8024           0 :     return fastEmitInst_rr(X86::VPADDSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8025             :   }
    8026           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8027           0 :     return fastEmitInst_rr(X86::PADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8028             :   }
    8029           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8030           0 :     return fastEmitInst_rr(X86::VPADDSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8031             :   }
    8032             :   return 0;
    8033             : }
    8034             : 
    8035           0 : unsigned fastEmit_X86ISD_ADDS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8036           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8037             :     return 0;
    8038           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8039           0 :     return fastEmitInst_rr(X86::VPADDSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8040             :   }
    8041           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8042           0 :     return fastEmitInst_rr(X86::VPADDSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8043             :   }
    8044             :   return 0;
    8045             : }
    8046             : 
    8047             : unsigned fastEmit_X86ISD_ADDS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8048           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8049             :     return 0;
    8050           0 :   if ((Subtarget->hasBWI())) {
    8051           0 :     return fastEmitInst_rr(X86::VPADDSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8052             :   }
    8053             :   return 0;
    8054             : }
    8055             : 
    8056           0 : unsigned fastEmit_X86ISD_ADDS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8057           0 :   switch (VT.SimpleTy) {
    8058           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8059           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8060           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8061           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8062           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8063           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8064             :   default: return 0;
    8065             :   }
    8066             : }
    8067             : 
    8068             : // FastEmit functions for X86ISD::ADDSUB.
    8069             : 
    8070           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8071           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8072             :     return 0;
    8073           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8074           0 :     return fastEmitInst_rr(X86::ADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8075             :   }
    8076           0 :   if ((Subtarget->hasAVX())) {
    8077           0 :     return fastEmitInst_rr(X86::VADDSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8078             :   }
    8079             :   return 0;
    8080             : }
    8081             : 
    8082             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8083           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8084             :     return 0;
    8085           0 :   if ((Subtarget->hasAVX())) {
    8086           0 :     return fastEmitInst_rr(X86::VADDSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8087             :   }
    8088             :   return 0;
    8089             : }
    8090             : 
    8091           0 : unsigned fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8092           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8093             :     return 0;
    8094           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8095           0 :     return fastEmitInst_rr(X86::ADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8096             :   }
    8097           0 :   if ((Subtarget->hasAVX())) {
    8098           0 :     return fastEmitInst_rr(X86::VADDSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8099             :   }
    8100             :   return 0;
    8101             : }
    8102             : 
    8103             : unsigned fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8104           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8105             :     return 0;
    8106           0 :   if ((Subtarget->hasAVX())) {
    8107           0 :     return fastEmitInst_rr(X86::VADDSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8108             :   }
    8109             :   return 0;
    8110             : }
    8111             : 
    8112           0 : unsigned fastEmit_X86ISD_ADDSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8113           0 :   switch (VT.SimpleTy) {
    8114           0 :   case MVT::v4f32: return fastEmit_X86ISD_ADDSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8115           0 :   case MVT::v8f32: return fastEmit_X86ISD_ADDSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8116           0 :   case MVT::v2f64: return fastEmit_X86ISD_ADDSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8117           0 :   case MVT::v4f64: return fastEmit_X86ISD_ADDSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8118             :   default: return 0;
    8119             :   }
    8120             : }
    8121             : 
    8122             : // FastEmit functions for X86ISD::ADDUS.
    8123             : 
    8124           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8125           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8126             :     return 0;
    8127           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8128           0 :     return fastEmitInst_rr(X86::VPADDUSBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8129             :   }
    8130           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8131           0 :     return fastEmitInst_rr(X86::PADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8132             :   }
    8133           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8134           0 :     return fastEmitInst_rr(X86::VPADDUSBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8135             :   }
    8136             :   return 0;
    8137             : }
    8138             : 
    8139           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8140           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8141             :     return 0;
    8142           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8143           0 :     return fastEmitInst_rr(X86::VPADDUSBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8144             :   }
    8145           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8146           0 :     return fastEmitInst_rr(X86::VPADDUSBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8147             :   }
    8148             :   return 0;
    8149             : }
    8150             : 
    8151             : unsigned fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8152           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8153             :     return 0;
    8154           0 :   if ((Subtarget->hasBWI())) {
    8155           0 :     return fastEmitInst_rr(X86::VPADDUSBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8156             :   }
    8157             :   return 0;
    8158             : }
    8159             : 
    8160           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8161           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8162             :     return 0;
    8163           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8164           0 :     return fastEmitInst_rr(X86::VPADDUSWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8165             :   }
    8166           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8167           0 :     return fastEmitInst_rr(X86::PADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8168             :   }
    8169           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8170           0 :     return fastEmitInst_rr(X86::VPADDUSWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8171             :   }
    8172             :   return 0;
    8173             : }
    8174             : 
    8175           0 : unsigned fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8176           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8177             :     return 0;
    8178           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8179           0 :     return fastEmitInst_rr(X86::VPADDUSWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8180             :   }
    8181           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8182           0 :     return fastEmitInst_rr(X86::VPADDUSWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8183             :   }
    8184             :   return 0;
    8185             : }
    8186             : 
    8187             : unsigned fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8188           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8189             :     return 0;
    8190           0 :   if ((Subtarget->hasBWI())) {
    8191           0 :     return fastEmitInst_rr(X86::VPADDUSWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8192             :   }
    8193             :   return 0;
    8194             : }
    8195             : 
    8196           0 : unsigned fastEmit_X86ISD_ADDUS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8197           0 :   switch (VT.SimpleTy) {
    8198           0 :   case MVT::v16i8: return fastEmit_X86ISD_ADDUS_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8199           0 :   case MVT::v32i8: return fastEmit_X86ISD_ADDUS_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8200           0 :   case MVT::v64i8: return fastEmit_X86ISD_ADDUS_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8201           0 :   case MVT::v8i16: return fastEmit_X86ISD_ADDUS_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8202           0 :   case MVT::v16i16: return fastEmit_X86ISD_ADDUS_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8203           0 :   case MVT::v32i16: return fastEmit_X86ISD_ADDUS_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8204             :   default: return 0;
    8205             :   }
    8206             : }
    8207             : 
    8208             : // FastEmit functions for X86ISD::ANDNP.
    8209             : 
    8210           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8211           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    8212             :     return 0;
    8213           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8214           0 :     return fastEmitInst_rr(X86::VPANDNQZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8215             :   }
    8216           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8217           0 :     return fastEmitInst_rr(X86::PANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8218             :   }
    8219           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8220           0 :     return fastEmitInst_rr(X86::VPANDNrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8221             :   }
    8222             :   return 0;
    8223             : }
    8224             : 
    8225           0 : unsigned fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8226           0 :   if (RetVT.SimpleTy != MVT::v4i64)
    8227             :     return 0;
    8228           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX2())) {
    8229           0 :     return fastEmitInst_rr(X86::VANDNPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8230             :   }
    8231           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8232           0 :     return fastEmitInst_rr(X86::VPANDNQZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8233             :   }
    8234           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX())) {
    8235           0 :     return fastEmitInst_rr(X86::VPANDNYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8236             :   }
    8237             :   return 0;
    8238             : }
    8239             : 
    8240             : unsigned fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8241           0 :   if (RetVT.SimpleTy != MVT::v8i64)
    8242             :     return 0;
    8243           0 :   if ((Subtarget->hasAVX512())) {
    8244           0 :     return fastEmitInst_rr(X86::VPANDNQZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8245             :   }
    8246             :   return 0;
    8247             : }
    8248             : 
    8249           0 : unsigned fastEmit_X86ISD_ANDNP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8250           0 :   switch (VT.SimpleTy) {
    8251           0 :   case MVT::v2i64: return fastEmit_X86ISD_ANDNP_MVT_v2i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8252           0 :   case MVT::v4i64: return fastEmit_X86ISD_ANDNP_MVT_v4i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8253           0 :   case MVT::v8i64: return fastEmit_X86ISD_ANDNP_MVT_v8i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8254             :   default: return 0;
    8255             :   }
    8256             : }
    8257             : 
    8258             : // FastEmit functions for X86ISD::AVG.
    8259             : 
    8260           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8261           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    8262             :     return 0;
    8263           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8264           0 :     return fastEmitInst_rr(X86::VPAVGBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8265             :   }
    8266           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8267           0 :     return fastEmitInst_rr(X86::PAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8268             :   }
    8269           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8270           0 :     return fastEmitInst_rr(X86::VPAVGBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8271             :   }
    8272             :   return 0;
    8273             : }
    8274             : 
    8275           0 : unsigned fastEmit_X86ISD_AVG_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8276           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    8277             :     return 0;
    8278           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8279           0 :     return fastEmitInst_rr(X86::VPAVGBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8280             :   }
    8281           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8282           0 :     return fastEmitInst_rr(X86::VPAVGBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8283             :   }
    8284             :   return 0;
    8285             : }
    8286             : 
    8287             : unsigned fastEmit_X86ISD_AVG_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8288           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    8289             :     return 0;
    8290           0 :   if ((Subtarget->hasBWI())) {
    8291           0 :     return fastEmitInst_rr(X86::VPAVGBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8292             :   }
    8293             :   return 0;
    8294             : }
    8295             : 
    8296           0 : unsigned fastEmit_X86ISD_AVG_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8297           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    8298             :     return 0;
    8299           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8300           0 :     return fastEmitInst_rr(X86::VPAVGWZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8301             :   }
    8302           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8303           0 :     return fastEmitInst_rr(X86::PAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8304             :   }
    8305           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8306           0 :     return fastEmitInst_rr(X86::VPAVGWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8307             :   }
    8308             :   return 0;
    8309             : }
    8310             : 
    8311           0 : unsigned fastEmit_X86ISD_AVG_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8312           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    8313             :     return 0;
    8314           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasVLX())) {
    8315           0 :     return fastEmitInst_rr(X86::VPAVGWZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8316             :   }
    8317           0 :   if ((Subtarget->hasAVX2()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    8318           0 :     return fastEmitInst_rr(X86::VPAVGWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8319             :   }
    8320             :   return 0;
    8321             : }
    8322             : 
    8323             : unsigned fastEmit_X86ISD_AVG_MVT_v32i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8324           0 :   if (RetVT.SimpleTy != MVT::v32i16)
    8325             :     return 0;
    8326           0 :   if ((Subtarget->hasBWI())) {
    8327           0 :     return fastEmitInst_rr(X86::VPAVGWZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8328             :   }
    8329             :   return 0;
    8330             : }
    8331             : 
    8332           0 : unsigned fastEmit_X86ISD_AVG_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8333           0 :   switch (VT.SimpleTy) {
    8334           0 :   case MVT::v16i8: return fastEmit_X86ISD_AVG_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8335           0 :   case MVT::v32i8: return fastEmit_X86ISD_AVG_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8336           0 :   case MVT::v64i8: return fastEmit_X86ISD_AVG_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8337           0 :   case MVT::v8i16: return fastEmit_X86ISD_AVG_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8338           0 :   case MVT::v16i16: return fastEmit_X86ISD_AVG_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8339           0 :   case MVT::v32i16: return fastEmit_X86ISD_AVG_MVT_v32i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8340             :   default: return 0;
    8341             :   }
    8342             : }
    8343             : 
    8344             : // FastEmit functions for X86ISD::BT.
    8345             : 
    8346             : unsigned fastEmit_X86ISD_BT_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8347           0 :   if (RetVT.SimpleTy != MVT::i32)
    8348             :     return 0;
    8349           0 :   return fastEmitInst_rr(X86::BT16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8350             : }
    8351             : 
    8352             : unsigned fastEmit_X86ISD_BT_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8353           0 :   if (RetVT.SimpleTy != MVT::i32)
    8354             :     return 0;
    8355           0 :   return fastEmitInst_rr(X86::BT32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8356             : }
    8357             : 
    8358             : unsigned fastEmit_X86ISD_BT_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8359           0 :   if (RetVT.SimpleTy != MVT::i32)
    8360             :     return 0;
    8361           0 :   return fastEmitInst_rr(X86::BT64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8362             : }
    8363             : 
    8364           0 : unsigned fastEmit_X86ISD_BT_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8365           0 :   switch (VT.SimpleTy) {
    8366           0 :   case MVT::i16: return fastEmit_X86ISD_BT_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8367           0 :   case MVT::i32: return fastEmit_X86ISD_BT_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8368           0 :   case MVT::i64: return fastEmit_X86ISD_BT_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8369             :   default: return 0;
    8370             :   }
    8371             : }
    8372             : 
    8373             : // FastEmit functions for X86ISD::CMP.
    8374             : 
    8375             : unsigned fastEmit_X86ISD_CMP_MVT_i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8376           0 :   if (RetVT.SimpleTy != MVT::i32)
    8377             :     return 0;
    8378           0 :   return fastEmitInst_rr(X86::CMP8rr, &X86::GR8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8379             : }
    8380             : 
    8381             : unsigned fastEmit_X86ISD_CMP_MVT_i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8382           0 :   if (RetVT.SimpleTy != MVT::i32)
    8383             :     return 0;
    8384           0 :   return fastEmitInst_rr(X86::CMP16rr, &X86::GR16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8385             : }
    8386             : 
    8387             : unsigned fastEmit_X86ISD_CMP_MVT_i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8388           0 :   if (RetVT.SimpleTy != MVT::i32)
    8389             :     return 0;
    8390           0 :   return fastEmitInst_rr(X86::CMP32rr, &X86::GR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8391             : }
    8392             : 
    8393             : unsigned fastEmit_X86ISD_CMP_MVT_i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8394           0 :   if (RetVT.SimpleTy != MVT::i32)
    8395             :     return 0;
    8396           0 :   return fastEmitInst_rr(X86::CMP64rr, &X86::GR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8397             : }
    8398             : 
    8399           0 : unsigned fastEmit_X86ISD_CMP_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8400           0 :   if (RetVT.SimpleTy != MVT::i32)
    8401             :     return 0;
    8402           0 :   if ((Subtarget->hasAVX512())) {
    8403           0 :     return fastEmitInst_rr(X86::VUCOMISSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8404             :   }
    8405           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8406           0 :     return fastEmitInst_rr(X86::UCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8407             :   }
    8408           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8409           0 :     return fastEmitInst_rr(X86::VUCOMISSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8410             :   }
    8411           0 :   if ((!Subtarget->hasSSE1())) {
    8412           0 :     return fastEmitInst_rr(X86::UCOM_FpIr32, &X86::RFP32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8413             :   }
    8414             :   return 0;
    8415             : }
    8416             : 
    8417           0 : unsigned fastEmit_X86ISD_CMP_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8418           0 :   if (RetVT.SimpleTy != MVT::i32)
    8419             :     return 0;
    8420           0 :   if ((Subtarget->hasAVX512())) {
    8421           0 :     return fastEmitInst_rr(X86::VUCOMISDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8422             :   }
    8423           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8424           0 :     return fastEmitInst_rr(X86::UCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8425             :   }
    8426           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8427           0 :     return fastEmitInst_rr(X86::VUCOMISDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8428             :   }
    8429           0 :   if ((!Subtarget->hasSSE2())) {
    8430           0 :     return fastEmitInst_rr(X86::UCOM_FpIr64, &X86::RFP64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8431             :   }
    8432             :   return 0;
    8433             : }
    8434             : 
    8435             : unsigned fastEmit_X86ISD_CMP_MVT_f80_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8436           0 :   if (RetVT.SimpleTy != MVT::i32)
    8437             :     return 0;
    8438           0 :   return fastEmitInst_rr(X86::UCOM_FpIr80, &X86::RFP80RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8439             : }
    8440             : 
    8441           0 : unsigned fastEmit_X86ISD_CMP_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8442           0 :   switch (VT.SimpleTy) {
    8443           0 :   case MVT::i8: return fastEmit_X86ISD_CMP_MVT_i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8444           0 :   case MVT::i16: return fastEmit_X86ISD_CMP_MVT_i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8445           0 :   case MVT::i32: return fastEmit_X86ISD_CMP_MVT_i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8446           0 :   case MVT::i64: return fastEmit_X86ISD_CMP_MVT_i64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8447           0 :   case MVT::f32: return fastEmit_X86ISD_CMP_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8448           0 :   case MVT::f64: return fastEmit_X86ISD_CMP_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8449           0 :   case MVT::f80: return fastEmit_X86ISD_CMP_MVT_f80_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8450             :   default: return 0;
    8451             :   }
    8452             : }
    8453             : 
    8454             : // FastEmit functions for X86ISD::COMI.
    8455             : 
    8456           0 : unsigned fastEmit_X86ISD_COMI_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8457           0 :   if (RetVT.SimpleTy != MVT::i32)
    8458             :     return 0;
    8459           0 :   if ((Subtarget->hasAVX512())) {
    8460           0 :     return fastEmitInst_rr(X86::VCOMISSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8461             :   }
    8462           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8463           0 :     return fastEmitInst_rr(X86::COMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8464             :   }
    8465           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8466           0 :     return fastEmitInst_rr(X86::VCOMISSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8467             :   }
    8468             :   return 0;
    8469             : }
    8470             : 
    8471           0 : unsigned fastEmit_X86ISD_COMI_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8472           0 :   if (RetVT.SimpleTy != MVT::i32)
    8473             :     return 0;
    8474           0 :   if ((Subtarget->hasAVX512())) {
    8475           0 :     return fastEmitInst_rr(X86::VCOMISDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8476             :   }
    8477           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8478           0 :     return fastEmitInst_rr(X86::COMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8479             :   }
    8480           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8481           0 :     return fastEmitInst_rr(X86::VCOMISDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8482             :   }
    8483             :   return 0;
    8484             : }
    8485             : 
    8486           0 : unsigned fastEmit_X86ISD_COMI_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8487           0 :   switch (VT.SimpleTy) {
    8488           0 :   case MVT::v4f32: return fastEmit_X86ISD_COMI_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8489           0 :   case MVT::v2f64: return fastEmit_X86ISD_COMI_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8490             :   default: return 0;
    8491             :   }
    8492             : }
    8493             : 
    8494             : // FastEmit functions for X86ISD::FAND.
    8495             : 
    8496             : unsigned fastEmit_X86ISD_FAND_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8497           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8498             :     return 0;
    8499           0 :   return fastEmitInst_rr(X86::ANDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8500             : }
    8501             : 
    8502             : unsigned fastEmit_X86ISD_FAND_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8503           0 :   switch (VT.SimpleTy) {
    8504           0 :   case MVT::v4f32: return fastEmit_X86ISD_FAND_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8505             :   default: return 0;
    8506             :   }
    8507             : }
    8508             : 
    8509             : // FastEmit functions for X86ISD::FANDN.
    8510             : 
    8511             : unsigned fastEmit_X86ISD_FANDN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8512           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8513             :     return 0;
    8514           0 :   return fastEmitInst_rr(X86::ANDNPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8515             : }
    8516             : 
    8517             : unsigned fastEmit_X86ISD_FANDN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8518           0 :   switch (VT.SimpleTy) {
    8519           0 :   case MVT::v4f32: return fastEmit_X86ISD_FANDN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8520             :   default: return 0;
    8521             :   }
    8522             : }
    8523             : 
    8524             : // FastEmit functions for X86ISD::FHADD.
    8525             : 
    8526           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8527           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8528             :     return 0;
    8529           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8530           0 :     return fastEmitInst_rr(X86::HADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8531             :   }
    8532           0 :   if ((Subtarget->hasAVX())) {
    8533           0 :     return fastEmitInst_rr(X86::VHADDPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8534             :   }
    8535             :   return 0;
    8536             : }
    8537             : 
    8538             : unsigned fastEmit_X86ISD_FHADD_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8539           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8540             :     return 0;
    8541           0 :   if ((Subtarget->hasAVX())) {
    8542           0 :     return fastEmitInst_rr(X86::VHADDPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8543             :   }
    8544             :   return 0;
    8545             : }
    8546             : 
    8547           0 : unsigned fastEmit_X86ISD_FHADD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8548           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8549             :     return 0;
    8550           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8551           0 :     return fastEmitInst_rr(X86::HADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8552             :   }
    8553           0 :   if ((Subtarget->hasAVX())) {
    8554           0 :     return fastEmitInst_rr(X86::VHADDPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8555             :   }
    8556             :   return 0;
    8557             : }
    8558             : 
    8559             : unsigned fastEmit_X86ISD_FHADD_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8560           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8561             :     return 0;
    8562           0 :   if ((Subtarget->hasAVX())) {
    8563           0 :     return fastEmitInst_rr(X86::VHADDPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8564             :   }
    8565             :   return 0;
    8566             : }
    8567             : 
    8568           0 : unsigned fastEmit_X86ISD_FHADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8569           0 :   switch (VT.SimpleTy) {
    8570           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHADD_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8571           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHADD_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8572           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHADD_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8573           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHADD_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8574             :   default: return 0;
    8575             :   }
    8576             : }
    8577             : 
    8578             : // FastEmit functions for X86ISD::FHSUB.
    8579             : 
    8580           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8581           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8582             :     return 0;
    8583           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8584           0 :     return fastEmitInst_rr(X86::HSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8585             :   }
    8586           0 :   if ((Subtarget->hasAVX())) {
    8587           0 :     return fastEmitInst_rr(X86::VHSUBPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8588             :   }
    8589             :   return 0;
    8590             : }
    8591             : 
    8592             : unsigned fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8593           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8594             :     return 0;
    8595           0 :   if ((Subtarget->hasAVX())) {
    8596           0 :     return fastEmitInst_rr(X86::VHSUBPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8597             :   }
    8598             :   return 0;
    8599             : }
    8600             : 
    8601           0 : unsigned fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8602           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8603             :     return 0;
    8604           0 :   if ((Subtarget->hasSSE3() && !Subtarget->hasAVX())) {
    8605           0 :     return fastEmitInst_rr(X86::HSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8606             :   }
    8607           0 :   if ((Subtarget->hasAVX())) {
    8608           0 :     return fastEmitInst_rr(X86::VHSUBPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8609             :   }
    8610             :   return 0;
    8611             : }
    8612             : 
    8613             : unsigned fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8614           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8615             :     return 0;
    8616           0 :   if ((Subtarget->hasAVX())) {
    8617           0 :     return fastEmitInst_rr(X86::VHSUBPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8618             :   }
    8619             :   return 0;
    8620             : }
    8621             : 
    8622           0 : unsigned fastEmit_X86ISD_FHSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8623           0 :   switch (VT.SimpleTy) {
    8624           0 :   case MVT::v4f32: return fastEmit_X86ISD_FHSUB_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8625           0 :   case MVT::v8f32: return fastEmit_X86ISD_FHSUB_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8626           0 :   case MVT::v2f64: return fastEmit_X86ISD_FHSUB_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8627           0 :   case MVT::v4f64: return fastEmit_X86ISD_FHSUB_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8628             :   default: return 0;
    8629             :   }
    8630             : }
    8631             : 
    8632             : // FastEmit functions for X86ISD::FMAX.
    8633             : 
    8634           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8635           0 :   if (RetVT.SimpleTy != MVT::f32)
    8636             :     return 0;
    8637           0 :   if ((Subtarget->hasAVX512())) {
    8638           0 :     return fastEmitInst_rr(X86::VMAXSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8639             :   }
    8640           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8641           0 :     return fastEmitInst_rr(X86::MAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8642             :   }
    8643           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8644           0 :     return fastEmitInst_rr(X86::VMAXSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8645             :   }
    8646             :   return 0;
    8647             : }
    8648             : 
    8649           0 : unsigned fastEmit_X86ISD_FMAX_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8650           0 :   if (RetVT.SimpleTy != MVT::f64)
    8651             :     return 0;
    8652           0 :   if ((Subtarget->hasAVX512())) {
    8653           0 :     return fastEmitInst_rr(X86::VMAXSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8654             :   }
    8655           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8656           0 :     return fastEmitInst_rr(X86::MAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8657             :   }
    8658           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8659           0 :     return fastEmitInst_rr(X86::VMAXSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8660             :   }
    8661             :   return 0;
    8662             : }
    8663             : 
    8664           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8665           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8666             :     return 0;
    8667           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8668           0 :     return fastEmitInst_rr(X86::VMAXPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8669             :   }
    8670           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8671           0 :     return fastEmitInst_rr(X86::MAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8672             :   }
    8673           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8674           0 :     return fastEmitInst_rr(X86::VMAXPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8675             :   }
    8676             :   return 0;
    8677             : }
    8678             : 
    8679           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8680           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8681             :     return 0;
    8682           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8683           0 :     return fastEmitInst_rr(X86::VMAXPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8684             :   }
    8685           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8686           0 :     return fastEmitInst_rr(X86::VMAXPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8687             :   }
    8688             :   return 0;
    8689             : }
    8690             : 
    8691             : unsigned fastEmit_X86ISD_FMAX_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8692           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8693             :     return 0;
    8694           0 :   if ((Subtarget->hasAVX512())) {
    8695           0 :     return fastEmitInst_rr(X86::VMAXPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8696             :   }
    8697             :   return 0;
    8698             : }
    8699             : 
    8700           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8701           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8702             :     return 0;
    8703           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8704           0 :     return fastEmitInst_rr(X86::VMAXPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8705             :   }
    8706           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8707           0 :     return fastEmitInst_rr(X86::MAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8708             :   }
    8709           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8710           0 :     return fastEmitInst_rr(X86::VMAXPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8711             :   }
    8712             :   return 0;
    8713             : }
    8714             : 
    8715           0 : unsigned fastEmit_X86ISD_FMAX_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8716           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8717             :     return 0;
    8718           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8719           0 :     return fastEmitInst_rr(X86::VMAXPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8720             :   }
    8721           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8722           0 :     return fastEmitInst_rr(X86::VMAXPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8723             :   }
    8724             :   return 0;
    8725             : }
    8726             : 
    8727             : unsigned fastEmit_X86ISD_FMAX_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8728           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8729             :     return 0;
    8730           0 :   if ((Subtarget->hasAVX512())) {
    8731           0 :     return fastEmitInst_rr(X86::VMAXPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8732             :   }
    8733             :   return 0;
    8734             : }
    8735             : 
    8736           0 : unsigned fastEmit_X86ISD_FMAX_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8737           0 :   switch (VT.SimpleTy) {
    8738           0 :   case MVT::f32: return fastEmit_X86ISD_FMAX_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8739           0 :   case MVT::f64: return fastEmit_X86ISD_FMAX_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8740           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAX_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8741           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAX_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8742           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAX_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8743           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAX_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8744           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAX_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8745           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAX_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8746             :   default: return 0;
    8747             :   }
    8748             : }
    8749             : 
    8750             : // FastEmit functions for X86ISD::FMAXC.
    8751             : 
    8752           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8753           0 :   if (RetVT.SimpleTy != MVT::f32)
    8754             :     return 0;
    8755           0 :   if ((Subtarget->hasAVX512())) {
    8756           0 :     return fastEmitInst_rr(X86::VMAXCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8757             :   }
    8758           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8759           0 :     return fastEmitInst_rr(X86::MAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8760             :   }
    8761           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8762           0 :     return fastEmitInst_rr(X86::VMAXCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8763             :   }
    8764             :   return 0;
    8765             : }
    8766             : 
    8767           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8768           0 :   if (RetVT.SimpleTy != MVT::f64)
    8769             :     return 0;
    8770           0 :   if ((Subtarget->hasAVX512())) {
    8771           0 :     return fastEmitInst_rr(X86::VMAXCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8772             :   }
    8773           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8774           0 :     return fastEmitInst_rr(X86::MAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8775             :   }
    8776           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8777           0 :     return fastEmitInst_rr(X86::VMAXCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8778             :   }
    8779             :   return 0;
    8780             : }
    8781             : 
    8782           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8783           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8784             :     return 0;
    8785           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8786           0 :     return fastEmitInst_rr(X86::VMAXCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8787             :   }
    8788           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8789           0 :     return fastEmitInst_rr(X86::MAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8790             :   }
    8791           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8792           0 :     return fastEmitInst_rr(X86::VMAXCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8793             :   }
    8794             :   return 0;
    8795             : }
    8796             : 
    8797           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8798           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8799             :     return 0;
    8800           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8801           0 :     return fastEmitInst_rr(X86::VMAXCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8802             :   }
    8803           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8804           0 :     return fastEmitInst_rr(X86::VMAXCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8805             :   }
    8806             :   return 0;
    8807             : }
    8808             : 
    8809             : unsigned fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8810           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8811             :     return 0;
    8812           0 :   if ((Subtarget->hasAVX512())) {
    8813           0 :     return fastEmitInst_rr(X86::VMAXCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8814             :   }
    8815             :   return 0;
    8816             : }
    8817             : 
    8818           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8819           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8820             :     return 0;
    8821           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8822           0 :     return fastEmitInst_rr(X86::VMAXCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8823             :   }
    8824           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8825           0 :     return fastEmitInst_rr(X86::MAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8826             :   }
    8827           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8828           0 :     return fastEmitInst_rr(X86::VMAXCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8829             :   }
    8830             :   return 0;
    8831             : }
    8832             : 
    8833           0 : unsigned fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8834           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8835             :     return 0;
    8836           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8837           0 :     return fastEmitInst_rr(X86::VMAXCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8838             :   }
    8839           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8840           0 :     return fastEmitInst_rr(X86::VMAXCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8841             :   }
    8842             :   return 0;
    8843             : }
    8844             : 
    8845             : unsigned fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8846           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    8847             :     return 0;
    8848           0 :   if ((Subtarget->hasAVX512())) {
    8849           0 :     return fastEmitInst_rr(X86::VMAXCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8850             :   }
    8851             :   return 0;
    8852             : }
    8853             : 
    8854           0 : unsigned fastEmit_X86ISD_FMAXC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8855           0 :   switch (VT.SimpleTy) {
    8856           0 :   case MVT::f32: return fastEmit_X86ISD_FMAXC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8857           0 :   case MVT::f64: return fastEmit_X86ISD_FMAXC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8858           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8859           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMAXC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8860           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMAXC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8861           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8862           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMAXC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8863           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMAXC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8864             :   default: return 0;
    8865             :   }
    8866             : }
    8867             : 
    8868             : // FastEmit functions for X86ISD::FMAXS.
    8869             : 
    8870           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8871           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8872             :     return 0;
    8873           0 :   if ((Subtarget->hasAVX512())) {
    8874           0 :     return fastEmitInst_rr(X86::VMAXSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8875             :   }
    8876           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8877           0 :     return fastEmitInst_rr(X86::MAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8878             :   }
    8879           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8880           0 :     return fastEmitInst_rr(X86::VMAXSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8881             :   }
    8882             :   return 0;
    8883             : }
    8884             : 
    8885           0 : unsigned fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8886           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8887             :     return 0;
    8888           0 :   if ((Subtarget->hasAVX512())) {
    8889           0 :     return fastEmitInst_rr(X86::VMAXSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8890             :   }
    8891           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8892           0 :     return fastEmitInst_rr(X86::MAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8893             :   }
    8894           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8895           0 :     return fastEmitInst_rr(X86::VMAXSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8896             :   }
    8897             :   return 0;
    8898             : }
    8899             : 
    8900           0 : unsigned fastEmit_X86ISD_FMAXS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8901           0 :   switch (VT.SimpleTy) {
    8902           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMAXS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8903           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMAXS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    8904             :   default: return 0;
    8905             :   }
    8906             : }
    8907             : 
    8908             : // FastEmit functions for X86ISD::FMIN.
    8909             : 
    8910           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8911           0 :   if (RetVT.SimpleTy != MVT::f32)
    8912             :     return 0;
    8913           0 :   if ((Subtarget->hasAVX512())) {
    8914           0 :     return fastEmitInst_rr(X86::VMINSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8915             :   }
    8916           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8917           0 :     return fastEmitInst_rr(X86::MINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8918             :   }
    8919           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8920           0 :     return fastEmitInst_rr(X86::VMINSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8921             :   }
    8922             :   return 0;
    8923             : }
    8924             : 
    8925           0 : unsigned fastEmit_X86ISD_FMIN_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8926           0 :   if (RetVT.SimpleTy != MVT::f64)
    8927             :     return 0;
    8928           0 :   if ((Subtarget->hasAVX512())) {
    8929           0 :     return fastEmitInst_rr(X86::VMINSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8930             :   }
    8931           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8932           0 :     return fastEmitInst_rr(X86::MINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8933             :   }
    8934           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    8935           0 :     return fastEmitInst_rr(X86::VMINSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8936             :   }
    8937             :   return 0;
    8938             : }
    8939             : 
    8940           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8941           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    8942             :     return 0;
    8943           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8944           0 :     return fastEmitInst_rr(X86::VMINPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8945             :   }
    8946           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    8947           0 :     return fastEmitInst_rr(X86::MINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8948             :   }
    8949           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8950           0 :     return fastEmitInst_rr(X86::VMINPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8951             :   }
    8952             :   return 0;
    8953             : }
    8954             : 
    8955           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8956           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    8957             :     return 0;
    8958           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8959           0 :     return fastEmitInst_rr(X86::VMINPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8960             :   }
    8961           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8962           0 :     return fastEmitInst_rr(X86::VMINPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8963             :   }
    8964             :   return 0;
    8965             : }
    8966             : 
    8967             : unsigned fastEmit_X86ISD_FMIN_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8968           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    8969             :     return 0;
    8970           0 :   if ((Subtarget->hasAVX512())) {
    8971           0 :     return fastEmitInst_rr(X86::VMINPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8972             :   }
    8973             :   return 0;
    8974             : }
    8975             : 
    8976           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8977           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    8978             :     return 0;
    8979           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8980           0 :     return fastEmitInst_rr(X86::VMINPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8981             :   }
    8982           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    8983           0 :     return fastEmitInst_rr(X86::MINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8984             :   }
    8985           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8986           0 :     return fastEmitInst_rr(X86::VMINPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8987             :   }
    8988             :   return 0;
    8989             : }
    8990             : 
    8991           0 : unsigned fastEmit_X86ISD_FMIN_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    8992           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    8993             :     return 0;
    8994           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    8995           0 :     return fastEmitInst_rr(X86::VMINPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8996             :   }
    8997           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    8998           0 :     return fastEmitInst_rr(X86::VMINPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    8999             :   }
    9000             :   return 0;
    9001             : }
    9002             : 
    9003             : unsigned fastEmit_X86ISD_FMIN_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9004           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9005             :     return 0;
    9006           0 :   if ((Subtarget->hasAVX512())) {
    9007           0 :     return fastEmitInst_rr(X86::VMINPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9008             :   }
    9009             :   return 0;
    9010             : }
    9011             : 
    9012           0 : unsigned fastEmit_X86ISD_FMIN_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9013           0 :   switch (VT.SimpleTy) {
    9014           0 :   case MVT::f32: return fastEmit_X86ISD_FMIN_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9015           0 :   case MVT::f64: return fastEmit_X86ISD_FMIN_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9016           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMIN_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9017           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMIN_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9018           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMIN_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9019           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMIN_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9020           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMIN_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9021           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMIN_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9022             :   default: return 0;
    9023             :   }
    9024             : }
    9025             : 
    9026             : // FastEmit functions for X86ISD::FMINC.
    9027             : 
    9028           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9029           0 :   if (RetVT.SimpleTy != MVT::f32)
    9030             :     return 0;
    9031           0 :   if ((Subtarget->hasAVX512())) {
    9032           0 :     return fastEmitInst_rr(X86::VMINCSSZrr, &X86::FR32XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9033             :   }
    9034           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9035           0 :     return fastEmitInst_rr(X86::MINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9036             :   }
    9037           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9038           0 :     return fastEmitInst_rr(X86::VMINCSSrr, &X86::FR32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9039             :   }
    9040             :   return 0;
    9041             : }
    9042             : 
    9043           0 : unsigned fastEmit_X86ISD_FMINC_MVT_f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9044           0 :   if (RetVT.SimpleTy != MVT::f64)
    9045             :     return 0;
    9046           0 :   if ((Subtarget->hasAVX512())) {
    9047           0 :     return fastEmitInst_rr(X86::VMINCSDZrr, &X86::FR64XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9048             :   }
    9049           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9050           0 :     return fastEmitInst_rr(X86::MINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9051             :   }
    9052           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9053           0 :     return fastEmitInst_rr(X86::VMINCSDrr, &X86::FR64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9054             :   }
    9055             :   return 0;
    9056             : }
    9057             : 
    9058           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9059           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9060             :     return 0;
    9061           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9062           0 :     return fastEmitInst_rr(X86::VMINCPSZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9063             :   }
    9064           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9065           0 :     return fastEmitInst_rr(X86::MINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9066             :   }
    9067           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9068           0 :     return fastEmitInst_rr(X86::VMINCPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9069             :   }
    9070             :   return 0;
    9071             : }
    9072             : 
    9073           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v8f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9074           0 :   if (RetVT.SimpleTy != MVT::v8f32)
    9075             :     return 0;
    9076           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9077           0 :     return fastEmitInst_rr(X86::VMINCPSZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9078             :   }
    9079           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9080           0 :     return fastEmitInst_rr(X86::VMINCPSYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9081             :   }
    9082             :   return 0;
    9083             : }
    9084             : 
    9085             : unsigned fastEmit_X86ISD_FMINC_MVT_v16f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9086           0 :   if (RetVT.SimpleTy != MVT::v16f32)
    9087             :     return 0;
    9088           0 :   if ((Subtarget->hasAVX512())) {
    9089           0 :     return fastEmitInst_rr(X86::VMINCPSZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9090             :   }
    9091             :   return 0;
    9092             : }
    9093             : 
    9094           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9095           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9096             :     return 0;
    9097           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9098           0 :     return fastEmitInst_rr(X86::VMINCPDZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9099             :   }
    9100           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9101           0 :     return fastEmitInst_rr(X86::MINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9102             :   }
    9103           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9104           0 :     return fastEmitInst_rr(X86::VMINCPDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9105             :   }
    9106             :   return 0;
    9107             : }
    9108             : 
    9109           0 : unsigned fastEmit_X86ISD_FMINC_MVT_v4f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9110           0 :   if (RetVT.SimpleTy != MVT::v4f64)
    9111             :     return 0;
    9112           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasVLX())) {
    9113           0 :     return fastEmitInst_rr(X86::VMINCPDZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9114             :   }
    9115           0 :   if ((Subtarget->hasAVX()) && (!Subtarget->hasVLX())) {
    9116           0 :     return fastEmitInst_rr(X86::VMINCPDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9117             :   }
    9118             :   return 0;
    9119             : }
    9120             : 
    9121             : unsigned fastEmit_X86ISD_FMINC_MVT_v8f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9122           0 :   if (RetVT.SimpleTy != MVT::v8f64)
    9123             :     return 0;
    9124           0 :   if ((Subtarget->hasAVX512())) {
    9125           0 :     return fastEmitInst_rr(X86::VMINCPDZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9126             :   }
    9127             :   return 0;
    9128             : }
    9129             : 
    9130           0 : unsigned fastEmit_X86ISD_FMINC_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9131           0 :   switch (VT.SimpleTy) {
    9132           0 :   case MVT::f32: return fastEmit_X86ISD_FMINC_MVT_f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9133           0 :   case MVT::f64: return fastEmit_X86ISD_FMINC_MVT_f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9134           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINC_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9135           0 :   case MVT::v8f32: return fastEmit_X86ISD_FMINC_MVT_v8f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9136           0 :   case MVT::v16f32: return fastEmit_X86ISD_FMINC_MVT_v16f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9137           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINC_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9138           0 :   case MVT::v4f64: return fastEmit_X86ISD_FMINC_MVT_v4f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9139           0 :   case MVT::v8f64: return fastEmit_X86ISD_FMINC_MVT_v8f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9140             :   default: return 0;
    9141             :   }
    9142             : }
    9143             : 
    9144             : // FastEmit functions for X86ISD::FMINS.
    9145             : 
    9146           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9147           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9148             :     return 0;
    9149           0 :   if ((Subtarget->hasAVX512())) {
    9150           0 :     return fastEmitInst_rr(X86::VMINSSZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9151             :   }
    9152           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9153           0 :     return fastEmitInst_rr(X86::MINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9154             :   }
    9155           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9156           0 :     return fastEmitInst_rr(X86::VMINSSrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9157             :   }
    9158             :   return 0;
    9159             : }
    9160             : 
    9161           0 : unsigned fastEmit_X86ISD_FMINS_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9162           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9163             :     return 0;
    9164           0 :   if ((Subtarget->hasAVX512())) {
    9165           0 :     return fastEmitInst_rr(X86::VMINSDZrr_Int, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9166             :   }
    9167           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9168           0 :     return fastEmitInst_rr(X86::MINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9169             :   }
    9170           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9171           0 :     return fastEmitInst_rr(X86::VMINSDrr_Int, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9172             :   }
    9173             :   return 0;
    9174             : }
    9175             : 
    9176           0 : unsigned fastEmit_X86ISD_FMINS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9177           0 :   switch (VT.SimpleTy) {
    9178           0 :   case MVT::v4f32: return fastEmit_X86ISD_FMINS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9179           0 :   case MVT::v2f64: return fastEmit_X86ISD_FMINS_MVT_v2f64_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9180             :   default: return 0;
    9181             :   }
    9182             : }
    9183             : 
    9184             : // FastEmit functions for X86ISD::FOR.
    9185             : 
    9186             : unsigned fastEmit_X86ISD_FOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9187           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9188             :     return 0;
    9189           0 :   return fastEmitInst_rr(X86::ORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9190             : }
    9191             : 
    9192             : unsigned fastEmit_X86ISD_FOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9193           0 :   switch (VT.SimpleTy) {
    9194           0 :   case MVT::v4f32: return fastEmit_X86ISD_FOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9195             :   default: return 0;
    9196             :   }
    9197             : }
    9198             : 
    9199             : // FastEmit functions for X86ISD::FXOR.
    9200             : 
    9201             : unsigned fastEmit_X86ISD_FXOR_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9202           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9203             :     return 0;
    9204           0 :   return fastEmitInst_rr(X86::XORPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9205             : }
    9206             : 
    9207             : unsigned fastEmit_X86ISD_FXOR_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9208           0 :   switch (VT.SimpleTy) {
    9209           0 :   case MVT::v4f32: return fastEmit_X86ISD_FXOR_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9210             :   default: return 0;
    9211             :   }
    9212             : }
    9213             : 
    9214             : // FastEmit functions for X86ISD::GF2P8MULB.
    9215             : 
    9216           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9217           0 :   if (RetVT.SimpleTy != MVT::v16i8)
    9218             :     return 0;
    9219           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
    9220           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZ128rr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9221             :   }
    9222           0 :   if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9223           0 :     return fastEmitInst_rr(X86::VGF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9224             :   }
    9225           0 :   if ((Subtarget->hasGFNI()) && (Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9226           0 :     return fastEmitInst_rr(X86::GF2P8MULBrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9227             :   }
    9228             :   return 0;
    9229             : }
    9230             : 
    9231           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9232           0 :   if (RetVT.SimpleTy != MVT::v32i8)
    9233             :     return 0;
    9234           0 :   if ((Subtarget->hasBWI()) && (Subtarget->hasGFNI()) && (Subtarget->hasVLX())) {
    9235           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZ256rr, &X86::VR256XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9236             :   }
    9237           0 :   if ((Subtarget->hasAVX()) && (Subtarget->hasGFNI()) && (!Subtarget->hasVLX() || !Subtarget->hasBWI())) {
    9238           0 :     return fastEmitInst_rr(X86::VGF2P8MULBYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9239             :   }
    9240             :   return 0;
    9241             : }
    9242             : 
    9243           0 : unsigned fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9244           0 :   if (RetVT.SimpleTy != MVT::v64i8)
    9245             :     return 0;
    9246           0 :   if ((Subtarget->hasAVX512()) && (Subtarget->hasBWI()) && (Subtarget->hasGFNI())) {
    9247           0 :     return fastEmitInst_rr(X86::VGF2P8MULBZrr, &X86::VR512RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9248             :   }
    9249             :   return 0;
    9250             : }
    9251             : 
    9252           0 : unsigned fastEmit_X86ISD_GF2P8MULB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9253           0 :   switch (VT.SimpleTy) {
    9254           0 :   case MVT::v16i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v16i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9255           0 :   case MVT::v32i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v32i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9256           0 :   case MVT::v64i8: return fastEmit_X86ISD_GF2P8MULB_MVT_v64i8_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9257             :   default: return 0;
    9258             :   }
    9259             : }
    9260             : 
    9261             : // FastEmit functions for X86ISD::HADD.
    9262             : 
    9263           0 : unsigned fastEmit_X86ISD_HADD_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9264           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9265             :     return 0;
    9266           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9267           0 :     return fastEmitInst_rr(X86::PHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9268             :   }
    9269           0 :   if ((Subtarget->hasAVX())) {
    9270           0 :     return fastEmitInst_rr(X86::VPHADDWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9271             :   }
    9272             :   return 0;
    9273             : }
    9274             : 
    9275             : unsigned fastEmit_X86ISD_HADD_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9276           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9277             :     return 0;
    9278           0 :   if ((Subtarget->hasAVX2())) {
    9279           0 :     return fastEmitInst_rr(X86::VPHADDWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9280             :   }
    9281             :   return 0;
    9282             : }
    9283             : 
    9284           0 : unsigned fastEmit_X86ISD_HADD_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9285           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9286             :     return 0;
    9287           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9288           0 :     return fastEmitInst_rr(X86::PHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9289             :   }
    9290           0 :   if ((Subtarget->hasAVX())) {
    9291           0 :     return fastEmitInst_rr(X86::VPHADDDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9292             :   }
    9293             :   return 0;
    9294             : }
    9295             : 
    9296             : unsigned fastEmit_X86ISD_HADD_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9297           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    9298             :     return 0;
    9299           0 :   if ((Subtarget->hasAVX2())) {
    9300           0 :     return fastEmitInst_rr(X86::VPHADDDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9301             :   }
    9302             :   return 0;
    9303             : }
    9304             : 
    9305           0 : unsigned fastEmit_X86ISD_HADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9306           0 :   switch (VT.SimpleTy) {
    9307           0 :   case MVT::v8i16: return fastEmit_X86ISD_HADD_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9308           0 :   case MVT::v16i16: return fastEmit_X86ISD_HADD_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9309           0 :   case MVT::v4i32: return fastEmit_X86ISD_HADD_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9310           0 :   case MVT::v8i32: return fastEmit_X86ISD_HADD_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9311             :   default: return 0;
    9312             :   }
    9313             : }
    9314             : 
    9315             : // FastEmit functions for X86ISD::HSUB.
    9316             : 
    9317           0 : unsigned fastEmit_X86ISD_HSUB_MVT_v8i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9318           0 :   if (RetVT.SimpleTy != MVT::v8i16)
    9319             :     return 0;
    9320           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9321           0 :     return fastEmitInst_rr(X86::PHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9322             :   }
    9323           0 :   if ((Subtarget->hasAVX())) {
    9324           0 :     return fastEmitInst_rr(X86::VPHSUBWrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9325             :   }
    9326             :   return 0;
    9327             : }
    9328             : 
    9329             : unsigned fastEmit_X86ISD_HSUB_MVT_v16i16_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9330           0 :   if (RetVT.SimpleTy != MVT::v16i16)
    9331             :     return 0;
    9332           0 :   if ((Subtarget->hasAVX2())) {
    9333           0 :     return fastEmitInst_rr(X86::VPHSUBWYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9334             :   }
    9335             :   return 0;
    9336             : }
    9337             : 
    9338           0 : unsigned fastEmit_X86ISD_HSUB_MVT_v4i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9339           0 :   if (RetVT.SimpleTy != MVT::v4i32)
    9340             :     return 0;
    9341           0 :   if ((Subtarget->hasSSSE3() && !Subtarget->hasAVX())) {
    9342           0 :     return fastEmitInst_rr(X86::PHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9343             :   }
    9344           0 :   if ((Subtarget->hasAVX())) {
    9345           0 :     return fastEmitInst_rr(X86::VPHSUBDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9346             :   }
    9347             :   return 0;
    9348             : }
    9349             : 
    9350             : unsigned fastEmit_X86ISD_HSUB_MVT_v8i32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9351           0 :   if (RetVT.SimpleTy != MVT::v8i32)
    9352             :     return 0;
    9353           0 :   if ((Subtarget->hasAVX2())) {
    9354           0 :     return fastEmitInst_rr(X86::VPHSUBDYrr, &X86::VR256RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9355             :   }
    9356             :   return 0;
    9357             : }
    9358             : 
    9359           0 : unsigned fastEmit_X86ISD_HSUB_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9360           0 :   switch (VT.SimpleTy) {
    9361           0 :   case MVT::v8i16: return fastEmit_X86ISD_HSUB_MVT_v8i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9362           0 :   case MVT::v16i16: return fastEmit_X86ISD_HSUB_MVT_v16i16_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9363           0 :   case MVT::v4i32: return fastEmit_X86ISD_HSUB_MVT_v4i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9364           0 :   case MVT::v8i32: return fastEmit_X86ISD_HSUB_MVT_v8i32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9365             :   default: return 0;
    9366             :   }
    9367             : }
    9368             : 
    9369             : // FastEmit functions for X86ISD::KADD.
    9370             : 
    9371             : unsigned fastEmit_X86ISD_KADD_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9372           0 :   if (RetVT.SimpleTy != MVT::v8i1)
    9373             :     return 0;
    9374           0 :   if ((Subtarget->hasDQI())) {
    9375           0 :     return fastEmitInst_rr(X86::KADDBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9376             :   }
    9377             :   return 0;
    9378             : }
    9379             : 
    9380             : unsigned fastEmit_X86ISD_KADD_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9381           0 :   if (RetVT.SimpleTy != MVT::v16i1)
    9382             :     return 0;
    9383           0 :   if ((Subtarget->hasDQI())) {
    9384           0 :     return fastEmitInst_rr(X86::KADDWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9385             :   }
    9386             :   return 0;
    9387             : }
    9388             : 
    9389             : unsigned fastEmit_X86ISD_KADD_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9390           0 :   if (RetVT.SimpleTy != MVT::v32i1)
    9391             :     return 0;
    9392           0 :   if ((Subtarget->hasBWI())) {
    9393           0 :     return fastEmitInst_rr(X86::KADDDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9394             :   }
    9395             :   return 0;
    9396             : }
    9397             : 
    9398             : unsigned fastEmit_X86ISD_KADD_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9399           0 :   if (RetVT.SimpleTy != MVT::v64i1)
    9400             :     return 0;
    9401           0 :   if ((Subtarget->hasBWI())) {
    9402           0 :     return fastEmitInst_rr(X86::KADDQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9403             :   }
    9404             :   return 0;
    9405             : }
    9406             : 
    9407           0 : unsigned fastEmit_X86ISD_KADD_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9408           0 :   switch (VT.SimpleTy) {
    9409           0 :   case MVT::v8i1: return fastEmit_X86ISD_KADD_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9410           0 :   case MVT::v16i1: return fastEmit_X86ISD_KADD_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9411           0 :   case MVT::v32i1: return fastEmit_X86ISD_KADD_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9412           0 :   case MVT::v64i1: return fastEmit_X86ISD_KADD_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9413             :   default: return 0;
    9414             :   }
    9415             : }
    9416             : 
    9417             : // FastEmit functions for X86ISD::KORTEST.
    9418             : 
    9419             : unsigned fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9420           0 :   if (RetVT.SimpleTy != MVT::i32)
    9421             :     return 0;
    9422           0 :   if ((Subtarget->hasDQI())) {
    9423           0 :     return fastEmitInst_rr(X86::KORTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9424             :   }
    9425             :   return 0;
    9426             : }
    9427             : 
    9428             : unsigned fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9429           0 :   if (RetVT.SimpleTy != MVT::i32)
    9430             :     return 0;
    9431           0 :   if ((Subtarget->hasAVX512())) {
    9432           0 :     return fastEmitInst_rr(X86::KORTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9433             :   }
    9434             :   return 0;
    9435             : }
    9436             : 
    9437             : unsigned fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9438           0 :   if (RetVT.SimpleTy != MVT::i32)
    9439             :     return 0;
    9440           0 :   if ((Subtarget->hasBWI())) {
    9441           0 :     return fastEmitInst_rr(X86::KORTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9442             :   }
    9443             :   return 0;
    9444             : }
    9445             : 
    9446             : unsigned fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9447           0 :   if (RetVT.SimpleTy != MVT::i32)
    9448             :     return 0;
    9449           0 :   if ((Subtarget->hasBWI())) {
    9450           0 :     return fastEmitInst_rr(X86::KORTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9451             :   }
    9452             :   return 0;
    9453             : }
    9454             : 
    9455           0 : unsigned fastEmit_X86ISD_KORTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9456           0 :   switch (VT.SimpleTy) {
    9457           0 :   case MVT::v8i1: return fastEmit_X86ISD_KORTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9458           0 :   case MVT::v16i1: return fastEmit_X86ISD_KORTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9459           0 :   case MVT::v32i1: return fastEmit_X86ISD_KORTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9460           0 :   case MVT::v64i1: return fastEmit_X86ISD_KORTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9461             :   default: return 0;
    9462             :   }
    9463             : }
    9464             : 
    9465             : // FastEmit functions for X86ISD::KTEST.
    9466             : 
    9467             : unsigned fastEmit_X86ISD_KTEST_MVT_v8i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9468           0 :   if (RetVT.SimpleTy != MVT::i32)
    9469             :     return 0;
    9470           0 :   if ((Subtarget->hasDQI())) {
    9471           0 :     return fastEmitInst_rr(X86::KTESTBrr, &X86::VK8RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9472             :   }
    9473             :   return 0;
    9474             : }
    9475             : 
    9476             : unsigned fastEmit_X86ISD_KTEST_MVT_v16i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9477           0 :   if (RetVT.SimpleTy != MVT::i32)
    9478             :     return 0;
    9479           0 :   if ((Subtarget->hasDQI())) {
    9480           0 :     return fastEmitInst_rr(X86::KTESTWrr, &X86::VK16RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9481             :   }
    9482             :   return 0;
    9483             : }
    9484             : 
    9485             : unsigned fastEmit_X86ISD_KTEST_MVT_v32i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9486           0 :   if (RetVT.SimpleTy != MVT::i32)
    9487             :     return 0;
    9488           0 :   if ((Subtarget->hasBWI())) {
    9489           0 :     return fastEmitInst_rr(X86::KTESTDrr, &X86::VK32RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9490             :   }
    9491             :   return 0;
    9492             : }
    9493             : 
    9494             : unsigned fastEmit_X86ISD_KTEST_MVT_v64i1_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9495           0 :   if (RetVT.SimpleTy != MVT::i32)
    9496             :     return 0;
    9497           0 :   if ((Subtarget->hasBWI())) {
    9498           0 :     return fastEmitInst_rr(X86::KTESTQrr, &X86::VK64RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9499             :   }
    9500             :   return 0;
    9501             : }
    9502             : 
    9503           0 : unsigned fastEmit_X86ISD_KTEST_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9504           0 :   switch (VT.SimpleTy) {
    9505           0 :   case MVT::v8i1: return fastEmit_X86ISD_KTEST_MVT_v8i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9506           0 :   case MVT::v16i1: return fastEmit_X86ISD_KTEST_MVT_v16i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9507           0 :   case MVT::v32i1: return fastEmit_X86ISD_KTEST_MVT_v32i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9508           0 :   case MVT::v64i1: return fastEmit_X86ISD_KTEST_MVT_v64i1_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9509             :   default: return 0;
    9510             :   }
    9511             : }
    9512             : 
    9513             : // FastEmit functions for X86ISD::MOVHLPS.
    9514             : 
    9515           0 : unsigned fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9516           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9517             :     return 0;
    9518           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9519           0 :     return fastEmitInst_rr(X86::MOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9520             :   }
    9521           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9522           0 :     return fastEmitInst_rr(X86::VMOVHLPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9523             :   }
    9524           0 :   if ((Subtarget->hasAVX512())) {
    9525           0 :     return fastEmitInst_rr(X86::VMOVHLPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9526             :   }
    9527             :   return 0;
    9528             : }
    9529             : 
    9530             : unsigned fastEmit_X86ISD_MOVHLPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9531           0 :   switch (VT.SimpleTy) {
    9532           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVHLPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9533             :   default: return 0;
    9534             :   }
    9535             : }
    9536             : 
    9537             : // FastEmit functions for X86ISD::MOVLHPS.
    9538             : 
    9539           0 : unsigned fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9540           0 :   if (RetVT.SimpleTy != MVT::v4f32)
    9541             :     return 0;
    9542           0 :   if ((Subtarget->hasSSE1() && !Subtarget->hasAVX())) {
    9543           0 :     return fastEmitInst_rr(X86::MOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9544             :   }
    9545           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9546           0 :     return fastEmitInst_rr(X86::VMOVLHPSrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9547             :   }
    9548           0 :   if ((Subtarget->hasAVX512())) {
    9549           0 :     return fastEmitInst_rr(X86::VMOVLHPSZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9550             :   }
    9551             :   return 0;
    9552             : }
    9553             : 
    9554             : unsigned fastEmit_X86ISD_MOVLHPS_rr(MVT VT, MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9555           0 :   switch (VT.SimpleTy) {
    9556           0 :   case MVT::v4f32: return fastEmit_X86ISD_MOVLHPS_MVT_v4f32_rr(RetVT, Op0, Op0IsKill, Op1, Op1IsKill);
    9557             :   default: return 0;
    9558             :   }
    9559             : }
    9560             : 
    9561             : // FastEmit functions for X86ISD::MOVLPD.
    9562             : 
    9563           0 : unsigned fastEmit_X86ISD_MOVLPD_MVT_v2i64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9564           0 :   if (RetVT.SimpleTy != MVT::v2i64)
    9565             :     return 0;
    9566           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9567           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9568             :   }
    9569           0 :   if ((Subtarget->hasAVX() && !Subtarget->hasAVX512())) {
    9570           0 :     return fastEmitInst_rr(X86::VMOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9571             :   }
    9572             :   return 0;
    9573             : }
    9574             : 
    9575           0 : unsigned fastEmit_X86ISD_MOVLPD_MVT_v2f64_rr(MVT RetVT, unsigned Op0, bool Op0IsKill, unsigned Op1, bool Op1IsKill) {
    9576           0 :   if (RetVT.SimpleTy != MVT::v2f64)
    9577             :     return 0;
    9578           0 :   if ((Subtarget->hasAVX512())) {
    9579           0 :     return fastEmitInst_rr(X86::VMOVSDZrr, &X86::VR128XRegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9580             :   }
    9581           0 :   if ((Subtarget->hasSSE2() && !Subtarget->hasAVX())) {
    9582           0 :     return fastEmitInst_rr(X86::MOVSDrr, &X86::VR128RegClass, Op0, Op0IsKill, Op1, Op1IsKill);
    9583             :   }
    9584